Skip to content

Commit 1b2c520

Browse files
author
Val Brodsky
committed
Create live mmc project without autogenerated data rows
1 parent 3e74cd0 commit 1b2c520

File tree

7 files changed

+72
-62
lines changed

7 files changed

+72
-62
lines changed

libs/labelbox/src/labelbox/client.py

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import sys
88
import time
99
import urllib.parse
10+
import warnings
1011
from collections import defaultdict
1112
from datetime import datetime, timezone
1213
from types import MappingProxyType
@@ -910,6 +911,16 @@ def create_model_evaluation_project(
910911
) -> Project:
911912
pass
912913

914+
@overload
915+
def create_model_evaluation_project(
916+
self,
917+
dataset_id: str,
918+
dataset_name: str = None,
919+
data_row_count: Optional[int] = None,
920+
**kwargs,
921+
) -> Project:
922+
pass
923+
913924
def create_model_evaluation_project(
914925
self,
915926
dataset_id: Optional[str] = None,
@@ -942,24 +953,32 @@ def create_model_evaluation_project(
942953
943954
944955
"""
945-
if not dataset_id and not dataset_name:
946-
raise ValueError(
947-
"dataset_name or data_set_id must be present and not be an empty string."
948-
)
949-
if data_row_count <= 0:
956+
if data_row_count < 0:
950957
raise ValueError("data_row_count must be a positive integer.")
951958

959+
autogenerate_data_rows = False
952960
if dataset_id:
961+
autogenerate_data_rows = True
953962
append_to_existing_dataset = True
954963
dataset_name_or_id = dataset_id
955-
else:
964+
warnings.warn(
965+
"Automatic generation of data rows of live model evaluation projects is deprecated. dataset_name_or_id will be removed in a future version.",
966+
DeprecationWarning,
967+
)
968+
elif dataset_name:
969+
autogenerate_data_rows = True
956970
append_to_existing_dataset = False
957971
dataset_name_or_id = dataset_name
972+
warnings.warn(
973+
"Automatic generation of data rows of live model evaluation projects is deprecated. dataset_name_or_id will be removed in a future version.",
974+
DeprecationWarning,
975+
)
958976

959977
kwargs["media_type"] = MediaType.Conversational
960-
kwargs["dataset_name_or_id"] = dataset_name_or_id
961-
kwargs["append_to_existing_dataset"] = append_to_existing_dataset
962-
kwargs["data_row_count"] = data_row_count
978+
if autogenerate_data_rows:
979+
kwargs["dataset_name_or_id"] = dataset_name_or_id
980+
kwargs["append_to_existing_dataset"] = append_to_existing_dataset
981+
kwargs["data_row_count"] = data_row_count
963982
kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value
964983

965984
return self._create_project(**kwargs)

libs/labelbox/src/labelbox/data/annotation_types/collection.py

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
import logging
2-
from concurrent.futures import ThreadPoolExecutor, as_completed
3-
from typing import Callable, Generator, Iterable, Union, Optional
4-
from uuid import uuid4
52
import warnings
3+
from typing import Callable, Generator, Iterable, Union
64

7-
from tqdm import tqdm
8-
9-
from labelbox.schema import ontology
105
from labelbox.orm.model import Entity
11-
from ..ontology import get_classifications, get_tools
6+
from labelbox.schema import ontology
7+
128
from ..generator import PrefetchGenerator
139
from .label import Label
1410

libs/labelbox/tests/integration/conftest.py

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -646,11 +646,28 @@ def chat_evaluation_ontology(client, rand_gen):
646646

647647

648648
@pytest.fixture
649-
def live_chat_evaluation_project_with_new_dataset(client, rand_gen):
649+
def live_chat_evaluation_project(client, rand_gen):
650650
project_name = f"test-model-evaluation-project-{rand_gen(str)}"
651-
dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}"
652-
project = client.create_model_evaluation_project(
653-
name=project_name, dataset_name=dataset_name, data_row_count=1
651+
project = client.create_model_evaluation_project(name=project_name)
652+
653+
yield project
654+
655+
project.delete()
656+
657+
658+
@pytest.fixture
659+
def live_chat_evaluation_project_with_batch(
660+
client,
661+
rand_gen,
662+
live_chat_evaluation_project,
663+
offline_conversational_data_row,
664+
):
665+
project_name = f"test-model-evaluation-project-{rand_gen(str)}"
666+
project = client.create_model_evaluation_project(name=project_name)
667+
668+
project.create_batch(
669+
rand_gen(str),
670+
[offline_conversational_data_row.uid], # sample of data row objects
654671
)
655672

656673
yield project

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
import pytest
2-
from unittest.mock import patch
32

43
from labelbox import MediaType
54
from labelbox.schema.ontology_kind import OntologyKind
6-
from labelbox.exceptions import MalformedQueryException
75

86

97
def test_create_chat_evaluation_ontology_project(
10-
client,
118
chat_evaluation_ontology,
12-
live_chat_evaluation_project_with_new_dataset,
9+
live_chat_evaluation_project,
1310
offline_conversational_data_row,
1411
rand_gen,
1512
):
@@ -28,36 +25,19 @@ def test_create_chat_evaluation_ontology_project(
2825
assert classification.schema_id
2926
assert classification.feature_schema_id
3027

31-
project = live_chat_evaluation_project_with_new_dataset
28+
project = live_chat_evaluation_project
3229
assert project.model_setup_complete is None
3330

3431
project.connect_ontology(ontology)
3532

3633
assert project.labeling_frontend().name == "Editor"
3734
assert project.ontology().name == ontology.name
3835

39-
with pytest.raises(
40-
ValueError,
41-
match="Cannot create batches for auto data generation projects",
42-
):
43-
project.create_batch(
44-
rand_gen(str),
45-
[offline_conversational_data_row.uid], # sample of data row objects
46-
)
47-
48-
with pytest.raises(
49-
ValueError,
50-
match="Cannot create batches for auto data generation projects",
51-
):
52-
with patch(
53-
"labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT", new=0
54-
): # force to async
55-
project.create_batch(
56-
rand_gen(str),
57-
[
58-
offline_conversational_data_row.uid
59-
], # sample of data row objects
60-
)
36+
batch = project.create_batch(
37+
rand_gen(str),
38+
[offline_conversational_data_row.uid], # sample of data row objects
39+
)
40+
assert batch
6141

6242

6343
def test_create_chat_evaluation_ontology_project_existing_dataset(

libs/labelbox/tests/integration/test_labeling_service.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,11 @@ def test_request_labeling_service_moe_offline_project(
4242

4343

4444
def test_request_labeling_service_moe_project(
45-
rand_gen,
46-
live_chat_evaluation_project_with_new_dataset,
45+
live_chat_evaluation_project_with_batch,
4746
chat_evaluation_ontology,
4847
model_config,
4948
):
50-
project = live_chat_evaluation_project_with_new_dataset
49+
project = live_chat_evaluation_project_with_batch
5150
project.connect_ontology(chat_evaluation_ontology)
5251

5352
project.upsert_instructions("tests/integration/media/sample_pdf.pdf")

libs/labelbox/tests/integration/test_project_model_config.py

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
import pytest
2+
23
from labelbox.exceptions import ResourceNotFoundError
34

45

5-
def test_add_single_model_config(
6-
live_chat_evaluation_project_with_new_dataset, model_config
7-
):
8-
configured_project = live_chat_evaluation_project_with_new_dataset
6+
def test_add_single_model_config(live_chat_evaluation_project, model_config):
7+
configured_project = live_chat_evaluation_project
98
project_model_config_id = configured_project.add_model_config(
109
model_config.uid
1110
)
@@ -22,11 +21,11 @@ def test_add_single_model_config(
2221
def test_add_multiple_model_config(
2322
client,
2423
rand_gen,
25-
live_chat_evaluation_project_with_new_dataset,
24+
live_chat_evaluation_project,
2625
model_config,
2726
valid_model_id,
2827
):
29-
configured_project = live_chat_evaluation_project_with_new_dataset
28+
configured_project = live_chat_evaluation_project
3029
second_model_config = client.create_model_config(
3130
rand_gen(str), valid_model_id, {"param": "value"}
3231
)
@@ -52,9 +51,9 @@ def test_add_multiple_model_config(
5251

5352

5453
def test_delete_project_model_config(
55-
live_chat_evaluation_project_with_new_dataset, model_config
54+
live_chat_evaluation_project, model_config
5655
):
57-
configured_project = live_chat_evaluation_project_with_new_dataset
56+
configured_project = live_chat_evaluation_project
5857
assert configured_project.delete_project_model_config(
5958
configured_project.add_model_config(model_config.uid)
6059
)

libs/labelbox/tests/integration/test_project_set_model_setup_complete.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44

55

66
def test_live_chat_evaluation_project(
7-
live_chat_evaluation_project_with_new_dataset, model_config
7+
live_chat_evaluation_project, model_config
88
):
9-
project = live_chat_evaluation_project_with_new_dataset
9+
project = live_chat_evaluation_project
1010

1111
project.set_project_model_setup_complete()
1212
assert bool(project.model_setup_complete) is True
@@ -19,9 +19,9 @@ def test_live_chat_evaluation_project(
1919

2020

2121
def test_live_chat_evaluation_project_delete_cofig(
22-
live_chat_evaluation_project_with_new_dataset, model_config
22+
live_chat_evaluation_project, model_config
2323
):
24-
project = live_chat_evaluation_project_with_new_dataset
24+
project = live_chat_evaluation_project
2525
project_model_config_id = project.add_model_config(model_config.uid)
2626
assert project_model_config_id
2727

0 commit comments

Comments
 (0)