Create live mmc project without autogenerated data rows

Author: Val Brodsky

libs/labelbox/src/labelbox/client.py

@@ -7,6 +7,7 @@
 import sys
 import time
 import urllib.parse
+import warnings
 from collections import defaultdict
 from datetime import datetime, timezone
 from types import MappingProxyType
@@ -910,6 +911,16 @@ def create_model_evaluation_project(
     ) -> Project:
         pass
 
+    @overload
+    def create_model_evaluation_project(
+        self,
+        dataset_id: str,
+        dataset_name: str = None,
+        data_row_count: Optional[int] = None,
+        **kwargs,
+    ) -> Project:
+        pass
+
     def create_model_evaluation_project(
         self,
         dataset_id: Optional[str] = None,
@@ -942,24 +953,32 @@ def create_model_evaluation_project(
 
 
         """
-        if not dataset_id and not dataset_name:
-            raise ValueError(
-                "dataset_name or data_set_id must be present and not be an empty string."
-            )
-        if data_row_count <= 0:
+        if data_row_count < 0:
             raise ValueError("data_row_count must be a positive integer.")
 
+        autogenerate_data_rows = False
         if dataset_id:
+            autogenerate_data_rows = True
             append_to_existing_dataset = True
             dataset_name_or_id = dataset_id
-        else:
+            warnings.warn(
+                "Automatic generation of data rows of live model evaluation projects is deprecated. dataset_name_or_id will be removed in a future version.",
+                DeprecationWarning,
+            )
+        elif dataset_name:
+            autogenerate_data_rows = True
             append_to_existing_dataset = False
             dataset_name_or_id = dataset_name
+            warnings.warn(
+                "Automatic generation of data rows of live model evaluation projects is deprecated. dataset_name_or_id will be removed in a future version.",
+                DeprecationWarning,
+            )
 
         kwargs["media_type"] = MediaType.Conversational
-        kwargs["dataset_name_or_id"] = dataset_name_or_id
-        kwargs["append_to_existing_dataset"] = append_to_existing_dataset
-        kwargs["data_row_count"] = data_row_count
+        if autogenerate_data_rows:
+            kwargs["dataset_name_or_id"] = dataset_name_or_id
+            kwargs["append_to_existing_dataset"] = append_to_existing_dataset
+            kwargs["data_row_count"] = data_row_count
         kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value
 
         return self._create_project(**kwargs)

libs/labelbox/src/labelbox/data/annotation_types/collection.py

@@ -1,14 +1,10 @@
 import logging
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Callable, Generator, Iterable, Union, Optional
-from uuid import uuid4
 import warnings
+from typing import Callable, Generator, Iterable, Union
 
-from tqdm import tqdm
-
-from labelbox.schema import ontology
 from labelbox.orm.model import Entity
-from ..ontology import get_classifications, get_tools
+from labelbox.schema import ontology
+
 from ..generator import PrefetchGenerator
 from .label import Label
 

libs/labelbox/tests/integration/conftest.py

@@ -646,11 +646,28 @@ def chat_evaluation_ontology(client, rand_gen):
 
 
 @pytest.fixture
-def live_chat_evaluation_project_with_new_dataset(client, rand_gen):
+def live_chat_evaluation_project(client, rand_gen):
     project_name = f"test-model-evaluation-project-{rand_gen(str)}"
-    dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}"
-    project = client.create_model_evaluation_project(
-        name=project_name, dataset_name=dataset_name, data_row_count=1
+    project = client.create_model_evaluation_project(name=project_name)
+
+    yield project
+
+    project.delete()
+
+
+@pytest.fixture
+def live_chat_evaluation_project_with_batch(
+    client,
+    rand_gen,
+    live_chat_evaluation_project,
+    offline_conversational_data_row,
+):
+    project_name = f"test-model-evaluation-project-{rand_gen(str)}"
+    project = client.create_model_evaluation_project(name=project_name)
+
+    project.create_batch(
+        rand_gen(str),
+        [offline_conversational_data_row.uid],  # sample of data row objects
     )
 
     yield project

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

@@ -1,15 +1,12 @@
 import pytest
-from unittest.mock import patch
 
 from labelbox import MediaType
 from labelbox.schema.ontology_kind import OntologyKind
-from labelbox.exceptions import MalformedQueryException
 
 
 def test_create_chat_evaluation_ontology_project(
-    client,
     chat_evaluation_ontology,
-    live_chat_evaluation_project_with_new_dataset,
+    live_chat_evaluation_project,
     offline_conversational_data_row,
     rand_gen,
 ):
@@ -28,36 +25,19 @@ def test_create_chat_evaluation_ontology_project(
         assert classification.schema_id
         assert classification.feature_schema_id
 
-    project = live_chat_evaluation_project_with_new_dataset
+    project = live_chat_evaluation_project
     assert project.model_setup_complete is None
 
     project.connect_ontology(ontology)
 
     assert project.labeling_frontend().name == "Editor"
     assert project.ontology().name == ontology.name
 
-    with pytest.raises(
-        ValueError,
-        match="Cannot create batches for auto data generation projects",
-    ):
-        project.create_batch(
-            rand_gen(str),
-            [offline_conversational_data_row.uid],  # sample of data row objects
-        )
-
-    with pytest.raises(
-        ValueError,
-        match="Cannot create batches for auto data generation projects",
-    ):
-        with patch(
-            "labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT", new=0
-        ):  # force to async
-            project.create_batch(
-                rand_gen(str),
-                [
-                    offline_conversational_data_row.uid
-                ],  # sample of data row objects
-            )
+    batch = project.create_batch(
+        rand_gen(str),
+        [offline_conversational_data_row.uid],  # sample of data row objects
+    )
+    assert batch
 
 
 def test_create_chat_evaluation_ontology_project_existing_dataset(

libs/labelbox/tests/integration/test_labeling_service.py

@@ -42,12 +42,11 @@ def test_request_labeling_service_moe_offline_project(
 
 
 def test_request_labeling_service_moe_project(
-    rand_gen,
-    live_chat_evaluation_project_with_new_dataset,
+    live_chat_evaluation_project_with_batch,
     chat_evaluation_ontology,
     model_config,
 ):
-    project = live_chat_evaluation_project_with_new_dataset
+    project = live_chat_evaluation_project_with_batch
     project.connect_ontology(chat_evaluation_ontology)
 
     project.upsert_instructions("tests/integration/media/sample_pdf.pdf")

libs/labelbox/tests/integration/test_project_model_config.py

@@ -1,11 +1,10 @@
 import pytest
+
 from labelbox.exceptions import ResourceNotFoundError
 
 
-def test_add_single_model_config(
-    live_chat_evaluation_project_with_new_dataset, model_config
-):
-    configured_project = live_chat_evaluation_project_with_new_dataset
+def test_add_single_model_config(live_chat_evaluation_project, model_config):
+    configured_project = live_chat_evaluation_project
     project_model_config_id = configured_project.add_model_config(
         model_config.uid
     )
@@ -22,11 +21,11 @@ def test_add_single_model_config(
 def test_add_multiple_model_config(
     client,
     rand_gen,
-    live_chat_evaluation_project_with_new_dataset,
+    live_chat_evaluation_project,
     model_config,
     valid_model_id,
 ):
-    configured_project = live_chat_evaluation_project_with_new_dataset
+    configured_project = live_chat_evaluation_project
     second_model_config = client.create_model_config(
         rand_gen(str), valid_model_id, {"param": "value"}
     )
@@ -52,9 +51,9 @@ def test_add_multiple_model_config(
 
 
 def test_delete_project_model_config(
-    live_chat_evaluation_project_with_new_dataset, model_config
+    live_chat_evaluation_project, model_config
 ):
-    configured_project = live_chat_evaluation_project_with_new_dataset
+    configured_project = live_chat_evaluation_project
     assert configured_project.delete_project_model_config(
         configured_project.add_model_config(model_config.uid)
     )

libs/labelbox/tests/integration/test_project_set_model_setup_complete.py

@@ -4,9 +4,9 @@
 
 
 def test_live_chat_evaluation_project(
-    live_chat_evaluation_project_with_new_dataset, model_config
+    live_chat_evaluation_project, model_config
 ):
-    project = live_chat_evaluation_project_with_new_dataset
+    project = live_chat_evaluation_project
 
     project.set_project_model_setup_complete()
     assert bool(project.model_setup_complete) is True
@@ -19,9 +19,9 @@ def test_live_chat_evaluation_project(
 
 
 def test_live_chat_evaluation_project_delete_cofig(
-    live_chat_evaluation_project_with_new_dataset, model_config
+    live_chat_evaluation_project, model_config
 ):
-    project = live_chat_evaluation_project_with_new_dataset
+    project = live_chat_evaluation_project
     project_model_config_id = project.add_model_config(model_config.uid)
     assert project_model_config_id