Improve usability of create_project for chat evaluation and add basic validations (#1599)

Author: vbrodsky

.github/workflows/python-package-develop.yml

@@ -10,6 +10,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  id-token: write
+
 jobs:
   build:
     strategy:

libs/labelbox/src/labelbox/client.py

@@ -9,7 +9,7 @@
 import urllib.parse
 from collections import defaultdict
 from datetime import datetime, timezone
-from typing import Any, List, Dict, Union, Optional
+from typing import Any, List, Dict, Union, Optional, overload
 
 import requests
 import requests.exceptions
@@ -310,9 +310,10 @@ def get_error_status_code(error: dict) -> int:
         if internal_server_error is not None:
             message = internal_server_error.get("message")
             error_status_code = get_error_status_code(internal_server_error)
-
             if error_status_code == 400:
                 raise labelbox.exceptions.InvalidQueryError(message)
+            elif error_status_code == 422:
+                raise labelbox.exceptions.UnprocessableEntityError(message)
             elif error_status_code == 426:
                 raise labelbox.exceptions.OperationNotAllowedException(message)
             elif error_status_code == 500:
@@ -596,7 +597,8 @@ def _create(self, db_object_type, data, extra_params={}):
         res = res["create%s" % db_object_type.type_name()]
         return db_object_type(self, res)
 
-    def create_model_config(self, name: str, model_id: str, inference_params: dict) -> ModelConfig:
+    def create_model_config(self, name: str, model_id: str,
+                            inference_params: dict) -> ModelConfig:
         """ Creates a new model config with the given params.
             Model configs are scoped to organizations, and can be reused between projects.
 
@@ -608,6 +610,8 @@ def create_model_config(self, name: str, model_id: str, inference_params: dict)
         Returns:
             str, id of the created model config
         """
+        if not name:
+            raise ValueError("Model config name must not be an empty string.")
 
         query = """mutation CreateModelConfigPyApi($modelId: ID!, $inferenceParams: Json!, $name: String!)  {
                     createModelConfig(input: {modelId: $modelId, inferenceParams: $inferenceParams, name: $name}) {
@@ -640,9 +644,7 @@ def delete_model_config(self, id: str) -> bool:
                         success
                     }
                 }"""
-        params = {
-            "id": id
-        }
+        params = {"id": id}
         result = self.execute(query, params)
         return result['deleteModelConfig']['success']
 
@@ -811,22 +813,66 @@ def create_project(self, **kwargs) -> Project:
 
         return self._create(Entity.Project, params, extra_params)
 
-    def create_model_evaluation_project(
-            self,
-            dataset_name_or_id: str,
-            append_to_existing_dataset: bool = False,
-            data_row_count: int = 100,
-            **kwargs) -> Project:
+    @overload
+    def create_model_evaluation_project(self,
+                                        dataset_name: str,
+                                        dataset_id: str = None,
+                                        data_row_count: int = 100,
+                                        **kwargs) -> Project:
+        pass
+
+    @overload
+    def create_model_evaluation_project(self,
+                                        dataset_id: str,
+                                        dataset_name: str = None,
+                                        data_row_count: int = 100,
+                                        **kwargs) -> Project:
+        pass
+
+    def create_model_evaluation_project(self,
+                                        dataset_id: Optional[str] = None,
+                                        dataset_name: Optional[str] = None,
+                                        data_row_count: int = 100,
+                                        **kwargs) -> Project:
         """
         Use this method exclusively to create a chat model evaluation project.
         Args:
-            dataset_name_or_id: The name or id of the dataset to use for the project
-            append_to_existing_dataset: If True, the project will append assets (data rows) to the existing dataset
+            dataset_name: When creating a new dataset, pass the name
+            dataset_id: When using an existing dataset, pass the id
             data_row_count: The number of data row assets to use for the project
             **kwargs: Additional parameters to pass to the the create_project method
         Returns:
             Project: The created project
+
+        Examples:
+            >>> client.create_model_evaluation_project(name=project_name, dataset_name="new data set")
+            >>>     This creates a new dataset with a default number of rows (100), creates new project and assigns a batch of the newly created datarows to the project.
+
+            >>> client.create_model_evaluation_project(name=project_name, dataset_name="new data set", data_row_count=10)
+            >>>     This creates a new dataset with 10 data rows, creates new project and assigns a batch of the newly created datarows to the project.
+
+            >>> client.create_model_evaluation_project(name=project_name, dataset_id="clr00u8j0j0j0")
+            >>>     This creates a new project, and adds 100 datarows to the dataset with id "clr00u8j0j0j0" and assigns a batch of the newly created data rows to the project.
+
+            >>> client.create_model_evaluation_project(name=project_name, dataset_id="clr00u8j0j0j0", data_row_count=10)
+            >>>     This creates a new project, and adds 100 datarows to the dataset with id "clr00u8j0j0j0" and assigns a batch of the newly created 10 data rows to the project.
+
+
         """
+        if not dataset_id and not dataset_name:
+            raise ValueError(
+                "dataset_name or data_set_id must be present and not be an empty string."
+            )
+        if data_row_count <= 0:
+            raise ValueError("data_row_count must be a positive integer.")
+
+        if dataset_id:
+            append_to_existing_dataset = True
+            dataset_name_or_id = dataset_id
+        else:
+            append_to_existing_dataset = False
+            dataset_name_or_id = dataset_name
+
         kwargs["media_type"] = MediaType.Conversational
         kwargs["ontology_kind"] = OntologyKind.ModelEvaluation
         kwargs["dataset_name_or_id"] = dataset_name_or_id

libs/labelbox/src/labelbox/exceptions.py

@@ -71,6 +71,12 @@ class InvalidQueryError(LabelboxError):
     pass
 
 
+class UnprocessableEntityError(LabelboxError):
+    """ Indicates that a resource could not be created in the server side
+    due to a validation or transaction error"""
+    pass
+
+
 class ResourceCreationError(LabelboxError):
     """ Indicates that a resource could not be created in the server side
     due to a validation or transaction error"""

libs/labelbox/tests/integration/conftest.py

@@ -363,11 +363,25 @@ def chat_evaluation_ontology(client, rand_gen):
 
 
 @pytest.fixture
-def chat_evaluation_project(client, rand_gen):
+def chat_evaluation_project_create_dataset(client, rand_gen):
     project_name = f"test-model-evaluation-project-{rand_gen(str)}"
-    dataset_name_or_id = f"test-model-evaluation-dataset-{rand_gen(str)}"
-    project = client.create_model_evaluation_project(
-        name=project_name, dataset_name_or_id=dataset_name_or_id)
+    dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}"
+    project = client.create_model_evaluation_project(name=project_name,
+                                                     dataset_name=dataset_name,
+                                                     data_row_count=1)
+
+    yield project
+
+    project.delete()
+
+
+@pytest.fixture
+def chat_evaluation_project_append_to_dataset(client, dataset, rand_gen):
+    project_name = f"test-model-evaluation-project-{rand_gen(str)}"
+    dataset_id = dataset.uid
+    project = client.create_model_evaluation_project(name=project_name,
+                                                     dataset_id=dataset_id,
+                                                     data_row_count=1)
 
     yield project
 

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

@@ -5,9 +5,9 @@
 from labelbox.schema.labeling_frontend import LabelingFrontend
 
 
-def test_create_chat_evaluation_ontology_project(client,
-                                                 chat_evaluation_ontology,
-                                                 chat_evaluation_project):
+def test_create_chat_evaluation_ontology_project(
+        client, chat_evaluation_ontology,
+        chat_evaluation_project_create_dataset):
     ontology = chat_evaluation_ontology
 
     # here we are essentially testing the ontology creation which is a fixture
@@ -18,7 +18,20 @@ def test_create_chat_evaluation_ontology_project(client,
         assert tool.schema_id
         assert tool.feature_schema_id
 
-    project = chat_evaluation_project
+    project = chat_evaluation_project_create_dataset
+    project.setup_editor(ontology)
+
+    assert project.labeling_frontend().name == "Editor"
+    assert project.ontology().name == ontology.name
+
+
+def test_create_chat_evaluation_ontology_project_existing_dataset(
+        client, chat_evaluation_ontology,
+        chat_evaluation_project_append_to_dataset):
+    ontology = chat_evaluation_ontology
+
+    project = chat_evaluation_project_append_to_dataset
+    assert project
     project.setup_editor(ontology)
 
     assert project.labeling_frontend().name == "Editor"