Use _CoreProjectInput in create_model_evaluation_project

Author: Val Brodsky

libs/labelbox/src/labelbox/client.py

@@ -8,7 +8,7 @@
 import urllib.parse
 from collections import defaultdict
 from types import MappingProxyType
-from typing import Any, Callable, Dict, List, Optional, Union, overload
+from typing import Any, Callable, Dict, List, Optional, Set, Union, overload
 
 import lbox.exceptions
 import requests
@@ -597,7 +597,16 @@ def create_dataset(
             raise e
         return dataset
 
-    def create_project(self, **kwargs) -> Project:
+    def create_project(
+        self,
+        name: str,
+        media_type: MediaType,
+        description: Optional[str] = None,
+        auto_audit_percentage: Optional[float] = None,
+        auto_audit_number_of_labels: Optional[int] = None,
+        quality_modes: Optional[Set[QualityMode]] = {QualityMode.Benchmark, QualityMode.Consensus},
+        is_benchmark_enabled: Optional[bool] = None,
+        is_consensus_enabled: Optional[bool] = None) -> Project:
         """Creates a Project object on the server.
 
         Attribute values are passed as keyword arguments.
@@ -628,39 +637,30 @@ def create_project(self, **kwargs) -> Project:
             dataset_name_or_id, append_to_existing_dataset, data_row_count, editor_task_type
             They are not used for general projects and not supported in this method
         """
-        #  The following arguments are not supported for general projects, only for chat model evaluation projects
-        kwargs.pop("dataset_name_or_id", None)
-        kwargs.pop("append_to_existing_dataset", None)
-        kwargs.pop("data_row_count", None)
-        kwargs.pop("editor_task_type", None)
-        return self._create_project(_CoreProjectInput(**kwargs))
-
-    @overload
-    def create_model_evaluation_project(
-        self,
-        dataset_name: str,
-        dataset_id: str = None,
-        data_row_count: int = 100,
-        **kwargs,
-    ) -> Project:
-        pass
-
-    @overload
-    def create_model_evaluation_project(
-        self,
-        dataset_id: str,
-        dataset_name: str = None,
-        data_row_count: int = 100,
-        **kwargs,
-    ) -> Project:
-        pass
+        input = {
+            "name": name,
+            "description": description,
+            "media_type": media_type,
+            "auto_audit_percentage": auto_audit_percentage,
+            "auto_audit_number_of_labels": auto_audit_number_of_labels,
+            "quality_modes": quality_modes,
+            "is_benchmark_enabled": is_benchmark_enabled,
+            "is_consensus_enabled": is_consensus_enabled,
+        }
+        return self._create_project(_CoreProjectInput(**input))
 
     def create_model_evaluation_project(
         self,
+        name: str,
+        description: Optional[str] = None,
+        auto_audit_percentage: Optional[float] = None,
+        auto_audit_number_of_labels: Optional[int] = None,
+        quality_modes: Optional[Set[QualityMode]] = {QualityMode.Benchmark, QualityMode.Consensus},
+        is_benchmark_enabled: Optional[bool] = None,
+        is_consensus_enabled: Optional[bool] = None,
         dataset_id: Optional[str] = None,
         dataset_name: Optional[str] = None,
         data_row_count: int = 100,
-        **kwargs,
     ) -> Project:
         """
         Use this method exclusively to create a chat model evaluation project.
@@ -691,8 +691,6 @@ def create_model_evaluation_project(
             raise ValueError(
                 "dataset_name or data_set_id must be present and not be an empty string."
             )
-        if data_row_count <= 0:
-            raise ValueError("data_row_count must be a positive integer.")
 
         if dataset_id:
             append_to_existing_dataset = True
@@ -701,13 +699,24 @@ def create_model_evaluation_project(
             append_to_existing_dataset = False
             dataset_name_or_id = dataset_name
 
-        kwargs["media_type"] = MediaType.Conversational
-        kwargs["dataset_name_or_id"] = dataset_name_or_id
-        kwargs["append_to_existing_dataset"] = append_to_existing_dataset
-        kwargs["data_row_count"] = data_row_count
-        kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value
+        media_type = MediaType.Conversational
+        editor_task_type = EditorTaskType.ModelChatEvaluation
 
-        return self._create_project(_CoreProjectInput(**kwargs))
+        input = {
+            "name": name,
+            "description": description,
+            "media_type": media_type,
+            "auto_audit_percentage": auto_audit_percentage,
+            "auto_audit_number_of_labels": auto_audit_number_of_labels,
+            "quality_modes": quality_modes,
+            "is_benchmark_enabled": is_benchmark_enabled,
+            "is_consensus_enabled": is_consensus_enabled,
+            "dataset_name_or_id": dataset_name_or_id,
+            "append_to_existing_dataset": append_to_existing_dataset,
+            "data_row_count": data_row_count,
+            "editor_task_type": editor_task_type,
+        }
+        return self._create_project(_CoreProjectInput(**input))
 
     def create_offline_model_evaluation_project(self, **kwargs) -> Project:
         """

libs/labelbox/src/labelbox/project_validation.py

@@ -1,8 +1,10 @@
 from typing import Optional, Set
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Annotated
 
 from labelbox.schema.media_type import MediaType
+from labelbox.schema.ontology_kind import EditorTaskType
 from labelbox.schema.quality_mode import (
     BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS,
     BENCHMARK_AUTO_AUDIT_PERCENTAGE,
@@ -12,6 +14,8 @@
 )
 from labelbox.schema.queue_mode import QueueMode
 
+PositiveInt = Annotated[int, Field(gt=0)]
+
 
 class _CoreProjectInput(BaseModel):
     name: str
@@ -27,6 +31,8 @@ class _CoreProjectInput(BaseModel):
     is_consensus_enabled: Optional[bool] = None
     dataset_name_or_id: Optional[str] = None
     append_to_existing_dataset: Optional[bool] = None
+    data_row_count: Optional[PositiveInt] = None
+    editor_task_type: Optional[EditorTaskType] = None
 
     model_config = ConfigDict(extra="forbid")
 

libs/labelbox/src/labelbox/schema/ontology_kind.py

@@ -53,7 +53,7 @@ def evaluate_ontology_kind_with_media_type(
         return media_type
 
 
-class EditorTaskType(Enum):
+class EditorTaskType(str, Enum):
     ModelChatEvaluation = "MODEL_CHAT_EVALUATION"
     ResponseCreation = "RESPONSE_CREATION"
     OfflineModelChatEvaluation = "OFFLINE_MODEL_CHAT_EVALUATION"