Skip setup_ editor and ontology for chat evaluation projects

Author: Val Brodsky

libs/labelbox/src/labelbox/schema/ontology_kind.py

@@ -0,0 +1,53 @@
+from enum import Enum
+from typing import Optional
+
+from labelbox.schema.media_type import MediaType
+
+
+class OntologyKind(Enum):
+    ModelEvaluation = "MODEL_EVALUATION"
+    Missing = None
+
+    @classmethod
+    def is_supported(cls, value):
+        return isinstance(value, cls)
+
+    @classmethod
+    def get_ontology_kind_validation_error(cls, ontology_kind):
+        return TypeError(f"{ontology_kind}: is not a valid ontology kind. Use"
+                         f" any of {OntologyKind.__members__.items()}"
+                         " from OntologyKind.")
+
+
+class EditorTaskType(Enum):
+    ModelChatEvaluation = "MODEL_CHAT_EVALUATION"
+    Missing = None
+
+    @classmethod
+    def is_supported(cls, value):
+        return isinstance(value, cls)
+
+
+class EditorTaskTypeMapper:
+
+    @staticmethod
+    def to_editor_task_type(ontology_kind: OntologyKind,
+                            media_type: MediaType) -> EditorTaskType:
+        if ontology_kind and OntologyKind.is_supported(
+                ontology_kind) and media_type and MediaType.is_supported(
+                    media_type):
+            editor_task_type = EditorTaskTypeMapper.map_to_editor_task_type(
+                ontology_kind, media_type)
+        else:
+            editor_task_type = EditorTaskType.Missing
+
+        return editor_task_type
+
+    @staticmethod
+    def map_to_editor_task_type(
+            onotology_kind: OntologyKind,
+            media_type: MediaType) -> Optional[EditorTaskType]:
+        if onotology_kind == OntologyKind.ModelEvaluation and media_type == MediaType.Conversational:
+            return EditorTaskType.ModelChatEvaluation
+        else:
+            return EditorTaskType.Missing

libs/labelbox/src/labelbox/schema/project.py

@@ -37,7 +37,7 @@
 from labelbox.schema.resource_tag import ResourceTag
 from labelbox.schema.task import Task
 from labelbox.schema.task_queue import TaskQueue
-from labelbox.schema.ontology_kind import (EditorTaskType)
+from labelbox.schema.ontology_kind import (EditorTaskType, OntologyKind)
 
 if TYPE_CHECKING:
     from labelbox import BulkImportRequest
@@ -133,6 +133,9 @@ class Project(DbObject, Updateable, Deletable):
     #
     _wait_processing_max_seconds = 3600
 
+    def is_chat_evalution(self) -> bool:
+        return self.media_type == MediaType.Conversational and self.editor_task_type == EditorTaskType.ModelChatEvaluation
+
     def update(self, **kwargs):
         """ Updates this project with the specified attributes
 
@@ -734,27 +737,35 @@ def setup_editor(self, ontology) -> None:
         Args:
             ontology (Ontology): The ontology to attach to the project
         """
-        if self.labeling_frontend() is not None:
-            raise ResourceConflict("Editor is already set up.")
 
-        labeling_frontend = next(
-            self.client.get_labeling_frontends(
-                where=Entity.LabelingFrontend.name == "Editor"))
-        self.labeling_frontend.connect(labeling_frontend)
+        if self.labeling_frontend(
+        ) is not None and not self.is_chat_evalution():
+            raise ResourceConflict("Editor is already set up.")
 
-        LFO = Entity.LabelingFrontendOptions
-        self.client._create(
-            LFO, {
-                LFO.project:
-                    self,
-                LFO.labeling_frontend:
-                    labeling_frontend,
-                LFO.customization_options:
-                    json.dumps({
-                        "tools": [],
-                        "classifications": []
-                    })
-            })
+        if not self.is_chat_evalution():
+            labeling_frontend = next(
+                self.client.get_labeling_frontends(
+                    where=Entity.LabelingFrontend.name == "Editor"))
+            self.labeling_frontend.connect(labeling_frontend)
+
+            LFO = Entity.LabelingFrontendOptions
+            self.client._create(
+                LFO, {
+                    LFO.project:
+                        self,
+                    LFO.labeling_frontend:
+                        labeling_frontend,
+                    LFO.customization_options:
+                        json.dumps({
+                            "tools": [],
+                            "classifications": []
+                        })
+                })
+        else:
+            warnings.warn("""
+            Skipping editor setup for a chat evaluation project.
+            Editor was setup automatically.
+            """)
 
         query_str = """mutation ConnectOntologyPyApi($projectId: ID!, $ontologyId: ID!){
             project(where: {id: $projectId}) {connectOntology(ontologyId: $ontologyId) {id}}}"""
@@ -776,6 +787,14 @@ def setup(self, labeling_frontend, labeling_frontend_options) -> None:
                 to `str` using `json.dumps`.
         """
 
+        if self.is_chat_evalution():
+            warnings.warn("""
+            This project is a chat evaluation project.
+            Editor was setup automatically.
+            No need to call this method.
+            """)
+            return
+
         if self.labeling_frontend() is not None:
             raise ResourceConflict("Editor is already set up.")
 

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

@@ -0,0 +1,105 @@
+import pytest
+from labelbox import OntologyBuilder, Tool
+from labelbox import MediaType
+from labelbox.schema.ontology_kind import OntologyKind
+from labelbox.schema.labeling_frontend import LabelingFrontend
+
+
+def test_create_chat_evaluation_ontology_project(client,
+                                                 chat_evaluation_ontology,
+                                                 chat_evaluation_project):
+    ontology = chat_evaluation_ontology
+
+    # here we are essentially testing the ontology creation which is a fixture
+    assert ontology
+    assert ontology.name
+    assert len(ontology.tools()) == 3
+    for tool in ontology.tools():
+        assert tool.schema_id
+        assert tool.feature_schema_id
+
+    project = chat_evaluation_project
+    # editor = list(
+    #     client.get_labeling_frontends(
+    #         where=LabelingFrontend.name == "editor"))[0]
+    # project.setup(editor, ontology)
+
+    project.setup_editor(ontology)
+
+    assert project.labeling_frontend().name == "Editor"
+    assert project.ontology().name == ontology.name
+
+
+@pytest.fixture
+def tools_json():
+    tools = [{
+        'tool': 'message-single-selection',
+        'name': 'model output single selection',
+        'required': False,
+        'color': '#ff0000',
+        'classifications': [],
+        'schemaNodeId': None,
+        'featureSchemaId': None
+    }, {
+        'tool': 'message-multi-selection',
+        'name': 'model output multi selection',
+        'required': False,
+        'color': '#00ff00',
+        'classifications': [],
+        'schemaNodeId': None,
+        'featureSchemaId': None
+    }, {
+        'tool': 'message-ranking',
+        'name': 'model output multi ranking',
+        'required': False,
+        'color': '#0000ff',
+        'classifications': [],
+        'schemaNodeId': None,
+        'featureSchemaId': None
+    }]
+
+    return tools
+
+
+@pytest.fixture
+def features_from_json(client, tools_json):
+    tools = tools_json
+    features = {client.create_feature_schema(t) for t in tools if t}
+
+    yield features
+
+    for f in features:
+        client.delete_unused_feature_schema(f.uid)
+
+
+@pytest.fixture
+def ontology_from_feature_ids(client, features_from_json):
+    feature_ids = {f.uid for f in features_from_json}
+    ontology = client.create_ontology_from_feature_schemas(
+        name="test-model-chat-evaluation-ontology{rand_gen(str)}",
+        feature_schema_ids=feature_ids,
+        media_type=MediaType.Conversational,
+        ontology_kind=OntologyKind.ModelEvaluation,
+    )
+
+    yield ontology
+
+    client.delete_unused_ontology(ontology.uid)
+
+
+def test_ontology_create_feature_schema(ontology_from_feature_ids,
+                                        features_from_json, tools_json):
+    created_ontology = ontology_from_feature_ids
+    feature_schema_ids = {f.uid for f in features_from_json}
+    tools_normalized = created_ontology.normalized['tools']
+    tools = tools_json
+
+    for tool in tools:
+        generated_tool = next(
+            t for t in tools_normalized if t['name'] == tool['name'])
+        assert generated_tool['schemaNodeId'] is not None
+        assert generated_tool['featureSchemaId'] in feature_schema_ids
+        assert generated_tool['tool'] == tool['tool']
+        assert generated_tool['name'] == tool['name']
+        assert generated_tool['required'] == tool['required']
+        assert generated_tool['color'] == tool['color']

libs/labelbox/tests/unit/test_unit_ontology_kind.py

@@ -0,0 +1,22 @@
+from labelbox.schema.ontology_kind import OntologyKind, EditorTaskType, EditorTaskTypeMapper
+from labelbox.schema.media_type import MediaType
+
+
+def test_ontology_kind_conversions_from_editor_task_type():
+    ontology_kind = OntologyKind.ModelEvaluation
+    media_type = MediaType.Conversational
+    editor_task_type = EditorTaskTypeMapper.to_editor_task_type(
+        ontology_kind, media_type)
+    assert editor_task_type == EditorTaskType.ModelChatEvaluation
+
+    ontology_kind = OntologyKind.Missing
+    media_type = MediaType.Image
+    editor_task_type = EditorTaskTypeMapper.to_editor_task_type(
+        ontology_kind, media_type)
+    assert editor_task_type == EditorTaskType.Missing
+
+    ontology_kind = OntologyKind.ModelEvaluation
+    media_type = MediaType.Video
+    editor_task_type = EditorTaskTypeMapper.to_editor_task_type(
+        ontology_kind, media_type)
+    assert editor_task_type == EditorTaskType.Missing