Merge develop

Author: kozikkamil

examples/README.md

@@ -109,6 +109,11 @@
       <td><a href="https://github.com/Labelbox/labelbox-python/tree/develop/examples/project_configuration/queue_management.ipynb" target="_blank"><img src="https://img.shields.io/badge/GitHub-100000?logo=github&logoColor=white" alt="Open In Github"></a></td>
       <td><a href="https://colab.research.google.com/github/Labelbox/labelbox-python/blob/develop/examples/project_configuration/queue_management.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a></td>
     </tr>
+    <tr>
+      <td>Multimodal Chat Project</td>
+      <td><a href="https://github.com/Labelbox/labelbox-python/tree/develop/examples/project_configuration/multimodal_chat_project.ipynb" target="_blank"><img src="https://img.shields.io/badge/GitHub-100000?logo=github&logoColor=white" alt="Open In Github"></a></td>
+      <td><a href="https://colab.research.google.com/github/Labelbox/labelbox-python/blob/develop/examples/project_configuration/multimodal_chat_project.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a></td>
+    </tr>
     <tr>
       <td>Project Setup</td>
       <td><a href="https://github.com/Labelbox/labelbox-python/tree/develop/examples/project_configuration/project_setup.ipynb" target="_blank"><img src="https://img.shields.io/badge/GitHub-100000?logo=github&logoColor=white" alt="Open In Github"></a></td>

examples/project_configuration/live_multimodal_chat_project.ipynb renamed to examples/project_configuration/multimodal_chat_project.ipynb

@@ -16,12 +16,12 @@
       "metadata": {},
       "source": [
         "<td>\n",
-        "<a href=\"https://colab.research.google.com/github/Labelbox/labelbox-python/blob/develop/examples/project_configuration/live_multimodal_chat_project.ipynb\" target=\"_blank\"><img\n",
+        "<a href=\"https://colab.research.google.com/github/Labelbox/labelbox-python/blob/develop/examples/project_configuration/multimodal_chat_project.ipynb\" target=\"_blank\"><img\n",
         "src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>\n",
         "</td>\n",
         "\n",
         "<td>\n",
-        "<a href=\"https://github.com/Labelbox/labelbox-python/tree/develop/examples/project_configuration/live_multimodal_chat_project.ipynb\" target=\"_blank\"><img\n",
+        "<a href=\"https://github.com/Labelbox/labelbox-python/tree/develop/examples/project_configuration/multimodal_chat_project.ipynb\" target=\"_blank\"><img\n",
         "src=\"https://img.shields.io/badge/GitHub-100000?logo=github&logoColor=white\" alt=\"GitHub\"></a>\n",
         "</td>"
       ],
@@ -30,14 +30,16 @@
     {
       "metadata": {},
       "source": [
-        "# Live Multimodal Chat project setup\n",
+        "# Multimodal chat project setup\n",
         "\n",
-        "This notebook will provide an example workflow of setting up a Live Multimodal Chat (LMC) Project with the Labelbox-Python SDK.\n",
-        "Live Multimodal Chat Projects are set up differently than other projects with its own unique method and modifications to existing methods:\n",
+        "This notebook will provide an example workflow of setting up a multimodal Chat (MMC) Project with the Labelbox-Python SDK.\n",
+        "Multimodal Chat Projects are set up differently than other projects with its own unique method and modifications to existing methods:\n",
         "\n",
-        "- `client.create_model_evaluation_project`: The main method used to create a Live Multimodal Chat project\n",
+        "- `client.create_model_evaluation_project`: The main method used to create a live multimodal Chat project.\n",
+        "  \n",
+        "- `client.create_offline_model_evaluation_project`: The main method used to create a offline multimodal Chat project.\n",
         "\n",
-        "- `client.create_ontology`: Methods used to create Labelbox ontologies for LMC project this requires an `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n",
+        "- `client.create_ontology`: Methods used to create Labelbox ontologies for LMC project this requires an `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`.\n",
         "\n",
         "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` designed to allow you to use existing features instead of creating new features. This also requires an `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`."
       ],
@@ -68,7 +70,7 @@
       "metadata": {},
       "source": [
         "## API key and client\n",
-        "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information."
+        "Please provide a valid API key below to connect to the Labelbox client properly. For more information, please review the [Create API key guide](https://docs.labelbox.com/reference/create-api-key)."
       ],
       "cell_type": "markdown"
     },
@@ -82,18 +84,18 @@
     {
       "metadata": {},
       "source": [
-        "## Example: Create Live Multimodal Chat project\n",
+        "## Example: Create multimodal Chat project\n",
         "\n",
-        "The steps to creating a Live Multimodal Chat Project through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow."
+        "The steps to creating a multimodal Chat Projects through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow."
       ],
       "cell_type": "markdown"
     },
     {
       "metadata": {},
       "source": [
-        "### Create a Live Multimodal Chat ontology\n",
+        "### Create a multimodal chat ontology\n",
         "\n",
-        "You can create ontologies for Model Evaluation projects the same way as creating ontologies for other projects with the only requirement of passing in a `ontology_kind` parameter which needs set to `lb.OntologyKind.ModelEvaluation`. You can create ontologies with two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`."
+        "You can create ontologies for multimodal chat projects in the same way as other project ontologies using two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`. The only additional requirement is to pass an ontology_kind parameter, which needs to be set to `lb.OntologyKind.ModelEvaluation`."
       ],
       "cell_type": "markdown"
     },
@@ -102,7 +104,7 @@
       "source": [
         "#### Option A: `client.create_ontology`\n",
         "\n",
-        "Typically, you create ontologies and generate the associated features at the same time. Below is an example of creating an ontology for your Live Multimodal Chat project using supported tools and classifications. For information on supported annotation types visit our [Live Multimodal Chat](https://docs.labelbox.com/docs/live-multimodal-chat#supported-annotation-types) guide."
+        "Typically, you create ontologies and generate the associated features simultaneously. Below is an example of creating an ontology for your multimodal chat project using supported tools and classifications; for information on supported annotation types, visit our [multimodal chat evaluation guide](https://docs.labelbox.com/docs/multimodal-chat#supported-annotation-types) guide."
       ],
       "cell_type": "markdown"
     },
@@ -128,6 +130,37 @@
       "outputs": [],
       "execution_count": null
     },
+    {
+      "metadata": {},
+      "source": [
+        "## Creating multimodal chat evaluation projects\n",
+        "\n",
+        "There are two versions of a multimodal chat evaluation projects:\n",
+        "\n",
+        "1. Offline multimodal chat evaluation projects: Data rows will need to be imported manually and have no live model invocation.\n",
+        "\n",
+        "2. Live multimodal chat evaluation projects: Empty data rows are generated on project creation and are filled out with live model invocation.\n",
+        "\n",
+        "We will discuss creating both types of projects with the Labelbox SDK."
+      ],
+      "cell_type": "markdown"
+    },
+    {
+      "metadata": {},
+      "source": [
+        "## Set up offline multimodal chat evaluation project\n",
+        "\n",
+        "For an offline multimodal chat evaluation project, you must import conversational version 2 data rows. For more information, please visit our [import multimodal chat evaluation data](https://docs.labelbox.com/reference/import-multimodal-chat-data) guide. Offline multimodal chat evaluation projects are created through the SDK with `client.create_offline_model_evaluation_project`. This method uses the same parameters as `client.create_project` but provides better validation to ensure the project is set up correctly."
+      ],
+      "cell_type": "markdown"
+    },
+    {
+      "metadata": {},
+      "source": "project = client.create_offline_model_evaluation_project(\n    name=\"<project_name>\",\n    description=\"<project_description>\",  # optional\n)",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
     {
       "metadata": {},
       "source": [
@@ -224,10 +257,21 @@
     {
       "metadata": {},
       "source": [
-        "**To finish setting up your LMC project, you will need to navigate to your project overview inside the Labelbox platform and select _Complete setup_ on the left side panel**"
+        "### Mark project setup as completed\n",
+        "\n",
+        "Once you have finalized your project and set up your model configs, you must mark the project setup as completed.\n",
+        "\n",
+        "**Once the project is marked as \"setup complete\", a user can not add, modify, or delete existing project model configs.**"
       ],
       "cell_type": "markdown"
     },
+    {
+      "metadata": {},
+      "source": "project.set_project_model_setup_complete()",
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
     {
       "metadata": {},
       "source": [

libs/labelbox/src/labelbox/schema/project.py

@@ -149,6 +149,12 @@ def is_chat_evaluation(self) -> bool:
     def is_auto_data_generation(self) -> bool:
         return (self.upload_type == UploadType.Auto)  # type: ignore
 
+    # we test not only the project ontology is None, but also a default empty ontology that we create when we attach a labeling front end in createLabelingFrontendOptions
+    def is_empty_ontology(self) -> bool:
+        ontology = self.ontology()  # type: ignore
+        return ontology is None or (len(ontology.tools()) == 0 and
+                                    len(ontology.classifications()) == 0)
+
     def project_model_configs(self):
         query_str = """query ProjectModelConfigsPyApi($id: ID!) {
             project(where: {id : $id}) {
@@ -772,35 +778,27 @@ def setup_editor(self, ontology) -> None:
         Args:
             ontology (Ontology): The ontology to attach to the project
         """
+        warnings.warn("This method is deprecated use connect_ontology instead.")
+        self.connect_ontology(ontology)
 
-        if self.labeling_frontend() is not None and not self.is_chat_evaluation(
-        ):  # Chat evaluation projects are automatically set up via the same api that creates a project
-            raise ResourceConflict("Editor is already set up.")
-
-        if not self.is_chat_evaluation():
-            labeling_frontend = next(
-                self.client.get_labeling_frontends(
-                    where=Entity.LabelingFrontend.name == "Editor"))
-            self.labeling_frontend.connect(labeling_frontend)
-
-            LFO = Entity.LabelingFrontendOptions
-            self.client._create(
-                LFO, {
-                    LFO.project:
-                        self,
-                    LFO.labeling_frontend:
-                        labeling_frontend,
-                    LFO.customization_options:
-                        json.dumps({
-                            "tools": [],
-                            "classifications": []
-                        })
-                })
-        else:
-            warnings.warn("""
-            Skipping editor setup for a chat evaluation project.
-            Editor was setup automatically.
-            """)
+    def connect_ontology(self, ontology) -> None:
+        """
+        Connects the ontology to the project. If an editor is not setup, it will be connected as well.
+
+        Note: For live chat model evaluation projects, the editor setup is skipped becase it is automatically setup when the project is created.
+
+        Args:
+            ontology (Ontology): The ontology to attach to the project
+        """
+        if self.labeling_frontend(
+        ) is None:  # Chat evaluation projects are automatically set up via the same api that creates a project
+            self._connect_default_labeling_front_end(ontology_as_dict={
+                "tools": [],
+                "classifications": []
+            })
+
+        if not self.is_empty_ontology():
+            raise ValueError("Ontology already connected to project.")
 
         query_str = """mutation ConnectOntologyPyApi($projectId: ID!, $ontologyId: ID!){
             project(where: {id: $projectId}) {connectOntology(ontologyId: $ontologyId) {id}}}"""
@@ -812,43 +810,55 @@ def setup_editor(self, ontology) -> None:
         self.update(setup_complete=timestamp)
 
     def setup(self, labeling_frontend, labeling_frontend_options) -> None:
-        """ Finalizes the Project setup.
+        """ This method will associate default labeling frontend with the project and create an ontology based on labeling_frontend_options.
 
         Args:
-            labeling_frontend (LabelingFrontend): Which UI to use to label the
-                data.
+            labeling_frontend (LabelingFrontend): Do not use, this parameter is deprecated. We now associate the default labeling frontend with the project.
             labeling_frontend_options (dict or str): Labeling frontend options,
                 a.k.a. project ontology. If given a `dict` it will be converted
                 to `str` using `json.dumps`.
         """
 
+        warnings.warn("This method is deprecated use connect_ontology instead.")
+        if labeling_frontend is not None:
+            warnings.warn(
+                "labeling_frontend parameter will not be used to create a new labeling frontend."
+            )
+
         if self.is_chat_evaluation():
             warnings.warn("""
-            This project is a chat evaluation project.
+            This project is a live chat evaluation project.
             Editor was setup automatically.
-            No need to call this method.
             """)
             return
 
-        if self.labeling_frontend() is not None:
-            raise ResourceConflict("Editor is already set up.")
+        if self.labeling_frontend(
+        ) is None:  # Chat evaluation projects are automatically set up via the same api that creates a project
+            self._connect_default_labeling_front_end(labeling_frontend_options)
 
-        if not isinstance(labeling_frontend_options, str):
-            labeling_frontend_options = json.dumps(labeling_frontend_options)
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        self.update(setup_complete=timestamp)
 
+    def _connect_default_labeling_front_end(self, ontology_as_dict: dict):
+        warnings.warn("Connecting default labeling editor for the project.")
+        labeling_frontend = next(
+            self.client.get_labeling_frontends(
+                where=Entity.LabelingFrontend.name == "Editor"))
         self.labeling_frontend.connect(labeling_frontend)
 
+        if not isinstance(ontology_as_dict, str):
+            labeling_frontend_options_str = json.dumps(ontology_as_dict)
+        else:
+            labeling_frontend_options_str = ontology_as_dict
+
         LFO = Entity.LabelingFrontendOptions
         self.client._create(
             LFO, {
                 LFO.project: self,
                 LFO.labeling_frontend: labeling_frontend,
-                LFO.customization_options: labeling_frontend_options
+                LFO.customization_options: labeling_frontend_options_str
             })
 
-        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-        self.update(setup_complete=timestamp)
-
     def create_batch(
         self,
         name: str,

libs/labelbox/tests/data/annotation_import/conftest.py

@@ -608,7 +608,8 @@ def get_data_row_id(indx=0):
 
     yield get_data_row_id
 
-#TODO: Switch to setup_editor, setup might get removed in later releases
+
+#TODO: Switch to connect_ontology, setup might get removed in later releases
 @pytest.fixture
 def configured_project(client, initial_dataset, ontology, rand_gen, image_url):
     dataset = initial_dataset
@@ -645,7 +646,8 @@ def configured_project(client, initial_dataset, ontology, rand_gen, image_url):
 
     project.delete()
 
-#TODO: Switch to setup_editor, setup might get removed in later releases
+
+#TODO: Switch to connect_ontology, setup might get removed in later releases
 @pytest.fixture
 def project_with_ontology(client, configured_project, ontology, rand_gen):
     project = client.create_project(name=rand_gen(str),
@@ -660,7 +662,8 @@ def project_with_ontology(client, configured_project, ontology, rand_gen):
 
     project.delete()
 
-#TODO: Switch to setup_editor, setup might get removed in later releases
+
+#TODO: Switch to connect_ontology, setup might get removed in later releases
 @pytest.fixture
 def configured_project_pdf(client, ontology, rand_gen, pdf_url):
     project = client.create_project(name=rand_gen(str),

libs/labelbox/tests/data/annotation_import/test_send_to_annotate_mea.py

@@ -14,7 +14,7 @@ def test_send_to_annotate_from_model(client, configured_project,
     destination_project = project
     model = client.get_model(model_run.model_id)
     ontology = client.get_ontology(model.ontology_id)
-    destination_project.setup_editor(ontology)
+    destination_project.connect_ontology(ontology)
 
     queues = destination_project.task_queues()
     initial_review_task = next(
@@ -55,13 +55,13 @@ def test_send_to_annotate_from_model(client, configured_project,
     # Check that the data row was sent to the new project
     destination_batches = list(destination_project.batches())
     assert len(destination_batches) == 1
-    
+
     export_task = destination_project.export()
     export_task.wait_till_done()
     stream = export_task.get_buffered_stream()
-    
+
     destination_data_rows = [dr.json["data_row"]["id"] for dr in stream]
-    
+
     assert len(destination_data_rows) == len(data_row_ids)
     assert all([dr in data_row_ids for dr in destination_data_rows])
 

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

@@ -28,7 +28,7 @@ def test_create_chat_evaluation_ontology_project(
     project = live_chat_evaluation_project_with_new_dataset
     assert project.model_setup_complete is None
 
-    project.setup_editor(ontology)
+    project.connect_ontology(ontology)
 
     assert project.labeling_frontend().name == "Editor"
     assert project.ontology().name == ontology.name
@@ -61,7 +61,7 @@ def test_create_chat_evaluation_ontology_project_existing_dataset(
 
     project = chat_evaluation_project_append_to_dataset
     assert project
-    project.setup_editor(ontology)
+    project.connect_ontology(ontology)
 
     assert project.labeling_frontend().name == "Editor"
     assert project.ontology().name == ontology.name

libs/labelbox/tests/integration/test_offline_chat_evaluation_project.py

@@ -1,5 +1,6 @@
 import pytest
 
+
 def test_create_offline_chat_evaluation_project(client, rand_gen,
                                                 offline_chat_evaluation_project,
                                                 chat_evaluation_ontology,
@@ -9,7 +10,7 @@ def test_create_offline_chat_evaluation_project(client, rand_gen,
     assert project
 
     ontology = chat_evaluation_ontology
-    project.setup_editor(ontology)
+    project.connect_ontology(ontology)
 
     assert project.labeling_frontend().name == "Editor"
     assert project.ontology().name == ontology.name