[MODEL-1474] Quality modes as list

examples/project_configuration/project_setup.ipynb

@@ -118,7 +118,7 @@
     },
     {
       "metadata": {},
-      "source": "batch_project = client.create_project(\n    name=\"Project Setup Demo\",\n    quality_mode=QualityMode.\n    Consensus,  # For benchmarks use quality_mode = QualityMode.Benchmark\n    media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)",
+      "source": "batch_project = client.create_project(\n    name=\"Project Setup Demo\",\n    quality_modes=[QualityMode.Consensus\n                  ],  # For benchmarks use quality_mode = QualityMode.Benchmark\n    media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)",
       "cell_type": "code",
       "outputs": [],
       "execution_count": null

examples/project_configuration/queue_management.ipynb

@@ -90,7 +90,7 @@
     },
     {
       "metadata": {},
-      "source": "# Create Labelbox  project\n\nproject = client.create_project(\n    name=\"batch-test-project\",\n    description=\"a description\",\n    quality_mode=QualityMode.\n    Benchmark,  # For Consensus projects use quality_mode = QualityMode.Consensus\n    media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")",
+      "source": "# Create Labelbox  project\n\nproject = client.create_project(\n    name=\"batch-test-project\",\n    description=\"a description\",\n    quality_modes=[\n        QualityMode.Benchmark\n    ],  # For Consensus projects use quality_mode = QualityMode.Consensus\n    media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")",
       "cell_type": "code",
       "outputs": [],
       "execution_count": null

libs/labelbox/src/labelbox/client.py

@@ -768,6 +768,8 @@ def create_project(self, **kwargs) -> Project:
             queue_mode (Optional[QueueMode]): The queue mode to use
             quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to
                 Benchmark
+            quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to
+                Benchmark.
         Returns:
             A new Project object.
         Raises:
@@ -878,7 +880,7 @@ def _create_project(self, **kwargs) -> Project:
         auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
         if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
             raise ValueError(
-                "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
+                "quality_modes must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
             )
 
         name = kwargs.get("name")
@@ -908,22 +910,54 @@ def _create_project(self, **kwargs) -> Project:
                 " through this method will soon no longer be supported.")
             media_type_value = None
 
+        quality_modes = kwargs.get("quality_modes")
         quality_mode = kwargs.get("quality_mode")
-        if not quality_mode:
-            logger.info("Defaulting quality mode to Benchmark.")
+        if quality_mode:
+            logger.warning(
+                "Passing quality_mode is deprecated and will soon no longer be supported. Use quality_modes instead."
+            )
+
+        if quality_modes and quality_mode:
+            raise ValueError(
+                "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.")
+
+        if not quality_modes and not quality_mode:
+            logger.info("Defaulting quality modes to Benchmark and Consensus.")
 
         data = kwargs
+        data.pop("quality_modes", None)
         data.pop("quality_mode", None)
-        if quality_mode is None or quality_mode is QualityMode.Benchmark:
+
+        # check if quality_modes is a set, if not, convert to set
+        quality_modes_set = quality_modes
+        if quality_modes and not isinstance(quality_modes, set):
+            quality_modes_set = set(quality_modes)
+        if quality_mode:
+            quality_modes_set = {quality_mode}
+
+        if (
+            quality_modes_set is None
+            or len(quality_modes_set) == 0
+            or quality_modes_set == {QualityMode.Benchmark, QualityMode.Consensus}
+        ):
+            data["auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
+            data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
+            data["is_benchmark_enabled"] = True
+            data["is_consensus_enabled"] = True
+        elif quality_modes_set == {QualityMode.Benchmark}:
             data[
                 "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
             data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
-        elif quality_mode is QualityMode.Consensus:
+            data["is_benchmark_enabled"] = True
+        elif quality_modes_set == {QualityMode.Consensus}:
             data[
                 "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
             data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
+            data["is_consensus_enabled"] = True
         else:
-            raise ValueError(f"{quality_mode} is not a valid quality mode.")
+            raise ValueError(
+                f"{quality_modes_set} is not a valid quality modes set. Allowed values are [Benchmark, Consensus]"
+            )
 
         params = {**data}
         if media_type_value:

libs/labelbox/tests/integration/test_project.py

@@ -255,17 +255,17 @@ def test_media_type(client, project: Project, rand_gen):
 
 
 def test_queue_mode(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))  # defaults to benchmark
-    assert project.auto_audit_number_of_labels == 1
-    assert project.auto_audit_percentage == 1
+    project = client.create_project(name=rand_gen(str))  # defaults to benchmark and consensus
+    assert project.auto_audit_number_of_labels == 3
+    assert project.auto_audit_percentage == 0
 
-    project = client.create_project(name=rand_gen(str),
-                                    quality_mode=QualityMode.Benchmark)
+    project = client.create_project(name=rand_gen(str), quality_modes=[QualityMode.Benchmark])
     assert project.auto_audit_number_of_labels == 1
     assert project.auto_audit_percentage == 1
 
-    project = client.create_project(name=rand_gen(str),
-                                    quality_mode=QualityMode.Consensus)
+    project = client.create_project(
+        name=rand_gen(str), quality_modes=[QualityMode.Benchmark, QualityMode.Consensus]
+    )
     assert project.auto_audit_number_of_labels == 3
     assert project.auto_audit_percentage == 0