- | Geospatial Predictions |
-  |
-  |
+ Video Predictions |
+ |
+ |
| HTML Predictions |
@@ -296,14 +296,9 @@
|
- | Conversational LLM Predictions |
- |
- |
-
-
- | Image Predictions |
- |
- |
+ Geospatial Predictions |
+ |
+ |
| Conversational Predictions |
@@ -316,15 +311,20 @@
|
- | Video Predictions |
- |
- |
+ Conversational LLM Predictions |
+ |
+ |
| PDF Predictions |
|
|
+
+ | Image Predictions |
+ |
+ |
+
diff --git a/examples/project_configuration/project_setup.ipynb b/examples/project_configuration/project_setup.ipynb
index 2733332b1..1e0a7a478 100644
--- a/examples/project_configuration/project_setup.ipynb
+++ b/examples/project_configuration/project_setup.ipynb
@@ -118,7 +118,7 @@
},
{
"metadata": {},
- "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_mode=QualityMode.\n Consensus, # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)",
+ "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.Consensus\n ], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)",
"cell_type": "code",
"outputs": [],
"execution_count": null
diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb
index a4125386f..30a6e7342 100644
--- a/examples/project_configuration/queue_management.ipynb
+++ b/examples/project_configuration/queue_management.ipynb
@@ -90,7 +90,7 @@
},
{
"metadata": {},
- "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_mode=QualityMode.\n Benchmark, # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")",
+ "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[\n QualityMode.Benchmark\n ], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")",
"cell_type": "code",
"outputs": [],
"execution_count": null
diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py
index 63320e45d..1764ee034 100644
--- a/libs/labelbox/src/labelbox/client.py
+++ b/libs/labelbox/src/labelbox/client.py
@@ -768,6 +768,8 @@ def create_project(self, **kwargs) -> Project:
queue_mode (Optional[QueueMode]): The queue mode to use
quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to
Benchmark
+ quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to
+ Benchmark.
Returns:
A new Project object.
Raises:
@@ -878,7 +880,7 @@ def _create_project(self, **kwargs) -> Project:
auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
raise ValueError(
- "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
+ "quality_modes must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
)
name = kwargs.get("name")
@@ -908,22 +910,54 @@ def _create_project(self, **kwargs) -> Project:
" through this method will soon no longer be supported.")
media_type_value = None
+ quality_modes = kwargs.get("quality_modes")
quality_mode = kwargs.get("quality_mode")
- if not quality_mode:
- logger.info("Defaulting quality mode to Benchmark.")
+ if quality_mode:
+ logger.warning(
+ "Passing quality_mode is deprecated and will soon no longer be supported. Use quality_modes instead."
+ )
+
+ if quality_modes and quality_mode:
+ raise ValueError(
+ "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.")
+
+ if not quality_modes and not quality_mode:
+ logger.info("Defaulting quality modes to Benchmark and Consensus.")
data = kwargs
+ data.pop("quality_modes", None)
data.pop("quality_mode", None)
- if quality_mode is None or quality_mode is QualityMode.Benchmark:
+
+ # check if quality_modes is a set, if not, convert to set
+ quality_modes_set = quality_modes
+ if quality_modes and not isinstance(quality_modes, set):
+ quality_modes_set = set(quality_modes)
+ if quality_mode:
+ quality_modes_set = {quality_mode}
+
+ if (
+ quality_modes_set is None
+ or len(quality_modes_set) == 0
+ or quality_modes_set == {QualityMode.Benchmark, QualityMode.Consensus}
+ ):
+ data["auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
+ data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
+ data["is_benchmark_enabled"] = True
+ data["is_consensus_enabled"] = True
+ elif quality_modes_set == {QualityMode.Benchmark}:
data[
"auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
- elif quality_mode is QualityMode.Consensus:
+ data["is_benchmark_enabled"] = True
+ elif quality_modes_set == {QualityMode.Consensus}:
data[
"auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
+ data["is_consensus_enabled"] = True
else:
- raise ValueError(f"{quality_mode} is not a valid quality mode.")
+ raise ValueError(
+ f"{quality_modes_set} is not a valid quality modes set. Allowed values are [Benchmark, Consensus]"
+ )
params = {**data}
if media_type_value:
diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py
index eb083e52e..7b63ee391 100644
--- a/libs/labelbox/tests/integration/test_project.py
+++ b/libs/labelbox/tests/integration/test_project.py
@@ -255,17 +255,17 @@ def test_media_type(client, project: Project, rand_gen):
def test_queue_mode(client, rand_gen):
- project = client.create_project(name=rand_gen(str)) # defaults to benchmark
- assert project.auto_audit_number_of_labels == 1
- assert project.auto_audit_percentage == 1
+ project = client.create_project(name=rand_gen(str)) # defaults to benchmark and consensus
+ assert project.auto_audit_number_of_labels == 3
+ assert project.auto_audit_percentage == 0
- project = client.create_project(name=rand_gen(str),
- quality_mode=QualityMode.Benchmark)
+ project = client.create_project(name=rand_gen(str), quality_modes=[QualityMode.Benchmark])
assert project.auto_audit_number_of_labels == 1
assert project.auto_audit_percentage == 1
- project = client.create_project(name=rand_gen(str),
- quality_mode=QualityMode.Consensus)
+ project = client.create_project(
+ name=rand_gen(str), quality_modes=[QualityMode.Benchmark, QualityMode.Consensus]
+ )
assert project.auto_audit_number_of_labels == 3
assert project.auto_audit_percentage == 0