From ca57091e8217d25f40e340720ce692805ebcda54 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 24 Jul 2024 17:54:41 -0700 Subject: [PATCH 1/3] Revert "[MODEL-1474] Quality modes as list (#1683)" This reverts commit 5bf3de0f35f4ad9f7178342e88d8fa597e455978. --- examples/README.md | 186 +++++++++--------- .../project_configuration/project_setup.ipynb | 2 +- .../queue_management.ipynb | 2 +- libs/labelbox/src/labelbox/client.py | 46 +---- .../tests/integration/test_project.py | 14 +- 5 files changed, 108 insertions(+), 142 deletions(-) diff --git a/examples/README.md b/examples/README.md index 1bc102947..1523e1737 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,49 +17,49 @@ - Basics - Open In Github - Open In Colab + Custom Embeddings + Open In Github + Open In Colab - Projects - Open In Github - Open In Colab + Batches + Open In Github + Open In Colab Ontologies Open In Github Open In Colab + + Basics + Open In Github + Open In Colab + Quick Start Open In Github Open In Colab - - Data Rows - Open In Github - Open In Colab - Data Row Metadata Open In Github Open In Colab - User Management - Open In Github - Open In Colab + Projects + Open In Github + Open In Colab - Batches - Open In Github - Open In Colab + Data Rows + Open In Github + Open In Colab - Custom Embeddings - Open In Github - Open In Colab + User Management + Open In Github + Open In Colab @@ -76,25 +76,25 @@ - Exporting to CSV - Open In Github - Open In Colab + Composite Mask Export + Open In Github + Open In Colab Export Data Open In Github Open In Colab + + Exporting to CSV + Open In Github + Open In Colab + Export V1 to V2 Migration Support Open In Github Open In Colab - - Composite Mask Export - Open In Github - Open In Colab - @@ -119,16 +119,16 @@ Open In Github Open In Colab - - Webhooks - Open In Github - Open In Colab - Multimodal Chat Project Open In Github Open In Colab + + Webhooks + Open In Github + Open In Colab + @@ -144,24 +144,34 @@ - Tiled - Open In Github - Open In Colab + PDF + Open In Github + Open In Colab - Conversational LLM - Open In Github - Open In Colab + Conversational + Open In Github + Open In Colab - HTML - Open In Github - Open In Colab + Video + Open In Github + Open In Colab - Conversational LLM Data Generation - Open In Github - Open In Colab + Text + Open In Github + Open In Colab + + + DICOM + Open In Github + Open In Colab + + + Tiled + Open In Github + Open In Colab Image @@ -169,19 +179,14 @@ Open In Colab - PDF - Open In Github - Open In Colab - - - DICOM - Open In Github - Open In Colab + Conversational LLM Data Generation + Open In Github + Open In Colab - Text - Open In Github - Open In Colab + Conversational LLM + Open In Github + Open In Colab Audio @@ -189,14 +194,9 @@ Open In Colab - Conversational - Open In Github - Open In Colab - - - Video - Open In Github - Open In Colab + HTML + Open In Github + Open In Colab @@ -212,31 +212,31 @@ + + Huggingface Custom Embeddings + Open In Github + Open In Colab + Import YOLOv8 Annotations Open In Github Open In Colab - - Meta SAM Video - Open In Github - Open In Colab - Meta SAM Open In Github Open In Colab + + Meta SAM Video + Open In Github + Open In Colab + Langchain Open In Github Open In Colab - - Huggingface Custom Embeddings - Open In Github - Open In Colab - @@ -251,6 +251,11 @@ + + Model Predictions to Project + Open In Github + Open In Colab + Custom Metrics Demo Open In Github @@ -266,11 +271,6 @@ Open In Github Open In Colab - - Model Predictions to Project - Open In Github - Open In Colab - @@ -286,9 +286,9 @@ - Video Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab HTML Predictions @@ -296,9 +296,14 @@ Open In Colab - Geospatial Predictions - Open In Github - Open In Colab + Conversational LLM Predictions + Open In Github + Open In Colab + + + Image Predictions + Open In Github + Open In Colab Conversational Predictions @@ -311,20 +316,15 @@ Open In Colab - Conversational LLM Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab PDF Predictions Open In Github Open In Colab - - Image Predictions - Open In Github - Open In Colab - diff --git a/examples/project_configuration/project_setup.ipynb b/examples/project_configuration/project_setup.ipynb index 1e0a7a478..2733332b1 100644 --- a/examples/project_configuration/project_setup.ipynb +++ b/examples/project_configuration/project_setup.ipynb @@ -118,7 +118,7 @@ }, { "metadata": {}, - "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.Consensus\n ], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", + "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_mode=QualityMode.\n Consensus, # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb index 30a6e7342..a4125386f 100644 --- a/examples/project_configuration/queue_management.ipynb +++ b/examples/project_configuration/queue_management.ipynb @@ -90,7 +90,7 @@ }, { "metadata": {}, - "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[\n QualityMode.Benchmark\n ], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", + "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_mode=QualityMode.\n Benchmark, # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 1764ee034..63320e45d 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -768,8 +768,6 @@ def create_project(self, **kwargs) -> Project: queue_mode (Optional[QueueMode]): The queue mode to use quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to Benchmark - quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to - Benchmark. Returns: A new Project object. Raises: @@ -880,7 +878,7 @@ def _create_project(self, **kwargs) -> Project: auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels") if auto_audit_percentage is not None or auto_audit_number_of_labels is not None: raise ValueError( - "quality_modes must be set instead of auto_audit_percentage or auto_audit_number_of_labels." + "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels." ) name = kwargs.get("name") @@ -910,54 +908,22 @@ def _create_project(self, **kwargs) -> Project: " through this method will soon no longer be supported.") media_type_value = None - quality_modes = kwargs.get("quality_modes") quality_mode = kwargs.get("quality_mode") - if quality_mode: - logger.warning( - "Passing quality_mode is deprecated and will soon no longer be supported. Use quality_modes instead." - ) - - if quality_modes and quality_mode: - raise ValueError( - "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.") - - if not quality_modes and not quality_mode: - logger.info("Defaulting quality modes to Benchmark and Consensus.") + if not quality_mode: + logger.info("Defaulting quality mode to Benchmark.") data = kwargs - data.pop("quality_modes", None) data.pop("quality_mode", None) - - # check if quality_modes is a set, if not, convert to set - quality_modes_set = quality_modes - if quality_modes and not isinstance(quality_modes, set): - quality_modes_set = set(quality_modes) - if quality_mode: - quality_modes_set = {quality_mode} - - if ( - quality_modes_set is None - or len(quality_modes_set) == 0 - or quality_modes_set == {QualityMode.Benchmark, QualityMode.Consensus} - ): - data["auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS - data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE - data["is_benchmark_enabled"] = True - data["is_consensus_enabled"] = True - elif quality_modes_set == {QualityMode.Benchmark}: + if quality_mode is None or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE - data["is_benchmark_enabled"] = True - elif quality_modes_set == {QualityMode.Consensus}: + elif quality_mode is QualityMode.Consensus: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE - data["is_consensus_enabled"] = True else: - raise ValueError( - f"{quality_modes_set} is not a valid quality modes set. Allowed values are [Benchmark, Consensus]" - ) + raise ValueError(f"{quality_mode} is not a valid quality mode.") params = {**data} if media_type_value: diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index 7b63ee391..eb083e52e 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -255,17 +255,17 @@ def test_media_type(client, project: Project, rand_gen): def test_queue_mode(client, rand_gen): - project = client.create_project(name=rand_gen(str)) # defaults to benchmark and consensus - assert project.auto_audit_number_of_labels == 3 - assert project.auto_audit_percentage == 0 + project = client.create_project(name=rand_gen(str)) # defaults to benchmark + assert project.auto_audit_number_of_labels == 1 + assert project.auto_audit_percentage == 1 - project = client.create_project(name=rand_gen(str), quality_modes=[QualityMode.Benchmark]) + project = client.create_project(name=rand_gen(str), + quality_mode=QualityMode.Benchmark) assert project.auto_audit_number_of_labels == 1 assert project.auto_audit_percentage == 1 - project = client.create_project( - name=rand_gen(str), quality_modes=[QualityMode.Benchmark, QualityMode.Consensus] - ) + project = client.create_project(name=rand_gen(str), + quality_mode=QualityMode.Consensus) assert project.auto_audit_number_of_labels == 3 assert project.auto_audit_percentage == 0 From 6ae0ab76f3d95c1930f5753899d43181f35d8773 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Jul 2024 00:57:58 +0000 Subject: [PATCH 2/3] :memo: README updated --- examples/README.md | 186 ++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/examples/README.md b/examples/README.md index 1523e1737..1bc102947 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,49 +17,49 @@ - Custom Embeddings - Open In Github - Open In Colab + Basics + Open In Github + Open In Colab - Batches - Open In Github - Open In Colab + Projects + Open In Github + Open In Colab Ontologies Open In Github Open In Colab - - Basics - Open In Github - Open In Colab - Quick Start Open In Github Open In Colab + + Data Rows + Open In Github + Open In Colab + Data Row Metadata Open In Github Open In Colab - Projects - Open In Github - Open In Colab + User Management + Open In Github + Open In Colab - Data Rows - Open In Github - Open In Colab + Batches + Open In Github + Open In Colab - User Management - Open In Github - Open In Colab + Custom Embeddings + Open In Github + Open In Colab @@ -76,25 +76,25 @@ - Composite Mask Export - Open In Github - Open In Colab + Exporting to CSV + Open In Github + Open In Colab Export Data Open In Github Open In Colab - - Exporting to CSV - Open In Github - Open In Colab - Export V1 to V2 Migration Support Open In Github Open In Colab + + Composite Mask Export + Open In Github + Open In Colab + @@ -119,16 +119,16 @@ Open In Github Open In Colab - - Multimodal Chat Project - Open In Github - Open In Colab - Webhooks Open In Github Open In Colab + + Multimodal Chat Project + Open In Github + Open In Colab + @@ -144,34 +144,24 @@ - PDF - Open In Github - Open In Colab - - - Conversational - Open In Github - Open In Colab - - - Video - Open In Github - Open In Colab + Tiled + Open In Github + Open In Colab - Text - Open In Github - Open In Colab + Conversational LLM + Open In Github + Open In Colab - DICOM - Open In Github - Open In Colab + HTML + Open In Github + Open In Colab - Tiled - Open In Github - Open In Colab + Conversational LLM Data Generation + Open In Github + Open In Colab Image @@ -179,14 +169,19 @@ Open In Colab - Conversational LLM Data Generation - Open In Github - Open In Colab + PDF + Open In Github + Open In Colab - Conversational LLM - Open In Github - Open In Colab + DICOM + Open In Github + Open In Colab + + + Text + Open In Github + Open In Colab Audio @@ -194,9 +189,14 @@ Open In Colab - HTML - Open In Github - Open In Colab + Conversational + Open In Github + Open In Colab + + + Video + Open In Github + Open In Colab @@ -212,31 +212,31 @@ - - Huggingface Custom Embeddings - Open In Github - Open In Colab - Import YOLOv8 Annotations Open In Github Open In Colab - - Meta SAM - Open In Github - Open In Colab - Meta SAM Video Open In Github Open In Colab + + Meta SAM + Open In Github + Open In Colab + Langchain Open In Github Open In Colab + + Huggingface Custom Embeddings + Open In Github + Open In Colab + @@ -251,11 +251,6 @@ - - Model Predictions to Project - Open In Github - Open In Colab - Custom Metrics Demo Open In Github @@ -271,6 +266,11 @@ Open In Github Open In Colab + + Model Predictions to Project + Open In Github + Open In Colab + @@ -286,9 +286,9 @@ - Geospatial Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab HTML Predictions @@ -296,14 +296,9 @@ Open In Colab - Conversational LLM Predictions - Open In Github - Open In Colab - - - Image Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab Conversational Predictions @@ -316,15 +311,20 @@ Open In Colab - Video Predictions - Open In Github - Open In Colab + Conversational LLM Predictions + Open In Github + Open In Colab PDF Predictions Open In Github Open In Colab + + Image Predictions + Open In Github + Open In Colab + From 292d61b0316ba3a061eaf8b7eb2323954f6e6eba Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 24 Jul 2024 18:03:29 -0700 Subject: [PATCH 3/3] Trigger tests