From 1c866e328cc8564e03405a778f4d0c00a784a07a Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Wed, 19 Jun 2024 15:12:42 +0200 Subject: [PATCH 01/12] [MODEL-1474] Quality modes as list --- .../project_configuration/project_setup.ipynb | 2 +- .../queue_management.ipynb | 2 +- lbx_prompt.txt | 86 ++++++++++++------- libs/labelbox/src/labelbox/client.py | 20 +++-- 4 files changed, 67 insertions(+), 43 deletions(-) diff --git a/examples/project_configuration/project_setup.ipynb b/examples/project_configuration/project_setup.ipynb index 2733332b1..42d9e9337 100644 --- a/examples/project_configuration/project_setup.ipynb +++ b/examples/project_configuration/project_setup.ipynb @@ -118,7 +118,7 @@ }, { "metadata": {}, - "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_mode=QualityMode.\n Consensus, # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", + "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.\n Consensus], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb index a4125386f..c3cbca3e6 100644 --- a/examples/project_configuration/queue_management.ipynb +++ b/examples/project_configuration/queue_management.ipynb @@ -90,7 +90,7 @@ }, { "metadata": {}, - "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_mode=QualityMode.\n Benchmark, # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", + "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[QualityMode.\n Benchmark], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/lbx_prompt.txt b/lbx_prompt.txt index 19873114d..0ea61acc3 100644 --- a/lbx_prompt.txt +++ b/lbx_prompt.txt @@ -3360,7 +3360,7 @@ def conversation_entity_data_row(client, rand_gen): @pytest.fixture def project(client, rand_gen): project = client.create_project(name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) yield project project.delete() @@ -3370,7 +3370,7 @@ def project(client, rand_gen): def consensus_project(client, rand_gen): project = client.create_project(name=rand_gen(str), quality_mode=QualityMode.Consensus, - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) yield project project.delete() @@ -3560,7 +3560,7 @@ def project_based_user(client, rand_gen): def project_pack(client): projects = [ client.create_project(name=f"user-proj-{idx}", - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) for idx in range(2) ] yield projects @@ -3729,7 +3729,7 @@ def _setup_ontology(project): def configured_project_with_complex_ontology(client, initial_dataset, rand_gen, image_url): project = client.create_project(name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) dataset = initial_dataset data_row = dataset.create_data_row(row_data=image_url) @@ -6650,7 +6650,7 @@ def test_project_dataset(client, rand_gen): ): client.create_project( name=rand_gen(str), - queue_mode=QueueMode.Dataset, + queue_modes=[QueueMode.Dataset], ) @@ -8348,9 +8348,9 @@ def project_to_test_where(client, rand_gen): p_b_name = f"b-{rand_gen(str)}" p_c_name = f"c-{rand_gen(str)}" - p_a = client.create_project(name=p_a_name, queue_mode=QueueMode.Batch) - p_b = client.create_project(name=p_b_name, queue_mode=QueueMode.Batch) - p_c = client.create_project(name=p_c_name, queue_mode=QueueMode.Batch) + p_a = client.create_project(name=p_a_name, queue_modes=[QueueMode.Batch]) + p_b = client.create_project(name=p_b_name, queue_modes=[QueueMode.Batch]) + p_c = client.create_project(name=p_c_name, queue_modes=[QueueMode.Batch]) yield p_a, p_b, p_c @@ -10053,7 +10053,7 @@ def hardcoded_datarow_id(): def configured_project_with_ontology(client, ontology, rand_gen): project = client.create_project( name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image, ) editor = list( @@ -12803,7 +12803,7 @@ def configured_project_with_ontology(client, initial_dataset, ontology, dataset = initial_dataset project = client.create_project( name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], ) editor = list( client.get_labeling_frontends( @@ -12827,7 +12827,7 @@ def configured_project_with_ontology(client, initial_dataset, ontology, def configured_project_without_data_rows(client, ontology, rand_gen): project = client.create_project(name=rand_gen(str), description=rand_gen(str), - queue_mode=QueueMode.Batch) + queue_modes=[QueueMode.Batch]) editor = list( client.get_labeling_frontends( where=LabelingFrontend.name == "editor"))[0] @@ -22430,7 +22430,7 @@ class Client: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use - quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to + quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to Benchmark Returns: A new Project object. @@ -22461,41 +22461,63 @@ class Client: ) media_type = kwargs.get("media_type") - if media_type: - if MediaType.is_supported(media_type): - media_type = media_type.value - else: - raise TypeError(f"{media_type} is not a valid media type. Use" - f" any of {MediaType.get_supported_members()}" - " from MediaType. Example: MediaType.Image.") + if media_type and MediaType.is_supported(media_type): + media_type_value = media_type.value + elif media_type: + raise TypeError(f"{media_type} is not a valid media type. Use" + f" any of {MediaType.get_supported_members()}" + " from MediaType. Example: MediaType.Image.") else: logger.warning( "Creating a project without specifying media_type" " through this method will soon no longer be supported.") + media_type_value = None + + ontology_kind = kwargs.pop("ontology_kind", None) + if ontology_kind and OntologyKind.is_supported(ontology_kind): + editor_task_type_value = EditorTaskTypeMapper.to_editor_task_type( + ontology_kind, media_type).value + elif ontology_kind: + raise OntologyKind.get_ontology_kind_validation_error(ontology_kind) + else: + editor_task_type_value = None - quality_mode = kwargs.get("quality_mode") - if not quality_mode: - logger.info("Defaulting quality mode to Benchmark.") + quality_modes = kwargs.get("quality_modes") + if not quality_modes: + logger.info("Defaulting quality modes to Benchmark.") data = kwargs - data.pop("quality_mode", None) - if quality_mode is None or quality_mode is QualityMode.Benchmark: + data.pop("quality_modes", None) + if quality_modes is None or quality_modes == [QualityMode.Benchmark]: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE - elif quality_mode is QualityMode.Consensus: + data["is_benchmark_enabled"] = True + elif QualityMode.Consensus in quality_modes: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE + data["is_consensus_enabled"] = True else: - raise ValueError(f"{quality_mode} is not a valid quality mode.") + raise ValueError(f"{quality_modes} is not a valid quality modes array. Allowed values are [Benchmark, Consensus]") + + params = {**data} + if media_type_value: + params["media_type"] = media_type_value + if editor_task_type_value: + params["editor_task_type"] = editor_task_type_value + + extra_params = { + Field.String("dataset_name_or_id"): + params.pop("dataset_name_or_id", None), + Field.Boolean("append_to_existing_dataset"): + params.pop("append_to_existing_dataset", None), + Field.Int("data_row_count"): + params.pop("data_row_count", None), + } + extra_params = {k: v for k, v in extra_params.items() if v is not None} - return self._create(Entity.Project, { - **data, - **({ - "media_type": media_type - } if media_type else {}) - }) + return self._create(Entity.Project, params, extra_params) def get_roles(self) -> List[Role]: """ diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index a2fb09186..e1e979075 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -733,8 +733,8 @@ def create_project(self, **kwargs) -> Project: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use - quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to - Benchmark + quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to + Benchmark. Returns: A new Project object. Raises: @@ -785,22 +785,24 @@ def create_project(self, **kwargs) -> Project: else: editor_task_type_value = None - quality_mode = kwargs.get("quality_mode") - if not quality_mode: - logger.info("Defaulting quality mode to Benchmark.") + quality_modes = kwargs.get("quality_modes") + if not quality_modes: + logger.info("Defaulting quality modes to Benchmark.") data = kwargs - data.pop("quality_mode", None) - if quality_mode is None or quality_mode is QualityMode.Benchmark: + data.pop("quality_modes", None) + if quality_modes is None or quality_modes == [QualityMode.Benchmark]: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE - elif quality_mode is QualityMode.Consensus: + data["is_benchmark_enabled"] = True + elif QualityMode.Consensus in quality_modes: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE + data["is_consensus_enabled"] = True else: - raise ValueError(f"{quality_mode} is not a valid quality mode.") + raise ValueError(f"{quality_modes} is not a valid quality modes array. Allowed values are [Benchmark, Consensus]") params = {**data} if media_type_value: From d469a20bd54e850efd9c2e6d1ab96bab55c81ebf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 19 Jun 2024 13:47:34 +0000 Subject: [PATCH 02/12] :art: Cleaned --- examples/project_configuration/project_setup.ipynb | 2 +- examples/project_configuration/queue_management.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/project_configuration/project_setup.ipynb b/examples/project_configuration/project_setup.ipynb index 42d9e9337..1e0a7a478 100644 --- a/examples/project_configuration/project_setup.ipynb +++ b/examples/project_configuration/project_setup.ipynb @@ -118,7 +118,7 @@ }, { "metadata": {}, - "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.\n Consensus], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", + "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.Consensus\n ], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb index c3cbca3e6..30a6e7342 100644 --- a/examples/project_configuration/queue_management.ipynb +++ b/examples/project_configuration/queue_management.ipynb @@ -90,7 +90,7 @@ }, { "metadata": {}, - "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[QualityMode.\n Benchmark], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", + "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[\n QualityMode.Benchmark\n ], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", "cell_type": "code", "outputs": [], "execution_count": null From 3336e2b22e4825fbf8102afdd0a5be2115c94351 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 19 Jun 2024 13:48:14 +0000 Subject: [PATCH 03/12] :memo: README updated --- examples/README.md | 180 ++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/examples/README.md b/examples/README.md index 30f615bc2..8aca50612 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,14 +17,9 @@ - Ontologies - Open In Github - Open In Colab - - - Data Rows - Open In Github - Open In Colab + Custom Embeddings + Open In Github + Open In Colab Batches @@ -32,34 +27,39 @@ Open In Colab - Projects - Open In Github - Open In Colab + User Management + Open In Github + Open In Colab - Custom Embeddings - Open In Github - Open In Colab + Basics + Open In Github + Open In Colab Data Row Metadata Open In Github Open In Colab + + Data Rows + Open In Github + Open In Colab + Quick Start Open In Github Open In Colab - Basics - Open In Github - Open In Colab + Ontologies + Open In Github + Open In Colab - User Management - Open In Github - Open In Colab + Projects + Open In Github + Open In Colab @@ -75,16 +75,16 @@ - - Composite Mask Export - Open In Github - Open In Colab - Export Data Open In Github Open In Colab + + Composite Mask Export + Open In Github + Open In Colab + Exporting to CSV Open In Github @@ -105,25 +105,25 @@ - Live Multimodal Chat Project - Open In Github - Open In Colab + Queue Management + Open In Github + Open In Colab Project Setup Open In Github Open In Colab + + Live Multimodal Chat Project + Open In Github + Open In Colab + Webhooks Open In Github Open In Colab - - Queue Management - Open In Github - Open In Colab - @@ -138,6 +138,11 @@ + + Conversational + Open In Github + Open In Colab + Conversational LLM Data Generation Open In Github @@ -154,19 +159,9 @@ Open In Colab - Audio - Open In Github - Open In Colab - - - Conversational - Open In Github - Open In Colab - - - PDF - Open In Github - Open In Colab + DICOM + Open In Github + Open In Colab Image @@ -174,14 +169,14 @@ Open In Colab - DICOM - Open In Github - Open In Colab + Tiled + Open In Github + Open In Colab - Conversational LLM - Open In Github - Open In Colab + Audio + Open In Github + Open In Colab HTML @@ -189,9 +184,14 @@ Open In Colab - Tiled - Open In Github - Open In Colab + Conversational LLM + Open In Github + Open In Colab + + + PDF + Open In Github + Open In Colab @@ -217,16 +217,16 @@ Open In Github Open In Colab - - Meta SAM Video - Open In Github - Open In Colab - Meta SAM Open In Github Open In Colab + + Meta SAM Video + Open In Github + Open In Colab + Import YOLOv8 Annotations Open In Github @@ -247,14 +247,9 @@ - Custom Metrics Demo - Open In Github - Open In Colab - - - Model Slices - Open In Github - Open In Colab + Model Predictions to Project + Open In Github + Open In Colab Custom Metrics Basics @@ -262,9 +257,14 @@ Open In Colab - Model Predictions to Project - Open In Github - Open In Colab + Model Slices + Open In Github + Open In Colab + + + Custom Metrics Demo + Open In Github + Open In Colab @@ -281,19 +281,9 @@ - PDF Predictions - Open In Github - Open In Colab - - - HTML Predictions - Open In Github - Open In Colab - - - Conversational Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab Image Predictions @@ -306,9 +296,14 @@ Open In Colab - Geospatial Predictions - Open In Github - Open In Colab + HTML Predictions + Open In Github + Open In Colab + + + Conversational Predictions + Open In Github + Open In Colab Conversational LLM Predictions @@ -316,9 +311,14 @@ Open In Colab - Video Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab + + + PDF Predictions + Open In Github + Open In Colab From 4fe10b2c733da7054ac006b3cda361fe10186f67 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Thu, 4 Jul 2024 13:04:21 +0200 Subject: [PATCH 04/12] Backwards compatible interface --- libs/labelbox/src/labelbox/client.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index e1e979075..d34fb1107 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -733,6 +733,8 @@ def create_project(self, **kwargs) -> Project: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use + quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to + Benchmark quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to Benchmark. Returns: @@ -786,17 +788,23 @@ def create_project(self, **kwargs) -> Project: editor_task_type_value = None quality_modes = kwargs.get("quality_modes") - if not quality_modes: + quality_mode = kwargs.get("quality_mode") + if quality_modes and quality_mode: + raise ValueError( + "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.") + + if not quality_modes and not quality_mode: logger.info("Defaulting quality modes to Benchmark.") data = kwargs data.pop("quality_modes", None) - if quality_modes is None or quality_modes == [QualityMode.Benchmark]: + data.pop("quality_mode", None) + if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode == QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif QualityMode.Consensus in quality_modes: + elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode == QualityMode.Consensus: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE From 13c3f6a9c05e430b75e77e709f05ed7ca007e8cf Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Thu, 4 Jul 2024 13:06:02 +0200 Subject: [PATCH 05/12] Change comparison --- libs/labelbox/src/labelbox/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index d34fb1107..0b50a6d6d 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -799,12 +799,12 @@ def create_project(self, **kwargs) -> Project: data = kwargs data.pop("quality_modes", None) data.pop("quality_mode", None) - if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode == QualityMode.Benchmark: + if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode == QualityMode.Consensus: + elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode is QualityMode.Consensus: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE From 79e9c763618b74544b2c1688bef03ec8200917c9 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Thu, 4 Jul 2024 13:09:54 +0200 Subject: [PATCH 06/12] Develop fix --- libs/labelbox/src/labelbox/client.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 6522a883c..6ebcff9fd 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -893,15 +893,6 @@ def _create_project(self, **kwargs) -> Project: " through this method will soon no longer be supported.") media_type_value = None - ontology_kind = kwargs.pop("ontology_kind", None) - if ontology_kind and OntologyKind.is_supported(ontology_kind): - editor_task_type_value = EditorTaskTypeMapper.to_editor_task_type( - ontology_kind, media_type).value - elif ontology_kind: - raise OntologyKind.get_ontology_kind_validation_error(ontology_kind) - else: - editor_task_type_value = None - quality_modes = kwargs.get("quality_modes") quality_mode = kwargs.get("quality_mode") if quality_modes and quality_mode: @@ -930,16 +921,12 @@ def _create_project(self, **kwargs) -> Project: params = {**data} if media_type_value: params["media_type"] = media_type_value - if editor_task_type_value: - params["editor_task_type"] = editor_task_type_value extra_params = { Field.String("dataset_name_or_id"): params.pop("dataset_name_or_id", None), Field.Boolean("append_to_existing_dataset"): params.pop("append_to_existing_dataset", None), - Field.Int("data_row_count"): - params.pop("data_row_count", None), } extra_params = {k: v for k, v in extra_params.items() if v is not None} From 0d714b5fda9cd97bbef1117507c688c26fffca23 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Fri, 5 Jul 2024 15:51:35 +0200 Subject: [PATCH 07/12] CR --- libs/labelbox/src/labelbox/client.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 6ebcff9fd..7aba023f5 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -863,7 +863,7 @@ def _create_project(self, **kwargs) -> Project: auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels") if auto_audit_percentage is not None or auto_audit_number_of_labels is not None: raise ValueError( - "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels." + "quality_modes must be set instead of auto_audit_percentage or auto_audit_number_of_labels." ) name = kwargs.get("name") @@ -895,6 +895,11 @@ def _create_project(self, **kwargs) -> Project: quality_modes = kwargs.get("quality_modes") quality_mode = kwargs.get("quality_mode") + if quality_mode: + logger.warning( + "Passing quality_mode is deprecated and will soon no longer be supported. Use quality_modes instead." + ) + if quality_modes and quality_mode: raise ValueError( "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.") @@ -905,7 +910,7 @@ def _create_project(self, **kwargs) -> Project: data = kwargs data.pop("quality_modes", None) data.pop("quality_mode", None) - if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: + if quality_modes is None or len(quality_modes) == 0 or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE From 4ccdf2b196c5bf271030d5add5429cbc809d9f75 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 8 Jul 2024 18:53:49 +0000 Subject: [PATCH 08/12] :memo: README updated --- examples/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/README.md b/examples/README.md index 9ca7ce6a0..38452d9d5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -119,11 +119,6 @@ Open In Github Open In Colab - - Live Multimodal Chat Project - Open In Github - Open In Colab - Webhooks Open In Github From d1173a419351d51ce6916c89ec369fff4d482e2a Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Mon, 8 Jul 2024 16:05:52 +0500 Subject: [PATCH 09/12] MODEL-1489: Allow marking Label with "is_benchmark_reference" flag --- .../labelbox/data/annotation_types/label.py | 13 ++++++------- .../data/serialization/ndjson/converter.py | 8 +++++--- .../data/annotation_import/test_data_types.py | 18 +++++++++++++++++- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index cd209a493..c7a0cb7b8 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -50,10 +50,10 @@ class Label(pydantic_compat.BaseModel): data: DataType annotations: List[Union[ClassificationAnnotation, ObjectAnnotation, VideoMaskAnnotation, ScalarMetric, - ConfusionMatrixMetric, - RelationshipAnnotation, + ConfusionMatrixMetric, RelationshipAnnotation, PromptClassificationAnnotation]] = [] extra: Dict[str, Any] = {} + is_benchmark_reference: Optional[bool] = False @pydantic_compat.root_validator(pre=True) def validate_data(cls, label): @@ -219,9 +219,8 @@ def validate_union(cls, value): ) # Validates only one prompt annotation is included if isinstance(v, PromptClassificationAnnotation): - prompt_count+=1 - if prompt_count > 1: - raise TypeError( - f"Only one prompt annotation is allowed per label" - ) + prompt_count += 1 + if prompt_count > 1: + raise TypeError( + f"Only one prompt annotation is allowed per label") return value diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py index a7c54b109..2ffeb9727 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py @@ -106,14 +106,16 @@ def serialize( if not isinstance(annotation, RelationshipAnnotation): uuid_safe_annotations.append(annotation) label.annotations = uuid_safe_annotations - for example in NDLabel.from_common([label]): - annotation_uuid = getattr(example, "uuid", None) + for annotation in NDLabel.from_common([label]): + annotation_uuid = getattr(annotation, "uuid", None) - res = example.dict( + res = annotation.dict( by_alias=True, exclude={"uuid"} if annotation_uuid == "None" else None, ) for k, v in list(res.items()): if k in IGNORE_IF_NONE and v is None: del res[k] + if getattr(label, 'is_benchmark_reference'): + res['isBenchmarkReferenceLabel'] = True yield res diff --git a/libs/labelbox/tests/data/annotation_import/test_data_types.py b/libs/labelbox/tests/data/annotation_import/test_data_types.py index d607c4a3c..58a16def9 100644 --- a/libs/labelbox/tests/data/annotation_import/test_data_types.py +++ b/libs/labelbox/tests/data/annotation_import/test_data_types.py @@ -198,6 +198,7 @@ def validate_iso_format(date_string: str): assert parsed_t.minute is not None assert parsed_t.second is not None + @pytest.mark.order(1) @pytest.mark.parametrize( "data_type_class", @@ -333,6 +334,22 @@ def test_import_label_annotations( data_row.delete() +@pytest.mark.parametrize("_, data_class, annotations", test_params) +def test_import_label_annotations_with_is_benchmark_reference_flag( + data_class, annotations, _): + labels = [ + lb_types.Label(data=data_class(uid=str(uuid.uuid4()), + url="http://test.com"), + annotations=annotations, + is_benchmark_reference=True) + ] + serialized_annotations = get_annotation_comparison_dicts_from_labels(labels) + + assert len(serialized_annotations) == len(annotations) + for serialized_annotation in serialized_annotations: + assert serialized_annotation["isBenchmarkReferenceLabel"] + + @pytest.mark.parametrize("data_type, data_class, annotations", test_params) @pytest.fixture def one_datarow(client, rand_gen, data_row_json_by_data_type, data_type): @@ -423,4 +440,3 @@ def test_import_mal_annotations_global_key(client, assert import_annotations.errors == [] # MAL Labels cannot be exported and compared to input labels - \ No newline at end of file From 16772695a0d8b7c0a171297b09ba47b8c389c40e Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Mon, 29 Jul 2024 14:06:44 +0500 Subject: [PATCH 10/12] MODEL-1489: Rewrote tests --- .../data/annotation_import/test_data_types.py | 24 ------------------- .../serialization/ndjson/test_conversation.py | 15 ++++++++++++ .../serialization/ndjson/test_rectangle.py | 20 ++++++++++++++++ 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/libs/labelbox/tests/data/annotation_import/test_data_types.py b/libs/labelbox/tests/data/annotation_import/test_data_types.py index 8f3e08faa..d7b3ef825 100644 --- a/libs/labelbox/tests/data/annotation_import/test_data_types.py +++ b/libs/labelbox/tests/data/annotation_import/test_data_types.py @@ -1,12 +1,8 @@ -import datetime -from labelbox.schema.label import Label import pytest -import uuid from labelbox.data.annotation_types.data import ( AudioData, ConversationData, - DicomData, DocumentData, HTMLData, ImageData, @@ -15,11 +11,8 @@ from labelbox.data.serialization import NDJsonConverter from labelbox.data.annotation_types.data.video import VideoData -import labelbox as lb import labelbox.types as lb_types from labelbox.schema.media_type import MediaType -from labelbox.schema.annotation_import import AnnotationImportState -from labelbox import Project, Client # Unit test for label based on data type. # TODO: Dicom removed it is unstable when you deserialize and serialize on label import. If we intend to keep this library this needs add generic data types tests work with this data type. @@ -84,20 +77,3 @@ def test_data_row_type_by_global_key( assert data_label.data.global_key == label.data.global_key assert label.annotations == data_label.annotations - - -@pytest.mark.parametrize("_, data_class, annotations", test_params) -def test_import_label_annotations_with_is_benchmark_reference_flag( - data_class, annotations, _): - labels = [ - lb_types.Label(data=data_class(uid=str(uuid.uuid4()), - url="http://test.com"), - annotations=annotations, - is_benchmark_reference=True) - ] - serialized_annotations = get_annotation_comparison_dicts_from_labels(labels) - - assert len(serialized_annotations) == len(annotations) - for serialized_annotation in serialized_annotations: - assert serialized_annotation["isBenchmarkReferenceLabel"] - diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py b/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py index acf21cc21..612660d89 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py @@ -101,3 +101,18 @@ def test_conversation_entity_import(filename: str): res = list(NDJsonConverter.deserialize(data)) res = list(NDJsonConverter.serialize(res)) assert res == data + + +def test_benchmark_reference_label_flag(): + label = lb_types.Label(data=lb_types.ConversationData(global_key='my_global_key'), + annotations=[ + lb_types.ClassificationAnnotation( + name='free_text', + message_id="0", + value=lb_types.Text(answer="sample text")) + ], + is_benchmark_reference=True + ) + + res = list(NDJsonConverter.serialize([label])) + assert res[0]["isBenchmarkReferenceLabel"] diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py b/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py index 73099c12f..efc5f5a86 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py @@ -85,3 +85,23 @@ def test_rectangle_mixed_start_end_points(): res = list(NDJsonConverter.deserialize(res)) assert res == [label] + + +def test_benchmark_reference_label_flag(): + bbox = lb_types.ObjectAnnotation( + name="bbox", + value=lb_types.Rectangle( + start=lb_types.Point(x=81, y=28), + end=lb_types.Point(x=38, y=69), + ), + extra={"uuid": "c1be3a57-597e-48cb-8d8d-a852665f9e72"} + ) + + label = lb_types.Label( + data={"uid":DATAROW_ID}, + annotations=[bbox], + is_benchmark_reference=True + ) + + res = list(NDJsonConverter.serialize([label])) + assert res[0]["isBenchmarkReferenceLabel"] From 9f81f3ca67a837a0d0ab66d4a929ea9ee67733fe Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Tue, 30 Jul 2024 12:22:51 +0500 Subject: [PATCH 11/12] MODEL-1489: Fixed metric tests --- .../labelbox/tests/data/annotation_types/test_metrics.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/tests/data/annotation_types/test_metrics.py b/libs/labelbox/tests/data/annotation_types/test_metrics.py index c68324842..db771f806 100644 --- a/libs/labelbox/tests/data/annotation_types/test_metrics.py +++ b/libs/labelbox/tests/data/annotation_types/test_metrics.py @@ -31,7 +31,8 @@ def test_legacy_scalar_metric(): 'extra': {}, }], 'extra': {}, - 'uid': None + 'uid': None, + 'is_benchmark_reference': False } assert label.dict() == expected @@ -92,7 +93,8 @@ def test_custom_scalar_metric(feature_name, subclass_name, aggregation, value): 'extra': {} }], 'extra': {}, - 'uid': None + 'uid': None, + 'is_benchmark_reference': False } assert label.dict() == expected @@ -149,7 +151,8 @@ def test_custom_confusison_matrix_metric(feature_name, subclass_name, 'extra': {} }], 'extra': {}, - 'uid': None + 'uid': None, + 'is_benchmark_reference': False } assert label.dict() == expected From 3d0f526efdce1ff5ad27ca23ca3a66f2c8007d95 Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Tue, 30 Jul 2024 22:07:59 +0500 Subject: [PATCH 12/12] MODEL-1489: Added inverted test cases --- .../serialization/ndjson/test_conversation.py | 17 +++++++++++++- .../serialization/ndjson/test_rectangle.py | 22 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py b/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py index 612660d89..33804ee32 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_conversation.py @@ -103,7 +103,7 @@ def test_conversation_entity_import(filename: str): assert res == data -def test_benchmark_reference_label_flag(): +def test_benchmark_reference_label_flag_enabled(): label = lb_types.Label(data=lb_types.ConversationData(global_key='my_global_key'), annotations=[ lb_types.ClassificationAnnotation( @@ -116,3 +116,18 @@ def test_benchmark_reference_label_flag(): res = list(NDJsonConverter.serialize([label])) assert res[0]["isBenchmarkReferenceLabel"] + + +def test_benchmark_reference_label_flag_disabled(): + label = lb_types.Label(data=lb_types.ConversationData(global_key='my_global_key'), + annotations=[ + lb_types.ClassificationAnnotation( + name='free_text', + message_id="0", + value=lb_types.Text(answer="sample text")) + ], + is_benchmark_reference=False + ) + + res = list(NDJsonConverter.serialize([label])) + assert not res[0].get("isBenchmarkReferenceLabel") diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py b/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py index efc5f5a86..c07dcc66d 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_rectangle.py @@ -87,7 +87,7 @@ def test_rectangle_mixed_start_end_points(): assert res == [label] -def test_benchmark_reference_label_flag(): +def test_benchmark_reference_label_flag_enabled(): bbox = lb_types.ObjectAnnotation( name="bbox", value=lb_types.Rectangle( @@ -105,3 +105,23 @@ def test_benchmark_reference_label_flag(): res = list(NDJsonConverter.serialize([label])) assert res[0]["isBenchmarkReferenceLabel"] + + +def test_benchmark_reference_label_flag_disabled(): + bbox = lb_types.ObjectAnnotation( + name="bbox", + value=lb_types.Rectangle( + start=lb_types.Point(x=81, y=28), + end=lb_types.Point(x=38, y=69), + ), + extra={"uuid": "c1be3a57-597e-48cb-8d8d-a852665f9e72"} + ) + + label = lb_types.Label( + data={"uid":DATAROW_ID}, + annotations=[bbox], + is_benchmark_reference=False + ) + + res = list(NDJsonConverter.serialize([label])) + assert not res[0].get("isBenchmarkReferenceLabel")