From 1c866e328cc8564e03405a778f4d0c00a784a07a Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Wed, 19 Jun 2024 15:12:42 +0200 Subject: [PATCH 01/17] [MODEL-1474] Quality modes as list --- .../project_configuration/project_setup.ipynb | 2 +- .../queue_management.ipynb | 2 +- lbx_prompt.txt | 86 ++++++++++++------- libs/labelbox/src/labelbox/client.py | 20 +++-- 4 files changed, 67 insertions(+), 43 deletions(-) diff --git a/examples/project_configuration/project_setup.ipynb b/examples/project_configuration/project_setup.ipynb index 2733332b1..42d9e9337 100644 --- a/examples/project_configuration/project_setup.ipynb +++ b/examples/project_configuration/project_setup.ipynb @@ -118,7 +118,7 @@ }, { "metadata": {}, - "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_mode=QualityMode.\n Consensus, # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", + "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.\n Consensus], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb index a4125386f..c3cbca3e6 100644 --- a/examples/project_configuration/queue_management.ipynb +++ b/examples/project_configuration/queue_management.ipynb @@ -90,7 +90,7 @@ }, { "metadata": {}, - "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_mode=QualityMode.\n Benchmark, # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", + "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[QualityMode.\n Benchmark], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/lbx_prompt.txt b/lbx_prompt.txt index 19873114d..0ea61acc3 100644 --- a/lbx_prompt.txt +++ b/lbx_prompt.txt @@ -3360,7 +3360,7 @@ def conversation_entity_data_row(client, rand_gen): @pytest.fixture def project(client, rand_gen): project = client.create_project(name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) yield project project.delete() @@ -3370,7 +3370,7 @@ def project(client, rand_gen): def consensus_project(client, rand_gen): project = client.create_project(name=rand_gen(str), quality_mode=QualityMode.Consensus, - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) yield project project.delete() @@ -3560,7 +3560,7 @@ def project_based_user(client, rand_gen): def project_pack(client): projects = [ client.create_project(name=f"user-proj-{idx}", - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) for idx in range(2) ] yield projects @@ -3729,7 +3729,7 @@ def _setup_ontology(project): def configured_project_with_complex_ontology(client, initial_dataset, rand_gen, image_url): project = client.create_project(name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image) dataset = initial_dataset data_row = dataset.create_data_row(row_data=image_url) @@ -6650,7 +6650,7 @@ def test_project_dataset(client, rand_gen): ): client.create_project( name=rand_gen(str), - queue_mode=QueueMode.Dataset, + queue_modes=[QueueMode.Dataset], ) @@ -8348,9 +8348,9 @@ def project_to_test_where(client, rand_gen): p_b_name = f"b-{rand_gen(str)}" p_c_name = f"c-{rand_gen(str)}" - p_a = client.create_project(name=p_a_name, queue_mode=QueueMode.Batch) - p_b = client.create_project(name=p_b_name, queue_mode=QueueMode.Batch) - p_c = client.create_project(name=p_c_name, queue_mode=QueueMode.Batch) + p_a = client.create_project(name=p_a_name, queue_modes=[QueueMode.Batch]) + p_b = client.create_project(name=p_b_name, queue_modes=[QueueMode.Batch]) + p_c = client.create_project(name=p_c_name, queue_modes=[QueueMode.Batch]) yield p_a, p_b, p_c @@ -10053,7 +10053,7 @@ def hardcoded_datarow_id(): def configured_project_with_ontology(client, ontology, rand_gen): project = client.create_project( name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], media_type=MediaType.Image, ) editor = list( @@ -12803,7 +12803,7 @@ def configured_project_with_ontology(client, initial_dataset, ontology, dataset = initial_dataset project = client.create_project( name=rand_gen(str), - queue_mode=QueueMode.Batch, + queue_modes=[QueueMode.Batch], ) editor = list( client.get_labeling_frontends( @@ -12827,7 +12827,7 @@ def configured_project_with_ontology(client, initial_dataset, ontology, def configured_project_without_data_rows(client, ontology, rand_gen): project = client.create_project(name=rand_gen(str), description=rand_gen(str), - queue_mode=QueueMode.Batch) + queue_modes=[QueueMode.Batch]) editor = list( client.get_labeling_frontends( where=LabelingFrontend.name == "editor"))[0] @@ -22430,7 +22430,7 @@ class Client: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use - quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to + quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to Benchmark Returns: A new Project object. @@ -22461,41 +22461,63 @@ class Client: ) media_type = kwargs.get("media_type") - if media_type: - if MediaType.is_supported(media_type): - media_type = media_type.value - else: - raise TypeError(f"{media_type} is not a valid media type. Use" - f" any of {MediaType.get_supported_members()}" - " from MediaType. Example: MediaType.Image.") + if media_type and MediaType.is_supported(media_type): + media_type_value = media_type.value + elif media_type: + raise TypeError(f"{media_type} is not a valid media type. Use" + f" any of {MediaType.get_supported_members()}" + " from MediaType. Example: MediaType.Image.") else: logger.warning( "Creating a project without specifying media_type" " through this method will soon no longer be supported.") + media_type_value = None + + ontology_kind = kwargs.pop("ontology_kind", None) + if ontology_kind and OntologyKind.is_supported(ontology_kind): + editor_task_type_value = EditorTaskTypeMapper.to_editor_task_type( + ontology_kind, media_type).value + elif ontology_kind: + raise OntologyKind.get_ontology_kind_validation_error(ontology_kind) + else: + editor_task_type_value = None - quality_mode = kwargs.get("quality_mode") - if not quality_mode: - logger.info("Defaulting quality mode to Benchmark.") + quality_modes = kwargs.get("quality_modes") + if not quality_modes: + logger.info("Defaulting quality modes to Benchmark.") data = kwargs - data.pop("quality_mode", None) - if quality_mode is None or quality_mode is QualityMode.Benchmark: + data.pop("quality_modes", None) + if quality_modes is None or quality_modes == [QualityMode.Benchmark]: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE - elif quality_mode is QualityMode.Consensus: + data["is_benchmark_enabled"] = True + elif QualityMode.Consensus in quality_modes: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE + data["is_consensus_enabled"] = True else: - raise ValueError(f"{quality_mode} is not a valid quality mode.") + raise ValueError(f"{quality_modes} is not a valid quality modes array. Allowed values are [Benchmark, Consensus]") + + params = {**data} + if media_type_value: + params["media_type"] = media_type_value + if editor_task_type_value: + params["editor_task_type"] = editor_task_type_value + + extra_params = { + Field.String("dataset_name_or_id"): + params.pop("dataset_name_or_id", None), + Field.Boolean("append_to_existing_dataset"): + params.pop("append_to_existing_dataset", None), + Field.Int("data_row_count"): + params.pop("data_row_count", None), + } + extra_params = {k: v for k, v in extra_params.items() if v is not None} - return self._create(Entity.Project, { - **data, - **({ - "media_type": media_type - } if media_type else {}) - }) + return self._create(Entity.Project, params, extra_params) def get_roles(self) -> List[Role]: """ diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index a2fb09186..e1e979075 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -733,8 +733,8 @@ def create_project(self, **kwargs) -> Project: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use - quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to - Benchmark + quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to + Benchmark. Returns: A new Project object. Raises: @@ -785,22 +785,24 @@ def create_project(self, **kwargs) -> Project: else: editor_task_type_value = None - quality_mode = kwargs.get("quality_mode") - if not quality_mode: - logger.info("Defaulting quality mode to Benchmark.") + quality_modes = kwargs.get("quality_modes") + if not quality_modes: + logger.info("Defaulting quality modes to Benchmark.") data = kwargs - data.pop("quality_mode", None) - if quality_mode is None or quality_mode is QualityMode.Benchmark: + data.pop("quality_modes", None) + if quality_modes is None or quality_modes == [QualityMode.Benchmark]: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE - elif quality_mode is QualityMode.Consensus: + data["is_benchmark_enabled"] = True + elif QualityMode.Consensus in quality_modes: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE + data["is_consensus_enabled"] = True else: - raise ValueError(f"{quality_mode} is not a valid quality mode.") + raise ValueError(f"{quality_modes} is not a valid quality modes array. Allowed values are [Benchmark, Consensus]") params = {**data} if media_type_value: From d469a20bd54e850efd9c2e6d1ab96bab55c81ebf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 19 Jun 2024 13:47:34 +0000 Subject: [PATCH 02/17] :art: Cleaned --- examples/project_configuration/project_setup.ipynb | 2 +- examples/project_configuration/queue_management.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/project_configuration/project_setup.ipynb b/examples/project_configuration/project_setup.ipynb index 42d9e9337..1e0a7a478 100644 --- a/examples/project_configuration/project_setup.ipynb +++ b/examples/project_configuration/project_setup.ipynb @@ -118,7 +118,7 @@ }, { "metadata": {}, - "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.\n Consensus], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", + "source": "batch_project = client.create_project(\n name=\"Project Setup Demo\",\n quality_modes=[QualityMode.Consensus\n ], # For benchmarks use quality_mode = QualityMode.Benchmark\n media_type=lb.MediaType.Image,\n)\n\nbatch_project.setup_editor(ontology)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb index c3cbca3e6..30a6e7342 100644 --- a/examples/project_configuration/queue_management.ipynb +++ b/examples/project_configuration/queue_management.ipynb @@ -90,7 +90,7 @@ }, { "metadata": {}, - "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[QualityMode.\n Benchmark], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", + "source": "# Create Labelbox project\n\nproject = client.create_project(\n name=\"batch-test-project\",\n description=\"a description\",\n quality_modes=[\n QualityMode.Benchmark\n ], # For Consensus projects use quality_mode = QualityMode.Consensus\n media_type=lb.MediaType.Image,\n)\n\ndataset = client.create_dataset(name=\"queue_dataset\")", "cell_type": "code", "outputs": [], "execution_count": null From 3336e2b22e4825fbf8102afdd0a5be2115c94351 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 19 Jun 2024 13:48:14 +0000 Subject: [PATCH 03/17] :memo: README updated --- examples/README.md | 180 ++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/examples/README.md b/examples/README.md index 30f615bc2..8aca50612 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,14 +17,9 @@ - Ontologies - Open In Github - Open In Colab - - - Data Rows - Open In Github - Open In Colab + Custom Embeddings + Open In Github + Open In Colab Batches @@ -32,34 +27,39 @@ Open In Colab - Projects - Open In Github - Open In Colab + User Management + Open In Github + Open In Colab - Custom Embeddings - Open In Github - Open In Colab + Basics + Open In Github + Open In Colab Data Row Metadata Open In Github Open In Colab + + Data Rows + Open In Github + Open In Colab + Quick Start Open In Github Open In Colab - Basics - Open In Github - Open In Colab + Ontologies + Open In Github + Open In Colab - User Management - Open In Github - Open In Colab + Projects + Open In Github + Open In Colab @@ -75,16 +75,16 @@ - - Composite Mask Export - Open In Github - Open In Colab - Export Data Open In Github Open In Colab + + Composite Mask Export + Open In Github + Open In Colab + Exporting to CSV Open In Github @@ -105,25 +105,25 @@ - Live Multimodal Chat Project - Open In Github - Open In Colab + Queue Management + Open In Github + Open In Colab Project Setup Open In Github Open In Colab + + Live Multimodal Chat Project + Open In Github + Open In Colab + Webhooks Open In Github Open In Colab - - Queue Management - Open In Github - Open In Colab - @@ -138,6 +138,11 @@ + + Conversational + Open In Github + Open In Colab + Conversational LLM Data Generation Open In Github @@ -154,19 +159,9 @@ Open In Colab - Audio - Open In Github - Open In Colab - - - Conversational - Open In Github - Open In Colab - - - PDF - Open In Github - Open In Colab + DICOM + Open In Github + Open In Colab Image @@ -174,14 +169,14 @@ Open In Colab - DICOM - Open In Github - Open In Colab + Tiled + Open In Github + Open In Colab - Conversational LLM - Open In Github - Open In Colab + Audio + Open In Github + Open In Colab HTML @@ -189,9 +184,14 @@ Open In Colab - Tiled - Open In Github - Open In Colab + Conversational LLM + Open In Github + Open In Colab + + + PDF + Open In Github + Open In Colab @@ -217,16 +217,16 @@ Open In Github Open In Colab - - Meta SAM Video - Open In Github - Open In Colab - Meta SAM Open In Github Open In Colab + + Meta SAM Video + Open In Github + Open In Colab + Import YOLOv8 Annotations Open In Github @@ -247,14 +247,9 @@ - Custom Metrics Demo - Open In Github - Open In Colab - - - Model Slices - Open In Github - Open In Colab + Model Predictions to Project + Open In Github + Open In Colab Custom Metrics Basics @@ -262,9 +257,14 @@ Open In Colab - Model Predictions to Project - Open In Github - Open In Colab + Model Slices + Open In Github + Open In Colab + + + Custom Metrics Demo + Open In Github + Open In Colab @@ -281,19 +281,9 @@ - PDF Predictions - Open In Github - Open In Colab - - - HTML Predictions - Open In Github - Open In Colab - - - Conversational Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab Image Predictions @@ -306,9 +296,14 @@ Open In Colab - Geospatial Predictions - Open In Github - Open In Colab + HTML Predictions + Open In Github + Open In Colab + + + Conversational Predictions + Open In Github + Open In Colab Conversational LLM Predictions @@ -316,9 +311,14 @@ Open In Colab - Video Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab + + + PDF Predictions + Open In Github + Open In Colab From 4fe10b2c733da7054ac006b3cda361fe10186f67 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Thu, 4 Jul 2024 13:04:21 +0200 Subject: [PATCH 04/17] Backwards compatible interface --- libs/labelbox/src/labelbox/client.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index e1e979075..d34fb1107 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -733,6 +733,8 @@ def create_project(self, **kwargs) -> Project: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use + quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to + Benchmark quality_modes (Optional[List[QualityMode]]): The quality modes to use (e.g. Benchmark, Consensus). Defaults to Benchmark. Returns: @@ -786,17 +788,23 @@ def create_project(self, **kwargs) -> Project: editor_task_type_value = None quality_modes = kwargs.get("quality_modes") - if not quality_modes: + quality_mode = kwargs.get("quality_mode") + if quality_modes and quality_mode: + raise ValueError( + "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.") + + if not quality_modes and not quality_mode: logger.info("Defaulting quality modes to Benchmark.") data = kwargs data.pop("quality_modes", None) - if quality_modes is None or quality_modes == [QualityMode.Benchmark]: + data.pop("quality_mode", None) + if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode == QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif QualityMode.Consensus in quality_modes: + elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode == QualityMode.Consensus: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE From 13c3f6a9c05e430b75e77e709f05ed7ca007e8cf Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Thu, 4 Jul 2024 13:06:02 +0200 Subject: [PATCH 05/17] Change comparison --- libs/labelbox/src/labelbox/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index d34fb1107..0b50a6d6d 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -799,12 +799,12 @@ def create_project(self, **kwargs) -> Project: data = kwargs data.pop("quality_modes", None) data.pop("quality_mode", None) - if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode == QualityMode.Benchmark: + if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode == QualityMode.Consensus: + elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode is QualityMode.Consensus: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE From 79e9c763618b74544b2c1688bef03ec8200917c9 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Thu, 4 Jul 2024 13:09:54 +0200 Subject: [PATCH 06/17] Develop fix --- libs/labelbox/src/labelbox/client.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 6522a883c..6ebcff9fd 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -893,15 +893,6 @@ def _create_project(self, **kwargs) -> Project: " through this method will soon no longer be supported.") media_type_value = None - ontology_kind = kwargs.pop("ontology_kind", None) - if ontology_kind and OntologyKind.is_supported(ontology_kind): - editor_task_type_value = EditorTaskTypeMapper.to_editor_task_type( - ontology_kind, media_type).value - elif ontology_kind: - raise OntologyKind.get_ontology_kind_validation_error(ontology_kind) - else: - editor_task_type_value = None - quality_modes = kwargs.get("quality_modes") quality_mode = kwargs.get("quality_mode") if quality_modes and quality_mode: @@ -930,16 +921,12 @@ def _create_project(self, **kwargs) -> Project: params = {**data} if media_type_value: params["media_type"] = media_type_value - if editor_task_type_value: - params["editor_task_type"] = editor_task_type_value extra_params = { Field.String("dataset_name_or_id"): params.pop("dataset_name_or_id", None), Field.Boolean("append_to_existing_dataset"): params.pop("append_to_existing_dataset", None), - Field.Int("data_row_count"): - params.pop("data_row_count", None), } extra_params = {k: v for k, v in extra_params.items() if v is not None} From 0d714b5fda9cd97bbef1117507c688c26fffca23 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Fri, 5 Jul 2024 15:51:35 +0200 Subject: [PATCH 07/17] CR --- libs/labelbox/src/labelbox/client.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 6ebcff9fd..7aba023f5 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -863,7 +863,7 @@ def _create_project(self, **kwargs) -> Project: auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels") if auto_audit_percentage is not None or auto_audit_number_of_labels is not None: raise ValueError( - "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels." + "quality_modes must be set instead of auto_audit_percentage or auto_audit_number_of_labels." ) name = kwargs.get("name") @@ -895,6 +895,11 @@ def _create_project(self, **kwargs) -> Project: quality_modes = kwargs.get("quality_modes") quality_mode = kwargs.get("quality_mode") + if quality_mode: + logger.warning( + "Passing quality_mode is deprecated and will soon no longer be supported. Use quality_modes instead." + ) + if quality_modes and quality_mode: raise ValueError( "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.") @@ -905,7 +910,7 @@ def _create_project(self, **kwargs) -> Project: data = kwargs data.pop("quality_modes", None) data.pop("quality_mode", None) - if quality_modes is None or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: + if quality_modes is None or len(quality_modes) == 0 or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE From 4ccdf2b196c5bf271030d5add5429cbc809d9f75 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 8 Jul 2024 18:53:49 +0000 Subject: [PATCH 08/17] :memo: README updated --- examples/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/README.md b/examples/README.md index 9ca7ce6a0..38452d9d5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -119,11 +119,6 @@ Open In Github Open In Colab - - Live Multimodal Chat Project - Open In Github - Open In Colab - Webhooks Open In Github From 232e7bf15a07196d32aaeb46dfa4331b25cf7c01 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Mon, 22 Jul 2024 12:48:48 +0200 Subject: [PATCH 09/17] B&C default --- libs/labelbox/src/labelbox/client.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 7aba023f5..2934f40ae 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -905,17 +905,22 @@ def _create_project(self, **kwargs) -> Project: "Cannot use both quality_modes and quality_mode at the same time. Use one or the other.") if not quality_modes and not quality_mode: - logger.info("Defaulting quality modes to Benchmark.") + logger.info("Defaulting quality modes to Benchmark and Consensus.") data = kwargs data.pop("quality_modes", None) data.pop("quality_mode", None) - if quality_modes is None or len(quality_modes) == 0 or quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: + if quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode is QualityMode.Consensus: + elif ( + quality_modes is None + or len(quality_modes) == 0 + or QualityMode.Consensus in (quality_modes if quality_modes else []) + or quality_mode is QualityMode.Consensus + ): data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE From 264d1602f32a5d66b66830d1ae5e798fd286970d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 22 Jul 2024 10:53:32 +0000 Subject: [PATCH 10/17] :memo: README updated --- examples/README.md | 148 ++++++++++++++++++++++----------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/examples/README.md b/examples/README.md index 38452d9d5..387840707 100644 --- a/examples/README.md +++ b/examples/README.md @@ -27,40 +27,40 @@ Open In Colab - User Management - Open In Github - Open In Colab + Ontologies + Open In Github + Open In Colab Basics Open In Github Open In Colab - - Data Row Metadata - Open In Github - Open In Colab - - - Data Rows - Open In Github - Open In Colab - Quick Start Open In Github Open In Colab - Ontologies - Open In Github - Open In Colab + Data Row Metadata + Open In Github + Open In Colab Projects Open In Github Open In Colab + + Data Rows + Open In Github + Open In Colab + + + User Management + Open In Github + Open In Colab + @@ -75,16 +75,16 @@ - - Export Data - Open In Github - Open In Colab - Composite Mask Export Open In Github Open In Colab + + Export Data + Open In Github + Open In Colab + Exporting to CSV Open In Github @@ -104,6 +104,11 @@ + + Project Setup + Open In Github + Open In Colab + Queue Management Open In Github @@ -114,11 +119,6 @@ Open In Github Open In Colab - - Project Setup - Open In Github - Open In Colab - Webhooks Open In Github @@ -138,16 +138,16 @@ + + PDF + Open In Github + Open In Colab + Conversational Open In Github Open In Colab - - Conversational LLM Data Generation - Open In Github - Open In Colab - Video Open In Github @@ -163,15 +163,25 @@ Open In Github Open In Colab + + Tiled + Open In Github + Open In Colab + Image Open In Github Open In Colab - Tiled - Open In Github - Open In Colab + Conversational LLM Data Generation + Open In Github + Open In Colab + + + Conversational LLM + Open In Github + Open In Colab Audio @@ -183,16 +193,6 @@ Open In Github Open In Colab - - Conversational LLM - Open In Github - Open In Colab - - - PDF - Open In Github - Open In Colab - @@ -213,9 +213,9 @@ Open In Colab - Langchain - Open In Github - Open In Colab + Import YOLOv8 Annotations + Open In Github + Open In Colab Meta SAM @@ -228,9 +228,9 @@ Open In Colab - Import YOLOv8 Annotations - Open In Github - Open In Colab + Langchain + Open In Github + Open In Colab @@ -252,9 +252,9 @@ Open In Colab - Custom Metrics Basics - Open In Github - Open In Colab + Custom Metrics Demo + Open In Github + Open In Colab Model Slices @@ -262,9 +262,9 @@ Open In Colab - Custom Metrics Demo - Open In Github - Open In Colab + Custom Metrics Basics + Open In Github + Open In Colab @@ -281,24 +281,24 @@ - Video Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab - Image Predictions - Open In Github - Open In Colab + HTML Predictions + Open In Github + Open In Colab - Text Predictions - Open In Github - Open In Colab + Conversational LLM Predictions + Open In Github + Open In Colab - HTML Predictions - Open In Github - Open In Colab + Image Predictions + Open In Github + Open In Colab Conversational Predictions @@ -306,14 +306,14 @@ Open In Colab - Conversational LLM Predictions - Open In Github - Open In Colab + Text Predictions + Open In Github + Open In Colab - Geospatial Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab PDF Predictions From a19e4185ddfb66d048cc8e36ec1f78436b2b0fd3 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Mon, 22 Jul 2024 12:55:58 +0200 Subject: [PATCH 11/17] B&C default --- libs/labelbox/src/labelbox/client.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 2934f40ae..d2ec9a62b 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -910,15 +910,23 @@ def _create_project(self, **kwargs) -> Project: data = kwargs data.pop("quality_modes", None) data.pop("quality_mode", None) - if quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: + + if ( + quality_modes is None + or len(quality_modes) == 0 + or quality_modes == [QualityMode.Benchmark, QualityMode.Consensus] + ): + data["auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS + data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE + data["is_benchmark_enabled"] = True + data["is_consensus_enabled"] = True + elif quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True elif ( - quality_modes is None - or len(quality_modes) == 0 - or QualityMode.Consensus in (quality_modes if quality_modes else []) + QualityMode.Consensus in (quality_modes if quality_modes else []) or quality_mode is QualityMode.Consensus ): data[ From 32684f8b0ce7ee024f1d1064b53e8e37c1e4f279 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Mon, 22 Jul 2024 12:57:10 +0200 Subject: [PATCH 12/17] B&C default --- libs/labelbox/src/labelbox/client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index d2ec9a62b..3574634c5 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -925,10 +925,7 @@ def _create_project(self, **kwargs) -> Project: "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif ( - QualityMode.Consensus in (quality_modes if quality_modes else []) - or quality_mode is QualityMode.Consensus - ): + elif quality_modes == [QualityMode.Consensus] or quality_mode is QualityMode.Consensus: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE From e1bf0a9f3eb4858ddb8293110f265be6e17adf55 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Tue, 23 Jul 2024 19:36:41 +0200 Subject: [PATCH 13/17] Use set --- libs/labelbox/src/labelbox/client.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 3574634c5..f84dd1cfa 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -911,27 +911,36 @@ def _create_project(self, **kwargs) -> Project: data.pop("quality_modes", None) data.pop("quality_mode", None) + # check if quality_modes is a set, if not, convert to set + quality_modes_set = quality_modes + if quality_modes and not isinstance(quality_modes, set): + quality_modes_set = set(quality_modes) + if quality_mode: + quality_modes_set = {quality_mode} + if ( - quality_modes is None - or len(quality_modes) == 0 - or quality_modes == [QualityMode.Benchmark, QualityMode.Consensus] + quality_modes_set is None + or len(quality_modes_set) == 0 + or quality_modes_set == {QualityMode.Benchmark, QualityMode.Consensus} ): data["auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True data["is_consensus_enabled"] = True - elif quality_modes == [QualityMode.Benchmark] or quality_mode is QualityMode.Benchmark: + elif quality_modes_set == {QualityMode.Benchmark}: data[ "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE data["is_benchmark_enabled"] = True - elif quality_modes == [QualityMode.Consensus] or quality_mode is QualityMode.Consensus: + elif quality_modes_set == {QualityMode.Consensus}: data[ "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE data["is_consensus_enabled"] = True else: - raise ValueError(f"{quality_modes} is not a valid quality modes array. Allowed values are [Benchmark, Consensus]") + raise ValueError( + f"{quality_modes_set} is not a valid quality modes set. Allowed values are [Benchmark, Consensus]" + ) params = {**data} if media_type_value: From 8776f1cd2586c9a3a649405fa3b086f5fe24ad72 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 23 Jul 2024 17:38:34 +0000 Subject: [PATCH 14/17] :memo: README updated --- examples/README.md | 182 ++++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/examples/README.md b/examples/README.md index 387840707..769876f92 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,49 +17,49 @@ - Custom Embeddings - Open In Github - Open In Colab + Basics + Open In Github + Open In Colab - Batches - Open In Github - Open In Colab + Projects + Open In Github + Open In Colab Ontologies Open In Github Open In Colab - - Basics - Open In Github - Open In Colab - Quick Start Open In Github Open In Colab + + Data Rows + Open In Github + Open In Colab + Data Row Metadata Open In Github Open In Colab - Projects - Open In Github - Open In Colab + User Management + Open In Github + Open In Colab - Data Rows - Open In Github - Open In Colab + Batches + Open In Github + Open In Colab - User Management - Open In Github - Open In Colab + Custom Embeddings + Open In Github + Open In Colab @@ -76,9 +76,9 @@ - Composite Mask Export - Open In Github - Open In Colab + Exporting to CSV + Open In Github + Open In Colab Export Data @@ -86,9 +86,9 @@ Open In Colab - Exporting to CSV - Open In Github - Open In Colab + Composite Mask Export + Open In Github + Open In Colab @@ -114,16 +114,16 @@ Open In Github Open In Colab - - Multimodal Chat Project - Open In Github - Open In Colab - Webhooks Open In Github Open In Colab + + Multimodal Chat Project + Open In Github + Open In Colab + @@ -139,34 +139,24 @@ - PDF - Open In Github - Open In Colab - - - Conversational - Open In Github - Open In Colab - - - Video - Open In Github - Open In Colab + Tiled + Open In Github + Open In Colab - Text - Open In Github - Open In Colab + Conversational LLM + Open In Github + Open In Colab - DICOM - Open In Github - Open In Colab + HTML + Open In Github + Open In Colab - Tiled - Open In Github - Open In Colab + Conversational LLM Data Generation + Open In Github + Open In Colab Image @@ -174,14 +164,19 @@ Open In Colab - Conversational LLM Data Generation - Open In Github - Open In Colab + PDF + Open In Github + Open In Colab - Conversational LLM - Open In Github - Open In Colab + DICOM + Open In Github + Open In Colab + + + Text + Open In Github + Open In Colab Audio @@ -189,9 +184,14 @@ Open In Colab - HTML - Open In Github - Open In Colab + Conversational + Open In Github + Open In Colab + + + Video + Open In Github + Open In Colab @@ -207,31 +207,31 @@ - - Huggingface Custom Embeddings - Open In Github - Open In Colab - Import YOLOv8 Annotations Open In Github Open In Colab - - Meta SAM - Open In Github - Open In Colab - Meta SAM Video Open In Github Open In Colab + + Meta SAM + Open In Github + Open In Colab + Langchain Open In Github Open In Colab + + Huggingface Custom Embeddings + Open In Github + Open In Colab + @@ -246,11 +246,6 @@ - - Model Predictions to Project - Open In Github - Open In Colab - Custom Metrics Demo Open In Github @@ -266,6 +261,11 @@ Open In Github Open In Colab + + Model Predictions to Project + Open In Github + Open In Colab + @@ -281,9 +281,9 @@ - Geospatial Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab HTML Predictions @@ -291,14 +291,9 @@ Open In Colab - Conversational LLM Predictions - Open In Github - Open In Colab - - - Image Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab Conversational Predictions @@ -311,15 +306,20 @@ Open In Colab - Video Predictions - Open In Github - Open In Colab + Conversational LLM Predictions + Open In Github + Open In Colab PDF Predictions Open In Github Open In Colab + + Image Predictions + Open In Github + Open In Colab + From 9c1e8a660cd1c8c24763436111a05b03b6d3ce10 Mon Sep 17 00:00:00 2001 From: kozikkamil Date: Tue, 23 Jul 2024 19:39:38 +0200 Subject: [PATCH 15/17] Fix test --- libs/labelbox/tests/integration/test_project.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index 07a512836..07e10ac02 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -257,17 +257,17 @@ def test_media_type(client, project: Project, rand_gen): def test_queue_mode(client, rand_gen): - project = client.create_project(name=rand_gen(str)) # defaults to benchmark - assert project.auto_audit_number_of_labels == 1 - assert project.auto_audit_percentage == 1 + project = client.create_project(name=rand_gen(str)) # defaults to benchmark and consensus + assert project.auto_audit_number_of_labels == 3 + assert project.auto_audit_percentage == 0 - project = client.create_project(name=rand_gen(str), - quality_mode=QualityMode.Benchmark) + project = client.create_project(name=rand_gen(str), quality_modes=[QualityMode.Benchmark]) assert project.auto_audit_number_of_labels == 1 assert project.auto_audit_percentage == 1 - project = client.create_project(name=rand_gen(str), - quality_mode=QualityMode.Consensus) + project = client.create_project( + name=rand_gen(str), quality_modes=[QualityMode.Benchmark, QualityMode.Consensus] + ) assert project.auto_audit_number_of_labels == 3 assert project.auto_audit_percentage == 0 @@ -295,4 +295,4 @@ def test_clone(client, project, rand_gen): assert cloned_project.get_label_count() == 0 project.delete() - cloned_project.delete() \ No newline at end of file + cloned_project.delete() From 340bdf4b0e41074a9cd45babe11638889edbfba8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 23 Jul 2024 17:43:02 +0000 Subject: [PATCH 16/17] :memo: README updated --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 1912b2eba..1523e1737 100644 --- a/examples/README.md +++ b/examples/README.md @@ -345,4 +345,4 @@ Open In Colab - \ No newline at end of file + From 16a7390eeb506827f451811c7194631b3583f524 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 24 Jul 2024 17:03:29 +0000 Subject: [PATCH 17/17] :memo: README updated --- examples/README.md | 186 ++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/examples/README.md b/examples/README.md index 1523e1737..1bc102947 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,49 +17,49 @@ - Custom Embeddings - Open In Github - Open In Colab + Basics + Open In Github + Open In Colab - Batches - Open In Github - Open In Colab + Projects + Open In Github + Open In Colab Ontologies Open In Github Open In Colab - - Basics - Open In Github - Open In Colab - Quick Start Open In Github Open In Colab + + Data Rows + Open In Github + Open In Colab + Data Row Metadata Open In Github Open In Colab - Projects - Open In Github - Open In Colab + User Management + Open In Github + Open In Colab - Data Rows - Open In Github - Open In Colab + Batches + Open In Github + Open In Colab - User Management - Open In Github - Open In Colab + Custom Embeddings + Open In Github + Open In Colab @@ -76,25 +76,25 @@ - Composite Mask Export - Open In Github - Open In Colab + Exporting to CSV + Open In Github + Open In Colab Export Data Open In Github Open In Colab - - Exporting to CSV - Open In Github - Open In Colab - Export V1 to V2 Migration Support Open In Github Open In Colab + + Composite Mask Export + Open In Github + Open In Colab + @@ -119,16 +119,16 @@ Open In Github Open In Colab - - Multimodal Chat Project - Open In Github - Open In Colab - Webhooks Open In Github Open In Colab + + Multimodal Chat Project + Open In Github + Open In Colab + @@ -144,34 +144,24 @@ - PDF - Open In Github - Open In Colab - - - Conversational - Open In Github - Open In Colab - - - Video - Open In Github - Open In Colab + Tiled + Open In Github + Open In Colab - Text - Open In Github - Open In Colab + Conversational LLM + Open In Github + Open In Colab - DICOM - Open In Github - Open In Colab + HTML + Open In Github + Open In Colab - Tiled - Open In Github - Open In Colab + Conversational LLM Data Generation + Open In Github + Open In Colab Image @@ -179,14 +169,19 @@ Open In Colab - Conversational LLM Data Generation - Open In Github - Open In Colab + PDF + Open In Github + Open In Colab - Conversational LLM - Open In Github - Open In Colab + DICOM + Open In Github + Open In Colab + + + Text + Open In Github + Open In Colab Audio @@ -194,9 +189,14 @@ Open In Colab - HTML - Open In Github - Open In Colab + Conversational + Open In Github + Open In Colab + + + Video + Open In Github + Open In Colab @@ -212,31 +212,31 @@ - - Huggingface Custom Embeddings - Open In Github - Open In Colab - Import YOLOv8 Annotations Open In Github Open In Colab - - Meta SAM - Open In Github - Open In Colab - Meta SAM Video Open In Github Open In Colab + + Meta SAM + Open In Github + Open In Colab + Langchain Open In Github Open In Colab + + Huggingface Custom Embeddings + Open In Github + Open In Colab + @@ -251,11 +251,6 @@ - - Model Predictions to Project - Open In Github - Open In Colab - Custom Metrics Demo Open In Github @@ -271,6 +266,11 @@ Open In Github Open In Colab + + Model Predictions to Project + Open In Github + Open In Colab + @@ -286,9 +286,9 @@ - Geospatial Predictions - Open In Github - Open In Colab + Video Predictions + Open In Github + Open In Colab HTML Predictions @@ -296,14 +296,9 @@ Open In Colab - Conversational LLM Predictions - Open In Github - Open In Colab - - - Image Predictions - Open In Github - Open In Colab + Geospatial Predictions + Open In Github + Open In Colab Conversational Predictions @@ -316,15 +311,20 @@ Open In Colab - Video Predictions - Open In Github - Open In Colab + Conversational LLM Predictions + Open In Github + Open In Colab PDF Predictions Open In Github Open In Colab + + Image Predictions + Open In Github + Open In Colab +