diff --git a/docs/labelbox/datarow_payload_templates.rst b/docs/labelbox/datarow_payload_templates.rst new file mode 100644 index 000000000..34dac6111 --- /dev/null +++ b/docs/labelbox/datarow_payload_templates.rst @@ -0,0 +1,6 @@ +Datarow payload templates +=============================================================================================== + +.. automodule:: labelbox.schema.data_row_payload_templates + :members: + :show-inheritance: \ No newline at end of file diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 02c93850e..d842b6d54 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -7,6 +7,7 @@ import sys import time import urllib.parse +import warnings from collections import defaultdict from datetime import datetime, timezone from types import MappingProxyType @@ -910,11 +911,21 @@ def create_model_evaluation_project( ) -> Project: pass + @overload def create_model_evaluation_project( self, dataset_id: Optional[str] = None, dataset_name: Optional[str] = None, - data_row_count: int = 100, + data_row_count: Optional[int] = None, + **kwargs, + ) -> Project: + pass + + def create_model_evaluation_project( + self, + dataset_id: Optional[str] = None, + dataset_name: Optional[str] = None, + data_row_count: Optional[int] = None, **kwargs, ) -> Project: """ @@ -940,26 +951,38 @@ def create_model_evaluation_project( >>> client.create_model_evaluation_project(name=project_name, dataset_id="clr00u8j0j0j0", data_row_count=10) >>> This creates a new project, and adds 100 datarows to the dataset with id "clr00u8j0j0j0" and assigns a batch of the newly created 10 data rows to the project. + >>> client.create_model_evaluation_project(name=project_name) + >>> This creates a new project with no data rows. """ - if not dataset_id and not dataset_name: - raise ValueError( - "dataset_name or data_set_id must be present and not be an empty string." - ) - if data_row_count <= 0: - raise ValueError("data_row_count must be a positive integer.") + autogenerate_data_rows = False + dataset_name_or_id = None + append_to_existing_dataset = None + + if dataset_id or dataset_name: + autogenerate_data_rows = True if dataset_id: append_to_existing_dataset = True dataset_name_or_id = dataset_id - else: + elif dataset_name: append_to_existing_dataset = False dataset_name_or_id = dataset_name + if autogenerate_data_rows: + kwargs["dataset_name_or_id"] = dataset_name_or_id + kwargs["append_to_existing_dataset"] = append_to_existing_dataset + if data_row_count is None: + data_row_count = 100 + if data_row_count < 0: + raise ValueError("data_row_count must be a positive integer.") + kwargs["data_row_count"] = data_row_count + warnings.warn( + "Automatic generation of data rows of live model evaluation projects is deprecated. dataset_name_or_id, append_to_existing_dataset, data_row_count will be removed in a future version.", + DeprecationWarning, + ) + kwargs["media_type"] = MediaType.Conversational - kwargs["dataset_name_or_id"] = dataset_name_or_id - kwargs["append_to_existing_dataset"] = append_to_existing_dataset - kwargs["data_row_count"] = data_row_count kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value return self._create_project(**kwargs) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/collection.py b/libs/labelbox/src/labelbox/data/annotation_types/collection.py index d90204309..a636a3b3a 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/collection.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/collection.py @@ -1,14 +1,10 @@ import logging -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Callable, Generator, Iterable, Union, Optional -from uuid import uuid4 import warnings +from typing import Callable, Generator, Iterable, Union -from tqdm import tqdm - -from labelbox.schema import ontology from labelbox.orm.model import Entity -from ..ontology import get_classifications, get_tools +from labelbox.schema import ontology + from ..generator import PrefetchGenerator from .label import Label diff --git a/libs/labelbox/src/labelbox/schema/data_row_payload_templates.py b/libs/labelbox/src/labelbox/schema/data_row_payload_templates.py new file mode 100644 index 000000000..bf64e055f --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/data_row_payload_templates.py @@ -0,0 +1,40 @@ +from typing import Dict, List + +from pydantic import BaseModel, Field + +from labelbox.schema.data_row import DataRowMetadataField + + +class ModelEvalutationTemlateRowData(BaseModel): + type: str = Field( + default="application/vnd.labelbox.conversational.model-chat-evaluation", + frozen=True, + ) + draft: bool = Field(default=True, frozen=True) + rootMessageIds: List[str] = Field(default=[]) + actors: Dict = Field(default={}) + version: int = Field(default=2, frozen=True) + messages: Dict = Field(default={}) + + +class ModelEvaluationTemplate(BaseModel): + """ + Use this class to create a model evaluation data row. + + Examples: + >>> data = ModelEvaluationTemplate() + >>> data.row_data.rootMessageIds = ["root1"] + >>> vector = [random.uniform(1.0, 2.0) for _ in range(embedding.dims)] + >>> data.embeddings = [...] + >>> data.metadata_fields = [...] + >>> data.attachments = [...] + >>> content = data.model_dump() + >>> task = dataset.create_data_rows([content]) + """ + + row_data: ModelEvalutationTemlateRowData = Field( + default=ModelEvalutationTemlateRowData() + ) + attachments: List[Dict] = Field(default=[]) + embeddings: List[Dict] = Field(default=[]) + metadata_fields: List[DataRowMetadataField] = Field(default=[]) diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index f8876f7c4..3d5f8ca92 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -1,11 +1,11 @@ import json import logging -from string import Template import time import warnings from collections import namedtuple from datetime import datetime, timezone from pathlib import Path +from string import Template from typing import ( TYPE_CHECKING, Any, @@ -14,28 +14,18 @@ List, Optional, Tuple, - TypeVar, Union, overload, ) from urllib.parse import urlparse -from labelbox.schema.labeling_service import ( - LabelingService, - LabelingServiceStatus, -) -from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard -import requests - -from labelbox import parser from labelbox import utils -from labelbox.exceptions import error_message_for_unparsed_graphql_error from labelbox.exceptions import ( InvalidQueryError, LabelboxError, ProcessingWaitTimeout, - ResourceConflict, ResourceNotFoundError, + error_message_for_unparsed_graphql_error, ) from labelbox.orm import query from labelbox.orm.db_object import DbObject, Deletable, Updateable, experimental @@ -46,7 +36,6 @@ from labelbox.schema.data_row import DataRow from labelbox.schema.export_filters import ( ProjectExportFilters, - validate_datetime, build_filters, ) from labelbox.schema.export_params import ProjectExportParams @@ -54,22 +43,26 @@ from labelbox.schema.id_type import IdType from labelbox.schema.identifiable import DataRowIdentifier, GlobalKey, UniqueId from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds +from labelbox.schema.labeling_service import ( + LabelingService, + LabelingServiceStatus, +) +from labelbox.schema.labeling_service_dashboard import LabelingServiceDashboard from labelbox.schema.media_type import MediaType from labelbox.schema.model_config import ModelConfig -from labelbox.schema.project_model_config import ProjectModelConfig -from labelbox.schema.queue_mode import QueueMode -from labelbox.schema.resource_tag import ResourceTag -from labelbox.schema.task import Task -from labelbox.schema.task_queue import TaskQueue from labelbox.schema.ontology_kind import ( EditorTaskType, - OntologyKind, UploadType, ) +from labelbox.schema.project_model_config import ProjectModelConfig from labelbox.schema.project_overview import ( ProjectOverview, ProjectOverviewDetailed, ) +from labelbox.schema.queue_mode import QueueMode +from labelbox.schema.resource_tag import ResourceTag +from labelbox.schema.task import Task +from labelbox.schema.task_queue import TaskQueue if TYPE_CHECKING: from labelbox import BulkImportRequest @@ -579,7 +572,7 @@ def upsert_instructions(self, instructions_file: str) -> None: if frontend.name != "Editor": logger.warning( - f"This function has only been tested to work with the Editor front end. Found %s", + "This function has only been tested to work with the Editor front end. Found %s", frontend.name, ) @@ -788,7 +781,9 @@ def create_batch( if self.queue_mode != QueueMode.Batch: raise ValueError("Project must be in batch mode") - if self.is_auto_data_generation(): + if ( + self.is_auto_data_generation() and not self.is_chat_evaluation() + ): # NOTE live chat evaluatiuon projects in sdk do not pre-generate data rows, but use batch as all other projects raise ValueError( "Cannot create batches for auto data generation projects" ) @@ -814,7 +809,7 @@ def create_batch( if row_count > 100_000: raise ValueError( - f"Batch exceeds max size, break into smaller batches" + "Batch exceeds max size, break into smaller batches" ) if not row_count: raise ValueError("You need at least one data row in a batch") @@ -1088,8 +1083,7 @@ def _create_batch_async( task = self._wait_for_task(task_id) if task.status != "COMPLETE": raise LabelboxError( - f"Batch was not created successfully: " - + json.dumps(task.errors) + "Batch was not created successfully: " + json.dumps(task.errors) ) return self.client.get_batch(self.uid, batch_id) @@ -1436,7 +1430,7 @@ def update_data_row_labeling_priority( task = self._wait_for_task(task_id) if task.status != "COMPLETE": raise LabelboxError( - f"Priority was not updated successfully: " + "Priority was not updated successfully: " + json.dumps(task.errors) ) return True @@ -1629,7 +1623,7 @@ def move_data_rows_to_task_queue(self, data_row_ids, task_queue_id: str): task = self._wait_for_task(task_id) if task.status != "COMPLETE": raise LabelboxError( - f"Data rows were not moved successfully: " + "Data rows were not moved successfully: " + json.dumps(task.errors) ) diff --git a/libs/labelbox/tests/integration/conftest.py b/libs/labelbox/tests/integration/conftest.py index 10b05681e..e73fef920 100644 --- a/libs/labelbox/tests/integration/conftest.py +++ b/libs/labelbox/tests/integration/conftest.py @@ -646,11 +646,28 @@ def chat_evaluation_ontology(client, rand_gen): @pytest.fixture -def live_chat_evaluation_project_with_new_dataset(client, rand_gen): +def live_chat_evaluation_project(client, rand_gen): project_name = f"test-model-evaluation-project-{rand_gen(str)}" - dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}" - project = client.create_model_evaluation_project( - name=project_name, dataset_name=dataset_name, data_row_count=1 + project = client.create_model_evaluation_project(name=project_name) + + yield project + + project.delete() + + +@pytest.fixture +def live_chat_evaluation_project_with_batch( + client, + rand_gen, + live_chat_evaluation_project, + offline_conversational_data_row, +): + project_name = f"test-model-evaluation-project-{rand_gen(str)}" + project = client.create_model_evaluation_project(name=project_name) + + project.create_batch( + rand_gen(str), + [offline_conversational_data_row.uid], # sample of data row objects ) yield project diff --git a/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py b/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py index 47e39e2cf..2c02b77ac 100644 --- a/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py +++ b/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py @@ -1,15 +1,12 @@ import pytest -from unittest.mock import patch from labelbox import MediaType from labelbox.schema.ontology_kind import OntologyKind -from labelbox.exceptions import MalformedQueryException def test_create_chat_evaluation_ontology_project( - client, chat_evaluation_ontology, - live_chat_evaluation_project_with_new_dataset, + live_chat_evaluation_project, offline_conversational_data_row, rand_gen, ): @@ -28,7 +25,7 @@ def test_create_chat_evaluation_ontology_project( assert classification.schema_id assert classification.feature_schema_id - project = live_chat_evaluation_project_with_new_dataset + project = live_chat_evaluation_project assert project.model_setup_complete is None project.connect_ontology(ontology) @@ -36,28 +33,11 @@ def test_create_chat_evaluation_ontology_project( assert project.labeling_frontend().name == "Editor" assert project.ontology().name == ontology.name - with pytest.raises( - ValueError, - match="Cannot create batches for auto data generation projects", - ): - project.create_batch( - rand_gen(str), - [offline_conversational_data_row.uid], # sample of data row objects - ) - - with pytest.raises( - ValueError, - match="Cannot create batches for auto data generation projects", - ): - with patch( - "labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT", new=0 - ): # force to async - project.create_batch( - rand_gen(str), - [ - offline_conversational_data_row.uid - ], # sample of data row objects - ) + batch = project.create_batch( + rand_gen(str), + [offline_conversational_data_row.uid], # sample of data row objects + ) + assert batch def test_create_chat_evaluation_ontology_project_existing_dataset( diff --git a/libs/labelbox/tests/integration/test_data_rows.py b/libs/labelbox/tests/integration/test_data_rows.py index 9f0429269..baa65db69 100644 --- a/libs/labelbox/tests/integration/test_data_rows.py +++ b/libs/labelbox/tests/integration/test_data_rows.py @@ -405,7 +405,7 @@ def test_create_data_row_with_metadata_dict( row_data=image_url, metadata_fields=make_metadata_fields_dict ) - assert len(list(dataset.data_rows())) == 1 + assert len([dr for dr in dataset.data_rows()]) == 1 assert data_row.dataset() == dataset assert data_row.created_by() == client.get_user() assert data_row.organization() == client.get_organization() diff --git a/libs/labelbox/tests/integration/test_labeling_service.py b/libs/labelbox/tests/integration/test_labeling_service.py index 04a1cb507..03a5694a7 100644 --- a/libs/labelbox/tests/integration/test_labeling_service.py +++ b/libs/labelbox/tests/integration/test_labeling_service.py @@ -42,12 +42,11 @@ def test_request_labeling_service_moe_offline_project( def test_request_labeling_service_moe_project( - rand_gen, - live_chat_evaluation_project_with_new_dataset, + live_chat_evaluation_project_with_batch, chat_evaluation_ontology, model_config, ): - project = live_chat_evaluation_project_with_new_dataset + project = live_chat_evaluation_project_with_batch project.connect_ontology(chat_evaluation_ontology) project.upsert_instructions("tests/integration/media/sample_pdf.pdf") diff --git a/libs/labelbox/tests/integration/test_mmc_data_rows.py b/libs/labelbox/tests/integration/test_mmc_data_rows.py index ee457a7fe..3b4f95530 100644 --- a/libs/labelbox/tests/integration/test_mmc_data_rows.py +++ b/libs/labelbox/tests/integration/test_mmc_data_rows.py @@ -3,26 +3,35 @@ import pytest +from labelbox.schema.data_row_payload_templates import ModelEvaluationTemplate + @pytest.fixture -def mmc_data_row(dataset, make_metadata_fields, embedding): - row_data = { - "type": "application/vnd.labelbox.conversational.model-chat-evaluation", - "draft": True, - "rootMessageIds": ["root1"], - "actors": {}, - "messages": {}, - } +def mmc_data_row(dataset): + data = ModelEvaluationTemplate() + + content_all = data.model_dump() + task = dataset.create_data_rows([content_all]) + task.wait_till_done() + assert task.status == "COMPLETE" + + data_row = list(dataset.data_rows())[0] + + yield data_row + + data_row.delete() + +@pytest.fixture +def mmc_data_row_all(dataset, make_metadata_fields, embedding): + data = ModelEvaluationTemplate() + data.row_data.rootMessageIds = ["root1"] vector = [random.uniform(1.0, 2.0) for _ in range(embedding.dims)] - embeddings = [{"embedding_id": embedding.id, "vector": vector}] + data.embeddings = [{"embedding_id": embedding.id, "vector": vector}] + data.metadata_fields = make_metadata_fields + data.attachments = [{"type": "RAW_TEXT", "value": "attachment value"}] - content_all = { - "row_data": row_data, - "attachments": [{"type": "RAW_TEXT", "value": "attachment value"}], - "metadata_fields": make_metadata_fields, - "embeddings": embeddings, - } + content_all = data.model_dump() task = dataset.create_data_rows([content_all]) task.wait_till_done() assert task.status == "COMPLETE" @@ -34,14 +43,27 @@ def mmc_data_row(dataset, make_metadata_fields, embedding): data_row.delete() -def test_mmc(mmc_data_row, embedding, constants): +def test_mmc(mmc_data_row): data_row = mmc_data_row + assert json.loads(data_row.row_data) == { + "type": "application/vnd.labelbox.conversational.model-chat-evaluation", + "draft": True, + "rootMessageIds": [], + "actors": {}, + "messages": {}, + "version": 2, + } + + +def test_mmc_all(mmc_data_row_all, embedding, constants): + data_row = mmc_data_row_all assert json.loads(data_row.row_data) == { "type": "application/vnd.labelbox.conversational.model-chat-evaluation", "draft": True, "rootMessageIds": ["root1"], "actors": {}, "messages": {}, + "version": 2, } metadata_fields = data_row.metadata_fields diff --git a/libs/labelbox/tests/integration/test_project_model_config.py b/libs/labelbox/tests/integration/test_project_model_config.py index 2d783f62b..975a39afe 100644 --- a/libs/labelbox/tests/integration/test_project_model_config.py +++ b/libs/labelbox/tests/integration/test_project_model_config.py @@ -1,11 +1,10 @@ import pytest + from labelbox.exceptions import ResourceNotFoundError -def test_add_single_model_config( - live_chat_evaluation_project_with_new_dataset, model_config -): - configured_project = live_chat_evaluation_project_with_new_dataset +def test_add_single_model_config(live_chat_evaluation_project, model_config): + configured_project = live_chat_evaluation_project project_model_config_id = configured_project.add_model_config( model_config.uid ) @@ -22,11 +21,11 @@ def test_add_single_model_config( def test_add_multiple_model_config( client, rand_gen, - live_chat_evaluation_project_with_new_dataset, + live_chat_evaluation_project, model_config, valid_model_id, ): - configured_project = live_chat_evaluation_project_with_new_dataset + configured_project = live_chat_evaluation_project second_model_config = client.create_model_config( rand_gen(str), valid_model_id, {"param": "value"} ) @@ -52,9 +51,9 @@ def test_add_multiple_model_config( def test_delete_project_model_config( - live_chat_evaluation_project_with_new_dataset, model_config + live_chat_evaluation_project, model_config ): - configured_project = live_chat_evaluation_project_with_new_dataset + configured_project = live_chat_evaluation_project assert configured_project.delete_project_model_config( configured_project.add_model_config(model_config.uid) ) diff --git a/libs/labelbox/tests/integration/test_project_set_model_setup_complete.py b/libs/labelbox/tests/integration/test_project_set_model_setup_complete.py index 1c3e68c9a..16a124945 100644 --- a/libs/labelbox/tests/integration/test_project_set_model_setup_complete.py +++ b/libs/labelbox/tests/integration/test_project_set_model_setup_complete.py @@ -4,9 +4,9 @@ def test_live_chat_evaluation_project( - live_chat_evaluation_project_with_new_dataset, model_config + live_chat_evaluation_project, model_config ): - project = live_chat_evaluation_project_with_new_dataset + project = live_chat_evaluation_project project.set_project_model_setup_complete() assert bool(project.model_setup_complete) is True @@ -19,9 +19,9 @@ def test_live_chat_evaluation_project( def test_live_chat_evaluation_project_delete_cofig( - live_chat_evaluation_project_with_new_dataset, model_config + live_chat_evaluation_project, model_config ): - project = live_chat_evaluation_project_with_new_dataset + project = live_chat_evaluation_project project_model_config_id = project.add_model_config(model_config.uid) assert project_model_config_id