From ae9012dbe03b76cc9e4651374ef2a40b11043c63 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 21 Aug 2024 16:06:02 -0700 Subject: [PATCH 1/6] Test improvements --- libs/labelbox/tests/conftest.py | 8 +++--- .../integration/test_labeling_dashboard.py | 25 +++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/libs/labelbox/tests/conftest.py b/libs/labelbox/tests/conftest.py index 0ee53a97c..a69556c04 100644 --- a/libs/labelbox/tests/conftest.py +++ b/libs/labelbox/tests/conftest.py @@ -1099,9 +1099,9 @@ def valid_model_id(): @pytest.fixture -def requested_labeling_service( - rand_gen, live_chat_evaluation_project_with_new_dataset, - chat_evaluation_ontology, model_config): +def requested_labeling_service(rand_gen, + live_chat_evaluation_project_with_new_dataset, + chat_evaluation_ontology, model_config): project = live_chat_evaluation_project_with_new_dataset project.connect_ontology(chat_evaluation_ontology) @@ -1113,4 +1113,4 @@ def requested_labeling_service( labeling_service.request() - yield project, labeling_service + yield project, project.get_labeling_service() diff --git a/libs/labelbox/tests/integration/test_labeling_dashboard.py b/libs/labelbox/tests/integration/test_labeling_dashboard.py index 11cd3ad19..8abfb6722 100644 --- a/libs/labelbox/tests/integration/test_labeling_dashboard.py +++ b/libs/labelbox/tests/integration/test_labeling_dashboard.py @@ -1,21 +1,15 @@ from datetime import datetime, timedelta +from labelbox.exceptions import ResourceNotFoundError from labelbox.schema.labeling_service import LabelingServiceStatus from labelbox.schema.ontology_kind import EditorTaskType from labelbox.schema.media_type import MediaType from labelbox.schema.search_filters import IntegerValue, RangeOperatorWithSingleValue, DateRange, RangeOperatorWithValue, DateRangeValue, DateValue, IdOperator, OperationType, OrganizationFilter, TaskCompletedCountFilter, WorkforceRequestedDateFilter, WorkforceRequestedDateRangeFilter, WorkspaceFilter, TaskRemainingCountFilter +import pytest -def test_request_labeling_service_dashboard(rand_gen, - offline_chat_evaluation_project, - chat_evaluation_ontology, - offline_conversational_data_row): - project = offline_chat_evaluation_project - project.connect_ontology(chat_evaluation_ontology) +def test_request_labeling_service_dashboard(requested_labeling_service): + project, _ = requested_labeling_service - project.create_batch( - rand_gen(str), - [offline_conversational_data_row.uid], # sample of data row objects - ) labeling_service_dashboard = project.labeling_service_dashboard() assert labeling_service_dashboard.status == LabelingServiceStatus.Missing assert labeling_service_dashboard.tasks_completed == 0 @@ -32,6 +26,17 @@ def test_request_labeling_service_dashboard(rand_gen, assert labeling_service_dashboard.tasks_remaining == 0 +def test_request_labeling_service_dashboard_not_started( + offline_chat_evaluation_project): + project = offline_chat_evaluation_project + + with pytest.raises(ResourceNotFoundError) as e: + project.labeling_service_dashboard() + + with pytest.raises(ResourceNotFoundError) as e: + [ld for ld in project.client.get_labeling_service_dashboards()][0] + + def test_request_labeling_service_dashboard_filters(requested_labeling_service): project, _ = requested_labeling_service From 92271f014f3a6448f876486602cb636694ba87fc Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 22 Aug 2024 16:24:09 -0700 Subject: [PATCH 2/6] Make sure labeling data do not fail due to media type issues and improve error message --- .../labelbox/src/labelbox/schema/labeling_service_dashboard.py | 3 ++- libs/labelbox/src/labelbox/schema/media_type.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index b7147e98a..d02ddabbc 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -121,7 +121,8 @@ def get(cls, client, project_id: str) -> 'LabelingServiceDashboard': result = client.execute(query, {"id": project_id}, experimental=True) if result["getProjectById"] is None: raise ResourceNotFoundError( - message="The project does not have a labeling service.") + message="The project does not have a labeling service data yet." + ) data = result["getProjectById"] data["client"] = client return cls(**data) diff --git a/libs/labelbox/src/labelbox/schema/media_type.py b/libs/labelbox/src/labelbox/schema/media_type.py index 266e2a0e3..99807522b 100644 --- a/libs/labelbox/src/labelbox/schema/media_type.py +++ b/libs/labelbox/src/labelbox/schema/media_type.py @@ -54,6 +54,8 @@ def matches(value, name): if matches(value, name): return member + return cls.Unsupported + @classmethod def is_supported(cls, value): return isinstance(value, From 81a22ddc23dcb3c53ba0dda502f0554becc3d982 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 22 Aug 2024 16:27:58 -0700 Subject: [PATCH 3/6] Add more user-friendly labeling dashboard data for print --- .../src/labelbox/schema/labeling_service_dashboard.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index d02ddabbc..dcf0a87b3 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -191,3 +191,11 @@ def convert_boost_data(cls, data): data['created_by_id'] = data.pop('boostRequestedBy') return data + + def __repr__(self): + row = self.dict() + row.pop('client') + row['tasks_completed'] = self.tasks_completed + row['tasks_remaining'] = self.tasks_remaining + row['service_type'] = self.service_type + return row From e00847b6dd4756ef609b3f6116d452831368086b Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 22 Aug 2024 16:32:10 -0700 Subject: [PATCH 4/6] Update organization filter due to api change --- libs/labelbox/src/labelbox/schema/search_filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/schema/search_filters.py b/libs/labelbox/src/labelbox/schema/search_filters.py index 7f3aefd89..e3e7d0e4b 100644 --- a/libs/labelbox/src/labelbox/schema/search_filters.py +++ b/libs/labelbox/src/labelbox/schema/search_filters.py @@ -30,7 +30,7 @@ class OperationType(Enum): """ Supported search entity types """ - Organization = 'organization' + Organization = 'organization_id' Workspace = 'workspace' Tag = 'tag' Stage = 'stage' From 15551ca01b7f265a29c974f0c00c74dd23272e8c Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 22 Aug 2024 16:46:53 -0700 Subject: [PATCH 5/6] Support tags --- .../src/labelbox/schema/labeling_service_dashboard.py | 8 ++++++++ libs/labelbox/tests/unit/test_unit_search_filters.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index dcf0a87b3..9bb96f62f 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -25,9 +25,16 @@ dataRowsDoneCount mediaType editorTaskType + tags """ +class LabelingServiceDashboardTags(BaseModel): + name: str + color: str + type: str + + class LabelingServiceDashboard(BaseModel): """ Represent labeling service data for a project @@ -52,6 +59,7 @@ class LabelingServiceDashboard(BaseModel): data_rows_done_count: int = Field(frozen=True) media_type: Optional[MediaType] = Field(frozen=True, default=None) editor_task_type: EditorTaskType = Field(frozen=True, default=None) + tags: List[LabelingServiceDashboardTags] = Field(frozen=True, default=None) client: Any # type Any to avoid circular import from client diff --git a/libs/labelbox/tests/unit/test_unit_search_filters.py b/libs/labelbox/tests/unit/test_unit_search_filters.py index 90b09ad67..404ba7b34 100644 --- a/libs/labelbox/tests/unit/test_unit_search_filters.py +++ b/libs/labelbox/tests/unit/test_unit_search_filters.py @@ -15,7 +15,7 @@ def test_id_filters(): assert build_search_filter( filters - ) == '[{operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "organization"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "workspace"}, {operator: "is", values: ["tag"], type: "tag"}, {operator: "is", values: ["requested"], type: "stage"}]' + ) == '[{operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "organization_id"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "workspace"}, {operator: "is", values: ["tag"], type: "tag"}, {operator: "is", values: ["requested"], type: "stage"}]' def test_date_filters(): From e2a4290074ff547122b18c74cd9102bf2e405f50 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 22 Aug 2024 20:11:06 -0700 Subject: [PATCH 6/6] Fix tests --- .../schema/labeling_service_dashboard.py | 4 +- .../integration/test_labeling_dashboard.py | 57 ++++++------------- 2 files changed, 18 insertions(+), 43 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index 9bb96f62f..efbd740fb 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -200,8 +200,8 @@ def convert_boost_data(cls, data): return data - def __repr__(self): - row = self.dict() + def dict(self, *args, **kwargs): + row = super().dict(*args, **kwargs) row.pop('client') row['tasks_completed'] = self.tasks_completed row['tasks_remaining'] = self.tasks_remaining diff --git a/libs/labelbox/tests/integration/test_labeling_dashboard.py b/libs/labelbox/tests/integration/test_labeling_dashboard.py index 8abfb6722..803293497 100644 --- a/libs/labelbox/tests/integration/test_labeling_dashboard.py +++ b/libs/labelbox/tests/integration/test_labeling_dashboard.py @@ -4,37 +4,22 @@ from labelbox.schema.ontology_kind import EditorTaskType from labelbox.schema.media_type import MediaType from labelbox.schema.search_filters import IntegerValue, RangeOperatorWithSingleValue, DateRange, RangeOperatorWithValue, DateRangeValue, DateValue, IdOperator, OperationType, OrganizationFilter, TaskCompletedCountFilter, WorkforceRequestedDateFilter, WorkforceRequestedDateRangeFilter, WorkspaceFilter, TaskRemainingCountFilter -import pytest def test_request_labeling_service_dashboard(requested_labeling_service): project, _ = requested_labeling_service labeling_service_dashboard = project.labeling_service_dashboard() - assert labeling_service_dashboard.status == LabelingServiceStatus.Missing + assert labeling_service_dashboard.status == LabelingServiceStatus.Requested assert labeling_service_dashboard.tasks_completed == 0 assert labeling_service_dashboard.tasks_remaining == 0 assert labeling_service_dashboard.media_type == MediaType.Conversational - assert labeling_service_dashboard.editor_task_type == EditorTaskType.OfflineModelChatEvaluation - assert labeling_service_dashboard.service_type == "Offline chat evaluation" + assert labeling_service_dashboard.editor_task_type == EditorTaskType.ModelChatEvaluation + assert labeling_service_dashboard.service_type == "Live chat evaluation" - labeling_service_dashboard = [ - ld for ld in project.client.get_labeling_service_dashboards() - ][0] - assert labeling_service_dashboard.status == LabelingServiceStatus.Missing - assert labeling_service_dashboard.tasks_completed == 0 - assert labeling_service_dashboard.tasks_remaining == 0 - - -def test_request_labeling_service_dashboard_not_started( - offline_chat_evaluation_project): - project = offline_chat_evaluation_project - - with pytest.raises(ResourceNotFoundError) as e: - project.labeling_service_dashboard() - - with pytest.raises(ResourceNotFoundError) as e: - [ld for ld in project.client.get_labeling_service_dashboards()][0] + labeling_service_dashboard = project.client.get_labeling_service_dashboards( + ).get_one() + assert labeling_service_dashboard def test_request_labeling_service_dashboard_filters(requested_labeling_service): @@ -45,10 +30,8 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): operator=IdOperator.Is, values=[organization.uid]) - labeling_service_dashboard = [ - ld for ld in project.client.get_labeling_service_dashboards( - search_query=[org_filter]) - ][0] + labeling_service_dashboard = project.client.get_labeling_service_dashboards( + search_query=[org_filter]).get_one() assert labeling_service_dashboard is not None workforce_requested_filter_before = WorkforceRequestedDateFilter( @@ -62,12 +45,10 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): value=DateValue(operator=RangeOperatorWithSingleValue.LessThanOrEqual, value=year_from_now)) - labeling_service_dashboard = [ - ld - for ld in project.client.get_labeling_service_dashboards(search_query=[ + labeling_service_dashboard = project.client.get_labeling_service_dashboards( + search_query=[ workforce_requested_filter_after, workforce_requested_filter_before - ]) - ][0] + ]).get_one() assert labeling_service_dashboard is not None workforce_date_range_filter = WorkforceRequestedDateRangeFilter( @@ -76,10 +57,8 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): value=DateRange(min="2024-01-01T00:00:00-0800", max=year_from_now))) - labeling_service_dashboard = [ - ld for ld in project.client.get_labeling_service_dashboards( - search_query=[workforce_date_range_filter]) - ][0] + labeling_service_dashboard = project.client.get_labeling_service_dashboards( + search_query=[workforce_date_range_filter]).get_one() assert labeling_service_dashboard is not None # with non existing data @@ -93,9 +72,6 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): ] assert len(labeling_service_dashboard) == 0 assert labeling_service_dashboard == [] - labeling_service_dashboard = project.client.get_labeling_service_dashboards( - ).get_one() - assert labeling_service_dashboard task_done_count_filter = TaskCompletedCountFilter( operation=OperationType.TaskCompletedCount, @@ -106,8 +82,7 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): value=IntegerValue( operator=RangeOperatorWithSingleValue.GreaterThanOrEqual, value=0)) - labeling_service_dashboard = [ - ld for ld in project.client.get_labeling_service_dashboards( - search_query=[task_done_count_filter, task_remaining_count_filter]) - ][0] + labeling_service_dashboard = project.client.get_labeling_service_dashboards( + search_query=[task_done_count_filter, task_remaining_count_filter + ]).get_one() assert labeling_service_dashboard is not None