From 41597e0855e2eeb658ebe61e0e4e44457ece52b1 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 28 Aug 2024 09:38:45 -0700 Subject: [PATCH 1/4] Add ProjectStageFilter validations --- .../labelbox/src/labelbox/schema/search_filters.py | 12 +++++++++++- .../tests/unit/test_unit_search_filters.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/schema/search_filters.py b/libs/labelbox/src/labelbox/schema/search_filters.py index 176405dd1..b4a81acb9 100644 --- a/libs/labelbox/src/labelbox/schema/search_filters.py +++ b/libs/labelbox/src/labelbox/schema/search_filters.py @@ -2,7 +2,7 @@ from enum import Enum from typing import List, Literal, Union -from labelbox.pydantic_compat import BaseModel +from labelbox.pydantic_compat import BaseModel, validator from labelbox.schema.labeling_service_status import LabelingServiceStatus from labelbox.utils import format_iso_datetime @@ -121,6 +121,16 @@ class ProjectStageFilter(BaseSearchFilter): operator: IdOperator values: List[LabelingServiceStatus] + @validator('values', pre=True) + def validate_values(cls, values): + disallowed_values = [LabelingServiceStatus.Missing] + for value in values: + if value in disallowed_values: + raise ValueError( + f"{value} is not a valid value for ProjectStageFilter") + + return values + class DateValue(BaseSearchFilter): """ diff --git a/libs/labelbox/tests/unit/test_unit_search_filters.py b/libs/labelbox/tests/unit/test_unit_search_filters.py index 6977e4af1..9a4b2bac9 100644 --- a/libs/labelbox/tests/unit/test_unit_search_filters.py +++ b/libs/labelbox/tests/unit/test_unit_search_filters.py @@ -2,6 +2,7 @@ from labelbox.schema.labeling_service import LabelingServiceStatus from labelbox.schema.search_filters import IntegerValue, RangeDateTimeOperatorWithSingleValue, RangeOperatorWithSingleValue, DateRange, RangeOperatorWithValue, DateRangeValue, DateValue, IdOperator, OperationType, OrganizationFilter, ProjectStageFilter, SharedWithOrganizationFilter, TagFilter, TaskCompletedCountFilter, TaskRemainingCountFilter, WorkforceRequestedDateFilter, WorkforceRequestedDateRangeFilter, WorkforceStageUpdatedFilter, WorkforceStageUpdatedRangeFilter, WorkspaceFilter, build_search_filter from labelbox.utils import format_iso_datetime +import pytest def test_id_filters(): @@ -22,6 +23,19 @@ def test_id_filters(): ) == '[{operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "organization_id"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "shared_with_organizations"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "workspace"}, {operator: "is", values: ["tag"], type: "tag"}, {operator: "is", values: ["REQUESTED"], type: "stage"}]' +def test_stage_filter_with_invalid_values(): + with pytest.raises( + ValueError, + match="is not a valid value for ProjectStageFilter") as e: + _ = [ + ProjectStageFilter(operator=IdOperator.Is, + values=[ + LabelingServiceStatus.Requested, + LabelingServiceStatus.Missing + ]), + ] + + def test_date_filters(): local_time_start = datetime.strptime("2024-01-01", "%Y-%m-%d") local_time_end = datetime.strptime("2025-01-01", "%Y-%m-%d") From 7f554a9b95b715f4ba6f051ff68ee3b4acd9b8d5 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 28 Aug 2024 10:06:00 -0700 Subject: [PATCH 2/4] 'Unflake' test --- .../integration/test_labeling_dashboard.py | 63 +++++++++---------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/libs/labelbox/tests/integration/test_labeling_dashboard.py b/libs/labelbox/tests/integration/test_labeling_dashboard.py index a464bd19f..c2b0fda43 100644 --- a/libs/labelbox/tests/integration/test_labeling_dashboard.py +++ b/libs/labelbox/tests/integration/test_labeling_dashboard.py @@ -1,28 +1,19 @@ from datetime import datetime, timedelta -from time import sleep -from labelbox.schema.labeling_service import LabelingServiceStatus -from labelbox.schema.ontology_kind import EditorTaskType -from labelbox.schema.media_type import MediaType from labelbox.schema.search_filters import IntegerValue, RangeDateTimeOperatorWithSingleValue, RangeOperatorWithSingleValue, DateRange, RangeOperatorWithValue, DateRangeValue, DateValue, IdOperator, OperationType, OrganizationFilter, TaskCompletedCountFilter, WorkforceRequestedDateFilter, WorkforceRequestedDateRangeFilter, WorkspaceFilter, TaskRemainingCountFilter -ALLOW_TIME_TO_CREATE_DASHBOARD = 5 ## seconds - def test_request_labeling_service_dashboard(requested_labeling_service): project, _ = requested_labeling_service - labeling_service_dashboard = project.get_labeling_service_dashboard() - assert labeling_service_dashboard.status == LabelingServiceStatus.Requested - assert labeling_service_dashboard.tasks_completed_count == 0 - assert labeling_service_dashboard.tasks_remaining_count == 0 - assert labeling_service_dashboard.media_type == MediaType.Conversational - assert labeling_service_dashboard.editor_task_type == EditorTaskType.ModelChatEvaluation - assert labeling_service_dashboard.service_type == "Live chat evaluation" + try: + project.get_labeling_service_dashboard() + except Exception as e: + assert False, f"An exception was raised: {e}" - sleep(ALLOW_TIME_TO_CREATE_DASHBOARD) - labeling_service_dashboard = project.client.get_labeling_service_dashboards( - ).get_one() - assert labeling_service_dashboard + try: + project.client.get_labeling_service_dashboards().get_one() + except Exception as e: + assert False, f"An exception was raised: {e}" def test_request_labeling_service_dashboard_filters(requested_labeling_service): @@ -33,28 +24,30 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): operator=IdOperator.Is, values=[organization.uid]) - sleep(ALLOW_TIME_TO_CREATE_DASHBOARD) - labeling_service_dashboard = project.client.get_labeling_service_dashboards( - search_query=[org_filter]).get_one() - assert labeling_service_dashboard is not None + try: + project.client.get_labeling_service_dashboards( + search_query=[org_filter]).get_one() + except Exception as e: + assert False, f"An exception was raised: {e}" - workforce_requested_filter_before = WorkforceRequestedDateFilter( + workforce_requested_filter_after = WorkforceRequestedDateFilter( operation=OperationType.WorforceRequestedDate, value=DateValue( operator=RangeDateTimeOperatorWithSingleValue.GreaterThanOrEqual, value=datetime.strptime("2024-01-01", "%Y-%m-%d"))) year_from_now = (datetime.now() + timedelta(days=365)) - workforce_requested_filter_after = WorkforceRequestedDateFilter( + workforce_requested_filter_before = WorkforceRequestedDateFilter( operation=OperationType.WorforceRequestedDate, value=DateValue( operator=RangeDateTimeOperatorWithSingleValue.LessThanOrEqual, value=year_from_now)) - labeling_service_dashboard = project.client.get_labeling_service_dashboards( - search_query=[ + try: + project.client.get_labeling_service_dashboards(search_query=[ workforce_requested_filter_after, workforce_requested_filter_before ]).get_one() - assert labeling_service_dashboard is not None + except Exception as e: + assert False, f"An exception was raised: {e}" workforce_date_range_filter = WorkforceRequestedDateRangeFilter( operation=OperationType.WorforceRequestedDate, @@ -62,9 +55,11 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): value=DateRange(min="2024-01-01T00:00:00-0800", max=year_from_now))) - labeling_service_dashboard = project.client.get_labeling_service_dashboards( - search_query=[workforce_date_range_filter]).get_one() - assert labeling_service_dashboard is not None + try: + project.client.get_labeling_service_dashboards( + search_query=[workforce_date_range_filter]).get_one() + except Exception as e: + assert False, f"An exception was raised: {e}" # with non existing data workspace_id = "clzzu4rme000008l42vnl4kre" @@ -87,7 +82,9 @@ def test_request_labeling_service_dashboard_filters(requested_labeling_service): value=IntegerValue( operator=RangeOperatorWithSingleValue.GreaterThanOrEqual, value=0)) - labeling_service_dashboard = project.client.get_labeling_service_dashboards( - search_query=[task_done_count_filter, task_remaining_count_filter - ]).get_one() - assert labeling_service_dashboard is not None + try: + project.client.get_labeling_service_dashboards( + search_query=[task_done_count_filter, task_remaining_count_filter + ]).get_one() + except Exception as e: + assert False, f"An exception was raised: {e}" From 591a3db21fe3c290177c4b6e9951050fd7a21962 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 28 Aug 2024 11:37:50 -0700 Subject: [PATCH 3/4] Update TagFilter to use id --- .../src/labelbox/schema/labeling_service_dashboard.py | 1 + libs/labelbox/src/labelbox/schema/search_filters.py | 2 ++ libs/labelbox/tests/unit/test_unit_search_filters.py | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index 22e3e363f..ae90473c1 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -30,6 +30,7 @@ class LabelingServiceDashboardTags(BaseModel): + id: str text: str color: str type: str diff --git a/libs/labelbox/src/labelbox/schema/search_filters.py b/libs/labelbox/src/labelbox/schema/search_filters.py index b4a81acb9..2badd5c47 100644 --- a/libs/labelbox/src/labelbox/schema/search_filters.py +++ b/libs/labelbox/src/labelbox/schema/search_filters.py @@ -106,6 +106,8 @@ class WorkspaceFilter(BaseSearchFilter): class TagFilter(BaseSearchFilter): """ Filter for project tags + + values are tag ids """ operation: Literal[OperationType.Tag] = OperationType.Tag operator: IdOperator diff --git a/libs/labelbox/tests/unit/test_unit_search_filters.py b/libs/labelbox/tests/unit/test_unit_search_filters.py index 9a4b2bac9..df8c92f1d 100644 --- a/libs/labelbox/tests/unit/test_unit_search_filters.py +++ b/libs/labelbox/tests/unit/test_unit_search_filters.py @@ -13,14 +13,14 @@ def test_id_filters(): values=["clphb4vd7000cd2wv1ktu5cwa"]), WorkspaceFilter(operator=IdOperator.Is, values=["clphb4vd7000cd2wv1ktu5cwa"]), - TagFilter(operator=IdOperator.Is, values=["tag"]), + TagFilter(operator=IdOperator.Is, values=["cls1vkrw401ab072vg2pq3t5d"]), ProjectStageFilter(operator=IdOperator.Is, values=[LabelingServiceStatus.Requested]), ] assert build_search_filter( filters - ) == '[{operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "organization_id"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "shared_with_organizations"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "workspace"}, {operator: "is", values: ["tag"], type: "tag"}, {operator: "is", values: ["REQUESTED"], type: "stage"}]' + ) == '[{operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "organization_id"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "shared_with_organizations"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "workspace"}, {operator: "is", values: ["cls1vkrw401ab072vg2pq3t5d"], type: "tag"}, {operator: "is", values: ["REQUESTED"], type: "stage"}]' def test_stage_filter_with_invalid_values(): From 03522ddc8fc7c3802ed7b93bf55a590af4c740ea Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 28 Aug 2024 16:06:55 -0700 Subject: [PATCH 4/4] Switch from task* fields to dataRows* fields --- .../schema/labeling_service_dashboard.py | 20 +++++++++---------- .../tests/unit/test_unit_search_filters.py | 4 +--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index ae90473c1..b8061b1e4 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -19,10 +19,10 @@ boostUpdatedAt boostRequestedBy boostStatus - tasksCompletedCount - tasksPercentCompleted - tasksRemainingCount - tasksTotalCount + dataRowsCount + dataRowsDoneCount + dataRowsInReviewCount + dataRowsInReworkCount mediaType editorTaskType tags @@ -44,9 +44,8 @@ class LabelingServiceDashboard(BaseModel): id (str): project id name (str): project name status (LabelingServiceStatus): status of the labeling service - tasks_completed_count (int): number of data rows completed - tasks_remaining_count (int): number of data rows that have not started - tasks_total_count (int): total number of data rows in the project + data_rows_count (int): total number of data rows in the project + data_rows_done_count (int): number of data rows completed tags (List[LabelingServiceDashboardTags]): tags associated with the project media_type (MediaType): media type of the project editor_task_type (EditorTaskType): editor task type of the project @@ -58,9 +57,10 @@ class LabelingServiceDashboard(BaseModel): updated_at: Optional[datetime] = Field(frozen=True, default=None) created_by_id: Optional[str] = Field(frozen=True, default=None) status: LabelingServiceStatus = Field(frozen=True, default=None) - tasks_completed_count: int = Field(frozen=True) - tasks_remaining_count: int = Field(frozen=True) - tasks_total_count: int = Field(frozen=True) + data_rows_count: int = Field(frozen=True) + data_rows_done_count: int = Field(frozen=True) + data_rows_in_review_count: int = Field(frozen=True) + data_rows_in_rework_count: int = Field(frozen=True) media_type: Optional[MediaType] = Field(frozen=True, default=None) editor_task_type: EditorTaskType = Field(frozen=True, default=None) tags: List[LabelingServiceDashboardTags] = Field(frozen=True, default=None) diff --git a/libs/labelbox/tests/unit/test_unit_search_filters.py b/libs/labelbox/tests/unit/test_unit_search_filters.py index df8c92f1d..4ad2156f3 100644 --- a/libs/labelbox/tests/unit/test_unit_search_filters.py +++ b/libs/labelbox/tests/unit/test_unit_search_filters.py @@ -27,13 +27,11 @@ def test_stage_filter_with_invalid_values(): with pytest.raises( ValueError, match="is not a valid value for ProjectStageFilter") as e: - _ = [ - ProjectStageFilter(operator=IdOperator.Is, + _ = ProjectStageFilter(operator=IdOperator.Is, values=[ LabelingServiceStatus.Requested, LabelingServiceStatus.Missing ]), - ] def test_date_filters():