diff --git a/docs/labelbox/index.rst b/docs/labelbox/index.rst index 35bcd03a1..35118f56f 100644 --- a/docs/labelbox/index.rst +++ b/docs/labelbox/index.rst @@ -30,6 +30,7 @@ Labelbox Python SDK Documentation labeling-frontend-options labeling-parameter-override labeling-service + labeling-service-dashboard model model-config model-run @@ -42,6 +43,7 @@ Labelbox Python SDK Documentation quality-mode resource-tag review + search-filters send-to-annotate-params slice task diff --git a/docs/labelbox/labeling-service-dashboard.rst b/docs/labelbox/labeling-service-dashboard.rst new file mode 100644 index 000000000..5cf5c8cb7 --- /dev/null +++ b/docs/labelbox/labeling-service-dashboard.rst @@ -0,0 +1,5 @@ +Labeling Service Dashboard +=============================================================================================== + +.. automodule:: labelbox.schema.labeling_service_dashboard + :show-inheritance: diff --git a/docs/labelbox/labeling-service-status.rst b/docs/labelbox/labeling-service-status.rst new file mode 100644 index 000000000..f2998052d --- /dev/null +++ b/docs/labelbox/labeling-service-status.rst @@ -0,0 +1,6 @@ +Labeling Service Status +=============================================================================================== + +.. automodule:: labelbox.schema.labeling_service_status + :members: + :show-inheritance: diff --git a/docs/labelbox/search-filters.rst b/docs/labelbox/search-filters.rst new file mode 100644 index 000000000..01ff0b2c8 --- /dev/null +++ b/docs/labelbox/search-filters.rst @@ -0,0 +1,7 @@ +Search Filters +=============================================================================================== + +.. automodule:: labelbox.schema.search_filters + :members: + :exclude-members: _dict_to_graphql_string + :show-inheritance: diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index dd70baaef..b92ae9b05 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -11,6 +11,7 @@ from datetime import datetime, timezone from typing import Any, List, Dict, Union, Optional, overload, Callable +from labelbox.schema.search_filters import SearchFilter import requests import requests.exceptions from google.api_core import retry @@ -2409,21 +2410,38 @@ def upsert_label_feedback(self, label_id: str, feedback: str, def get_labeling_service_dashboards( self, - after: Optional[str] = None, - search_query: Optional[List[Dict]] = None, + search_query: Optional[List[SearchFilter]] = None, ) -> PaginatedCollection: """ Get all labeling service dashboards for a given org. Optional parameters: - after: The cursor to use for pagination. - where: A filter to apply to the query. - - NOTE: support for after and search_query are not yet implemented. - """ - return LabelingServiceDashboard.get_all(self, - after, - search_query=search_query) + search_query: A list of search filters representing the search + + NOTE: + - Retrieves all projects for the organization or as filtered by the search query. + - Sorted by project created date in ascending order. + + Examples: + Retrieves all labeling service dashboards for a given workspace id: + >>> workspace_filter = WorkspaceFilter( + >>> operation=OperationType.Workspace, + >>> operator=IdOperator.Is, + >>> values=[workspace_id]) + >>> labeling_service_dashboard = [ + >>> ld for ld in project.client.get_labeling_service_dashboards(search_query=[workspace_filter])] + + Retrieves all labeling service dashboards requested less than 7 days ago: + >>> seven_days_ago = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") + >>> workforce_requested_filter_before = WorkforceRequestedDateFilter( + >>> operation=OperationType.WorforceRequestedDate, + >>> value=DateValue(operator=DateOperator.GreaterThanOrEqual, + >>> value=seven_days_ago)) + >>> labeling_service_dashboard = [ld for ld in project.client.get_labeling_service_dashboards(search_query=[workforce_requested_filter_before])] + + See libs/labelbox/src/labelbox/schema/search_filters.py and libs/labelbox/tests/unit/test_unit_search_filters.py for more examples. + """ + return LabelingServiceDashboard.get_all(self, search_query=search_query) def get_task_by_id(self, task_id: str) -> Union[Task, DataUpsertTask]: """ diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index 56f28c865..9621008e3 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -5,6 +5,7 @@ from labelbox.exceptions import ResourceNotFoundError from labelbox.pagination import PaginatedCollection from labelbox.pydantic_compat import BaseModel, root_validator, Field +from labelbox.schema.search_filters import SearchFilter, build_search_filter from labelbox.utils import _CamelCaseMixin from labelbox.schema.labeling_service_status import LabelingServiceStatus @@ -94,28 +95,36 @@ def get(cls, client, project_id: str) -> 'LabelingServiceDashboard': def get_all( cls, client, - after: Optional[str] = None, - search_query: Optional[List[Dict]] = None, + search_query: Optional[List[SearchFilter]] = None, ) -> PaginatedCollection: - template = Template( - """query SearchProjectsPyApi($$first: Int, $$from: String) { - searchProjects(input: {after: $$from, searchQuery: $search_query, size: $$first}) - { - nodes { $labeling_dashboard_selections } - pageInfo { endCursor } + + if search_query is not None: + template = Template( + """query SearchProjectsPyApi($$first: Int, $$from: String) { + searchProjects(input: {after: $$from, searchQuery: $search_query, size: $$first}) + { + nodes { $labeling_dashboard_selections } + pageInfo { endCursor } + } + } + """) + else: + template = Template( + """query SearchProjectsPyApi($$first: Int, $$from: String) { + searchProjects(input: {after: $$from, size: $$first}) + { + nodes { $labeling_dashboard_selections } + pageInfo { endCursor } + } } - } - """) - organization_id = client.get_organization().uid + """) query_str = template.substitute( labeling_dashboard_selections=GRAPHQL_QUERY_SELECTIONS, - search_query= - f"[{{type: \"organization\", operator: \"is\", values: [\"{organization_id}\"]}}]" + search_query=build_search_filter(search_query) + if search_query else None, ) params: Dict[str, Union[str, int]] = {} - if after: - params = {"from": after} def convert_to_labeling_service_dashboard(client, data): data['client'] = client diff --git a/libs/labelbox/src/labelbox/schema/search_filters.py b/libs/labelbox/src/labelbox/schema/search_filters.py new file mode 100644 index 000000000..976071086 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/search_filters.py @@ -0,0 +1,187 @@ +import datetime +from enum import Enum +from typing import List, Literal, Union + +from labelbox.pydantic_compat import BaseModel +from labelbox.utils import format_iso_datetime + + +class BaseSearchFilter(BaseModel): + """ + Shared code for all search filters + """ + + class Config: + use_enum_values = True + + def dict(self, *args, **kwargs): + res = super().dict(*args, **kwargs) + if 'operation' in res: + res['type'] = res.pop('operation') + + # go through all the keys and convert date to string + for key in res: + if isinstance(res[key], datetime.datetime): + res[key] = format_iso_datetime(res[key]) + return res + + +class OperationType(Enum): + """ + Supported search entity types + """ + Organization = 'organization' + Workspace = 'workspace' + Tag = 'tag' + Stage = 'stage' + WorforceRequestedDate = 'workforce_requested_at' + WorkforceStageUpdatedDate = 'workforce_stage_updated_at' + + +class IdOperator(Enum): + """ + Supported operators for ids + """ + Is = 'is' + + +class DateOperator(Enum): + """ + Supported operators for dates + """ + Equals = 'EQUALS' + GreaterThanOrEqual = 'GREATER_THAN_OR_EQUAL' + LessThanOrEqual = 'LESS_THAN_OR_EQUAL' + + +class DateRangeOperator(Enum): + """ + Supported operators for date ranges + """ + Between = 'BETWEEN' + + +class OrganizationFilter(BaseSearchFilter): + """ + Filter for organization + """ + operation: Literal[OperationType.Organization] + operator: IdOperator + values: List[str] + + +class WorkspaceFilter(BaseSearchFilter): + """ + Filter for workspace + """ + operation: Literal[OperationType.Workspace] + operator: IdOperator + values: List[str] + + +class TagFilter(BaseSearchFilter): + """ + Filter for project tags + """ + operation: Literal[OperationType.Tag] + operator: IdOperator + values: List[str] + + +class ProjectStageFilter(BaseSearchFilter): + """ + Filter labelbox service / aka project stages + """ + operation: Literal[OperationType.Stage] + operator: IdOperator + values: List[str] + + +class DateValue(BaseSearchFilter): + """ + Date value for a search filter + + Date formats: + datetime: an existing datetime object + str the following formats are accepted: YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM] + NOTE + if a date / datetime string is passed without a timezone, we will assume the time is UTC and convert it to a local timezone + so for a string '2024-01-01' that is run on a computer in PST, we would convert it to '2024-01-01T08:00:00Z' + while the same string in EST will get converted to '2024-01-01T05:00:00Z' + """ + operator: DateOperator + value: datetime.datetime + + +class WorkforceStageUpdatedFilter(BaseSearchFilter): + """ + Filter for workforce stage updated date + """ + operation: Literal[OperationType.WorkforceStageUpdatedDate] + value: DateValue + + +class WorkforceRequestedDateFilter(BaseSearchFilter): + """ + Filter for workforce requested date + """ + operation: Literal[OperationType.WorforceRequestedDate] + value: DateValue + + +class DateRange(BaseSearchFilter): + """ + Date range for a search filter + """ + min: datetime.datetime + max: datetime.datetime + + +class DateRangeValue(BaseSearchFilter): + """ + Date range value for a search filter + """ + operator: DateRangeOperator + value: DateRange + + +class WorkforceRequestedDateRangeFilter(BaseSearchFilter): + """ + Filter for workforce requested date range + """ + operation: Literal[OperationType.WorforceRequestedDate] + value: DateRangeValue + + +class WorkforceStageUpdatedRangeFilter(BaseSearchFilter): + """ + Filter for workforce stage updated date range + """ + operation: Literal[OperationType.WorkforceStageUpdatedDate] + value: DateRangeValue + + +SearchFilter = Union[OrganizationFilter, WorkspaceFilter, TagFilter, + ProjectStageFilter, WorkforceRequestedDateFilter, + WorkforceStageUpdatedFilter, + WorkforceRequestedDateRangeFilter, + WorkforceStageUpdatedRangeFilter] + + +def _dict_to_graphql_string(d: Union[dict, list, str, int]) -> str: + if isinstance(d, dict): + return "{" + ", ".join( + f'{k}: {_dict_to_graphql_string(v)}' for k, v in d.items()) + "}" + elif isinstance(d, list): + return "[" + ", ".join( + _dict_to_graphql_string(item) for item in d) + "]" + else: + return f'"{d}"' if isinstance(d, str) else str(d) + + +def build_search_filter(filter: List[SearchFilter]): + """ + Converts a list of search filters to a graphql string + """ + filters = [_dict_to_graphql_string(f.dict()) for f in filter] + return "[" + ", ".join(filters) + "]" diff --git a/libs/labelbox/tests/conftest.py b/libs/labelbox/tests/conftest.py index eea9851be..0ee53a97c 100644 --- a/libs/labelbox/tests/conftest.py +++ b/libs/labelbox/tests/conftest.py @@ -1096,3 +1096,21 @@ def embedding(client: Client, environ): @pytest.fixture def valid_model_id(): return "2c903542-d1da-48fd-9db1-8c62571bd3d2" + + +@pytest.fixture +def requested_labeling_service( + rand_gen, live_chat_evaluation_project_with_new_dataset, + chat_evaluation_ontology, model_config): + project = live_chat_evaluation_project_with_new_dataset + project.connect_ontology(chat_evaluation_ontology) + + project.upsert_instructions('tests/integration/media/sample_pdf.pdf') + + labeling_service = project.get_labeling_service() + project.add_model_config(model_config.uid) + project.set_project_model_setup_complete() + + labeling_service.request() + + yield project, labeling_service diff --git a/libs/labelbox/tests/integration/test_labeling_dashboard.py b/libs/labelbox/tests/integration/test_labeling_dashboard.py index a45334bcc..8289be90d 100644 --- a/libs/labelbox/tests/integration/test_labeling_dashboard.py +++ b/libs/labelbox/tests/integration/test_labeling_dashboard.py @@ -1,9 +1,12 @@ +from datetime import datetime, timedelta from labelbox.schema.labeling_service import LabelingServiceStatus +from labelbox.schema.search_filters import DateOperator, DateRange, DateRangeOperator, DateRangeValue, DateValue, IdOperator, OperationType, OrganizationFilter, WorkforceRequestedDateFilter, WorkforceRequestedDateRangeFilter, WorkspaceFilter -def test_request_labeling_service_moe_offline_project( - rand_gen, offline_chat_evaluation_project, chat_evaluation_ontology, - offline_conversational_data_row): +def test_request_labeling_service_dashboard(rand_gen, + offline_chat_evaluation_project, + chat_evaluation_ontology, + offline_conversational_data_row): project = offline_chat_evaluation_project project.connect_ontology(chat_evaluation_ontology) @@ -22,3 +25,60 @@ def test_request_labeling_service_moe_offline_project( assert labeling_service_dashboard.status == LabelingServiceStatus.Missing assert labeling_service_dashboard.tasks_completed == 0 assert labeling_service_dashboard.tasks_remaining == 0 + + +def test_request_labeling_service_dashboard_filters(requested_labeling_service): + project, _ = requested_labeling_service + + organization = project.client.get_organization() + org_filter = OrganizationFilter(operation=OperationType.Organization, + operator=IdOperator.Is, + values=[organization.uid]) + + labeling_service_dashboard = [ + ld for ld in project.client.get_labeling_service_dashboards( + search_query=[org_filter]) + ][0] + assert labeling_service_dashboard is not None + + workforce_requested_filter_before = WorkforceRequestedDateFilter( + operation=OperationType.WorforceRequestedDate, + value=DateValue(operator=DateOperator.GreaterThanOrEqual, + value=datetime.strptime("2024-01-01", "%Y-%m-%d"))) + year_from_now = (datetime.now() + timedelta(days=365)) + workforce_requested_filter_after = WorkforceRequestedDateFilter( + operation=OperationType.WorforceRequestedDate, + value=DateValue(operator=DateOperator.LessThanOrEqual, + value=year_from_now)) + + labeling_service_dashboard = [ + ld + for ld in project.client.get_labeling_service_dashboards(search_query=[ + workforce_requested_filter_after, workforce_requested_filter_before + ]) + ][0] + assert labeling_service_dashboard is not None + + workforce_date_range_filter = WorkforceRequestedDateRangeFilter( + operation=OperationType.WorforceRequestedDate, + value=DateRangeValue(operator=DateRangeOperator.Between, + value=DateRange(min="2024-01-01T00:00:00-0800", + max=year_from_now))) + + labeling_service_dashboard = [ + ld for ld in project.client.get_labeling_service_dashboards( + search_query=[workforce_date_range_filter]) + ][0] + assert labeling_service_dashboard is not None + + # with non existing data + workspace_id = "clzzu4rme000008l42vnl4kre" + workspace_filter = WorkspaceFilter(operation=OperationType.Workspace, + operator=IdOperator.Is, + values=[workspace_id]) + labeling_service_dashboard = [ + ld for ld in project.client.get_labeling_service_dashboards( + search_query=[workspace_filter]) + ] + assert len(labeling_service_dashboard) == 0 + assert labeling_service_dashboard == [] diff --git a/libs/labelbox/tests/integration/test_task.py b/libs/labelbox/tests/integration/test_task.py index 8ab4bd4e3..b0eac2fa1 100644 --- a/libs/labelbox/tests/integration/test_task.py +++ b/libs/labelbox/tests/integration/test_task.py @@ -1,6 +1,4 @@ import json -from labelbox.schema.disconnected_task import DisconnectedTask -import pytest import collections.abc from labelbox import DataRow from labelbox.schema.data_row_metadata import DataRowMetadataField diff --git a/libs/labelbox/tests/unit/test_unit_search_filters.py b/libs/labelbox/tests/unit/test_unit_search_filters.py new file mode 100644 index 000000000..5ea679a24 --- /dev/null +++ b/libs/labelbox/tests/unit/test_unit_search_filters.py @@ -0,0 +1,71 @@ +from datetime import datetime +from labelbox.schema.search_filters import DateOperator, DateRange, DateRangeOperator, DateRangeValue, DateValue, IdOperator, OperationType, OrganizationFilter, ProjectStageFilter, TagFilter, WorkforceRequestedDateFilter, WorkforceRequestedDateRangeFilter, WorkforceStageUpdatedFilter, WorkforceStageUpdatedRangeFilter, WorkspaceFilter, build_search_filter +from labelbox.utils import format_iso_datetime + + +def test_id_filters(): + filters = [ + OrganizationFilter(operation=OperationType.Organization, + operator=IdOperator.Is, + values=["clphb4vd7000cd2wv1ktu5cwa"]), + WorkspaceFilter(operation=OperationType.Workspace, + operator=IdOperator.Is, + values=["clphb4vd7000cd2wv1ktu5cwa"]), + TagFilter(operation=OperationType.Tag, + operator=IdOperator.Is, + values=["tag"]), + ProjectStageFilter(operation=OperationType.Stage, + operator=IdOperator.Is, + values=["requested"]), + ] + + assert build_search_filter( + filters + ) == '[{operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "organization"}, {operator: "is", values: ["clphb4vd7000cd2wv1ktu5cwa"], type: "workspace"}, {operator: "is", values: ["tag"], type: "tag"}, {operator: "is", values: ["requested"], type: "stage"}]' + + +def test_date_filters(): + local_time_start = datetime.strptime("2024-01-01", "%Y-%m-%d") + local_time_end = datetime.strptime("2025-01-01", "%Y-%m-%d") + + filters = [ + WorkforceRequestedDateFilter( + operation=OperationType.WorforceRequestedDate, + value=DateValue(operator=DateOperator.GreaterThanOrEqual, + value=local_time_start)), + WorkforceStageUpdatedFilter( + operation=OperationType.WorkforceStageUpdatedDate, + value=DateValue(operator=DateOperator.LessThanOrEqual, + value=local_time_end)), + ] + expected_start = format_iso_datetime(local_time_start) + expected_end = format_iso_datetime(local_time_end) + + expected = '[{value: {operator: "GREATER_THAN_OR_EQUAL", value: "' + expected_start + '"}, type: "workforce_requested_at"}, {value: {operator: "LESS_THAN_OR_EQUAL", value: "' + expected_end + '"}, type: "workforce_stage_updated_at"}]' + assert build_search_filter(filters) == expected + + +def test_date_range_filters(): + filters = [ + WorkforceRequestedDateRangeFilter( + operation=OperationType.WorforceRequestedDate, + value=DateRangeValue(operator=DateRangeOperator.Between, + value=DateRange(min=datetime.strptime( + "2024-01-01T00:00:00-0800", + "%Y-%m-%dT%H:%M:%S%z"), + max=datetime.strptime( + "2025-01-01T00:00:00-0800", + "%Y-%m-%dT%H:%M:%S%z")))), + WorkforceStageUpdatedRangeFilter( + operation=OperationType.WorkforceStageUpdatedDate, + value=DateRangeValue(operator=DateRangeOperator.Between, + value=DateRange(min=datetime.strptime( + "2024-01-01T00:00:00-0800", + "%Y-%m-%dT%H:%M:%S%z"), + max=datetime.strptime( + "2025-01-01T00:00:00-0800", + "%Y-%m-%dT%H:%M:%S%z")))), + ] + assert build_search_filter( + filters + ) == '[{value: {operator: "BETWEEN", value: {min: "2024-01-01T08:00:00Z", max: "2025-01-01T08:00:00Z"}}, type: "workforce_requested_at"}, {value: {operator: "BETWEEN", value: {min: "2024-01-01T08:00:00Z", max: "2025-01-01T08:00:00Z"}}, type: "workforce_stage_updated_at"}]'