From 30fe86c827e16f4dede5d75d2c8faddab6e0e5af Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 24 Jul 2024 12:08:52 -0700 Subject: [PATCH 1/4] Add Project get_labeling_service --- docs/labelbox/labeling-service.rst | 6 ++++ .../src/labelbox/schema/labeling_service.py | 29 ++++++++++++++++++ libs/labelbox/src/labelbox/schema/project.py | 30 +++++++++++++++++++ .../integration/test_labeling_service.py | 7 +++++ 4 files changed, 72 insertions(+) create mode 100644 docs/labelbox/labeling-service.rst create mode 100644 libs/labelbox/src/labelbox/schema/labeling_service.py create mode 100644 libs/labelbox/tests/integration/test_labeling_service.py diff --git a/docs/labelbox/labeling-service.rst b/docs/labelbox/labeling-service.rst new file mode 100644 index 000000000..03afb31e9 --- /dev/null +++ b/docs/labelbox/labeling-service.rst @@ -0,0 +1,6 @@ +Project Labeling Service +=============================================================================================== + +.. automodule:: labelbox.schema.labeling_service + :members: + :show-inheritance: diff --git a/libs/labelbox/src/labelbox/schema/labeling_service.py b/libs/labelbox/src/labelbox/schema/labeling_service.py new file mode 100644 index 000000000..e052c64c1 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/labeling_service.py @@ -0,0 +1,29 @@ +from datetime import datetime +from enum import Enum + +from labelbox.data.annotation_types.types import Cuid +from labelbox.orm.db_object import experimental +from labelbox.pydantic_compat import BaseModel +from labelbox.utils import _CamelCaseMixin + + +class LabelingServiceStatus(Enum): + Accepted = 'ACCEPTED', + Calibration = 'CALIBRATION', + Complete = 'COMPLETE', + Production = 'PRODUCTION', + Requested = 'REQUESTED', + SetUp = 'SET_UP' + + +@experimental +class LabelingService(_CamelCaseMixin, BaseModel): + id: Cuid + project_id: Cuid + created_at: datetime + updated_at: datetime + created_by_id: Cuid + status: LabelingServiceStatus + + def status_as_string(self): + return self.status.value diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index a07d8e19c..598222ef5 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, TypeVar, Union, overload from urllib.parse import urlparse +from labelbox.schema.labeling_service import LabelingService import requests from labelbox import parser @@ -1915,6 +1916,35 @@ def clone(self) -> "Project": result = self.client.execute(mutation, {"projectId": self.uid}) return self.client.get_project(result["cloneProject"]["id"]) + @experimental + def get_labeling_service(self) -> LabelingService: + """ + Returns the labeling service associated with the project. + + Returns: + LabelingService: The labeling service associated with the project. + + Raises: + ResourceNotFoundError: If the project does not have a labeling service. + """ + query = """ + query GetProjectBoostWorkforcePyApi($projectId: ID!) { + projectBoostWorkforce(data: { projectId: $projectId }) { + id + projectId + createdAt + updatedAt + createdById + status + } + } + """ + result = self.client.execute(query, {"projectId": self.uid}) + if result["projectBoostWorkforce"] is None: + raise ResourceNotFoundError( + message="The project does not have a labeling service.") + return LabelingService(**result["projectBoostWorkforce"]) + class ProjectMember(DbObject): user = Relationship.ToOne("User", cache=True) diff --git a/libs/labelbox/tests/integration/test_labeling_service.py b/libs/labelbox/tests/integration/test_labeling_service.py new file mode 100644 index 000000000..0cb11eb25 --- /dev/null +++ b/libs/labelbox/tests/integration/test_labeling_service.py @@ -0,0 +1,7 @@ +from labelbox.exceptions import ResourceNotFoundError +import pytest + + +def test_get_labeling_service_throws_exception(project): + with pytest.raises(ResourceNotFoundError): # No labeling service by default + project.get_labeling_service() From 29545c7a342ad21012942249f21e8ac59478a654 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 24 Jul 2024 17:53:15 -0700 Subject: [PATCH 2/4] Init checkin --- .../src/labelbox/schema/labeling_service.py | 30 +++++++++++++++++++ libs/labelbox/src/labelbox/schema/project.py | 1 - 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service.py b/libs/labelbox/src/labelbox/schema/labeling_service.py index e052c64c1..31de8d584 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service.py @@ -1,6 +1,7 @@ from datetime import datetime from enum import Enum +from ..client import Client from labelbox.data.annotation_types.types import Cuid from labelbox.orm.db_object import experimental from labelbox.pydantic_compat import BaseModel @@ -25,5 +26,34 @@ class LabelingService(_CamelCaseMixin, BaseModel): created_by_id: Cuid status: LabelingServiceStatus + @classmethod + def start(cls, client: Client, project_id: Cuid) -> 'LabelingService': + return cls._create(client=client, project_id=project_id) + +""" +mutation CreateProjectBoostWorkforce($projectId: ID!) { + upsertProjectBoostWorkforce(data: { projectId: $projectId }) { + success + __typename + } +} +{ + "projectId": "clz0b7jg901fh07zic3u67b7g" +} + + +{ + "data": { + "upsertProjectBoostWorkforce": { + "success": true, + "__typename": "ProjectBoostWorkforceResult" + } + } +} +""" + @classmethod + def _create(cls, client: Client, project_id: Cuid) -> 'LabelingService': + ... + def status_as_string(self): return self.status.value diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index 598222ef5..54641cae8 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -1945,7 +1945,6 @@ def get_labeling_service(self) -> LabelingService: message="The project does not have a labeling service.") return LabelingService(**result["projectBoostWorkforce"]) - class ProjectMember(DbObject): user = Relationship.ToOne("User", cache=True) role = Relationship.ToOne("Role", cache=True) From b2e2e15b6a247bb35c3584c136b0a62ed6834c14 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 25 Jul 2024 13:44:15 -0700 Subject: [PATCH 3/4] Add LabelingService start() --- libs/labelbox/src/labelbox/__init__.py | 1 + .../src/labelbox/schema/labeling_service.py | 93 ++++++++++++------- libs/labelbox/src/labelbox/schema/project.py | 29 +----- .../integration/test_labeling_service.py | 15 ++- 4 files changed, 80 insertions(+), 58 deletions(-) diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index bcd1f9fee..b1f477c6c 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -43,3 +43,4 @@ from labelbox.schema.identifiable import UniqueId, GlobalKey from labelbox.schema.ontology_kind import OntologyKind from labelbox.schema.project_overview import ProjectOverview, ProjectOverviewDetailed +from labelbox.schema.labeling_service import LabelingService, LabelingServiceStatus diff --git a/libs/labelbox/src/labelbox/schema/labeling_service.py b/libs/labelbox/src/labelbox/schema/labeling_service.py index 31de8d584..eab6b7b72 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service.py @@ -1,9 +1,10 @@ from datetime import datetime from enum import Enum +from typing import Any + +from labelbox.exceptions import ResourceNotFoundError -from ..client import Client from labelbox.data.annotation_types.types import Cuid -from labelbox.orm.db_object import experimental from labelbox.pydantic_compat import BaseModel from labelbox.utils import _CamelCaseMixin @@ -17,43 +18,69 @@ class LabelingServiceStatus(Enum): SetUp = 'SET_UP' -@experimental -class LabelingService(_CamelCaseMixin, BaseModel): +class LabelingService(BaseModel): id: Cuid project_id: Cuid created_at: datetime updated_at: datetime created_by_id: Cuid status: LabelingServiceStatus + client: Any # type Any to avoid circular import from client + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if not self.client.enable_experimental: + raise RuntimeError( + "Please enable experimental in client to use LabelingService") + + class Config(_CamelCaseMixin.Config): + ... @classmethod - def start(cls, client: Client, project_id: Cuid) -> 'LabelingService': - return cls._create(client=client, project_id=project_id) - -""" -mutation CreateProjectBoostWorkforce($projectId: ID!) { - upsertProjectBoostWorkforce(data: { projectId: $projectId }) { - success - __typename - } -} -{ - "projectId": "clz0b7jg901fh07zic3u67b7g" -} - - -{ - "data": { - "upsertProjectBoostWorkforce": { - "success": true, - "__typename": "ProjectBoostWorkforceResult" - } - } -} -""" + def start(cls, client, project_id: Cuid) -> 'LabelingService': + """ + Starts the labeling service for the project. This is equivalent to a UI acction to Request Specialized Labelers + + Returns: + LabelingService: The labeling service for the project. + Raises: + Exception: If the service fails to start. + """ + query_str = """mutation CreateProjectBoostWorkforcePyApi($projectId: ID!) { + upsertProjectBoostWorkforce(data: { projectId: $projectId }) { + success + } + }""" + result = client.execute(query_str, {"projectId": project_id}) + success = result["upsertProjectBoostWorkforce"]["success"] + if not success: + raise Exception("Failed to start labeling service") + return cls.get(client, project_id) + @classmethod - def _create(cls, client: Client, project_id: Cuid) -> 'LabelingService': - ... - - def status_as_string(self): - return self.status.value + def get(cls, client, project_id: Cuid) -> 'LabelingService': + """ + Returns the labeling service associated with the project. + + Raises: + ResourceNotFoundError: If the project does not have a labeling service. + """ + query = """ + query GetProjectBoostWorkforcePyApi($projectId: ID!) { + projectBoostWorkforce(data: { projectId: $projectId }) { + id + projectId + createdAt + updatedAt + createdById + status + } + } + """ + result = client.execute(query, {"projectId": project_id}) + if result["projectBoostWorkforce"] is None: + raise ResourceNotFoundError( + message="The project does not have a labeling service.") + data = result["projectBoostWorkforce"] + data["client"] = client + return LabelingService(**data) diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index 54641cae8..e1b23af12 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -150,7 +150,7 @@ def is_chat_evaluation(self) -> bool: True if this project is a live chat evaluation project, False otherwise """ return self.media_type == MediaType.Conversational and self.editor_task_type == EditorTaskType.ModelChatEvaluation - + def is_prompt_response(self) -> bool: """ Returns: @@ -1918,32 +1918,13 @@ def clone(self) -> "Project": @experimental def get_labeling_service(self) -> LabelingService: - """ - Returns the labeling service associated with the project. + """Get the labeling service for this project. Returns: - LabelingService: The labeling service associated with the project. - - Raises: - ResourceNotFoundError: If the project does not have a labeling service. + LabelingService: The labeling service for this project. """ - query = """ - query GetProjectBoostWorkforcePyApi($projectId: ID!) { - projectBoostWorkforce(data: { projectId: $projectId }) { - id - projectId - createdAt - updatedAt - createdById - status - } - } - """ - result = self.client.execute(query, {"projectId": self.uid}) - if result["projectBoostWorkforce"] is None: - raise ResourceNotFoundError( - message="The project does not have a labeling service.") - return LabelingService(**result["projectBoostWorkforce"]) + return LabelingService.get(self.client, self.uid) # type: ignore + class ProjectMember(DbObject): user = Relationship.ToOne("User", cache=True) diff --git a/libs/labelbox/tests/integration/test_labeling_service.py b/libs/labelbox/tests/integration/test_labeling_service.py index 0cb11eb25..589eca60a 100644 --- a/libs/labelbox/tests/integration/test_labeling_service.py +++ b/libs/labelbox/tests/integration/test_labeling_service.py @@ -1,7 +1,20 @@ -from labelbox.exceptions import ResourceNotFoundError import pytest +from labelbox.exceptions import ResourceNotFoundError +from labelbox.schema.labeling_service import LabelingService, LabelingServiceStatus + def test_get_labeling_service_throws_exception(project): with pytest.raises(ResourceNotFoundError): # No labeling service by default project.get_labeling_service() + + +def test_start_labeling_service(project): + labeling_service = LabelingService.start(project.client, project.uid) + assert labeling_service.status == LabelingServiceStatus.SetUp + assert labeling_service.project_id == project.uid + + # Check that the labeling service is now available + labeling_service = project.get_labeling_service() + assert labeling_service.status == LabelingServiceStatus.SetUp + assert labeling_service.project_id == project.uid From 6307e8dcd7f4ec5ee44ba42047866004bb7b81fe Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Mon, 29 Jul 2024 10:04:06 -0700 Subject: [PATCH 4/4] Add Project request_specialized_labelers --- libs/labelbox/src/labelbox/schema/project.py | 9 +++++++++ libs/labelbox/tests/integration/test_labeling_service.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index e1b23af12..62cf6b282 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -1925,6 +1925,15 @@ def get_labeling_service(self) -> LabelingService: """ return LabelingService.get(self.client, self.uid) # type: ignore + @experimental + def request_labeling_service(self) -> LabelingService: + """Get the labeling service for this project. + + Returns: + LabelingService: The labeling service for this project. + """ + return LabelingService.start(self.client, self.uid) # type: ignore + class ProjectMember(DbObject): user = Relationship.ToOne("User", cache=True) diff --git a/libs/labelbox/tests/integration/test_labeling_service.py b/libs/labelbox/tests/integration/test_labeling_service.py index 589eca60a..ff3b065a8 100644 --- a/libs/labelbox/tests/integration/test_labeling_service.py +++ b/libs/labelbox/tests/integration/test_labeling_service.py @@ -1,7 +1,7 @@ import pytest from labelbox.exceptions import ResourceNotFoundError -from labelbox.schema.labeling_service import LabelingService, LabelingServiceStatus +from labelbox.schema.labeling_service import LabelingServiceStatus def test_get_labeling_service_throws_exception(project): @@ -10,7 +10,7 @@ def test_get_labeling_service_throws_exception(project): def test_start_labeling_service(project): - labeling_service = LabelingService.start(project.client, project.uid) + labeling_service = project.request_labeling_service() assert labeling_service.status == LabelingServiceStatus.SetUp assert labeling_service.project_id == project.uid