From 8d5b97c8d0b734e2bd4a21d4b82abfc7508d8c3f Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Thu, 20 Jun 2024 17:46:35 +0500 Subject: [PATCH 1/6] MODEL-1448: Upsert label feedback method --- libs/labelbox/src/labelbox/__init__.py | 1 + libs/labelbox/src/labelbox/client.py | 24 ++++++++++++++++ libs/labelbox/src/labelbox/orm/model.py | 1 + .../src/labelbox/schema/label_score.py | 19 +++++++++++++ libs/labelbox/tests/integration/test_label.py | 28 ++++++++++++++----- 5 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 libs/labelbox/src/labelbox/schema/label_score.py diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index c163cde90..4ebebe441 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -36,6 +36,7 @@ from labelbox.schema.slice import Slice, CatalogSlice, ModelSlice from labelbox.schema.queue_mode import QueueMode from labelbox.schema.task_queue import TaskQueue +from labelbox.schema.label_score import LabelScore from labelbox.schema.identifiables import UniqueIds, GlobalKeys, DataRowIds from labelbox.schema.identifiable import UniqueId, GlobalKey from labelbox.schema.ontology_kind import OntologyKind diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index f0abd2bb5..be21228ec 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2216,3 +2216,27 @@ def get_embedding_by_name(self, name: str) -> Embedding: return e raise labelbox.exceptions.ResourceNotFoundError(Embedding, dict(name=name)) + + def upsert_label_feedback(self, label_id: str, feedback: str, + scores: Dict[str, float]) -> Entity.Label: + """ + + Args: + label_id: Target label ID + feedback: Free text comment regarding the label + scores: A dict of scores, the key is a score name and the value is the score value + Returns: A list of LabelScore instances + + """ + mutation_str = """mutation UpsertAutoQaLabelFeedbackPyApi($labelId: ID!, $feedback: String!, $scores: Json!){ + upsertAutoQaLabelFeedback(input: {labelId: $labelId, feedback: $feedback, scores: $scores}) { id scores {id name score} } + } + """ + res = self.execute(mutation_str, { + "labelId": label_id, + "feedback": feedback, + "scores": scores + }) + scores_raw = res["upsertAutoQaLabelFeedback"]["scores"] + + return [Entity.LabelScore(self, x) for x in scores_raw] diff --git a/libs/labelbox/src/labelbox/orm/model.py b/libs/labelbox/src/labelbox/orm/model.py index f3afa174e..dcca28fb7 100644 --- a/libs/labelbox/src/labelbox/orm/model.py +++ b/libs/labelbox/src/labelbox/orm/model.py @@ -382,6 +382,7 @@ class Entity(metaclass=EntityMeta): CatalogSlice: Type[labelbox.CatalogSlice] ModelSlice: Type[labelbox.ModelSlice] TaskQueue: Type[labelbox.TaskQueue] + LabelScore: Type[labelbox.LabelScore] @classmethod def _attributes_of_type(cls, attr_type): diff --git a/libs/labelbox/src/labelbox/schema/label_score.py b/libs/labelbox/src/labelbox/schema/label_score.py new file mode 100644 index 000000000..959288442 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/label_score.py @@ -0,0 +1,19 @@ +from labelbox.orm.db_object import DbObject +from labelbox.orm.model import Field + + +class LabelScore(DbObject): + """ + a label score + + Attributes + name + score + + """ + + name = Field.String("name") + data_row_count = Field.Float("score") + + def __init__(self, client, *args, **kwargs): + super().__init__(client, *args, **kwargs) diff --git a/libs/labelbox/tests/integration/test_label.py b/libs/labelbox/tests/integration/test_label.py index 266b5f9a8..6c7a76b10 100644 --- a/libs/labelbox/tests/integration/test_label.py +++ b/libs/labelbox/tests/integration/test_label.py @@ -1,7 +1,7 @@ import time +from labelbox import Client import pytest -import requests import os from labelbox import Label @@ -29,11 +29,13 @@ def test_labels(configured_project_with_label): # TODO: Skipping this test in staging due to label not updating -@pytest.mark.skipif(condition=os.environ['LABELBOX_TEST_ENVIRON'] == "onprem" or - os.environ['LABELBOX_TEST_ENVIRON'] == "staging" or - os.environ['LABELBOX_TEST_ENVIRON'] == "local" or - os.environ['LABELBOX_TEST_ENVIRON'] == "custom", - reason="does not work for onprem") +@pytest.mark.skipif( + condition=os.environ["LABELBOX_TEST_ENVIRON"] == "onprem" + or os.environ["LABELBOX_TEST_ENVIRON"] == "staging" + or os.environ["LABELBOX_TEST_ENVIRON"] == "local" + or os.environ["LABELBOX_TEST_ENVIRON"] == "custom", + reason="does not work for onprem", +) def test_label_update(configured_project_with_label): _, _, _, label = configured_project_with_label label.update(label="something else") @@ -57,7 +59,7 @@ def test_label_bulk_deletion(configured_project_with_label): project, _, _, _ = configured_project_with_label for _ in range(2): - #only run twice, already have one label in the fixture + # only run twice, already have one label in the fixture project.create_label() labels = project.labels() l1 = next(labels) @@ -74,3 +76,15 @@ def test_label_bulk_deletion(configured_project_with_label): time.sleep(5) assert set(project.labels()) == {l2} + + +def test_upsert_label_scores(configured_project_with_label, client: Client): + project, _, _, _ = configured_project_with_label + + label = next(project.labels()) + + scores = client.upsert_label_feedback( + label_id=label.uid, feedback="That's a great label!", scores={"overall": 5} + ) + assert len(scores) == 1 + assert scores[0].score == 5 From 1fd1d571e16599a663853ccca2646ff71d21dbbc Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Fri, 21 Jun 2024 14:09:07 +0500 Subject: [PATCH 2/6] MODEL-1448: Fixed formatting, switched to pydantic class approach --- libs/labelbox/src/labelbox/client.py | 41 +++++++++++++++---- libs/labelbox/src/labelbox/orm/model.py | 1 - .../src/labelbox/schema/label_score.py | 12 ++---- libs/labelbox/tests/integration/test_label.py | 19 ++++----- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index be21228ec..2087d7271 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2218,19 +2218,41 @@ def get_embedding_by_name(self, name: str) -> Embedding: dict(name=name)) def upsert_label_feedback(self, label_id: str, feedback: str, - scores: Dict[str, float]) -> Entity.Label: + scores: Dict[str, float]) -> List[Entity.Label]: """ + Submits the label feedback which is a free-form text and numeric + label scores. Args: label_id: Target label ID feedback: Free text comment regarding the label - scores: A dict of scores, the key is a score name and the value is the score value - Returns: A list of LabelScore instances + scores: A dict of scores, the key is a score name and the value is + the score value - """ - mutation_str = """mutation UpsertAutoQaLabelFeedbackPyApi($labelId: ID!, $feedback: String!, $scores: Json!){ - upsertAutoQaLabelFeedback(input: {labelId: $labelId, feedback: $feedback, scores: $scores}) { id scores {id name score} } - } + Returns: + A list of LabelScore instances + """ + mutation_str = """ + mutation UpsertAutoQaLabelFeedbackPyApi( + $labelId: ID! + $feedback: String! + $scores: Json! + ) { + upsertAutoQaLabelFeedback( + input: { + labelId: $labelId, + feedback: $feedback, + scores: $scores + } + ) { + id + scores { + id + name + score + } + } + } """ res = self.execute(mutation_str, { "labelId": label_id, @@ -2239,4 +2261,7 @@ def upsert_label_feedback(self, label_id: str, feedback: str, }) scores_raw = res["upsertAutoQaLabelFeedback"]["scores"] - return [Entity.LabelScore(self, x) for x in scores_raw] + return [ + labelbox.LabelScore(name=x['name'], score=x['score']) + for x in scores_raw + ] diff --git a/libs/labelbox/src/labelbox/orm/model.py b/libs/labelbox/src/labelbox/orm/model.py index dcca28fb7..f3afa174e 100644 --- a/libs/labelbox/src/labelbox/orm/model.py +++ b/libs/labelbox/src/labelbox/orm/model.py @@ -382,7 +382,6 @@ class Entity(metaclass=EntityMeta): CatalogSlice: Type[labelbox.CatalogSlice] ModelSlice: Type[labelbox.ModelSlice] TaskQueue: Type[labelbox.TaskQueue] - LabelScore: Type[labelbox.LabelScore] @classmethod def _attributes_of_type(cls, attr_type): diff --git a/libs/labelbox/src/labelbox/schema/label_score.py b/libs/labelbox/src/labelbox/schema/label_score.py index 959288442..71b90925e 100644 --- a/libs/labelbox/src/labelbox/schema/label_score.py +++ b/libs/labelbox/src/labelbox/schema/label_score.py @@ -1,8 +1,7 @@ -from labelbox.orm.db_object import DbObject -from labelbox.orm.model import Field +from labelbox import pydantic_compat -class LabelScore(DbObject): +class LabelScore(pydantic_compat.BaseModel): """ a label score @@ -12,8 +11,5 @@ class LabelScore(DbObject): """ - name = Field.String("name") - data_row_count = Field.Float("score") - - def __init__(self, client, *args, **kwargs): - super().__init__(client, *args, **kwargs) + name: str + score: float diff --git a/libs/labelbox/tests/integration/test_label.py b/libs/labelbox/tests/integration/test_label.py index 6c7a76b10..c7221553e 100644 --- a/libs/labelbox/tests/integration/test_label.py +++ b/libs/labelbox/tests/integration/test_label.py @@ -1,10 +1,9 @@ +import os import time -from labelbox import Client import pytest -import os -from labelbox import Label +from labelbox import Client, Label def test_labels(configured_project_with_label): @@ -30,10 +29,10 @@ def test_labels(configured_project_with_label): # TODO: Skipping this test in staging due to label not updating @pytest.mark.skipif( - condition=os.environ["LABELBOX_TEST_ENVIRON"] == "onprem" - or os.environ["LABELBOX_TEST_ENVIRON"] == "staging" - or os.environ["LABELBOX_TEST_ENVIRON"] == "local" - or os.environ["LABELBOX_TEST_ENVIRON"] == "custom", + condition=os.environ["LABELBOX_TEST_ENVIRON"] == "onprem" or + os.environ["LABELBOX_TEST_ENVIRON"] == "staging" or + os.environ["LABELBOX_TEST_ENVIRON"] == "local" or + os.environ["LABELBOX_TEST_ENVIRON"] == "custom", reason="does not work for onprem", ) def test_label_update(configured_project_with_label): @@ -83,8 +82,8 @@ def test_upsert_label_scores(configured_project_with_label, client: Client): label = next(project.labels()) - scores = client.upsert_label_feedback( - label_id=label.uid, feedback="That's a great label!", scores={"overall": 5} - ) + scores = client.upsert_label_feedback(label_id=label.uid, + feedback="That's a great label!", + scores={"overall": 5}) assert len(scores) == 1 assert scores[0].score == 5 From 2b9c0b6d8e378a697b8df370f9130107f62ade5c Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Fri, 21 Jun 2024 14:10:17 +0500 Subject: [PATCH 3/6] MODEL-1448: Fixed return type --- libs/labelbox/src/labelbox/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 2087d7271..f059e12d3 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -2217,8 +2217,9 @@ def get_embedding_by_name(self, name: str) -> Embedding: raise labelbox.exceptions.ResourceNotFoundError(Embedding, dict(name=name)) - def upsert_label_feedback(self, label_id: str, feedback: str, - scores: Dict[str, float]) -> List[Entity.Label]: + def upsert_label_feedback( + self, label_id: str, feedback: str, + scores: Dict[str, float]) -> List[labelbox.LabelScore]: """ Submits the label feedback which is a free-form text and numeric label scores. From 3dd32d4b8bf647383cf8ed358a99cec339057012 Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Fri, 21 Jun 2024 14:15:42 +0500 Subject: [PATCH 4/6] MODEL-1448: Updated docstring --- libs/labelbox/src/labelbox/schema/label_score.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/label_score.py b/libs/labelbox/src/labelbox/schema/label_score.py index 71b90925e..9197f6b55 100644 --- a/libs/labelbox/src/labelbox/schema/label_score.py +++ b/libs/labelbox/src/labelbox/schema/label_score.py @@ -3,11 +3,11 @@ class LabelScore(pydantic_compat.BaseModel): """ - a label score + A label score. - Attributes - name - score + Attributes: + name (str) + score (float) """ From 55c710bd5d95a526625bce4a165ca4ffc7969f6e Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Mon, 24 Jun 2024 12:59:47 +0500 Subject: [PATCH 5/6] MODEL-1448: Fixed import --- libs/labelbox/src/labelbox/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index f059e12d3..019285350 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -53,6 +53,7 @@ from labelbox.schema.slice import CatalogSlice, ModelSlice from labelbox.schema.task import Task from labelbox.schema.user import User +from labelbox.schema.label_score import LabelScore from labelbox.schema.ontology_kind import (OntologyKind, EditorTaskTypeMapper, EditorTaskType) @@ -2219,7 +2220,7 @@ def get_embedding_by_name(self, name: str) -> Embedding: def upsert_label_feedback( self, label_id: str, feedback: str, - scores: Dict[str, float]) -> List[labelbox.LabelScore]: + scores: Dict[str, float]) -> List[LabelScore]: """ Submits the label feedback which is a free-form text and numeric label scores. From a859280d924b9fc820c23d64cb9b784adb1261a6 Mon Sep 17 00:00:00 2001 From: Sergey Dubinin Date: Mon, 24 Jun 2024 20:12:32 +0500 Subject: [PATCH 6/6] MODEL-1448: Added docs --- docs/labelbox/index.rst | 1 + docs/labelbox/label-score.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 docs/labelbox/label-score.rst diff --git a/docs/labelbox/index.rst b/docs/labelbox/index.rst index cb965359f..1134179a3 100644 --- a/docs/labelbox/index.rst +++ b/docs/labelbox/index.rst @@ -25,6 +25,7 @@ Labelbox Python SDK Documentation identifiable identifiables label + label-score labeling-frontend labeling-frontend-options labeling-parameter-override diff --git a/docs/labelbox/label-score.rst b/docs/labelbox/label-score.rst new file mode 100644 index 000000000..b92880dbc --- /dev/null +++ b/docs/labelbox/label-score.rst @@ -0,0 +1,6 @@ +Label Score +=============================================================================================== + +.. automodule:: labelbox.schema.label_score + :members: + :show-inheritance: