Skip to content

Commit bc61745

Browse files
authored
[PLT-1151] Vb/model setup complete (#1685)
2 parents 10840ce + 63090c4 commit bc61745

File tree

10 files changed

+179
-27
lines changed

10 files changed

+179
-27
lines changed

libs/labelbox/src/labelbox/client.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -840,7 +840,7 @@ def create_offline_model_evaluation_project(self, **kwargs) -> Project:
840840
kwargs.pop("append_to_existing_dataset", None)
841841
kwargs.pop("data_row_count", None)
842842

843-
return self.create_project(**kwargs)
843+
return self._create_project(**kwargs)
844844

845845
def _create_project(self, **kwargs) -> Project:
846846
auto_audit_percentage = kwargs.get("auto_audit_percentage")

libs/labelbox/src/labelbox/exceptions.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import re
2+
3+
14
class LabelboxError(Exception):
25
"""Base class for exceptions."""
36

@@ -147,3 +150,21 @@ class CustomMetricsNotSupportedException(Exception):
147150

148151
class ProcessingWaitTimeout(Exception):
149152
"""Raised when waiting for the data rows to be processed takes longer than allowed"""
153+
154+
155+
def error_message_for_unparsed_graphql_error(error_string: str) -> str:
156+
"""
157+
Since our client only parses certain graphql errors, this function is used to
158+
extract the error message from the error string when the error is not
159+
parsed by the client.
160+
"""
161+
# Regex to find the message content
162+
pattern = r"'message': '([^']+)'"
163+
# Search for the pattern in the error string
164+
match = re.search(pattern, error_string)
165+
if match:
166+
error_content = match.group(1)
167+
else:
168+
error_content = "Unknown error"
169+
170+
return error_content

libs/labelbox/src/labelbox/schema/project.py

Lines changed: 51 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,10 @@
1313

1414
from labelbox import parser
1515
from labelbox import utils
16-
from labelbox.exceptions import (
17-
InvalidQueryError,
18-
LabelboxError,
19-
ProcessingWaitTimeout,
20-
ResourceConflict,
21-
ResourceNotFoundError
22-
)
16+
from labelbox.exceptions import error_message_for_unparsed_graphql_error
17+
from labelbox.exceptions import (InvalidQueryError, LabelboxError,
18+
ProcessingWaitTimeout, ResourceConflict,
19+
ResourceNotFoundError)
2320
from labelbox.orm import query
2421
from labelbox.orm.db_object import DbObject, Deletable, Updateable, experimental
2522
from labelbox.orm.model import Entity, Field, Relationship
@@ -122,6 +119,7 @@ class Project(DbObject, Updateable, Deletable):
122119
media_type = Field.Enum(MediaType, "media_type", "allowedMediaType")
123120
editor_task_type = Field.Enum(EditorTaskType, "editor_task_type")
124121
data_row_count = Field.Int("data_row_count")
122+
model_setup_complete: Field = Field.Boolean("model_setup_complete")
125123

126124
# Relationships
127125
created_by = Relationship.ToOne("User", False, "created_by")
@@ -1271,7 +1269,18 @@ def add_model_config(self, model_config_id: str) -> str:
12711269
"projectId": self.uid,
12721270
"modelConfigId": model_config_id,
12731271
}
1274-
result = self.client.execute(query, params)
1272+
try:
1273+
result = self.client.execute(query, params)
1274+
except LabelboxError as e:
1275+
if e.message.startswith(
1276+
"Unknown error: "
1277+
): # unfortunate hack to handle unparsed graphql errors
1278+
error_content = error_message_for_unparsed_graphql_error(
1279+
e.message)
1280+
else:
1281+
error_content = e.message
1282+
raise LabelboxError(message=error_content) from e
1283+
12751284
if not result:
12761285
raise ResourceNotFoundError(ModelConfig, params)
12771286
return result["createProjectModelConfig"]["projectModelConfigId"]
@@ -1299,6 +1308,29 @@ def delete_project_model_config(self, project_model_config_id: str) -> bool:
12991308
raise ResourceNotFoundError(ProjectModelConfig, params)
13001309
return result["deleteProjectModelConfig"]["success"]
13011310

1311+
def set_project_model_setup_complete(self) -> bool:
1312+
"""
1313+
Sets the model setup is complete for this project.
1314+
Once the project is marked as "setup complete", a user can not add / modify delete existing project model configs.
1315+
1316+
Returns:
1317+
bool, indicates if the model setup is complete.
1318+
1319+
NOTE: This method should only be used for live model evaluation projects.
1320+
It will throw exception for all other types of projects.
1321+
User Project is_chat_evaluation() method to check if the project is a live model evaluation project.
1322+
"""
1323+
query = """mutation SetProjectModelSetupCompletePyApi($projectId: ID!) {
1324+
setProjectModelSetupComplete(where: {id: $projectId}, data: {modelSetupComplete: true}) {
1325+
modelSetupComplete
1326+
}
1327+
}"""
1328+
1329+
result = self.client.execute(query, {"projectId": self.uid})
1330+
self.model_setup_complete = result["setProjectModelSetupComplete"][
1331+
"modelSetupComplete"]
1332+
return result["setProjectModelSetupComplete"]["modelSetupComplete"]
1333+
13021334
def set_labeling_parameter_overrides(
13031335
self, data: List[LabelingParameterOverrideInput]) -> bool:
13041336
""" Adds labeling parameter overrides to this project.
@@ -1752,7 +1784,9 @@ def __check_data_rows_have_been_processed(
17521784
return response["queryAllDataRowsHaveBeenProcessed"][
17531785
"allDataRowsHaveBeenProcessed"]
17541786

1755-
def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewDetailed]:
1787+
def get_overview(
1788+
self,
1789+
details=False) -> Union[ProjectOverview, ProjectOverviewDetailed]:
17561790
"""Return the overview of a project.
17571791
17581792
This method returns the number of data rows per task queue and issues of a project,
@@ -1792,7 +1826,7 @@ def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewD
17921826

17931827
# Must use experimental to access "issues"
17941828
result = self.client.execute(query, {"projectId": self.uid},
1795-
experimental=True)["project"]
1829+
experimental=True)["project"]
17961830

17971831
# Reformat category names
17981832
overview = {
@@ -1805,26 +1839,28 @@ def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewD
18051839

18061840
# Rename categories
18071841
overview["to_label"] = overview.pop("unlabeled")
1808-
overview["total_data_rows"] = overview.pop("all")
1842+
overview["total_data_rows"] = overview.pop("all")
18091843

18101844
if not details:
18111845
return ProjectOverview(**overview)
18121846
else:
18131847
# Build dictionary for queue details for review and rework queues
18141848
for category in ["rework", "review"]:
18151849
queues = [
1816-
{tq["name"]: tq.get("dataRowCount")}
1850+
{
1851+
tq["name"]: tq.get("dataRowCount")
1852+
}
18171853
for tq in result.get("taskQueues")
18181854
if tq.get("queueType") == f"MANUAL_{category.upper()}_QUEUE"
18191855
]
18201856

1821-
overview[f"in_{category}"] = {
1857+
overview[f"in_{category}"] = {
18221858
"data": queues,
18231859
"total": overview[f"in_{category}"]
18241860
}
1825-
1861+
18261862
return ProjectOverviewDetailed(**overview)
1827-
1863+
18281864
def clone(self) -> "Project":
18291865
"""
18301866
Clones the current project.

libs/labelbox/src/labelbox/schema/project_model_config.py

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
from labelbox.orm.db_object import DbObject, Deletable
1+
from labelbox.orm.db_object import DbObject
22
from labelbox.orm.model import Field, Relationship
3+
from labelbox.exceptions import LabelboxError, error_message_for_unparsed_graphql_error
34

45

56
class ProjectModelConfig(DbObject):
@@ -30,5 +31,17 @@ def delete(self) -> bool:
3031
params = {
3132
"id": self.uid,
3233
}
33-
result = self.client.execute(query, params)
34+
35+
try:
36+
result = self.client.execute(query, params)
37+
except LabelboxError as e:
38+
if e.message.startswith(
39+
"Unknown error: "
40+
): # unfortunate hack to handle unparsed graphql errors
41+
error_content = error_message_for_unparsed_graphql_error(
42+
e.message)
43+
else:
44+
error_content = e.message
45+
raise LabelboxError(message=error_content) from e
46+
3447
return result["deleteProjectModelConfig"]["success"]

libs/labelbox/tests/integration/conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ def chat_evaluation_ontology(client, rand_gen):
407407

408408

409409
@pytest.fixture
410-
def chat_evaluation_project_create_dataset(client, rand_gen):
410+
def live_chat_evaluation_project_with_new_dataset(client, rand_gen):
411411
project_name = f"test-model-evaluation-project-{rand_gen(str)}"
412412
dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}"
413413
project = client.create_model_evaluation_project(name=project_name,

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
def test_create_chat_evaluation_ontology_project(
88
client, chat_evaluation_ontology,
9-
chat_evaluation_project_create_dataset, conversation_data_row,
9+
live_chat_evaluation_project_with_new_dataset, conversation_data_row,
1010
rand_gen):
1111
ontology = chat_evaluation_ontology
1212

@@ -23,7 +23,9 @@ def test_create_chat_evaluation_ontology_project(
2323
assert classification.schema_id
2424
assert classification.feature_schema_id
2525

26-
project = chat_evaluation_project_create_dataset
26+
project = live_chat_evaluation_project_with_new_dataset
27+
assert project.model_setup_complete is None
28+
2729
project.setup_editor(ontology)
2830

2931
assert project.labeling_frontend().name == "Editor"

libs/labelbox/tests/integration/test_project_model_config.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@
33
from labelbox.exceptions import ResourceNotFoundError
44

55

6-
def test_add_single_model_config(chat_evaluation_project_create_dataset,
6+
def test_add_single_model_config(live_chat_evaluation_project_with_new_dataset,
77
model_config):
8-
configured_project = chat_evaluation_project_create_dataset
8+
configured_project = live_chat_evaluation_project_with_new_dataset
99
project_model_config_id = configured_project.add_model_config(
1010
model_config.uid)
1111

@@ -18,9 +18,9 @@ def test_add_single_model_config(chat_evaluation_project_create_dataset,
1818

1919

2020
def test_add_multiple_model_config(client, rand_gen,
21-
chat_evaluation_project_create_dataset,
21+
live_chat_evaluation_project_with_new_dataset,
2222
model_config, valid_model_id):
23-
configured_project = chat_evaluation_project_create_dataset
23+
configured_project = live_chat_evaluation_project_with_new_dataset
2424
second_model_config = client.create_model_config(rand_gen(str),
2525
valid_model_id,
2626
{"param": "value"})
@@ -40,9 +40,9 @@ def test_add_multiple_model_config(client, rand_gen,
4040
project_model_config_id)
4141

4242

43-
def test_delete_project_model_config(chat_evaluation_project_create_dataset,
43+
def test_delete_project_model_config(live_chat_evaluation_project_with_new_dataset,
4444
model_config):
45-
configured_project = chat_evaluation_project_create_dataset
45+
configured_project = live_chat_evaluation_project_with_new_dataset
4646
assert configured_project.delete_project_model_config(
4747
configured_project.add_model_config(model_config.uid))
4848
assert not len(configured_project.project_model_configs())
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import pytest
2+
3+
from labelbox.exceptions import LabelboxError, OperationNotAllowedException
4+
5+
6+
def test_live_chat_evaluation_project(
7+
live_chat_evaluation_project_with_new_dataset, model_config):
8+
9+
project = live_chat_evaluation_project_with_new_dataset
10+
11+
project.set_project_model_setup_complete()
12+
assert bool(project.model_setup_complete) is True
13+
14+
with pytest.raises(
15+
expected_exception=LabelboxError,
16+
match=
17+
"Cannot create model config for project because model setup is complete"
18+
):
19+
project.add_model_config(model_config.uid)
20+
21+
22+
def test_live_chat_evaluation_project_delete_cofig(
23+
live_chat_evaluation_project_with_new_dataset, model_config):
24+
25+
project = live_chat_evaluation_project_with_new_dataset
26+
project_model_config_id = project.add_model_config(model_config.uid)
27+
assert project_model_config_id
28+
29+
project_model_config = None
30+
for pmg in project.project_model_configs():
31+
if pmg.uid == project_model_config_id:
32+
project_model_config = pmg
33+
break
34+
assert project_model_config
35+
36+
project.set_project_model_setup_complete()
37+
assert bool(project.model_setup_complete) is True
38+
39+
with pytest.raises(
40+
expected_exception=LabelboxError,
41+
match=
42+
"Cannot create model config for project because model setup is complete"
43+
):
44+
project_model_config.delete()
45+
46+
47+
def test_offline_chat_evaluation_project(offline_chat_evaluation_project,
48+
model_config):
49+
50+
project = offline_chat_evaluation_project
51+
52+
with pytest.raises(
53+
expected_exception=OperationNotAllowedException,
54+
match=
55+
"Only live model chat evaluation projects can complete model setup"
56+
):
57+
project.set_project_model_setup_complete()
58+
59+
60+
def test_any_other_project(project, model_config):
61+
with pytest.raises(
62+
expected_exception=OperationNotAllowedException,
63+
match=
64+
"Only live model chat evaluation projects can complete model setup"
65+
):
66+
project.set_project_model_setup_complete()
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import pytest
2+
3+
from labelbox.exceptions import error_message_for_unparsed_graphql_error
4+
5+
6+
@pytest.mark.parametrize('exception_message, expected_result', [
7+
("Unparsed errors on query execution: [{'message': 'Cannot create model config for project because model setup is complete'}]",
8+
"Cannot create model config for project because model setup is complete"),
9+
("blah blah blah", "Unknown error"),
10+
])
11+
def test_client_unparsed_exception_messages(exception_message, expected_result):
12+
assert error_message_for_unparsed_graphql_error(
13+
exception_message) == expected_result

libs/labelbox/tests/unit/test_project.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def test_project_editor_task_type(api_editor_task_type,
3131
"allowedMediaType": "IMAGE",
3232
"queueMode": "BATCH",
3333
"setupComplete": "2021-06-01T00:00:00.000Z",
34+
"modelSetupComplete": None,
3435
})
3536

3637
assert project.editor_task_type == expected_editor_task_type

0 commit comments

Comments
 (0)