Skip to content

Commit ca7f114

Browse files
author
Val Brodsky
committed
Add wait for labels
Simplify test for user group creation / deletion Rewrite annotation upload tests for LLM projects
1 parent 8724901 commit ca7f114

File tree

4 files changed

+103
-51
lines changed

4 files changed

+103
-51
lines changed

libs/labelbox/tests/conftest.py

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,39 @@
1-
from datetime import datetime
2-
from random import randint
3-
from string import ascii_letters
4-
51
import json
62
import os
73
import re
8-
import uuid
94
import time
10-
from labelbox.schema.project import Project
11-
import requests
12-
from labelbox.schema.ontology import Ontology
13-
import pytest
14-
from types import SimpleNamespace
15-
from typing import Type
5+
import uuid
6+
from datetime import datetime
167
from enum import Enum
17-
from typing import Tuple
8+
from random import randint
9+
from string import ascii_letters
10+
from types import SimpleNamespace
11+
from typing import Tuple, Type
12+
13+
import pytest
14+
import requests
1815

19-
from labelbox import Dataset, DataRow
20-
from labelbox import MediaType
16+
from labelbox import (
17+
Classification,
18+
Client,
19+
DataRow,
20+
Dataset,
21+
LabelingFrontend,
22+
MediaType,
23+
OntologyBuilder,
24+
Option,
25+
Tool,
26+
)
27+
from labelbox.exceptions import LabelboxError
2128
from labelbox.orm import query
2229
from labelbox.pagination import PaginatedCollection
30+
from labelbox.schema.annotation_import import LabelImport
31+
from labelbox.schema.enums import AnnotationImportState
2332
from labelbox.schema.invite import Invite
33+
from labelbox.schema.ontology import Ontology
34+
from labelbox.schema.project import Project
2435
from labelbox.schema.quality_mode import QualityMode
2536
from labelbox.schema.queue_mode import QueueMode
26-
from labelbox import Client
27-
28-
from labelbox import LabelingFrontend
29-
from labelbox import OntologyBuilder, Tool, Option, Classification
30-
from labelbox.schema.annotation_import import LabelImport
31-
from labelbox.schema.enums import AnnotationImportState
32-
from labelbox.exceptions import LabelboxError
3337

3438
IMG_URL = "https://picsum.photos/200/300.jpg"
3539
MASKABLE_IMG_URL = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg"
@@ -1255,6 +1259,21 @@ def teardown_ontology_feature_schemas(ontology: Ontology):
12551259
class ModuleTearDownHelpers(TearDownHelpers): ...
12561260

12571261

1262+
class LabelHelpers:
1263+
def wait_for_labels(self, project, number_of_labels=1):
1264+
timeout_seconds = 10
1265+
while True:
1266+
labels = list(project.labels())
1267+
if len(labels) >= number_of_labels:
1268+
return labels
1269+
timeout_seconds -= 2
1270+
if timeout_seconds <= 0:
1271+
raise TimeoutError(
1272+
f"Timed out waiting for label for project '{project.uid}' to finish processing"
1273+
)
1274+
time.sleep(2)
1275+
1276+
12581277
@pytest.fixture
12591278
def teardown_helpers():
12601279
return TearDownHelpers()
@@ -1263,3 +1282,8 @@ def teardown_helpers():
12631282
@pytest.fixture(scope="module")
12641283
def module_teardown_helpers():
12651284
return TearDownHelpers()
1285+
1286+
1287+
@pytest.fixture
1288+
def label_helpers():
1289+
return LabelHelpers()

libs/labelbox/tests/data/annotation_import/test_generic_data_types.py

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,6 @@ def test_generic_data_row_type_by_global_key(
112112
(MediaType.Conversational, MediaType.Conversational),
113113
(MediaType.Document, MediaType.Document),
114114
(MediaType.Dicom, MediaType.Dicom),
115-
(
116-
MediaType.LLMPromptResponseCreation,
117-
MediaType.LLMPromptResponseCreation,
118-
),
119-
(MediaType.LLMPromptCreation, MediaType.LLMPromptCreation),
120115
(OntologyKind.ResponseCreation, OntologyKind.ResponseCreation),
121116
(OntologyKind.ModelEvaluation, OntologyKind.ModelEvaluation),
122117
],
@@ -186,6 +181,53 @@ def test_import_media_types(
186181
assert exported_annotations == expected_data
187182

188183

184+
@pytest.mark.parametrize(
185+
"configured_project, media_type",
186+
[
187+
(
188+
MediaType.LLMPromptResponseCreation,
189+
MediaType.LLMPromptResponseCreation,
190+
),
191+
(MediaType.LLMPromptCreation, MediaType.LLMPromptCreation),
192+
],
193+
indirect=["configured_project"],
194+
)
195+
def test_import_media_types_llm(
196+
client: Client,
197+
configured_project: Project,
198+
annotations_by_media_type,
199+
exports_v2_by_media_type,
200+
export_v2_test_helpers,
201+
helpers,
202+
media_type,
203+
wait_for_label_processing,
204+
):
205+
annotations_ndjson = list(
206+
itertools.chain.from_iterable(annotations_by_media_type[media_type])
207+
)
208+
209+
label_import = lb.LabelImport.create_from_objects(
210+
client,
211+
configured_project.uid,
212+
f"test-import-{media_type}",
213+
annotations_ndjson,
214+
)
215+
label_import.wait_until_done()
216+
217+
assert label_import.state == AnnotationImportState.FINISHED
218+
assert len(label_import.errors) == 0
219+
220+
all_annotations = sorted([a["uuid"] for a in annotations_ndjson])
221+
successful_annotations = sorted(
222+
[
223+
status["uuid"]
224+
for status in label_import.statuses
225+
if status["status"] == "SUCCESS"
226+
]
227+
)
228+
assert successful_annotations == all_annotations
229+
230+
189231
@pytest.mark.parametrize(
190232
"configured_project_by_global_key, media_type",
191233
[

libs/labelbox/tests/integration/schema/test_user_group.py

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
import pytest
2-
import faker
31
from uuid import uuid4
4-
from labelbox import Client
5-
from labelbox.schema.user_group import UserGroup, UserGroupColor
2+
3+
import faker
4+
import pytest
5+
66
from labelbox.exceptions import (
7-
ResourceNotFoundError,
87
ResourceCreationError,
9-
UnprocessableEntityError,
8+
ResourceNotFoundError,
109
)
10+
from labelbox.schema.user_group import UserGroup, UserGroupColor
1111

1212
data = faker.Faker()
1313

@@ -147,35 +147,19 @@ def test_cannot_update_group_id(user_group):
147147
def test_get_user_groups_with_creation_deletion(client):
148148
user_group = None
149149
try:
150-
# Get all user groups
151-
user_groups = list(UserGroup(client).get_user_groups())
152-
153150
# manual delete for iterators
154151
group_name = data.name()
155152
user_group = UserGroup(client)
156153
user_group.name = group_name
157154
user_group.create()
158155

159156
user_groups_post_creation = list(UserGroup(client).get_user_groups())
157+
assert user_group in user_groups_post_creation
160158

161-
# Verify that at least one user group is returned
162-
assert len(user_groups_post_creation) > 0
163-
assert len(user_groups_post_creation) == len(user_groups) + 1
164-
165-
# Verify that each user group has a valid ID and name
166-
for ug in user_groups_post_creation:
167-
assert ug.id is not None
168-
assert ug.name is not None
169-
170-
user_group.delete()
171159
user_group = None
172160

173161
user_groups_post_deletion = list(UserGroup(client).get_user_groups())
174-
175-
assert (
176-
len(user_groups_post_deletion) == len(user_groups_post_creation) - 1
177-
)
178-
162+
assert user_group not in user_groups_post_deletion
179163
finally:
180164
if user_group:
181165
user_group.delete()

libs/labelbox/tests/integration/test_label.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,13 @@ def test_label_update(configured_project_with_label):
4141
assert label.label == "something else"
4242

4343

44-
def test_label_filter_order(configured_project_with_label):
44+
def test_label_filter_order(configured_project_with_label, label_helpers):
4545
project, _, _, label = configured_project_with_label
4646

4747
l1 = label
4848
project.create_label()
49+
label_helpers.wait_for_labels(project, 2)
50+
4951
l2 = next(project.labels())
5052

5153
assert set(project.labels()) == {l1, l2}

0 commit comments

Comments
 (0)