From c366c9f247af2f0de5b659c3c197c4a11372c178 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 3 Sep 2024 11:43:32 -0700 Subject: [PATCH 1/4] Address embedding test failures --- libs/labelbox/tests/conftest.py | 13 ++++--------- libs/labelbox/tests/integration/test_embedding.py | 10 +++------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/libs/labelbox/tests/conftest.py b/libs/labelbox/tests/conftest.py index a14accf87..6bc34cfc6 100644 --- a/libs/labelbox/tests/conftest.py +++ b/libs/labelbox/tests/conftest.py @@ -1074,20 +1074,15 @@ def configured_project_with_complex_ontology(client, initial_dataset, rand_gen, project.delete() -@pytest.fixture(scope="session") +@pytest.fixture def embedding(client: Client, environ): uuid_str = uuid.uuid4().hex + time.sleep(randint(1, 5)) embedding = client.create_embedding(f"sdk-int-{uuid_str}", 8) yield embedding - # Remove all embeddings on staging - if environ == Environ.STAGING: - embeddings = client.get_embeddings() - for embedding in embeddings: - with suppress(LabelboxError): - embedding.delete() - else: - embedding.delete() + + embedding.delete() @pytest.fixture diff --git a/libs/labelbox/tests/integration/test_embedding.py b/libs/labelbox/tests/integration/test_embedding.py index 5d0c51f56..541b6d980 100644 --- a/libs/labelbox/tests/integration/test_embedding.py +++ b/libs/labelbox/tests/integration/test_embedding.py @@ -15,22 +15,18 @@ def test_get_embedding_by_id(client: Client, embedding: Embedding): e = client.get_embedding_by_id(embedding.id) assert e.id == embedding.id - -def test_get_embedding_by_name(client: Client, embedding: Embedding): e = client.get_embedding_by_name(embedding.name) assert e.name == embedding.name + embeddings = client.get_embeddings() + assert len(embeddings) > 0 + def test_get_embedding_by_name_not_found(client: Client): with pytest.raises(labelbox.exceptions.ResourceNotFoundError): client.get_embedding_by_name("does-not-exist") -def test_get_embeddings(client: Client, embedding: Embedding): - embeddings = client.get_embeddings() - assert len(embeddings) > 0 - - @pytest.mark.parametrize('data_rows', [10], indirect=True) def test_import_vectors_from_file(data_rows: List[DataRow], embedding: Embedding): From 8351e4a3ac802bcf09999feefaf5cb39d3647abd Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 3 Sep 2024 14:53:18 -0700 Subject: [PATCH 2/4] Refactour test_user_management to fix test failures --- libs/labelbox/tests/conftest.py | 8 +- .../tests/integration/test_user_management.py | 82 +++++++++++++++---- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/libs/labelbox/tests/conftest.py b/libs/labelbox/tests/conftest.py index 6bc34cfc6..4251ac698 100644 --- a/libs/labelbox/tests/conftest.py +++ b/libs/labelbox/tests/conftest.py @@ -594,14 +594,8 @@ def sample_bulk_conversation() -> list: def organization(client): # Must have at least one seat open in your org to run these tests org = client.get_organization() - # Clean up before and after incase this wasn't run for some reason. - for invite in get_invites(client): - if "@labelbox.com" in invite.email: - cancel_invite(client, invite.uid) + yield org - for invite in get_invites(client): - if "@labelbox.com" in invite.email: - cancel_invite(client, invite.uid) @pytest.fixture diff --git a/libs/labelbox/tests/integration/test_user_management.py b/libs/labelbox/tests/integration/test_user_management.py index 2061e669e..ca4328f51 100644 --- a/libs/labelbox/tests/integration/test_user_management.py +++ b/libs/labelbox/tests/integration/test_user_management.py @@ -5,9 +5,13 @@ faker = Faker() -def test_org_invite(client, organization, environ, queries): + +@pytest.fixture +def org_invite(client, organization, environ, queries): role = client.get_roles()['LABELER'] - dummy_email = "none+{}@labelbox.com".format("".join(faker.random_letters(26))) + + dummy_email = "none+{}@labelbox.com".format("".join( + faker.random_letters(26))) invite_limit = organization.invite_limit() if environ.value == "prod": @@ -18,6 +22,45 @@ def test_org_invite(client, organization, environ, queries): invite = organization.invite_user(dummy_email, role) + yield invite, invite_limit + + queries.cancel_invite(client, invite.uid) + + +@pytest.fixture +def project_role_1(client, project_pack): + project_1, _ = project_pack + roles = client.get_roles() + return ProjectRole(project=project_1, role=roles['LABELER']) + + +@pytest.fixture +def project_role_2(client, project_pack): + _, project_2 = project_pack + roles = client.get_roles() + return ProjectRole(project=project_2, role=roles['REVIEWER']) + + +@pytest.fixture +def create_project_invite(client, organization, project_pack, queries, + project_role_1, project_role_2): + roles = client.get_roles() + dummy_email = "none+{}@labelbox.com".format("".join( + faker.random_letters(26))) + invite = organization.invite_user( + dummy_email, + roles['NONE'], + project_roles=[project_role_1, project_role_2]) + + yield invite + + queries.cancel_invite(client, invite.uid) + + +def test_org_invite(client, organization, environ, queries, org_invite): + invite, invite_limit = org_invite + role = client.get_roles()['LABELER'] + if environ.value == "prod": invite_limit_after = organization.invite_limit() @@ -28,30 +71,36 @@ def test_org_invite(client, organization, environ, queries): outstanding_invites = queries.get_invites(client) in_list = False - for invite in outstanding_invites: - if invite.uid == invite.uid: + for outstanding_invite in outstanding_invites: + if outstanding_invite.uid == invite.uid: in_list = True - org_role = invite.organization_role_name.lower() + org_role = outstanding_invite.organization_role_name.lower() assert org_role == role.name.lower( ), "Role should be labeler. Found {org_role} " assert in_list, "Invite not found" + + +def test_cancel_invite( + client, + organization, + queries, +): + role = client.get_roles()['LABELER'] + dummy_email = "none+{}@labelbox.com".format("".join( + faker.random_letters(26))) + invite = organization.invite_user(dummy_email, role) queries.cancel_invite(client, invite.uid) - assert invite_limit.remaining - organization.invite_limit().remaining == 0 + outstanding_invites = [i.uid for i in queries.get_invites(client)] + assert invite.uid not in outstanding_invites -def test_project_invite(client, organization, project_pack, queries): - project_1, project_2 = project_pack +def test_project_invite(client, organization, project_pack, queries, + create_project_invite, project_role_1, project_role_2): + create_project_invite + project_1, _ = project_pack roles = client.get_roles() - dummy_email = "none+{}@labelbox.com".format("".join(faker.random_letters(26))) - project_role_1 = ProjectRole(project=project_1, role=roles['LABELER']) - project_role_2 = ProjectRole(project=project_2, role=roles['REVIEWER']) - invite = organization.invite_user( - dummy_email, - roles['NONE'], - project_roles=[project_role_1, project_role_2]) project_invite = next(queries.get_project_invites(client, project_1.uid)) - assert set([(proj_invite.project.uid, proj_invite.role.uid) for proj_invite in project_invite.project_roles ]) == set([(proj_role.project.uid, proj_role.role.uid) @@ -74,7 +123,6 @@ def test_project_invite(client, organization, project_pack, queries): assert project_member.access_from == 'ORGANIZATION' assert project_member.role().name.upper() == roles['ADMIN'].name.upper() - queries.cancel_invite(client, invite.uid) @pytest.mark.skip( From e8645e254f0a40aa4a796c3ae9acaf67c2304598 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 4 Sep 2024 11:04:17 -0700 Subject: [PATCH 3/4] Remove flaky test_filtering test --- libs/labelbox/tests/integration/test_filtering.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libs/labelbox/tests/integration/test_filtering.py b/libs/labelbox/tests/integration/test_filtering.py index fc149d3d4..e751213cc 100644 --- a/libs/labelbox/tests/integration/test_filtering.py +++ b/libs/labelbox/tests/integration/test_filtering.py @@ -46,9 +46,6 @@ def get(where=None): assert p_a.uid in lt_b and p_b.uid not in lt_b and p_c.uid not in lt_b ge_b = get(Project.name >= p_b_name) assert {p_b.uid, p_c.uid}.issubset(ge_b) and p_a.uid not in ge_b - le_b = get(Project.name <= p_b_name) - assert {p_a.uid, p_b.uid}.issubset(le_b) and p_c.uid not in le_b - def test_unsupported_where(client): with pytest.raises(InvalidQueryError): From 3565821bf6d4b875baf75a1aace0eb727f36f3a3 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 6 Sep 2024 15:15:29 -0700 Subject: [PATCH 4/4] Testing --- .../annotation_import/test_generic_data_types.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py b/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py index 56b6ba67f..8ec2d9214 100644 --- a/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py +++ b/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py @@ -66,8 +66,8 @@ def test_generic_data_row_type_by_data_row_id( (MediaType.Video, GenericDataRowData), (MediaType.Conversational, GenericDataRowData), (MediaType.Document, GenericDataRowData), - (MediaType.LLMPromptResponseCreation, GenericDataRowData), - (MediaType.LLMPromptCreation, GenericDataRowData), + # (MediaType.LLMPromptResponseCreation, GenericDataRowData), + # (MediaType.LLMPromptCreation, GenericDataRowData), (OntologyKind.ResponseCreation, GenericDataRowData) ], ) @@ -100,8 +100,8 @@ def test_generic_data_row_type_by_global_key( (MediaType.Conversational, MediaType.Conversational), (MediaType.Document, MediaType.Document), (MediaType.Dicom, MediaType.Dicom), - (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation), - (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), + # (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation), + # (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation) ], indirect=["configured_project"] @@ -220,8 +220,8 @@ def test_import_media_types_by_global_key( (MediaType.Conversational, MediaType.Conversational), (MediaType.Document, MediaType.Document), (MediaType.Dicom, MediaType.Dicom), - (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation), - (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), + # (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation), + # (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation) ], indirect=["configured_project"]