Skip to content

Vb/fix ontology leaks plt 1379 #1814

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 17, 2024
Merged

Vb/fix ontology leaks plt 1379 #1814

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cbdaf35
Followup update for the feature schema test
Sep 12, 2024
cc423a6
Fix to delete feature schema resources
Sep 12, 2024
46e6f82
Rye fmt fixes
Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 129 additions & 66 deletions libs/labelbox/tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
import re
import uuid
import time
from labelbox.schema.project import Project
import requests
from labelbox.schema.ontology import Ontology
import pytest
from types import SimpleNamespace
from typing import Type
Expand All @@ -23,21 +25,11 @@
from labelbox.schema.queue_mode import QueueMode
from labelbox import Client

from labelbox import Dataset, DataRow
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

used ruff to delete unused imports

from labelbox import LabelingFrontend
from labelbox import OntologyBuilder, Tool, Option, Classification, MediaType
from labelbox.orm import query
from labelbox.pagination import PaginatedCollection
from labelbox import OntologyBuilder, Tool, Option, Classification
from labelbox.schema.annotation_import import LabelImport
from labelbox.schema.catalog import Catalog
from labelbox.schema.enums import AnnotationImportState
from labelbox.schema.invite import Invite
from labelbox.schema.quality_mode import QualityMode
from labelbox.schema.queue_mode import QueueMode
from labelbox.schema.user import User
from labelbox.exceptions import LabelboxError
from contextlib import suppress
from labelbox import Client

IMG_URL = "https://picsum.photos/200/300.jpg"
MASKABLE_IMG_URL = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg"
Expand Down Expand Up @@ -635,20 +627,17 @@ def organization(client):


@pytest.fixture
def configured_project_with_label(
client,
rand_gen,
image_url,
project,
dataset,
data_row,
wait_for_label_processing,
):
def configured_project_with_label(client, rand_gen, dataset, data_row,
wait_for_label_processing, teardown_helpers):
"""Project with a connected dataset, having one datarow
Project contains an ontology with 1 bbox tool
Additionally includes a create_label method for any needed extra labels
One label is already created and yielded when using fixture
"""
project = client.create_project(name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image)
project._wait_until_data_rows_are_processed(
data_row_ids=[data_row.uid],
wait_processing_max_seconds=DATA_ROW_PROCESSING_WAIT_TIMEOUT_SECONDS,
Expand All @@ -666,8 +655,7 @@ def configured_project_with_label(
)
yield [project, dataset, data_row, label]

for label in project.labels():
label.delete()
teardown_helpers.teardown_project_labels_ontology_feature_schemas(project)


def _create_label(project, data_row, ontology, wait_for_label_processing):
Expand Down Expand Up @@ -735,14 +723,17 @@ def big_dataset(dataset: Dataset):


@pytest.fixture
def configured_batch_project_with_label(
project, dataset, data_row, wait_for_label_processing
):
def configured_batch_project_with_label(client, dataset, data_row,
wait_for_label_processing, rand_gen,
teardown_helpers):
"""Project with a batch having one datarow
Project contains an ontology with 1 bbox tool
Additionally includes a create_label method for any needed extra labels
One label is already created and yielded when using fixture
"""
project = client.create_project(name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image)
data_rows = [dr.uid for dr in list(dataset.data_rows())]
project._wait_until_data_rows_are_processed(
data_row_ids=data_rows, sleep_interval=3
Expand All @@ -757,18 +748,20 @@ def configured_batch_project_with_label(

yield [project, dataset, data_row, label]

for label in project.labels():
label.delete()
teardown_helpers.teardown_project_labels_ontology_feature_schemas(project)


@pytest.fixture
def configured_batch_project_with_multiple_datarows(
project, dataset, data_rows, wait_for_label_processing
):
def configured_batch_project_with_multiple_datarows(client, dataset, data_rows,
wait_for_label_processing,
rand_gen, teardown_helpers):
"""Project with a batch having multiple datarows
Project contains an ontology with 1 bbox tool
Additionally includes a create_label method for any needed extra labels
"""
project = client.create_project(name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image)
global_keys = [dr.global_key for dr in data_rows]

batch_name = f"batch {uuid.uuid4()}"
Expand All @@ -780,26 +773,7 @@ def configured_batch_project_with_multiple_datarows(

yield [project, dataset, data_rows]

for label in project.labels():
label.delete()


@pytest.fixture
def configured_batch_project_for_labeling_service(
project, data_row_and_global_key
):
"""Project with a batch having multiple datarows
Project contains an ontology with 1 bbox tool
Additionally includes a create_label method for any needed extra labels
"""
global_keys = [data_row_and_global_key[1]]

batch_name = f"batch {uuid.uuid4()}"
project.create_batch(batch_name, global_keys=global_keys)

_setup_ontology(project)

yield project
teardown_helpers.teardown_project_labels_ontology_feature_schemas(project)


# NOTE this is nice heuristics, also there is this logic _wait_until_data_rows_are_processed in Project
Expand Down Expand Up @@ -1061,14 +1035,11 @@ def project_with_empty_ontology(project):


@pytest.fixture
def configured_project_with_complex_ontology(
client, initial_dataset, rand_gen, image_url
):
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
def configured_project_with_complex_ontology(client, initial_dataset, rand_gen,
image_url, teardown_helpers):
project = client.create_project(name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image)
dataset = initial_dataset
data_row = dataset.create_data_row(row_data=image_url)
data_row_ids = [data_row.uid]
Expand Down Expand Up @@ -1127,7 +1098,7 @@ def configured_project_with_complex_ontology(
project.setup(editor, ontology.asdict())

yield [project, data_row]
project.delete()
teardown_helpers.teardown_project_labels_ontology_feature_schemas(project)


@pytest.fixture
Expand All @@ -1146,13 +1117,13 @@ def valid_model_id():


@pytest.fixture
def requested_labeling_service(
rand_gen,
live_chat_evaluation_project_with_new_dataset,
chat_evaluation_ontology,
model_config,
):
project = live_chat_evaluation_project_with_new_dataset
def requested_labeling_service(rand_gen, client, chat_evaluation_ontology,
model_config, teardown_helpers):
project_name = f"test-model-evaluation-project-{rand_gen(str)}"
dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}"
project = client.create_model_evaluation_project(name=project_name,
dataset_name=dataset_name,
data_row_count=1)
project.connect_ontology(chat_evaluation_ontology)

project.upsert_instructions("tests/integration/media/sample_pdf.pdf")
Expand All @@ -1164,3 +1135,95 @@ def requested_labeling_service(
labeling_service.request()

yield project, project.get_labeling_service()

teardown_helpers.teardown_project_labels_ontology_feature_schemas(project)


class TearDownHelpers:

@staticmethod
def teardown_project_labels_ontology_feature_schemas(project: Project):
"""
Call this function to release project, labels, ontology and feature schemas in fixture teardown
NOTE: exception handling is not required as this is a fixture teardown
"""
ontology = project.ontology()
ontology_id = ontology.uid
client = project.client
classification_feature_schema_ids = [
feature["featureSchemaId"]
for feature in ontology.normalized["classifications"]
]
tool_feature_schema_ids = [
feature["featureSchemaId"] for feature in ontology.normalized["tools"]
]

feature_schema_ids = classification_feature_schema_ids + tool_feature_schema_ids
labels = list(project.labels())
for label in labels:
label.delete()

project.delete()
client.delete_unused_ontology(ontology_id)
for feature_schema_id in feature_schema_ids:
try:
project.client.delete_unused_feature_schema(feature_schema_id)
except LabelboxError as e:
print(
f"Failed to delete feature schema {feature_schema_id}: {e}")

@staticmethod
def teardown_ontology_feature_schemas(ontology: Ontology):
"""
Call this function to release project, labels, ontology and feature schemas in fixture teardown
NOTE: exception handling is not required as this is a fixture teardown
"""
ontology_id = ontology.uid
client = ontology.client
classification_feature_schema_ids = [
feature["featureSchemaId"]
for feature in ontology.normalized["classifications"]
] + [
option["featureSchemaId"]
for feature in ontology.normalized["classifications"]
for option in feature.get("options", [])
]

tool_feature_schema_ids = [
feature["featureSchemaId"] for feature in ontology.normalized["tools"]
] + [
classification["featureSchemaId"]
for tool in ontology.normalized["tools"]
for classification in tool.get("classifications", [])
] + [
option["featureSchemaId"]
for tool in ontology.normalized["tools"]
for classification in tool.get("classifications", [])
for option in classification.get("options", [])
]

feature_schema_ids = classification_feature_schema_ids + tool_feature_schema_ids

client.delete_unused_ontology(ontology_id)
for feature_schema_id in feature_schema_ids:
try:
project.client.delete_unused_feature_schema(feature_schema_id)
except LabelboxError as e:
print(
f"Failed to delete feature schema {feature_schema_id}: {e}")


class ModuleTearDownHelpers(TearDownHelpers):
...


@pytest.fixture
def teardown_helpers():
return TearDownHelpers()


@pytest.fixture(scope='module')
def module_teardown_helpers():
return TearDownHelpers()
Loading
Loading