Skip to content

[PLT-1154] Vb/refactor project inputs plt 1154 #1844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Sep 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
311 changes: 126 additions & 185 deletions libs/labelbox/src/labelbox/client.py

Large diffs are not rendered by default.

87 changes: 87 additions & 0 deletions libs/labelbox/src/labelbox/project_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
from typing import Optional, Set

from pydantic import BaseModel, ConfigDict, Field, model_validator
from typing_extensions import Annotated

from labelbox.schema.media_type import MediaType
from labelbox.schema.ontology_kind import EditorTaskType
from labelbox.schema.quality_mode import (
BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS,
BENCHMARK_AUTO_AUDIT_PERCENTAGE,
CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS,
CONSENSUS_AUTO_AUDIT_PERCENTAGE,
QualityMode,
)
from labelbox.schema.queue_mode import QueueMode

PositiveInt = Annotated[int, Field(gt=0)]


class _CoreProjectInput(BaseModel):
name: str
description: Optional[str] = None
media_type: MediaType
queue_mode: QueueMode = Field(default=QueueMode.Batch, frozen=True)
auto_audit_percentage: Optional[float] = None
auto_audit_number_of_labels: Optional[int] = None
quality_modes: Optional[Set[QualityMode]] = Field(
default={QualityMode.Benchmark, QualityMode.Consensus}, exclude=True
)
is_benchmark_enabled: Optional[bool] = None
is_consensus_enabled: Optional[bool] = None
dataset_name_or_id: Optional[str] = None
append_to_existing_dataset: Optional[bool] = None
data_row_count: Optional[PositiveInt] = None
editor_task_type: Optional[EditorTaskType] = None

model_config = ConfigDict(extra="forbid")

@model_validator(mode="after")
def validate_fields(self):
if (
self.auto_audit_percentage is not None
or self.auto_audit_number_of_labels is not None
):
raise ValueError(
"quality_modes must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
)

if not self.name.strip():
raise ValueError("project name must be a valid string.")

if self.quality_modes == {
QualityMode.Benchmark,
QualityMode.Consensus,
}:
self._set_quality_mode_attributes(
CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS,
CONSENSUS_AUTO_AUDIT_PERCENTAGE,
is_benchmark_enabled=True,
is_consensus_enabled=True,
)
elif self.quality_modes == {QualityMode.Benchmark}:
self._set_quality_mode_attributes(
BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS,
BENCHMARK_AUTO_AUDIT_PERCENTAGE,
is_benchmark_enabled=True,
)
elif self.quality_modes == {QualityMode.Consensus}:
self._set_quality_mode_attributes(
number_of_labels=CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS,
percentage=CONSENSUS_AUTO_AUDIT_PERCENTAGE,
is_consensus_enabled=True,
)

return self

def _set_quality_mode_attributes(
self,
number_of_labels,
percentage,
is_benchmark_enabled=False,
is_consensus_enabled=False,
):
self.auto_audit_number_of_labels = number_of_labels
self.auto_audit_percentage = percentage
self.is_benchmark_enabled = is_benchmark_enabled
self.is_consensus_enabled = is_consensus_enabled
2 changes: 1 addition & 1 deletion libs/labelbox/src/labelbox/schema/ontology_kind.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def evaluate_ontology_kind_with_media_type(
return media_type


class EditorTaskType(Enum):
class EditorTaskType(str, Enum):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

needed to support json serializaiton

ModelChatEvaluation = "MODEL_CHAT_EVALUATION"
ResponseCreation = "RESPONSE_CREATION"
OfflineModelChatEvaluation = "OFFLINE_MODEL_CHAT_EVALUATION"
Expand Down
9 changes: 1 addition & 8 deletions libs/labelbox/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
from labelbox.schema.ontology import Ontology
from labelbox.schema.project import Project
from labelbox.schema.quality_mode import QualityMode
from labelbox.schema.queue_mode import QueueMode

IMG_URL = "https://picsum.photos/200/300.jpg"
MASKABLE_IMG_URL = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg"
Expand Down Expand Up @@ -444,7 +443,6 @@ def conversation_entity_data_row(client, rand_gen):
def project(client, rand_gen):
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
yield project
Expand All @@ -455,8 +453,7 @@ def project(client, rand_gen):
def consensus_project(client, rand_gen):
project = client.create_project(
name=rand_gen(str),
quality_mode=QualityMode.Consensus,
queue_mode=QueueMode.Batch,
quality_modes={QualityMode.Consensus},
media_type=MediaType.Image,
)
yield project
Expand Down Expand Up @@ -646,7 +643,6 @@ def configured_project_with_label(
"""
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
project._wait_until_data_rows_are_processed(
Expand Down Expand Up @@ -749,7 +745,6 @@ def configured_batch_project_with_label(
"""
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
data_rows = [dr.uid for dr in list(dataset.data_rows())]
Expand Down Expand Up @@ -784,7 +779,6 @@ def configured_batch_project_with_multiple_datarows(
"""
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
global_keys = [dr.global_key for dr in data_rows]
Expand Down Expand Up @@ -1065,7 +1059,6 @@ def configured_project_with_complex_ontology(
):
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
dataset = initial_dataset
Expand Down
29 changes: 24 additions & 5 deletions libs/labelbox/tests/data/export/conftest.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import uuid
import time
import uuid

import pytest
from labelbox.schema.queue_mode import QueueMode

from labelbox.schema.annotation_import import AnnotationImportState, LabelImport
from labelbox.schema.labeling_frontend import LabelingFrontend
from labelbox.schema.annotation_import import LabelImport, AnnotationImportState
from labelbox.schema.media_type import MediaType


@pytest.fixture
Expand Down Expand Up @@ -246,7 +248,7 @@ def configured_project_with_ontology(
dataset = initial_dataset
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
editor = list(
client.get_labeling_frontends(where=LabelingFrontend.name == "editor")
Expand All @@ -273,7 +275,24 @@ def configured_project_without_data_rows(
project = client.create_project(
name=rand_gen(str),
description=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
editor = list(
client.get_labeling_frontends(where=LabelingFrontend.name == "editor")
)[0]
project.setup(editor, ontology)
yield project
teardown_helpers.teardown_project_labels_ontology_feature_schemas(project)


@pytest.fixture
def configured_video_project_without_data_rows(
client, ontology, rand_gen, teardown_helpers
):
project = client.create_project(
name=rand_gen(str),
description=rand_gen(str),
media_type=MediaType.Video,
)
editor = list(
client.get_labeling_frontends(where=LabelingFrontend.name == "editor")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ def org_id(self, client):
def test_export(
self,
client,
configured_project_without_data_rows,
configured_video_project_without_data_rows,
video_data,
video_data_row,
bbox_video_annotation_objects,
rand_gen,
):
project = configured_project_without_data_rows
project = configured_video_project_without_data_rows
project_id = project.uid
labels = []

Expand Down
30 changes: 13 additions & 17 deletions libs/labelbox/tests/integration/conftest.py
Original file line number Diff line number Diff line change
@@ -1,42 +1,40 @@
from collections import defaultdict
from itertools import islice
import json
import os
import sys
import re
import sys
import time
import uuid
import requests
from types import SimpleNamespace
from typing import Type, List
from collections import defaultdict
from enum import Enum
from typing import Tuple
from itertools import islice
from types import SimpleNamespace
from typing import List, Tuple, Type

import pytest
import requests

from labelbox import Dataset, DataRow
from labelbox import LabelingFrontend
from labelbox import (
OntologyBuilder,
Tool,
Option,
Classification,
Client,
DataRow,
Dataset,
LabelingFrontend,
MediaType,
OntologyBuilder,
Option,
PromptResponseClassification,
ResponseOption,
Tool,
)
from labelbox.orm import query
from labelbox.pagination import PaginatedCollection
from labelbox.schema.annotation_import import LabelImport
from labelbox.schema.catalog import Catalog
from labelbox.schema.enums import AnnotationImportState
from labelbox.schema.invite import Invite
from labelbox.schema.ontology_kind import OntologyKind
from labelbox.schema.quality_mode import QualityMode
from labelbox.schema.queue_mode import QueueMode
from labelbox.schema.user import User
from labelbox import Client
from labelbox.schema.ontology_kind import OntologyKind


@pytest.fixture
Expand Down Expand Up @@ -69,7 +67,6 @@ def project_pack(client):
projects = [
client.create_project(
name=f"user-proj-{idx}",
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
for idx in range(2)
Expand Down Expand Up @@ -117,7 +114,6 @@ def configured_project_with_complex_ontology(
):
project = client.create_project(
name=rand_gen(str),
queue_mode=QueueMode.Batch,
media_type=MediaType.Image,
)
dataset = initial_dataset
Expand Down
29 changes: 3 additions & 26 deletions libs/labelbox/tests/integration/test_client_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import labelbox.client
from labelbox import Project, User
from labelbox.schema.media_type import MediaType


def test_missing_api_key():
Expand All @@ -29,7 +30,7 @@ def test_bad_key(rand_gen):
client = labelbox.client.Client(api_key=bad_key)

with pytest.raises(lbox.exceptions.AuthenticationError) as excinfo:
client.create_project(name=rand_gen(str))
client.create_project(name=rand_gen(str), media_type=MediaType.Image)


def test_syntax_error(client):
Expand Down Expand Up @@ -77,31 +78,7 @@ def test_network_error(client):
)

with pytest.raises(lbox.exceptions.NetworkError) as excinfo:
client.create_project(name="Project name")


def test_invalid_attribute_error(
client,
rand_gen,
):
# Creation
with pytest.raises(lbox.exceptions.InvalidAttributeError) as excinfo:
client.create_project(name="Name", invalid_field="Whatever")
assert excinfo.value.db_object_type == Project
assert excinfo.value.field == "invalid_field"

# Update
project = client.create_project(name=rand_gen(str))
with pytest.raises(lbox.exceptions.InvalidAttributeError) as excinfo:
project.update(invalid_field="Whatever")
assert excinfo.value.db_object_type == Project
assert excinfo.value.field == "invalid_field"

# Top-level-fetch
with pytest.raises(lbox.exceptions.InvalidAttributeError) as excinfo:
client.get_projects(where=User.email == "email")
assert excinfo.value.db_object_type == Project
assert excinfo.value.field == {User.email}
client.create_project(name="Project name", media_type=MediaType.Image)


@pytest.mark.skip("timeouts cause failure before rate limit")
Expand Down
8 changes: 4 additions & 4 deletions libs/labelbox/tests/integration/test_filtering.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from lbox.exceptions import InvalidQueryError

from labelbox import Project
from labelbox.schema.queue_mode import QueueMode
from labelbox.schema.media_type import MediaType


@pytest.fixture
Expand All @@ -11,9 +11,9 @@ def project_to_test_where(client, rand_gen):
p_b_name = f"b-{rand_gen(str)}"
p_c_name = f"c-{rand_gen(str)}"

p_a = client.create_project(name=p_a_name, queue_mode=QueueMode.Batch)
p_b = client.create_project(name=p_b_name, queue_mode=QueueMode.Batch)
p_c = client.create_project(name=p_c_name, queue_mode=QueueMode.Batch)
p_a = client.create_project(name=p_a_name, media_type=MediaType.Image)
p_b = client.create_project(name=p_b_name, media_type=MediaType.Image)
p_c = client.create_project(name=p_c_name, media_type=MediaType.Image)

yield p_a, p_b, p_c

Expand Down
Loading
Loading