From 0d729be7f97f8e4ee124cd1ccbf3b58da22b8979 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 4 Jun 2024 13:32:09 -0700 Subject: [PATCH 01/24] Start BaseModel conversion to v2 --- .../src/labelbox/data/annotation_types/video.py | 17 ++++++++++------- libs/labelbox/src/labelbox/utils.py | 7 +++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/video.py b/libs/labelbox/src/labelbox/data/annotation_types/video.py index 91b258de3..1c838161c 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/video.py @@ -1,6 +1,8 @@ from enum import Enum from typing import List, Optional, Tuple +from pydantic import BaseModel, model_validator + from labelbox import pydantic_compat from labelbox.data.annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation @@ -87,12 +89,13 @@ class DICOMObjectAnnotation(VideoObjectAnnotation): group_key: GroupKey -class MaskFrame(_CamelCaseMixin, pydantic_compat.BaseModel): +class MaskFrame(_CamelCaseMixin, BaseModel): index: int instance_uri: Optional[str] = None im_bytes: Optional[bytes] = None - @pydantic_compat.root_validator() + @model_validator(mode='before') + @classmethod def validate_args(cls, values): im_bytes = values.get("im_bytes") instance_uri = values.get("instance_uri") @@ -101,11 +104,11 @@ def validate_args(cls, values): raise ValueError("One of `instance_uri`, `im_bytes` required.") return values - @pydantic_compat.validator("instance_uri") - def validate_uri(cls, v): - if not is_valid_uri(v): - raise ValueError(f"{v} is not a valid uri") - return v + # @pydantic_compat.validator("instance_uri") + # def validate_uri(cls, v): + # if not is_valid_uri(v): + # raise ValueError(f"{v} is not a valid uri") + # return v class MaskInstance(_CamelCaseMixin, FeatureSchema): diff --git a/libs/labelbox/src/labelbox/utils.py b/libs/labelbox/src/labelbox/utils.py index f606932c7..f0fd1c1f4 100644 --- a/libs/labelbox/src/labelbox/utils.py +++ b/libs/labelbox/src/labelbox/utils.py @@ -6,6 +6,9 @@ from dateutil.utils import default_tzinfo from urllib.parse import urlparse + +from pydantic import BaseModel + from labelbox import pydantic_compat UPPERCASE_COMPONENTS = ['uri', 'rgb'] @@ -51,10 +54,10 @@ def is_valid_uri(uri): return False -class _CamelCaseMixin(pydantic_compat.BaseModel): +class _CamelCaseMixin(BaseModel): class Config: - allow_population_by_field_name = True + populate_by_name = True alias_generator = camel_case From b6059bb240191ee898d9be7e806a53f4af9e459a Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 4 Jun 2024 14:41:34 -0700 Subject: [PATCH 02/24] Dealing with constr --- .../src/labelbox/schema/data_row_metadata.py | 16 +++++++++------- libs/labelbox/src/labelbox/schema/export_task.py | 10 ++++++---- libs/labelbox/src/labelbox/schema/ontology.py | 11 +++++++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 3a6b58706..0998bfda3 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -5,7 +5,9 @@ from itertools import chain import warnings -from typing import List, Optional, Dict, Union, Callable, Type, Any, Generator, overload +from typing import List, Optional, Dict, Union, Callable, Type, Any, Generator, overload, Annotated + +from pydantic import StringConstraints from labelbox import pydantic_compat from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds @@ -124,17 +126,17 @@ def dict(self, *args, **kwargs): class _UpsertCustomMetadataSchemaEnumOptionInput(_CamelCaseMixin): id: Optional[SchemaId] - name: pydantic_compat.constr(strip_whitespace=True, - min_length=1, - max_length=100) + name: Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=100)] kind: str class _UpsertCustomMetadataSchemaInput(_CamelCaseMixin): id: Optional[SchemaId] - name: pydantic_compat.constr(strip_whitespace=True, - min_length=1, - max_length=100) + name: Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=100)] kind: str options: Optional[List[_UpsertCustomMetadataSchemaEnumOptionInput]] diff --git a/libs/labelbox/src/labelbox/schema/export_task.py b/libs/labelbox/src/labelbox/schema/export_task.py index b00c9166a..5d4bb5d28 100644 --- a/libs/labelbox/src/labelbox/schema/export_task.py +++ b/libs/labelbox/src/labelbox/schema/export_task.py @@ -23,6 +23,9 @@ import warnings import tempfile import os + +from pydantic import BaseModel + from labelbox import pydantic_compat from labelbox.schema.task import Task @@ -41,19 +44,19 @@ class StreamType(Enum): ERRORS = "ERRORS" -class Range(_CamelCaseMixin, pydantic_compat.BaseModel): # pylint: disable=too-few-public-methods +class Range(_CamelCaseMixin, BaseModel): # pylint: disable=too-few-public-methods """Represents a range.""" start: int end: int -class _MetadataHeader(_CamelCaseMixin, pydantic_compat.BaseModel): # pylint: disable=too-few-public-methods +class _MetadataHeader(_CamelCaseMixin, BaseModel): # pylint: disable=too-few-public-methods total_size: int total_lines: int -class _MetadataFileInfo(_CamelCaseMixin, pydantic_compat.BaseModel): # pylint: disable=too-few-public-methods +class _MetadataFileInfo(_CamelCaseMixin, BaseModel): # pylint: disable=too-few-public-methods offsets: Range lines: Range file: str @@ -920,4 +923,3 @@ def get_stream( def get_task(client, task_id): """Returns the task with the given id.""" return ExportTask(Task.get_task(client, task_id)) - \ No newline at end of file diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index f6c758faa..cf3e5ab4a 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -3,18 +3,21 @@ import colorsys from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Union, Type +from typing import Any, Dict, List, Optional, Union, Type, Annotated import warnings +from pydantic import StringConstraints + from labelbox.exceptions import InconsistentOntologyException from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship from labelbox import pydantic_compat import json -FeatureSchemaId: Type[str] = pydantic_compat.constr(min_length=25, - max_length=25) -SchemaId: Type[str] = pydantic_compat.constr(min_length=25, max_length=25) +FeatureSchemaId: Type[str] = Annotated[ + str, StringConstraints(min_length=25, max_length=25)] +SchemaId: Type[str] = Annotated[str, + StringConstraints(min_length=25, max_length=25)] class DeleteFeatureFromOntologyResult: From 80f7658bf2534061f00e285738b137b19457b0d7 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 4 Jun 2024 15:09:57 -0700 Subject: [PATCH 03/24] Dealing with BaseModel --- .../data/annotation_types/base_annotation.py | 2 +- .../classification/classification.py | 8 ++--- .../data/annotation_types/data/base_data.py | 4 +-- .../data/generic_data_row_data.py | 5 +-- .../data/annotation_types/data/raster.py | 7 ++-- .../data/annotation_types/data/text.py | 5 +-- .../data/annotation_types/data/tiled_image.py | 11 +++--- .../data/annotation_types/data/video.py | 5 +-- .../labelbox/data/annotation_types/feature.py | 7 ++-- .../annotation_types/geometry/geometry.py | 4 +-- .../data/annotation_types/geometry/line.py | 2 +- .../data/annotation_types/geometry/mask.py | 2 +- .../data/annotation_types/geometry/polygon.py | 2 +- .../labelbox/data/annotation_types/label.py | 7 ++-- .../data/annotation_types/metrics/base.py | 4 +-- .../metrics/confusion_matrix.py | 2 +- .../data/annotation_types/metrics/scalar.py | 2 +- .../annotation_types/ner/document_entity.py | 8 +++-- .../data/annotation_types/ner/text_entity.py | 7 ++-- .../data/annotation_types/relationship.py | 4 +-- .../labelbox/data/annotation_types/types.py | 2 +- .../labelbox/data/annotation_types/video.py | 14 ++++---- libs/labelbox/src/labelbox/data/mixins.py | 8 ++--- .../data/serialization/coco/annotation.py | 11 +++--- .../data/serialization/coco/categories.py | 2 +- .../serialization/coco/instance_dataset.py | 2 +- .../serialization/coco/panoptic_dataset.py | 4 +-- .../labelbox/data/serialization/coco/path.py | 4 +-- .../labelbox_v1/classification.py | 4 +-- .../data/serialization/labelbox_v1/feature.py | 7 ++-- .../data/serialization/labelbox_v1/label.py | 6 ++-- .../data/serialization/labelbox_v1/objects.py | 20 +++++------ .../data/serialization/ndjson/base.py | 8 +++-- .../serialization/ndjson/classification.py | 9 ++--- .../data/serialization/ndjson/label.py | 8 ++--- .../data/serialization/ndjson/objects.py | 20 +++++------ .../data/serialization/ndjson/relationship.py | 4 +-- .../labelbox/schema/bulk_import_request.py | 34 +++++++++---------- .../src/labelbox/schema/data_row_metadata.py | 4 +-- .../src/labelbox/schema/export_task.py | 2 +- .../src/labelbox/schema/foundry/app.py | 4 +-- .../src/labelbox/schema/foundry/model.py | 4 +-- libs/labelbox/src/labelbox/schema/ontology.py | 2 +- .../schema/send_to_annotate_params.py | 20 ++++++----- libs/labelbox/src/labelbox/utils.py | 2 +- 45 files changed, 160 insertions(+), 143 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py b/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py index 4a16b8b17..9c764daee 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py @@ -1,7 +1,7 @@ import abc from uuid import UUID, uuid4 from typing import Any, Dict, Optional -from labelbox import pydantic_compat +from pydantic import BaseModel from .feature import FeatureSchema diff --git a/libs/labelbox/src/labelbox/data/annotation_types/classification/classification.py b/libs/labelbox/src/labelbox/data/annotation_types/classification/classification.py index 9a1867ff2..f3b3ff66b 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/classification/classification.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/classification/classification.py @@ -9,12 +9,12 @@ except: from typing_extensions import Literal -from labelbox import pydantic_compat +from pydantic import BaseModel from ..feature import FeatureSchema # TODO: Replace when pydantic adds support for unions that don't coerce types -class _TempName(ConfidenceMixin, pydantic_compat.BaseModel): +class _TempName(ConfidenceMixin, BaseModel): name: str def dict(self, *args, **kwargs): @@ -47,7 +47,7 @@ def dict(self, *args, **kwargs) -> Dict[str, str]: return res -class Radio(ConfidenceMixin, CustomMetricsMixin, pydantic_compat.BaseModel): +class Radio(ConfidenceMixin, CustomMetricsMixin, BaseModel): """ A classification with only one selected option allowed >>> Radio(answer = ClassificationAnswer(name = "dog")) @@ -66,7 +66,7 @@ class Checklist(_TempName): answer: List[ClassificationAnswer] -class Text(ConfidenceMixin, CustomMetricsMixin, pydantic_compat.BaseModel): +class Text(ConfidenceMixin, CustomMetricsMixin, BaseModel): """ Free form text >>> Text(answer = "some text answer") diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/base_data.py b/libs/labelbox/src/labelbox/data/annotation_types/data/base_data.py index ab0dd1e53..2ccda34c3 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/base_data.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/base_data.py @@ -1,10 +1,10 @@ from abc import ABC from typing import Optional, Dict, List, Any -from labelbox import pydantic_compat +from pydantic import BaseModel -class BaseData(pydantic_compat.BaseModel, ABC): +class BaseData(BaseModel, ABC): """ Base class for objects representing data. This class shouldn't directly be used diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py b/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py index c4a68add6..1fabb0a9d 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py @@ -1,6 +1,6 @@ from typing import Callable, Literal, Optional -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from labelbox.data.annotation_types.data.base_data import BaseData from labelbox.utils import _NoCoercionMixin @@ -14,7 +14,8 @@ class GenericDataRowData(BaseData, _NoCoercionMixin): def create_url(self, signer: Callable[[bytes], str]) -> Optional[str]: return self.url - @pydantic_compat.root_validator(pre=True) + @model_validator(mode='before') + @classmethod(pre=True) def validate_one_datarow_key_present(cls, data): keys = ['external_id', 'global_key', 'uid'] count = sum([key in data for key in keys]) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py index 94b8a2a7e..481978eab 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py @@ -9,13 +9,13 @@ import requests import numpy as np -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from labelbox.exceptions import InternalServerError from .base_data import BaseData from ..types import TypedArray -class RasterData(pydantic_compat.BaseModel, ABC): +class RasterData(BaseModel, ABC): """Represents an image or segmentation mask. """ im_bytes: Optional[bytes] = None @@ -155,7 +155,8 @@ def create_url(self, signer: Callable[[bytes], str]) -> str: "One of url, im_bytes, file_path, arr must not be None.") return self.url - @pydantic_compat.root_validator() + @model_validator(mode='before') + @classmethod() def validate_args(cls, values): file_path = values.get("file_path") im_bytes = values.get("im_bytes") diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/text.py b/libs/labelbox/src/labelbox/data/annotation_types/data/text.py index e46eee507..aa78a7285 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/text.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/text.py @@ -4,7 +4,7 @@ from requests.exceptions import ConnectTimeout from google.api_core import retry -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from labelbox.exceptions import InternalServerError from labelbox.typing_imports import Literal from labelbox.utils import _NoCoercionMixin @@ -90,7 +90,8 @@ def create_url(self, signer: Callable[[bytes], str]) -> None: "One of url, im_bytes, file_path, numpy must not be None.") return self.url - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def validate_date(cls, values): file_path = values.get("file_path") text = values.get("text") diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py index 6a3bd6988..048b0c074 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py @@ -12,7 +12,7 @@ from PIL import Image from pyproj import Transformer from pygeotile.point import Point as PygeoPoint -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from labelbox.data.annotation_types import Rectangle, Point, Line, Polygon from .base_data import BaseData @@ -40,7 +40,7 @@ class EPSG(Enum): EPSG3857 = 3857 -class TiledBounds(pydantic_compat.BaseModel): +class TiledBounds(BaseModel): """ Bounds for a tiled image asset related to the relevant epsg. Bounds should be Point objects. @@ -66,7 +66,8 @@ def validate_bounds_not_equal(cls, bounds): return bounds #validate bounds are within lat,lng range if they are EPSG4326 - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def validate_bounds_lat_lng(cls, values): epsg = values.get('epsg') bounds = values.get('bounds') @@ -82,7 +83,7 @@ def validate_bounds_lat_lng(cls, values): return values -class TileLayer(pydantic_compat.BaseModel): +class TileLayer(BaseModel): """ Url that contains the tile layer. Must be in the format: https://c.tile.openstreetmap.org/{z}/{x}/{y}.png @@ -352,7 +353,7 @@ def validate_zoom_levels(cls, zoom_levels): return zoom_levels -class EPSGTransformer(pydantic_compat.BaseModel): +class EPSGTransformer(BaseModel): """Transformer class between different EPSG's. Useful when wanting to project in different formats. """ diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py index 3ebda5c4c..11fd71e27 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py @@ -12,7 +12,7 @@ from .base_data import BaseData from ..types import TypedArray -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator logger = logging.getLogger(__name__) @@ -148,7 +148,8 @@ def frames_to_video(self, out.release() return file_path - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def validate_data(cls, values): file_path = values.get("file_path") url = values.get("url") diff --git a/libs/labelbox/src/labelbox/data/annotation_types/feature.py b/libs/labelbox/src/labelbox/data/annotation_types/feature.py index 21e3eb413..e8db0bbfe 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/feature.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/feature.py @@ -1,11 +1,11 @@ from typing import Optional -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from .types import Cuid -class FeatureSchema(pydantic_compat.BaseModel): +class FeatureSchema(BaseModel): """ Class that represents a feature schema. Could be a annotation, a subclass, or an option. @@ -14,7 +14,8 @@ class FeatureSchema(pydantic_compat.BaseModel): name: Optional[str] = None feature_schema_id: Optional[Cuid] = None - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def must_set_one(cls, values): if values['feature_schema_id'] is None and values['name'] is None: raise ValueError( diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/geometry.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/geometry.py index 2394f011f..731e9a591 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/geometry.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/geometry.py @@ -3,12 +3,12 @@ import geojson import numpy as np -from labelbox import pydantic_compat +from pydantic import BaseModel from shapely import geometry as geom -class Geometry(pydantic_compat.BaseModel, ABC): +class Geometry(BaseModel, ABC): """Abstract base class for geometry objects """ extra: Dict[str, Any] = {} diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py index 0ae0c3b1d..ccbb401ce 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py @@ -9,7 +9,7 @@ from .point import Point from .geometry import Geometry -from labelbox import pydantic_compat +from pydantic import BaseModel class Line(Geometry): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py index 7c903b644..dd9eb22b7 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py @@ -8,7 +8,7 @@ from ..data import MaskData from .geometry import Geometry -from labelbox import pydantic_compat +from pydantic import BaseModel class Mask(Geometry): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py index 423861e31..b2e94ccbc 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py @@ -9,7 +9,7 @@ from .geometry import Geometry from .point import Point -from labelbox import pydantic_compat +from pydantic import BaseModel class Polygon(Geometry): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index f31dbdcda..4c4431c69 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -2,7 +2,7 @@ from typing import Any, Callable, Dict, List, Union, Optional import warnings -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator import labelbox from labelbox.data.annotation_types.data.generic_data_row_data import GenericDataRowData @@ -25,7 +25,7 @@ LlmResponseCreationData, GenericDataRowData] -class Label(pydantic_compat.BaseModel): +class Label(BaseModel): """Container for holding data and annotations >>> Label( @@ -53,7 +53,8 @@ class Label(pydantic_compat.BaseModel): RelationshipAnnotation]] = [] extra: Dict[str, Any] = {} - @pydantic_compat.root_validator(pre=True) + @model_validator(mode='before') + @classmethod def validate_data(cls, label): if isinstance(label.get("data"), Dict): label["data"]["class_name"] = "GenericDataRowData" diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py index 79cf22419..1608ad084 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py @@ -1,7 +1,7 @@ from abc import ABC from typing import Dict, Optional, Any, Union -from labelbox import pydantic_compat +from pydantic import BaseModel ConfidenceValue = pydantic_compat.confloat(ge=0, le=1) @@ -9,7 +9,7 @@ MAX_CONFIDENCE_SCORES = 15 -class BaseMetric(pydantic_compat.BaseModel, ABC): +class BaseMetric(BaseModel, ABC): value: Union[Any, Dict[float, Any]] feature_name: Optional[str] = None subclass_name: Optional[str] = None diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py index f915e2f25..58a52f1c9 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Tuple, Dict, Union -from labelbox import pydantic_compat +from pydantic import BaseModel from .base import ConfidenceValue, BaseMetric diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py index 5f1279fd6..d79624152 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py @@ -3,7 +3,7 @@ from .base import ConfidenceValue, BaseMetric -from labelbox import pydantic_compat +from pydantic import BaseModel ScalarMetricValue = pydantic_compat.confloat(ge=0, le=100_000_000) ScalarMetricConfidenceValue = Dict[ConfidenceValue, ScalarMetricValue] diff --git a/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py b/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py index 77141de06..a5db33f4e 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py @@ -1,10 +1,12 @@ from typing import List -from labelbox import pydantic_compat +from pydantic import BaseModel + +from pydantic import BaseModel from labelbox.utils import _CamelCaseMixin -class DocumentTextSelection(_CamelCaseMixin, pydantic_compat.BaseModel): +class DocumentTextSelection(_CamelCaseMixin, BaseModel): token_ids: List[str] group_id: str page: int @@ -16,6 +18,6 @@ def validate_page(cls, v): return v -class DocumentEntity(_CamelCaseMixin, pydantic_compat.BaseModel): +class DocumentEntity(_CamelCaseMixin, BaseModel): """ Represents a text entity """ text_selections: List[DocumentTextSelection] diff --git a/libs/labelbox/src/labelbox/data/annotation_types/ner/text_entity.py b/libs/labelbox/src/labelbox/data/annotation_types/ner/text_entity.py index 6f410987f..e2eac14c0 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/ner/text_entity.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/ner/text_entity.py @@ -1,15 +1,16 @@ from typing import Dict, Any -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator -class TextEntity(pydantic_compat.BaseModel): +class TextEntity(BaseModel): """ Represents a text entity """ start: int end: int extra: Dict[str, Any] = {} - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def validate_start_end(cls, values): if 'start' in values and 'end' in values: if (isinstance(values['start'], int) and diff --git a/libs/labelbox/src/labelbox/data/annotation_types/relationship.py b/libs/labelbox/src/labelbox/data/annotation_types/relationship.py index db61883d5..27a833830 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/relationship.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/relationship.py @@ -1,9 +1,9 @@ -from labelbox import pydantic_compat +from pydantic import BaseModel from enum import Enum from labelbox.data.annotation_types.annotation import BaseAnnotation, ObjectAnnotation -class Relationship(pydantic_compat.BaseModel): +class Relationship(BaseModel): class Type(Enum): UNIDIRECTIONAL = "unidirectional" diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index 3305462b1..aac119f20 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -5,7 +5,7 @@ from packaging import version import numpy as np -from labelbox import pydantic_compat +from pydantic import BaseModel Cuid = Annotated[str, pydantic_compat.Field(min_length=25, max_length=25)] diff --git a/libs/labelbox/src/labelbox/data/annotation_types/video.py b/libs/labelbox/src/labelbox/data/annotation_types/video.py index 1c838161c..3ebd9ff69 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/video.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, model_validator -from labelbox import pydantic_compat +from pydantic import BaseModel from labelbox.data.annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation from labelbox.data.annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation @@ -104,11 +104,11 @@ def validate_args(cls, values): raise ValueError("One of `instance_uri`, `im_bytes` required.") return values - # @pydantic_compat.validator("instance_uri") - # def validate_uri(cls, v): - # if not is_valid_uri(v): - # raise ValueError(f"{v} is not a valid uri") - # return v + @pydantic_compat.validator("instance_uri") + def validate_uri(cls, v): + if not is_valid_uri(v): + raise ValueError(f"{v} is not a valid uri") + return v class MaskInstance(_CamelCaseMixin, FeatureSchema): @@ -116,7 +116,7 @@ class MaskInstance(_CamelCaseMixin, FeatureSchema): name: str -class VideoMaskAnnotation(pydantic_compat.BaseModel): +class VideoMaskAnnotation(BaseModel): """Video mask annotation >>> VideoMaskAnnotation( >>> frames=[ diff --git a/libs/labelbox/src/labelbox/data/mixins.py b/libs/labelbox/src/labelbox/data/mixins.py index 36fd91671..d8d4456d6 100644 --- a/libs/labelbox/src/labelbox/data/mixins.py +++ b/libs/labelbox/src/labelbox/data/mixins.py @@ -1,11 +1,11 @@ from typing import Optional, List -from labelbox import pydantic_compat +from pydantic import BaseModel from labelbox.exceptions import ConfidenceNotSupportedException, CustomMetricsNotSupportedException -class ConfidenceMixin(pydantic_compat.BaseModel): +class ConfidenceMixin(BaseModel): confidence: Optional[float] = None @pydantic_compat.validator("confidence") @@ -32,7 +32,7 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls) -class CustomMetric(pydantic_compat.BaseModel): +class CustomMetric(BaseModel): name: str value: float @@ -49,7 +49,7 @@ def value_valid_float(cls, value): return value -class CustomMetricsMixin(pydantic_compat.BaseModel): +class CustomMetricsMixin(BaseModel): custom_metrics: Optional[List[CustomMetric]] = None def dict(self, *args, **kwargs): diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py b/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py index 64742c8e2..e0bce98fd 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py @@ -35,12 +35,13 @@ def get_annotation_lookup(annotations): annotation_lookup = defaultdict(list) for annotation in annotations: # Provide a default value of None if the attribute doesn't exist - attribute_value = getattr(annotation, 'image_id', None) or getattr(annotation, 'name', None) + attribute_value = getattr(annotation, 'image_id', None) or getattr( + annotation, 'name', None) annotation_lookup[attribute_value].append(annotation) - return annotation_lookup + return annotation_lookup -class SegmentInfo(pydantic_compat.BaseModel): +class SegmentInfo(BaseModel): id: int category_id: int area: int @@ -48,12 +49,12 @@ class SegmentInfo(pydantic_compat.BaseModel): iscrowd: int = 0 -class RLE(pydantic_compat.BaseModel): +class RLE(BaseModel): counts: List[int] size: Tuple[int, int] # h,w or w,h? -class COCOObjectAnnotation(pydantic_compat.BaseModel): +class COCOObjectAnnotation(BaseModel): # All segmentations for a particular class in an image... # So each image will have one of these for each class present in the image.. # Annotations only exist if there is data.. diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/categories.py b/libs/labelbox/src/labelbox/data/serialization/coco/categories.py index 167737c67..e7b05fbdc 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/categories.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/categories.py @@ -4,7 +4,7 @@ from .... import pydantic_compat -class Categories(pydantic_compat.BaseModel): +class Categories(BaseModel): id: int name: str supercategory: str diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py b/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py index 9a6b122f3..d57fe1ded 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py @@ -129,7 +129,7 @@ def process_label( return image, coco_annotations, categories -class CocoInstanceDataset(pydantic_compat.BaseModel): +class CocoInstanceDataset(BaseModel): info: Dict[str, Any] = {} images: List[CocoImage] annotations: List[COCOObjectAnnotation] diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/panoptic_dataset.py b/libs/labelbox/src/labelbox/data/serialization/coco/panoptic_dataset.py index b6d2c9ae6..d041c6e30 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/panoptic_dataset.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/panoptic_dataset.py @@ -2,7 +2,7 @@ from typing import Dict, Any, List, Union from pathlib import Path -from labelbox import pydantic_compat +from pydantic import BaseModel from tqdm import tqdm import numpy as np from PIL import Image @@ -115,7 +115,7 @@ def process_label(label: Label, segments_info=segments), categories, is_thing -class CocoPanopticDataset(pydantic_compat.BaseModel): +class CocoPanopticDataset(BaseModel): info: Dict[str, Any] = {} images: List[CocoImage] annotations: List[PanopticAnnotation] diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/path.py b/libs/labelbox/src/labelbox/data/serialization/coco/path.py index 6f523152c..ecd860417 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/path.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/path.py @@ -1,8 +1,8 @@ -from labelbox import pydantic_compat +from pydantic import BaseModel from pathlib import Path -class PathSerializerMixin(pydantic_compat.BaseModel): +class PathSerializerMixin(BaseModel): def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/classification.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/classification.py index 8600f08e3..40cccefcc 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/classification.py @@ -1,6 +1,6 @@ from typing import List, Union -from labelbox import pydantic_compat +from pydantic import BaseModel from .feature import LBV1Feature from ...annotation_types.annotation import ClassificationAnnotation @@ -90,7 +90,7 @@ def from_common(cls, text: Text, feature_schema_id: Cuid, return cls(schema_id=feature_schema_id, answer=text.answer, **extra) -class LBV1Classifications(pydantic_compat.BaseModel): +class LBV1Classifications(BaseModel): classifications: List[Union[LBV1Text, LBV1Radio, LBV1Dropdown, LBV1Checklist]] = [] diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py index cefddd079..f2f552dca 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py @@ -1,19 +1,20 @@ from typing import Optional -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from labelbox.utils import camel_case from ...annotation_types.types import Cuid -class LBV1Feature(pydantic_compat.BaseModel): +class LBV1Feature(BaseModel): keyframe: Optional[bool] = None title: str = None value: Optional[str] = None schema_id: Optional[Cuid] = None feature_id: Optional[Cuid] = None - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def check_ids(cls, values): if values.get('value') is None: values['value'] = values['title'] diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py index ee45bc8f0..2bc19a46a 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py @@ -2,7 +2,7 @@ from labelbox.utils import camel_case from typing import List, Optional, Union, Dict, Any -from labelbox import pydantic_compat +from pydantic import BaseModel from ...annotation_types.annotation import (ClassificationAnnotation, ObjectAnnotation) @@ -104,7 +104,7 @@ class Config: allow_population_by_field_name = True -class Review(pydantic_compat.BaseModel): +class Review(BaseModel): score: int id: str created_at: str @@ -118,7 +118,7 @@ class Config: Extra = lambda name: pydantic_compat.Field(None, alias=name, extra_field=True) -class LBV1Label(pydantic_compat.BaseModel): +class LBV1Label(BaseModel): label: Union[LBV1LabelAnnotations, List[LBV1LabelAnnotationsVideo]] = pydantic_compat.Field( ..., alias='Label') diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py index 19f6c0717..7ddd9817a 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py @@ -4,7 +4,7 @@ except: from typing_extensions import Literal -from labelbox import pydantic_compat +from pydantic import BaseModel import numpy as np from .classification import LBV1Checklist, LBV1Classifications, LBV1Radio, LBV1Text, LBV1Dropdown @@ -49,19 +49,19 @@ def validate_subclasses(cls, value, field): return value -class TIPointCoordinate(pydantic_compat.BaseModel): +class TIPointCoordinate(BaseModel): coordinates: List[float] -class TILineCoordinate(pydantic_compat.BaseModel): +class TILineCoordinate(BaseModel): coordinates: List[List[float]] -class TIPolygonCoordinate(pydantic_compat.BaseModel): +class TIPolygonCoordinate(BaseModel): coordinates: List[List[List[float]]] -class TIRectangleCoordinate(pydantic_compat.BaseModel): +class TIRectangleCoordinate(BaseModel): coordinates: List[List[List[float]]] @@ -111,12 +111,12 @@ def to_common(self) -> Rectangle: end=Point(x=end[0], y=end[1])) -class _Point(pydantic_compat.BaseModel): +class _Point(BaseModel): x: float y: float -class _Box(pydantic_compat.BaseModel): +class _Box(BaseModel): top: float left: float height: float @@ -230,12 +230,12 @@ def from_common(cls, mask: Mask, }) -class _TextPoint(pydantic_compat.BaseModel): +class _TextPoint(BaseModel): start: int end: int -class _Location(pydantic_compat.BaseModel): +class _Location(BaseModel): location: _TextPoint @@ -263,7 +263,7 @@ def from_common(cls, text_entity: TextEntity, **extra) -class LBV1Objects(pydantic_compat.BaseModel): +class LBV1Objects(BaseModel): objects: List[Union[ LBV1Line, LBV1Point, diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index 2a9186e02..f9d72fa59 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -2,7 +2,7 @@ from uuid import uuid4 from labelbox.utils import _CamelCaseMixin, is_exactly_one_set -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from ...annotation_types.types import Cuid @@ -10,7 +10,8 @@ class DataRow(_CamelCaseMixin): id: str = None global_key: str = None - @pydantic_compat.root_validator() + @model_validator(mode='before') + @classmethod() def must_set_one(cls, values): if not is_exactly_one_set(values.get('id'), values.get('global_key')): raise ValueError("Must set either id or global_key") @@ -42,7 +43,8 @@ class NDAnnotation(NDJsonBase): page: Optional[int] = None unit: Optional[str] = None - @pydantic_compat.root_validator() + @model_validator(mode='before') + @classmethod() def must_set_one(cls, values): if ('schema_id' not in values or values['schema_id'] is None) and ('name' not in values or values['name'] is None): diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index 028eeded8..6c3052451 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Union, Optional -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from labelbox.data.mixins import ConfidenceMixin, CustomMetric, CustomMetricsMixin from labelbox.data.serialization.ndjson.base import DataRow, NDAnnotation @@ -17,7 +17,8 @@ class NDAnswer(ConfidenceMixin, CustomMetricsMixin): schema_id: Optional[Cuid] = None classifications: Optional[List['NDSubclassificationType']] = [] - @pydantic_compat.root_validator() + @model_validator(mode='before') + @classmethod() def must_set_one(cls, values): if ('schema_id' not in values or values['schema_id'] is None) and ('name' not in values or values['name'] is None): @@ -43,12 +44,12 @@ class Config: alias_generator = camel_case -class FrameLocation(pydantic_compat.BaseModel): +class FrameLocation(BaseModel): end: int start: int -class VideoSupported(pydantic_compat.BaseModel): +class VideoSupported(BaseModel): # Note that frames are only allowed as top level inferences for video frames: Optional[List[FrameLocation]] = None diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py index 1b649a80e..4b8211d14 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py @@ -4,7 +4,7 @@ from collections import defaultdict import warnings -from labelbox import pydantic_compat +from pydantic import BaseModel from ...annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation from ...annotation_types.relationship import RelationshipAnnotation @@ -28,16 +28,16 @@ NDSegments, NDDicomMasks, NDVideoMasks, NDRelationship] -class NDLabel(pydantic_compat.BaseModel): +class NDLabel(BaseModel): annotations: List[AnnotationType] - class _Relationship(pydantic_compat.BaseModel): + class _Relationship(BaseModel): """This object holds information about the relationship""" ndjson: NDRelationship source: str target: str - class _AnnotationGroup(pydantic_compat.BaseModel): + class _AnnotationGroup(BaseModel): """Stores all the annotations and relationships per datarow""" data_row: DataRow = None ndjson_annotations: Dict[str, AnnotationType] = {} diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py index fd13a6bf6..91822f2ab 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py @@ -7,7 +7,7 @@ from labelbox.data.mixins import ConfidenceMixin, CustomMetricsMixin, CustomMetric, CustomMetricsNotSupportedMixin import numpy as np -from labelbox import pydantic_compat +from pydantic import BaseModel from PIL import Image from labelbox.data.annotation_types import feature @@ -27,21 +27,21 @@ class NDBaseObject(NDAnnotation): classifications: List[NDSubclassificationType] = [] -class VideoSupported(pydantic_compat.BaseModel): +class VideoSupported(BaseModel): # support for video for objects are per-frame basis frame: int -class DicomSupported(pydantic_compat.BaseModel): +class DicomSupported(BaseModel): group_key: str -class _Point(pydantic_compat.BaseModel): +class _Point(BaseModel): x: float y: float -class Bbox(pydantic_compat.BaseModel): +class Bbox(BaseModel): top: float left: float height: float @@ -328,7 +328,7 @@ def from_common( classifications=classifications) -class NDSegment(pydantic_compat.BaseModel): +class NDSegment(BaseModel): keyframes: List[Union[NDFrameRectangle, NDFramePoint, NDFrameLine]] @staticmethod @@ -454,12 +454,12 @@ def from_common(cls, segments: List[DICOMObjectAnnotation], data: VideoData, group_key=group_key) -class _URIMask(pydantic_compat.BaseModel): +class _URIMask(BaseModel): instanceURI: str colorRGB: Tuple[int, int, int] -class _PNGMask(pydantic_compat.BaseModel): +class _PNGMask(BaseModel): png: str @@ -512,7 +512,7 @@ def from_common( custom_metrics=custom_metrics) -class NDVideoMasksFramesInstances(pydantic_compat.BaseModel): +class NDVideoMasksFramesInstances(BaseModel): frames: List[MaskFrame] instances: List[MaskInstance] @@ -564,7 +564,7 @@ def from_common(cls, annotation, data): ) -class Location(pydantic_compat.BaseModel): +class Location(BaseModel): start: int end: int diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/relationship.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/relationship.py index 82976aedb..d95c1584f 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/relationship.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/relationship.py @@ -1,5 +1,5 @@ from typing import Union -from labelbox import pydantic_compat +from pydantic import BaseModel from .base import NDAnnotation, DataRow from ...annotation_types.data import ImageData, TextData from ...annotation_types.relationship import RelationshipAnnotation @@ -10,7 +10,7 @@ SUPPORTED_ANNOTATIONS = NDObjectType -class _Relationship(pydantic_compat.BaseModel): +class _Relationship(BaseModel): source: str target: str type: str diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index 65b71a310..cb9b8c13e 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -8,7 +8,7 @@ from google.api_core import retry from labelbox import parser import requests -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator from typing_extensions import Literal from typing import (Any, List, Optional, BinaryIO, Dict, Iterable, Tuple, Union, Type, Set, TYPE_CHECKING) @@ -505,24 +505,24 @@ def get_mal_schemas(ontology): LabelboxID: str = pydantic_compat.Field(..., min_length=25, max_length=25) -class Bbox(pydantic_compat.BaseModel): +class Bbox(BaseModel): top: float left: float height: float width: float -class Point(pydantic_compat.BaseModel): +class Point(BaseModel): x: float y: float -class FrameLocation(pydantic_compat.BaseModel): +class FrameLocation(BaseModel): end: int start: int -class VideoSupported(pydantic_compat.BaseModel): +class VideoSupported(BaseModel): #Note that frames are only allowed as top level inferences for video frames: Optional[List[FrameLocation]] @@ -554,18 +554,17 @@ def get_union_types(cls): return union_types[0].__args__[0].__args__ @classmethod - def build(cls: Any, data: Union[dict, - pydantic_compat.BaseModel]) -> "NDBase": + def build(cls: Any, data: Union[dict, BaseModel]) -> "NDBase": """ Checks through all objects in the union to see which matches the input data. Args: - data (Union[dict, pydantic_compat.BaseModel]) : The data for constructing one of the objects in the union + data (Union[dict, BaseModel]) : The data for constructing one of the objects in the union raises: KeyError: data does not contain the determinant fields for any of the types supported by this SpecialUnion pydantic_compat.ValidationError: Error while trying to construct a specific object in the union """ - if isinstance(data, pydantic_compat.BaseModel): + if isinstance(data, BaseModel): data = data.dict() top_level_fields = [] @@ -607,15 +606,16 @@ def schema(cls): return results -class DataRow(pydantic_compat.BaseModel): +class DataRow(BaseModel): id: str -class NDFeatureSchema(pydantic_compat.BaseModel): +class NDFeatureSchema(BaseModel): schemaId: Optional[str] = None name: Optional[str] = None - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def must_set_one(cls, values): if values['schemaId'] is None and values['name'] is None: raise ValueError( @@ -814,7 +814,7 @@ class NDPoint(NDBaseTool): #Could check if points are positive -class EntityLocation(pydantic_compat.BaseModel): +class EntityLocation(BaseModel): start: int end: int @@ -825,7 +825,7 @@ class NDTextEntity(NDBaseTool): @pydantic_compat.validator('location') def is_valid_location(cls, v): - if isinstance(v, pydantic_compat.BaseModel): + if isinstance(v, BaseModel): v = v.dict() if len(v) < 2: @@ -840,7 +840,7 @@ def is_valid_location(cls, v): return v -class RLEMaskFeatures(pydantic_compat.BaseModel): +class RLEMaskFeatures(BaseModel): counts: List[int] size: List[int] @@ -864,12 +864,12 @@ def validate_size(cls, size): return size -class PNGMaskFeatures(pydantic_compat.BaseModel): +class PNGMaskFeatures(BaseModel): # base64 encoded png bytes png: str -class URIMaskFeatures(pydantic_compat.BaseModel): +class URIMaskFeatures(BaseModel): instanceURI: str colorRGB: Union[List[int], Tuple[int, int, int]] diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 0998bfda3..13a35ce3d 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -9,7 +9,7 @@ from pydantic import StringConstraints -from labelbox import pydantic_compat +from pydantic import BaseModel from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds from labelbox.schema.identifiable import UniqueId, GlobalKey @@ -27,7 +27,7 @@ class DataRowMetadataKind(Enum): # Metadata schema -class DataRowMetadataSchema(pydantic_compat.BaseModel): +class DataRowMetadataSchema(BaseModel): uid: SchemaId name: pydantic_compat.constr(strip_whitespace=True, min_length=1, diff --git a/libs/labelbox/src/labelbox/schema/export_task.py b/libs/labelbox/src/labelbox/schema/export_task.py index 5d4bb5d28..545fd8ca9 100644 --- a/libs/labelbox/src/labelbox/schema/export_task.py +++ b/libs/labelbox/src/labelbox/schema/export_task.py @@ -26,7 +26,7 @@ from pydantic import BaseModel -from labelbox import pydantic_compat +from pydantic import BaseModel from labelbox.schema.task import Task from labelbox.utils import _CamelCaseMixin diff --git a/libs/labelbox/src/labelbox/schema/foundry/app.py b/libs/labelbox/src/labelbox/schema/foundry/app.py index eead39518..b31709f80 100644 --- a/libs/labelbox/src/labelbox/schema/foundry/app.py +++ b/libs/labelbox/src/labelbox/schema/foundry/app.py @@ -1,11 +1,11 @@ from labelbox.utils import _CamelCaseMixin -from labelbox import pydantic_compat +from pydantic import BaseModel from typing import Any, Dict, Optional -class App(_CamelCaseMixin, pydantic_compat.BaseModel): +class App(_CamelCaseMixin, BaseModel): id: Optional[str] model_id: str name: str diff --git a/libs/labelbox/src/labelbox/schema/foundry/model.py b/libs/labelbox/src/labelbox/schema/foundry/model.py index 16ccae422..3e7ebd6e7 100644 --- a/libs/labelbox/src/labelbox/schema/foundry/model.py +++ b/libs/labelbox/src/labelbox/schema/foundry/model.py @@ -1,12 +1,12 @@ from labelbox.utils import _CamelCaseMixin -from labelbox import pydantic_compat +from pydantic import BaseModel from datetime import datetime from typing import Dict -class Model(_CamelCaseMixin, pydantic_compat.BaseModel): +class Model(_CamelCaseMixin, BaseModel): id: str description: str inference_params_json_schema: Dict diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index cf3e5ab4a..c9a80813c 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -11,7 +11,7 @@ from labelbox.exceptions import InconsistentOntologyException from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship -from labelbox import pydantic_compat +from pydantic import BaseModel import json FeatureSchemaId: Type[str] = Annotated[ diff --git a/libs/labelbox/src/labelbox/schema/send_to_annotate_params.py b/libs/labelbox/src/labelbox/schema/send_to_annotate_params.py index f86844fda..6587ba88a 100644 --- a/libs/labelbox/src/labelbox/schema/send_to_annotate_params.py +++ b/libs/labelbox/src/labelbox/schema/send_to_annotate_params.py @@ -3,7 +3,7 @@ from typing import Optional, Dict from labelbox.schema.conflict_resolution_strategy import ConflictResolutionStrategy -from labelbox import pydantic_compat +from pydantic import BaseModel, model_validator if sys.version_info >= (3, 8): from typing import TypedDict @@ -11,7 +11,7 @@ from typing_extensions import TypedDict -class SendToAnnotateFromCatalogParams(pydantic_compat.BaseModel): +class SendToAnnotateFromCatalogParams(BaseModel): """ Extra parameters for sending data rows to a project through catalog. At least one of source_model_run_id or source_project_id must be provided. @@ -40,18 +40,20 @@ class SendToAnnotateFromCatalogParams(pydantic_compat.BaseModel): ConflictResolutionStrategy] = ConflictResolutionStrategy.KeepExisting batch_priority: Optional[int] = 5 - @pydantic_compat.root_validator + @model_validator(mode='before') + @classmethod def check_project_id_or_model_run_id(cls, values): - if not values.get("source_model_run_id") and not values.get("source_project_id"): + if not values.get("source_model_run_id") and not values.get( + "source_project_id"): raise ValueError( - 'Either source_project_id or source_model_id are required' - ) - if values.get("source_model_run_id") and values.get("source_project_id"): + 'Either source_project_id or source_model_id are required') + if values.get("source_model_run_id") and values.get( + "source_project_id"): raise ValueError( - 'Provide only a source_project_id or source_model_id not both' - ) + 'Provide only a source_project_id or source_model_id not both') return values + class SendToAnnotateFromModelParams(TypedDict): """ Extra parameters for sending data rows to a project through a model run. diff --git a/libs/labelbox/src/labelbox/utils.py b/libs/labelbox/src/labelbox/utils.py index f0fd1c1f4..f2d9b10ad 100644 --- a/libs/labelbox/src/labelbox/utils.py +++ b/libs/labelbox/src/labelbox/utils.py @@ -9,7 +9,7 @@ from pydantic import BaseModel -from labelbox import pydantic_compat +from pydantic import BaseModel UPPERCASE_COMPONENTS = ['uri', 'rgb'] ISO_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' From 4d4713b14778ac1db1bc98ea292f37f01493889c Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 4 Jun 2024 15:58:05 -0700 Subject: [PATCH 04/24] Dealing with constrained types --- libs/labelbox/src/labelbox/data/annotation_types/types.py | 6 ++++-- libs/labelbox/src/labelbox/schema/bulk_import_request.py | 6 +++--- libs/labelbox/src/labelbox/schema/ontology.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index aac119f20..aac1cdf7b 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -1,13 +1,15 @@ import sys from typing import Generic, TypeVar, Any +from pydantic import StringConstraints + from typing_extensions import Annotated from packaging import version import numpy as np -from pydantic import BaseModel +from pydantic import Field -Cuid = Annotated[str, pydantic_compat.Field(min_length=25, max_length=25)] +Cuid = Annotated[str, StringConstraints(min_length=25, max_length=25)] DType = TypeVar('DType') DShape = TypeVar('DShape') diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index cb9b8c13e..49ebaa51e 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -8,10 +8,10 @@ from google.api_core import retry from labelbox import parser import requests -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, StringConstraints from typing_extensions import Literal from typing import (Any, List, Optional, BinaryIO, Dict, Iterable, Tuple, Union, - Type, Set, TYPE_CHECKING) + Type, Set, TYPE_CHECKING, Annotated) from labelbox import exceptions as lb_exceptions from labelbox.orm.model import Entity @@ -502,7 +502,7 @@ def get_mal_schemas(ontology): return valid_feature_schemas_by_schema_id, valid_feature_schemas_by_name -LabelboxID: str = pydantic_compat.Field(..., min_length=25, max_length=25) +LabelboxID = Annotated[str, StringConstraints(min_length=25, max_length=25)] class Bbox(BaseModel): diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index c9a80813c..7081e86b8 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -11,7 +11,6 @@ from labelbox.exceptions import InconsistentOntologyException from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship -from pydantic import BaseModel import json FeatureSchemaId: Type[str] = Annotated[ From 59de9154805167519fcfba3d1548799353c96805 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 10:59:47 -0700 Subject: [PATCH 05/24] Support Field --- .../data/annotation_types/data/tiled_image.py | 2 +- .../labelbox/data/annotation_types/label.py | 2 +- .../metrics/confusion_matrix.py | 5 ++--- .../labelbox/data/annotation_types/video.py | 2 +- libs/labelbox/src/labelbox/data/mixins.py | 2 +- .../data/serialization/labelbox_v1/label.py | 20 +++++++++---------- .../data/serialization/labelbox_v1/objects.py | 10 +++++----- .../serialization/ndjson/classification.py | 4 ++-- .../labelbox/schema/bulk_import_request.py | 16 +++++++-------- 9 files changed, 31 insertions(+), 32 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py index 048b0c074..2362c66fc 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py @@ -12,7 +12,7 @@ from PIL import Image from pyproj import Transformer from pygeotile.point import Point as PygeoPoint -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, field_validator from labelbox.data.annotation_types import Rectangle, Point, Line, Polygon from .base_data import BaseData diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index 4c4431c69..59256613a 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -2,7 +2,7 @@ from typing import Any, Callable, Dict, List, Union, Optional import warnings -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, field_validator import labelbox from labelbox.data.annotation_types.data.generic_data_row_data import GenericDataRowData diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py index 58a52f1c9..dd53919a6 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Tuple, Dict, Union -from pydantic import BaseModel +from pydantic import Field from .base import ConfidenceValue, BaseMetric @@ -30,5 +30,4 @@ class ConfusionMatrixMetric(BaseMetric): metric_name: str value: Union[ConfusionMatrixMetricValue, ConfusionMatrixMetricConfidenceValue] - aggregation: ConfusionMatrixAggregation = pydantic_compat.Field( - ConfusionMatrixAggregation.CONFUSION_MATRIX, const=True) + aggregation: ConfusionMatrixAggregation = ConfusionMatrixAggregation.CONFUSION_MATRIX diff --git a/libs/labelbox/src/labelbox/data/annotation_types/video.py b/libs/labelbox/src/labelbox/data/annotation_types/video.py index 3ebd9ff69..0278cda11 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/video.py @@ -1,7 +1,7 @@ from enum import Enum from typing import List, Optional, Tuple -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, field_validator from pydantic import BaseModel from labelbox.data.annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation diff --git a/libs/labelbox/src/labelbox/data/mixins.py b/libs/labelbox/src/labelbox/data/mixins.py index d8d4456d6..d42ea8d73 100644 --- a/libs/labelbox/src/labelbox/data/mixins.py +++ b/libs/labelbox/src/labelbox/data/mixins.py @@ -1,6 +1,6 @@ from typing import Optional, List -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from labelbox.exceptions import ConfidenceNotSupportedException, CustomMetricsNotSupportedException diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py index 2bc19a46a..a711d2d25 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py @@ -2,7 +2,7 @@ from labelbox.utils import camel_case from typing import List, Optional, Union, Dict, Any -from pydantic import BaseModel +from pydantic import BaseModel, Field from ...annotation_types.annotation import (ClassificationAnnotation, ObjectAnnotation) @@ -35,7 +35,7 @@ def from_common( class LBV1LabelAnnotationsVideo(LBV1LabelAnnotations): - frame_number: int = pydantic_compat.Field(..., alias='frameNumber') + frame_number: int = Field(..., alias='frameNumber') def to_common( self @@ -115,21 +115,21 @@ class Config: alias_generator = camel_case -Extra = lambda name: pydantic_compat.Field(None, alias=name, extra_field=True) +Extra = lambda name: Field(None, alias=name, extra_field=True) class LBV1Label(BaseModel): label: Union[LBV1LabelAnnotations, - List[LBV1LabelAnnotationsVideo]] = pydantic_compat.Field( + List[LBV1LabelAnnotationsVideo]] = Field( ..., alias='Label') - data_row_id: str = pydantic_compat.Field(..., alias="DataRow ID") - row_data: str = pydantic_compat.Field(None, alias="Labeled Data") - id: Optional[str] = pydantic_compat.Field(None, alias='ID') - external_id: Optional[str] = pydantic_compat.Field(None, + data_row_id: str = Field(..., alias="DataRow ID") + row_data: str = Field(None, alias="Labeled Data") + id: Optional[str] = Field(None, alias='ID') + external_id: Optional[str] = Field(None, alias="External ID") - data_row_media_attributes: Optional[Dict[str, Any]] = pydantic_compat.Field( + data_row_media_attributes: Optional[Dict[str, Any]] = Field( {}, alias="Media Attributes") - data_row_metadata: Optional[List[Dict[str, Any]]] = pydantic_compat.Field( + data_row_metadata: Optional[List[Dict[str, Any]]] = Field( [], alias="DataRow Metadata") created_by: Optional[str] = Extra('Created By') diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py index 7ddd9817a..76198fa44 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py @@ -4,7 +4,7 @@ except: from typing_extensions import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field import numpy as np from .classification import LBV1Checklist, LBV1Classifications, LBV1Radio, LBV1Text, LBV1Dropdown @@ -66,7 +66,7 @@ class TIRectangleCoordinate(BaseModel): class LBV1TIPoint(LBV1ObjectBase): - object_type: Literal['point'] = pydantic_compat.Field(..., alias='type') + object_type: Literal['point'] = Field(..., alias='type') geometry: TIPointCoordinate def to_common(self) -> Point: @@ -75,7 +75,7 @@ def to_common(self) -> Point: class LBV1TILine(LBV1ObjectBase): - object_type: Literal['polyline'] = pydantic_compat.Field(..., alias='type') + object_type: Literal['polyline'] = Field(..., alias='type') geometry: TILineCoordinate def to_common(self) -> Line: @@ -85,7 +85,7 @@ def to_common(self) -> Line: class LBV1TIPolygon(LBV1ObjectBase): - object_type: Literal['polygon'] = pydantic_compat.Field(..., alias='type') + object_type: Literal['polygon'] = Field(..., alias='type') geometry: TIPolygonCoordinate def to_common(self) -> Polygon: @@ -95,7 +95,7 @@ def to_common(self) -> Polygon: class LBV1TIRectangle(LBV1ObjectBase): - object_type: Literal['rectangle'] = pydantic_compat.Field(..., alias='type') + object_type: Literal['rectangle'] = Field(..., alias='type') geometry: TIRectangleCoordinate def to_common(self) -> Rectangle: diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index 6c3052451..d2de3d7fb 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Union, Optional -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, Field from labelbox.data.mixins import ConfidenceMixin, CustomMetric, CustomMetricsMixin from labelbox.data.serialization.ndjson.base import DataRow, NDAnnotation @@ -82,7 +82,7 @@ def from_common(cls, text: Text, name: str, class NDChecklistSubclass(NDAnswer): - answer: List[NDAnswer] = pydantic_compat.Field(..., alias='answers') + answer: List[NDAnswer] = Field(..., alias='answers') def to_common(self) -> Checklist: diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index 49ebaa51e..585a99beb 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -8,7 +8,7 @@ from google.api_core import retry from labelbox import parser import requests -from pydantic import BaseModel, model_validator, StringConstraints +from pydantic import BaseModel, model_validator, StringConstraints, Field as PydanticField, field_validator from typing_extensions import Literal from typing import (Any, List, Optional, BinaryIO, Dict, Iterable, Tuple, Union, Type, Set, TYPE_CHECKING, Annotated) @@ -716,7 +716,7 @@ def validate_feature_schemas(self, valid_feature_schemas_by_id, class NDRadio(VideoSupported, NDBase): ontology_type: Literal["radio"] = "radio" - answer: NDFeatureSchema = pydantic_compat.Field(determinant=True) + answer: NDFeatureSchema = PydanticField(determinant=True) def validate_feature_schemas(self, valid_feature_schemas_by_id, valid_feature_schemas_by_name): @@ -780,7 +780,7 @@ def validate_subclasses(cls, value, field): class NDPolygon(NDBaseTool): ontology_type: Literal["polygon"] = "polygon" - polygon: List[Point] = pydantic_compat.Field(determinant=True) + polygon: List[Point] = PydanticField(determinant=True) @pydantic_compat.validator('polygon') def is_geom_valid(cls, v): @@ -792,7 +792,7 @@ def is_geom_valid(cls, v): class NDPolyline(NDBaseTool): ontology_type: Literal["line"] = "line" - line: List[Point] = pydantic_compat.Field(determinant=True) + line: List[Point] = PydanticField(determinant=True) @pydantic_compat.validator('line') def is_geom_valid(cls, v): @@ -804,13 +804,13 @@ def is_geom_valid(cls, v): class NDRectangle(NDBaseTool): ontology_type: Literal["rectangle"] = "rectangle" - bbox: Bbox = pydantic_compat.Field(determinant=True) + bbox: Bbox = PydanticField(determinant=True) #Could check if points are positive class NDPoint(NDBaseTool): ontology_type: Literal["point"] = "point" - point: Point = pydantic_compat.Field(determinant=True) + point: Point = PydanticField(determinant=True) #Could check if points are positive @@ -821,7 +821,7 @@ class EntityLocation(BaseModel): class NDTextEntity(NDBaseTool): ontology_type: Literal["named-entity"] = "named-entity" - location: EntityLocation = pydantic_compat.Field(determinant=True) + location: EntityLocation = PydanticField(determinant=True) @pydantic_compat.validator('location') def is_valid_location(cls, v): @@ -893,7 +893,7 @@ def validate_color(cls, colorRGB): class NDMask(NDBaseTool): ontology_type: Literal["superpixel"] = "superpixel" mask: Union[URIMaskFeatures, PNGMaskFeatures, - RLEMaskFeatures] = pydantic_compat.Field(determinant=True) + RLEMaskFeatures] = PydanticField(determinant=True) #A union with custom construction logic to improve error messages From af2e74f26e20ca6d8c47b487fe0fcc33de67d04f Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 11:32:37 -0700 Subject: [PATCH 06/24] Convert field validator --- .../data/annotation_types/data/tiled_image.py | 9 +++++--- .../data/annotation_types/data/video.py | 2 +- .../data/annotation_types/geometry/line.py | 5 ++-- .../data/annotation_types/geometry/mask.py | 5 ++-- .../data/annotation_types/geometry/polygon.py | 3 ++- .../labelbox/data/annotation_types/label.py | 6 ++--- .../data/annotation_types/metrics/base.py | 5 ++-- .../data/annotation_types/metrics/scalar.py | 4 +++- .../annotation_types/ner/document_entity.py | 5 ++-- .../labelbox/data/annotation_types/video.py | 3 ++- libs/labelbox/src/labelbox/data/mixins.py | 9 +++++--- .../data/serialization/labelbox_v1/objects.py | 7 +++--- .../data/serialization/ndjson/base.py | 7 +++--- .../labelbox/schema/bulk_import_request.py | 23 ++++++++++++------- 14 files changed, 58 insertions(+), 35 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py index 2362c66fc..4bfa27d7a 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py @@ -54,7 +54,8 @@ class TiledBounds(BaseModel): epsg: EPSG bounds: List[Point] - @pydantic_compat.validator('bounds') + @field_validator('bounds') + @classmethod def validate_bounds_not_equal(cls, bounds): first_bound = bounds[0] second_bound = bounds[1] @@ -99,7 +100,8 @@ class TileLayer(BaseModel): def asdict(self) -> Dict[str, str]: return {"tileLayerUrl": self.url, "name": self.name} - @pydantic_compat.validator('url') + @field_validator('url') + @classmethod def validate_url(cls, url): xyz_format = "/{z}/{x}/{y}" if xyz_format not in url: @@ -344,7 +346,8 @@ def _validate_num_tiles(self, xstart: float, ystart: float, xend: float, f"Max allowed tiles are {max_tiles}" f"Increase max tiles or reduce zoom level.") - @pydantic_compat.validator('zoom_levels') + @field_validator('zoom_levels') + @classmethod def validate_zoom_levels(cls, zoom_levels): if zoom_levels[0] > zoom_levels[1]: raise ValueError( diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py index 11fd71e27..523984fb3 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py @@ -12,7 +12,7 @@ from .base_data import BaseData from ..types import TypedArray -from pydantic import BaseModel, model_validator +from pydantic import model_validator, field_validator logger = logging.getLogger(__name__) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py index ccbb401ce..ed7194f9e 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/line.py @@ -9,7 +9,7 @@ from .point import Point from .geometry import Geometry -from pydantic import BaseModel +from pydantic import field_validator class Line(Geometry): @@ -65,7 +65,8 @@ def draw(self, color=color, thickness=thickness) - @pydantic_compat.validator('points') + @field_validator('points') + @classmethod def is_geom_valid(cls, points): if len(points) < 2: raise ValueError( diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py index dd9eb22b7..ac2966590 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/mask.py @@ -8,7 +8,7 @@ from ..data import MaskData from .geometry import Geometry -from pydantic import BaseModel +from pydantic import field_validator class Mask(Geometry): @@ -122,7 +122,8 @@ def create_url(self, signer: Callable[[bytes], str]) -> str: """ return self.mask.create_url(signer) - @pydantic_compat.validator('color') + @field_validator('color') + @classmethod def is_valid_color(cls, color): if isinstance(color, (tuple, list)): if len(color) == 1: diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py index b2e94ccbc..d19f7ca52 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py @@ -68,7 +68,8 @@ def draw(self, return cv2.fillPoly(canvas, pts, color) return cv2.polylines(canvas, pts, True, color, thickness) - @pydantic_compat.validator('points') + @field_validator('points') + @classmethod def is_geom_valid(cls, points): if len(points) < 3: raise ValueError( diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index 59256613a..8b2b203c2 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -202,11 +202,11 @@ def _assign_option(self, classification: ClassificationAnnotation, f"Unexpected type for answer found. {type(classification.value.answer)}" ) - @pydantic_compat.validator("annotations", pre=True) + @field_validator("annotations", mode='before') def validate_union(cls, value): supported = tuple([ - field.type_ - for field in cls.__fields__['annotations'].sub_fields[0].sub_fields + field.type_ for field in + cls.__model_fields__['annotations'].sub_fields[0].sub_fields ]) if not isinstance(value, list): raise TypeError(f"Annotations must be a list. Found {type(value)}") diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py index 1608ad084..61ad19cb8 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py @@ -1,7 +1,7 @@ from abc import ABC from typing import Dict, Optional, Any, Union -from pydantic import BaseModel +from pydantic import BaseModel, field_validator ConfidenceValue = pydantic_compat.confloat(ge=0, le=1) @@ -19,7 +19,8 @@ def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) return {k: v for k, v in res.items() if v is not None} - @pydantic_compat.validator('value') + @field_validator('value') + @classmethod def validate_value(cls, value): if isinstance(value, Dict): if not (MIN_CONFIDENCE_SCORES <= len(value) <= diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py index d79624152..0c4a9ae04 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py @@ -1,3 +1,4 @@ +from dataclasses import field from typing import Dict, Optional, Union from enum import Enum @@ -33,7 +34,8 @@ class ScalarMetric(BaseMetric): value: Union[ScalarMetricValue, ScalarMetricConfidenceValue] aggregation: ScalarMetricAggregation = ScalarMetricAggregation.ARITHMETIC_MEAN - @pydantic_compat.validator('metric_name') + @field_validator('metric_name') + @classmethod def validate_metric_name(cls, name: Union[str, None]): if name is None: return None diff --git a/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py b/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py index a5db33f4e..fa165b2e5 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from labelbox.utils import _CamelCaseMixin @@ -11,7 +11,8 @@ class DocumentTextSelection(_CamelCaseMixin, BaseModel): group_id: str page: int - @pydantic_compat.validator("page") + @field_validator("page") + @classmethod def validate_page(cls, v): if v < 1: raise ValueError("Page must be greater than 1") diff --git a/libs/labelbox/src/labelbox/data/annotation_types/video.py b/libs/labelbox/src/labelbox/data/annotation_types/video.py index 0278cda11..bebd5cd74 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/video.py @@ -104,7 +104,8 @@ def validate_args(cls, values): raise ValueError("One of `instance_uri`, `im_bytes` required.") return values - @pydantic_compat.validator("instance_uri") + @field_validator("instance_uri") + @classmethod def validate_uri(cls, v): if not is_valid_uri(v): raise ValueError(f"{v} is not a valid uri") diff --git a/libs/labelbox/src/labelbox/data/mixins.py b/libs/labelbox/src/labelbox/data/mixins.py index d42ea8d73..9234bb142 100644 --- a/libs/labelbox/src/labelbox/data/mixins.py +++ b/libs/labelbox/src/labelbox/data/mixins.py @@ -8,7 +8,8 @@ class ConfidenceMixin(BaseModel): confidence: Optional[float] = None - @pydantic_compat.validator("confidence") + @field_validator('confidence') + @classmethod def confidence_valid_float(cls, value): if value is None: return value @@ -36,13 +37,15 @@ class CustomMetric(BaseModel): name: str value: float - @pydantic_compat.validator("name") + @field_validator("name") + @classmethod def confidence_valid_float(cls, value): if not isinstance(value, str): raise ValueError("Name must be a string") return value - @pydantic_compat.validator("value") + @field_validator("value") + @classmethod def value_valid_float(cls, value): if not isinstance(value, (int, float)): raise ValueError("Value must be a number") diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py index 76198fa44..13090fdb7 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/objects.py @@ -4,7 +4,7 @@ except: from typing_extensions import Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator import numpy as np from .classification import LBV1Checklist, LBV1Classifications, LBV1Radio, LBV1Text, LBV1Dropdown @@ -32,8 +32,9 @@ def dict(self, *args, **kwargs) -> Dict[str, Any]: res.pop('instanceURI') return res - @pydantic_compat.validator('classifications', pre=True) - def validate_subclasses(cls, value, field): + @field_validator('classifications', mode='before') + @classmethod + def validate_subclasses(cls, value): # checklist subclasses create extra unessesary nesting. So we just remove it. if isinstance(value, list) and len(value): subclasses = [] diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index f9d72fa59..4c1454833 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -2,7 +2,7 @@ from uuid import uuid4 from labelbox.utils import _CamelCaseMixin, is_exactly_one_set -from pydantic import BaseModel, model_validator +from pydantic import model_validator, field_validator from ...annotation_types.types import Cuid @@ -11,7 +11,7 @@ class DataRow(_CamelCaseMixin): global_key: str = None @model_validator(mode='before') - @classmethod() + @classmethod def must_set_one(cls, values): if not is_exactly_one_set(values.get('id'), values.get('global_key')): raise ValueError("Must set either id or global_key") @@ -22,7 +22,8 @@ class NDJsonBase(_CamelCaseMixin): uuid: str = None data_row: DataRow - @pydantic_compat.validator('uuid', pre=True, always=True) + @field_validator('uuid', mode='before', validate_default=True) + @classmethod def set_id(cls, v): return v or str(uuid4()) diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index 585a99beb..823ce98a5 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -685,7 +685,7 @@ class NDChecklist(VideoSupported, NDBase): ontology_type: Literal["checklist"] = "checklist" answers: List[NDFeatureSchema] = pydantic_compat.Field(determinant=True) - @pydantic_compat.validator('answers', pre=True) + @field_validator('answers', mode='before') def validate_answers(cls, value, field): #constr not working with mypy. if not len(value): @@ -762,7 +762,8 @@ def validate_feature_schemas(self, valid_feature_schemas_by_id, if self.name else valid_feature_schemas_by_id[ self.schemaId]['classificationsByName']) - @pydantic_compat.validator('classifications', pre=True) + @field_validator('classifications', mode='before') + @classmethod def validate_subclasses(cls, value, field): #Create uuid and datarow id so we don't have to define classification objects twice #This is caused by the fact that we require these ids for top level classifications but not for subclasses @@ -782,7 +783,8 @@ class NDPolygon(NDBaseTool): ontology_type: Literal["polygon"] = "polygon" polygon: List[Point] = PydanticField(determinant=True) - @pydantic_compat.validator('polygon') + @field_validator('polygon') + @classmethod def is_geom_valid(cls, v): if len(v) < 3: raise ValueError( @@ -794,7 +796,8 @@ class NDPolyline(NDBaseTool): ontology_type: Literal["line"] = "line" line: List[Point] = PydanticField(determinant=True) - @pydantic_compat.validator('line') + @field_validator('line') + @classmethod def is_geom_valid(cls, v): if len(v) < 2: raise ValueError( @@ -823,7 +826,8 @@ class NDTextEntity(NDBaseTool): ontology_type: Literal["named-entity"] = "named-entity" location: EntityLocation = PydanticField(determinant=True) - @pydantic_compat.validator('location') + @field_validator('location') + @classmethod def is_valid_location(cls, v): if isinstance(v, BaseModel): v = v.dict() @@ -844,7 +848,8 @@ class RLEMaskFeatures(BaseModel): counts: List[int] size: List[int] - @pydantic_compat.validator('counts') + @field_validator('counts') + @classmethod def validate_counts(cls, counts): if not all([count >= 0 for count in counts]): raise ValueError( @@ -852,7 +857,8 @@ def validate_counts(cls, counts): ) return counts - @pydantic_compat.validator('size') + @field_validator('size') + @classmethod def validate_size(cls, size): if len(size) != 2: raise ValueError( @@ -873,7 +879,8 @@ class URIMaskFeatures(BaseModel): instanceURI: str colorRGB: Union[List[int], Tuple[int, int, int]] - @pydantic_compat.validator('colorRGB') + @field_validator('colorRGB') + @classmethod def validate_color(cls, colorRGB): #Does the dtype matter? Can it be a float? if not isinstance(colorRGB, (tuple, list)): From f182b0fb72d910f642b1944280ce927ac4d30eda Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 11:38:04 -0700 Subject: [PATCH 07/24] More Field conversions --- libs/labelbox/src/labelbox/schema/bulk_import_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index 823ce98a5..e935ba9ca 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -677,13 +677,13 @@ def determinants(parent_cls) -> List[str]: class NDText(NDBase): ontology_type: Literal["text"] = "text" - answer: str = pydantic_compat.Field(determinant=True) + answer: str = PydanticField(determinant=True) #No feature schema to check class NDChecklist(VideoSupported, NDBase): ontology_type: Literal["checklist"] = "checklist" - answers: List[NDFeatureSchema] = pydantic_compat.Field(determinant=True) + answers: List[NDFeatureSchema] = PydanticField(determinant=True) @field_validator('answers', mode='before') def validate_answers(cls, value, field): From e89ec37ace16ca431e48e8cb19f2b4c9fb861c2a Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 13:19:05 -0700 Subject: [PATCH 08/24] Dealing with constrained types --- .../labelbox/data/annotation_types/metrics/base.py | 6 +++--- .../annotation_types/metrics/confusion_matrix.py | 4 ++-- .../data/annotation_types/metrics/scalar.py | 6 +++--- .../src/labelbox/schema/data_row_metadata.py | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py index 61ad19cb8..c2c64f438 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py @@ -1,9 +1,9 @@ from abc import ABC -from typing import Dict, Optional, Any, Union +from typing import Annotated, Dict, Optional, Any, Union -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, Field -ConfidenceValue = pydantic_compat.confloat(ge=0, le=1) +ConfidenceValue = Annotated[float, Field(ge=0, le=1)] MIN_CONFIDENCE_SCORES = 2 MAX_CONFIDENCE_SCORES = 15 diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py index dd53919a6..1d818fb99 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py @@ -1,11 +1,11 @@ from enum import Enum -from typing import Tuple, Dict, Union +from typing import Tuple, Dict, Union, Annotated from pydantic import Field from .base import ConfidenceValue, BaseMetric -Count = pydantic_compat.conint(ge=0, le=1e10) +Count = Annotated[int, Field(ge=0, le=1e10)] ConfusionMatrixMetricValue = Tuple[Count, Count, Count, Count] ConfusionMatrixMetricConfidenceValue = Dict[ConfidenceValue, diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py index 0c4a9ae04..800d1c95d 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py @@ -1,12 +1,12 @@ from dataclasses import field -from typing import Dict, Optional, Union +from typing import Annotated, Dict, Optional, Union from enum import Enum from .base import ConfidenceValue, BaseMetric -from pydantic import BaseModel +from pydantic import Field -ScalarMetricValue = pydantic_compat.confloat(ge=0, le=100_000_000) +ScalarMetricValue = Annotated[float, Field(ge=0, le=100_000_000)] ScalarMetricConfidenceValue = Dict[ConfidenceValue, ScalarMetricValue] diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 13a35ce3d..edc13d263 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -7,7 +7,7 @@ from typing import List, Optional, Dict, Union, Callable, Type, Any, Generator, overload, Annotated -from pydantic import StringConstraints +from pydantic import StringConstraints, Field from pydantic import BaseModel from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds @@ -29,9 +29,9 @@ class DataRowMetadataKind(Enum): # Metadata schema class DataRowMetadataSchema(BaseModel): uid: SchemaId - name: pydantic_compat.constr(strip_whitespace=True, - min_length=1, - max_length=100) + name: Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=100)] reserved: bool kind: DataRowMetadataKind options: Optional[List["DataRowMetadataSchema"]] @@ -40,10 +40,10 @@ class DataRowMetadataSchema(BaseModel): DataRowMetadataSchema.update_forward_refs() -Embedding: Type[List[float]] = pydantic_compat.conlist(float, +Embedding: Type[List[float]] = Annotated[List[float], Field( min_items=128, - max_items=128) -String: Type[str] = pydantic_compat.constr(max_length=4096) + max_items=128)] +String: Type[str] = Annotated[List[str], Field(max_length=4096)] # Metadata base class From a4ebf10c0fadb1212080b06ee867bcdc8eb39a98 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 14:20:38 -0700 Subject: [PATCH 09/24] Deal with custom field validation --- .../labelbox/src/labelbox/data/annotation_types/types.py | 9 +++------ libs/labelbox/src/labelbox/schema/data_row_metadata.py | 5 ++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index aac1cdf7b..52f2b50e7 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -7,7 +7,7 @@ from packaging import version import numpy as np -from pydantic import Field +from pydantic import ValidationInfo Cuid = Annotated[str, StringConstraints(min_length=25, max_length=25)] @@ -22,14 +22,11 @@ def __get_validators__(cls): yield cls.validate @classmethod - def validate(cls, val, field: pydantic_compat.ModelField): + def validate(cls, val, field: ValidationInfo): if not isinstance(val, np.ndarray): raise TypeError(f"Expected numpy array. Found {type(val)}") - if sys.version_info.minor > 6: - actual_dtype = field.sub_fields[-1].type_.__args__[0] - else: - actual_dtype = field.sub_fields[-1].type_.__values__[0] + actual_dtype = field.sub_fields[-1].type_.__args__[0] if val.dtype != actual_dtype: raise TypeError( diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index edc13d263..143391377 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -40,9 +40,8 @@ class DataRowMetadataSchema(BaseModel): DataRowMetadataSchema.update_forward_refs() -Embedding: Type[List[float]] = Annotated[List[float], Field( - min_items=128, - max_items=128)] +Embedding: Type[List[float]] = Annotated[List[float], + Field(min_items=128, max_items=128)] String: Type[str] = Annotated[List[str], Field(max_length=4096)] From 31a272a795f1cea25782112317ac2a73bf41caeb Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 14:59:23 -0700 Subject: [PATCH 10/24] Attempt to deal with typed array validation for numpy --- .../labelbox/src/labelbox/data/annotation_types/types.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index 52f2b50e7..5b83ad49f 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -1,15 +1,14 @@ import sys from typing import Generic, TypeVar, Any -from pydantic import StringConstraints - from typing_extensions import Annotated from packaging import version import numpy as np from pydantic import ValidationInfo +from labelbox import pydantic_compat -Cuid = Annotated[str, StringConstraints(min_length=25, max_length=25)] +Cuid = Annotated[str, pydantic_compat.Field(min_length=25, max_length=25)] DType = TypeVar('DType') DShape = TypeVar('DShape') @@ -22,11 +21,11 @@ def __get_validators__(cls): yield cls.validate @classmethod - def validate(cls, val, field: ValidationInfo): + def validate(cls, val, field_info: ValidationInfo): if not isinstance(val, np.ndarray): raise TypeError(f"Expected numpy array. Found {type(val)}") - actual_dtype = field.sub_fields[-1].type_.__args__[0] + actual_dtype = cls.model_fields[field_info.name].type_.__args__[0] if val.dtype != actual_dtype: raise TypeError( From e0bccbd6e9acfbaf6f7a0bed0f7819e163c160da Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 16:18:20 -0700 Subject: [PATCH 11/24] Cleanup: removed all that I missed --- .../labelbox/data/annotation_types/base_annotation.py | 4 ++-- .../annotation_types/data/generic_data_row_data.py | 2 +- .../src/labelbox/data/annotation_types/data/raster.py | 10 +++------- .../labelbox/data/annotation_types/geometry/polygon.py | 2 +- .../labelbox/data/annotation_types/metrics/scalar.py | 2 +- .../src/labelbox/data/serialization/coco/annotation.py | 2 +- .../src/labelbox/data/serialization/coco/categories.py | 2 +- .../data/serialization/coco/instance_dataset.py | 2 +- .../labelbox/data/serialization/labelbox_v1/feature.py | 2 +- .../labelbox/data/serialization/labelbox_v1/label.py | 10 ++++------ .../src/labelbox/data/serialization/ndjson/base.py | 4 ++-- .../data/serialization/ndjson/classification.py | 4 ++-- 12 files changed, 20 insertions(+), 26 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py b/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py index 9c764daee..a85d1ea8c 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py @@ -1,7 +1,7 @@ import abc from uuid import UUID, uuid4 from typing import Any, Dict, Optional -from pydantic import BaseModel +from pydantic import PrivateAttr from .feature import FeatureSchema @@ -9,7 +9,7 @@ class BaseAnnotation(FeatureSchema, abc.ABC): """ Base annotation class. Shouldn't be directly instantiated """ - _uuid: Optional[UUID] = pydantic_compat.PrivateAttr() + _uuid: Optional[UUID] = PrivateAttr() extra: Dict[str, Any] = {} def __init__(self, **data): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py b/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py index 1fabb0a9d..881588058 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py @@ -15,7 +15,7 @@ def create_url(self, signer: Callable[[bytes], str]) -> Optional[str]: return self.url @model_validator(mode='before') - @classmethod(pre=True) + @classmethod def validate_one_datarow_key_present(cls, data): keys = ['external_id', 'global_key', 'uid'] count = sum([key in data for key in keys]) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py index 481978eab..bbfd74c11 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py @@ -9,7 +9,7 @@ import requests import numpy as np -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, ConfigDict, Extra from labelbox.exceptions import InternalServerError from .base_data import BaseData from ..types import TypedArray @@ -156,7 +156,7 @@ def create_url(self, signer: Callable[[bytes], str]) -> str: return self.url @model_validator(mode='before') - @classmethod() + @classmethod def validate_args(cls, values): file_path = values.get("file_path") im_bytes = values.get("im_bytes") @@ -186,11 +186,7 @@ def __repr__(self) -> str: f"url={self.url}," \ f"arr={symbol_or_none(self.arr)})" - class Config: - # Required for sharing references - copy_on_model_validation = 'none' - # Required for discriminating between data types - extra = 'forbid' + model_config = ConfigDict(extra=Extra.forbid,) class MaskData(RasterData): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py b/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py index d19f7ca52..a14783ea9 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/geometry/polygon.py @@ -9,7 +9,7 @@ from .geometry import Geometry from .point import Point -from pydantic import BaseModel +from pydantic import field_validator class Polygon(Geometry): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py index 800d1c95d..e2d503628 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py @@ -4,7 +4,7 @@ from .base import ConfidenceValue, BaseMetric -from pydantic import Field +from pydantic import Field, field_validator ScalarMetricValue = Annotated[float, Field(ge=0, le=100_000_000)] ScalarMetricConfidenceValue = Dict[ConfidenceValue, ScalarMetricValue] diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py b/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py index e0bce98fd..ab1e279fe 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/annotation.py @@ -10,7 +10,7 @@ from ...annotation_types.annotation import ObjectAnnotation from ...annotation_types.classification.classification import ClassificationAnnotation -from .... import pydantic_compat +from pydantic import BaseModel import numpy as np from .path import PathSerializerMixin diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/categories.py b/libs/labelbox/src/labelbox/data/serialization/coco/categories.py index e7b05fbdc..07ecacb03 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/categories.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/categories.py @@ -1,7 +1,7 @@ import sys from hashlib import md5 -from .... import pydantic_compat +from pydantic import BaseModel class Categories(BaseModel): diff --git a/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py b/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py index d57fe1ded..d5568f299 100644 --- a/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py +++ b/libs/labelbox/src/labelbox/data/serialization/coco/instance_dataset.py @@ -6,7 +6,7 @@ import numpy as np from tqdm import tqdm -from .... import pydantic_compat +from pydantic import BaseModel from ...annotation_types import ImageData, MaskData, Mask, ObjectAnnotation, Label, Polygon, Point, Rectangle from ...annotation_types.collection import LabelCollection diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py index f2f552dca..8933b5fa4 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py @@ -28,5 +28,5 @@ def dict(self, *args, **kwargs): return res class Config: - allow_population_by_field_name = True + populate_by_name = True alias_generator = camel_case diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py index a711d2d25..69b379bb0 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py @@ -101,7 +101,7 @@ def from_common( return result class Config: - allow_population_by_field_name = True + populate_by_name = True class Review(BaseModel): @@ -120,13 +120,11 @@ class Config: class LBV1Label(BaseModel): label: Union[LBV1LabelAnnotations, - List[LBV1LabelAnnotationsVideo]] = Field( - ..., alias='Label') + List[LBV1LabelAnnotationsVideo]] = Field(..., alias='Label') data_row_id: str = Field(..., alias="DataRow ID") row_data: str = Field(None, alias="Labeled Data") id: Optional[str] = Field(None, alias='ID') - external_id: Optional[str] = Field(None, - alias="External ID") + external_id: Optional[str] = Field(None, alias="External ID") data_row_media_attributes: Optional[Dict[str, Any]] = Field( {}, alias="Media Attributes") data_row_metadata: Optional[List[Dict[str, Any]]] = Field( @@ -248,4 +246,4 @@ def _is_url(self) -> bool: "s3://")) or "tileLayerUrl" in self.row_data class Config: - allow_population_by_field_name = True + populate_by_name = True diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index 4c1454833..c64ceb88f 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -22,7 +22,7 @@ class NDJsonBase(_CamelCaseMixin): uuid: str = None data_row: DataRow - @field_validator('uuid', mode='before', validate_default=True) + @field_validator('uuid', mode='before') @classmethod def set_id(cls, v): return v or str(uuid4()) @@ -45,7 +45,7 @@ class NDAnnotation(NDJsonBase): unit: Optional[str] = None @model_validator(mode='before') - @classmethod() + @classmethod def must_set_one(cls, values): if ('schema_id' not in values or values['schema_id'] is None) and ('name' not in values or values['name'] is None): diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index d2de3d7fb..58015ab0b 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -18,7 +18,7 @@ class NDAnswer(ConfidenceMixin, CustomMetricsMixin): classifications: Optional[List['NDSubclassificationType']] = [] @model_validator(mode='before') - @classmethod() + @classmethod def must_set_one(cls, values): if ('schema_id' not in values or values['schema_id'] is None) and ('name' not in values or values['name'] is None): @@ -40,7 +40,7 @@ def dict(self, *args, **kwargs): return res class Config: - allow_population_by_field_name = True + populate_by_name = True alias_generator = camel_case From 7971283a4c76fa542610fbe1a1c0ff88f3c9a291 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 5 Jun 2024 16:19:29 -0700 Subject: [PATCH 12/24] Remove pydantic_compat --- libs/labelbox/src/labelbox/pydantic_compat.py | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 libs/labelbox/src/labelbox/pydantic_compat.py diff --git a/libs/labelbox/src/labelbox/pydantic_compat.py b/libs/labelbox/src/labelbox/pydantic_compat.py deleted file mode 100644 index 51c082480..000000000 --- a/libs/labelbox/src/labelbox/pydantic_compat.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Optional - - -def pydantic_import(class_name, sub_module_path: Optional[str] = None): - import importlib - import importlib.metadata - - # Get the version of pydantic - pydantic_version = importlib.metadata.version("pydantic") - - # Determine the module name based on the version - module_name = "pydantic" if pydantic_version.startswith( - "1") else "pydantic.v1" - module_name = f"{module_name}.{sub_module_path}" if sub_module_path else module_name - - # Import the class from the module - klass = getattr(importlib.import_module(module_name), class_name) - - return klass - - -BaseModel = pydantic_import("BaseModel") -PrivateAttr = pydantic_import("PrivateAttr") -Field = pydantic_import("Field") -ModelField = pydantic_import("ModelField", "fields") -ValidationError = pydantic_import("ValidationError") -ErrorWrapper = pydantic_import("ErrorWrapper", "error_wrappers") - -validator = pydantic_import("validator") -root_validator = pydantic_import("root_validator") -conint = pydantic_import("conint") -conlist = pydantic_import("conlist") -constr = pydantic_import("constr") -confloat = pydantic_import("confloat") From 229df1af68648c077ba3806a0f4792a7769b7bae Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 6 Jun 2024 10:42:26 -0700 Subject: [PATCH 13/24] Convert config --- .../data/annotation_types/data/tiled_image.py | 5 ++--- .../data/annotation_types/data/video.py | 6 ++--- .../labelbox/data/annotation_types/label.py | 7 +++--- .../data/annotation_types/metrics/base.py | 7 +++--- .../labelbox/data/annotation_types/types.py | 5 ++--- .../data/serialization/labelbox_v1/feature.py | 7 +++--- .../data/serialization/labelbox_v1/label.py | 12 +++++----- .../serialization/ndjson/classification.py | 7 +++--- .../data/serialization/ndjson/metric.py | 5 +++-- .../labelbox/schema/bulk_import_request.py | 13 +++++------ .../src/labelbox/schema/data_row_metadata.py | 10 ++++----- libs/labelbox/src/labelbox/schema/dataset.py | 1 - .../labelbox/src/labelbox/schema/embedding.py | 2 +- .../schema/internal/data_row_uploader.py | 9 ++++---- .../schema/internal/data_row_upsert_item.py | 3 +-- .../src/labelbox/schema/project_overview.py | 11 +++++----- libs/labelbox/src/labelbox/utils.py | 10 +++------ .../classification/test_classification.py | 22 +++++++++---------- .../data/annotation_types/data/test_raster.py | 5 +++-- .../data/annotation_types/data/test_text.py | 5 +++-- .../data/annotation_types/data/test_video.py | 5 +++-- .../annotation_types/geometry/test_line.py | 7 +++--- .../annotation_types/geometry/test_mask.py | 5 +++-- .../annotation_types/geometry/test_point.py | 5 +++-- .../annotation_types/geometry/test_polygon.py | 9 ++++---- .../geometry/test_rectangle.py | 5 +++-- .../data/annotation_types/test_annotation.py | 9 ++++---- .../data/annotation_types/test_metrics.py | 19 ++++++++-------- .../data/annotation_types/test_tiled_image.py | 4 ++-- 29 files changed, 106 insertions(+), 114 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py index 4bfa27d7a..85d5c0e8f 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/tiled_image.py @@ -12,7 +12,7 @@ from PIL import Image from pyproj import Transformer from pygeotile.point import Point as PygeoPoint -from pydantic import BaseModel, model_validator, field_validator +from pydantic import BaseModel, model_validator, field_validator, ConfigDict from labelbox.data.annotation_types import Rectangle, Point, Line, Polygon from .base_data import BaseData @@ -361,8 +361,7 @@ class EPSGTransformer(BaseModel): in different formats. """ - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) transformer: Any diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py index 523984fb3..3a9aa9a05 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py @@ -12,7 +12,7 @@ from .base_data import BaseData from ..types import TypedArray -from pydantic import model_validator, field_validator +from pydantic import model_validator, ConfigDict, Extra logger = logging.getLogger(__name__) @@ -168,6 +168,4 @@ def __repr__(self) -> str: f"frames={'...' if self.frames is not None else None}," \ f"url={self.url})" - class Config: - # Required for discriminating between data types - extra = 'forbid' + model_config = ConfigDict(extra=Extra.forbid,) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index 8b2b203c2..eddae4a16 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -59,9 +59,10 @@ def validate_data(cls, label): if isinstance(label.get("data"), Dict): label["data"]["class_name"] = "GenericDataRowData" else: - warnings.warn( - f"Using {type(label['data']).__name__} class for label.data is deprecated. " - "Use a dict or an instance of GenericDataRowData instead.") + if not isinstance(label.get("data"), GenericDataRowData): + warnings.warn( + f"Using {type(label['data']).__name__} class for label.data is deprecated. " + "Use a dict or an instance of GenericDataRowData instead.") return label def object_annotations(self) -> List[ObjectAnnotation]: diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py index c2c64f438..8a66a1a83 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py @@ -1,7 +1,7 @@ from abc import ABC from typing import Annotated, Dict, Optional, Any, Union -from pydantic import BaseModel, field_validator, Field +from pydantic import BaseModel, field_validator, Field, ValidationError ConfidenceValue = Annotated[float, Field(ge=0, le=1)] @@ -25,12 +25,11 @@ def validate_value(cls, value): if isinstance(value, Dict): if not (MIN_CONFIDENCE_SCORES <= len(value) <= MAX_CONFIDENCE_SCORES): - raise pydantic_compat.ValidationError([ - pydantic_compat.ErrorWrapper(ValueError( + raise ValidationError([ + ValueError( "Number of confidence scores must be greater" f" than or equal to {MIN_CONFIDENCE_SCORES} and" f" less than or equal to {MAX_CONFIDENCE_SCORES}. Found {len(value)}" ), - loc='value') ], cls) return value diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index 5b83ad49f..5c3537924 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -5,10 +5,9 @@ from packaging import version import numpy as np -from pydantic import ValidationInfo -from labelbox import pydantic_compat +from pydantic import ValidationInfo, Field -Cuid = Annotated[str, pydantic_compat.Field(min_length=25, max_length=25)] +Cuid = Annotated[str, Field(min_length=25, max_length=25)] DType = TypeVar('DType') DShape = TypeVar('DShape') diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py index 8933b5fa4..ea2d14063 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, ConfigDict from labelbox.utils import camel_case from ...annotation_types.types import Cuid @@ -27,6 +27,5 @@ def dict(self, *args, **kwargs): res.pop('keyframe') return res - class Config: - populate_by_name = True - alias_generator = camel_case + model_config = ConfigDict(allow_population_by_field_name=True, + alias_generator=camel_case) diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py index 69b379bb0..e122df00f 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py @@ -1,8 +1,9 @@ +from pyexpat import model from labelbox.data.annotation_types.data.tiled_image import TiledImageData from labelbox.utils import camel_case from typing import List, Optional, Union, Dict, Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from ...annotation_types.annotation import (ClassificationAnnotation, ObjectAnnotation) @@ -100,8 +101,7 @@ def from_common( return result - class Config: - populate_by_name = True + model_config = ConfigDict(allow_population_by_field_name=True) class Review(BaseModel): @@ -111,8 +111,7 @@ class Review(BaseModel): created_by: str label_id: Optional[str] = None - class Config: - alias_generator = camel_case + model_config = ConfigDict(alias_generator=camel_case) Extra = lambda name: Field(None, alias=name, extra_field=True) @@ -245,5 +244,4 @@ def _is_url(self) -> bool: ("http://", "https://", "gs://", "s3://")) or "tileLayerUrl" in self.row_data - class Config: - populate_by_name = True + model_config = ConfigDict(allow_population_by_field_name=True) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index 58015ab0b..12b724201 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Union, Optional -from pydantic import BaseModel, model_validator, Field +from pydantic import BaseModel, ConfigDict, model_validator, Field from labelbox.data.mixins import ConfidenceMixin, CustomMetric, CustomMetricsMixin from labelbox.data.serialization.ndjson.base import DataRow, NDAnnotation @@ -39,9 +39,8 @@ def dict(self, *args, **kwargs): ] return res - class Config: - populate_by_name = True - alias_generator = camel_case + model_config = ConfigDict(allow_population_by_field_name=True, + alias_generator=camel_case) class FrameLocation(BaseModel): diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py index 5abbf2761..16e27c8b1 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py @@ -1,5 +1,7 @@ from typing import Optional, Union, Type +from pydantic import ConfigDict + from labelbox.data.annotation_types.data import ImageData, TextData from labelbox.data.serialization.ndjson.base import DataRow, NDJsonBase from labelbox.data.annotation_types.metrics.scalar import ( @@ -15,8 +17,7 @@ class BaseNDMetric(NDJsonBase): feature_name: Optional[str] = None subclass_name: Optional[str] = None - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index e935ba9ca..7d3547132 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -8,7 +8,7 @@ from google.api_core import retry from labelbox import parser import requests -from pydantic import BaseModel, model_validator, StringConstraints, Field as PydanticField, field_validator +from pydantic import BaseModel, model_validator, StringConstraints, Field as PydanticField, field_validator, ValidationError, Extra, ConfigDict from typing_extensions import Literal from typing import (Any, List, Optional, BinaryIO, Dict, Iterable, Tuple, Union, Type, Set, TYPE_CHECKING, Annotated) @@ -428,8 +428,7 @@ def _validate_ndjson(lines: Iterable[Dict[str, Any]], f'{uuid} already used in this import job, ' 'must be unique for the project.') uids.add(uuid) - except (pydantic_compat.ValidationError, ValueError, TypeError, - KeyError) as e: + except (ValidationError, ValueError, TypeError, KeyError) as e: raise lb_exceptions.MALValidationError( f"Invalid NDJson on line {idx}") from e @@ -528,7 +527,7 @@ class VideoSupported(BaseModel): #Base class for a special kind of union. -# Compatible with pydantic_compat. Improves error messages over a traditional union +# Improves error messages over a traditional union class SpecialUnion: def __new__(cls, **kwargs): @@ -561,7 +560,7 @@ def build(cls: Any, data: Union[dict, BaseModel]) -> "NDBase": data (Union[dict, BaseModel]) : The data for constructing one of the objects in the union raises: KeyError: data does not contain the determinant fields for any of the types supported by this SpecialUnion - pydantic_compat.ValidationError: Error while trying to construct a specific object in the union + ValidationError: Error while trying to construct a specific object in the union """ if isinstance(data, BaseModel): @@ -659,9 +658,7 @@ def validate_instance(self, valid_feature_schemas_by_id, self.validate_feature_schemas(valid_feature_schemas_by_id, valid_feature_schemas_by_name) - class Config: - #Users shouldn't to add extra data to the payload - extra = 'forbid' + model_config = ConfigDict(extra=Extra.forbid,) @staticmethod def determinants(parent_cls) -> List[str]: diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 143391377..335ba537e 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -7,7 +7,7 @@ from typing import List, Optional, Dict, Union, Callable, Type, Any, Generator, overload, Annotated -from pydantic import StringConstraints, Field +from pydantic import StringConstraints, Field, ConfigDict from pydantic import BaseModel from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds @@ -66,8 +66,7 @@ class DeleteDataRowMetadata(_CamelCaseMixin): data_row_id: Union[str, UniqueId, GlobalKey] fields: List[SchemaId] - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) class DataRowMetadataBatchResponse(_CamelCaseMixin): @@ -98,9 +97,8 @@ class _DeleteBatchDataRowMetadata(_CamelCaseMixin): data_row_identifier: Union[UniqueId, GlobalKey] schema_ids: List[SchemaId] - class Config: - arbitrary_types_allowed = True - alias_generator = camel_case + model_config = ConfigDict(arbitrary_types_allowed=True, + alias_generator=camel_case) def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) diff --git a/libs/labelbox/src/labelbox/schema/dataset.py b/libs/labelbox/src/labelbox/schema/dataset.py index 081bc949a..1f725b07d 100644 --- a/libs/labelbox/src/labelbox/schema/dataset.py +++ b/libs/labelbox/src/labelbox/schema/dataset.py @@ -22,7 +22,6 @@ from labelbox.orm import query from labelbox.exceptions import MalformedQueryException from labelbox.pagination import PaginatedCollection -from labelbox.pydantic_compat import BaseModel from labelbox.schema.data_row import DataRow from labelbox.schema.embedding import EmbeddingVector from labelbox.schema.export_filters import DatasetExportFilters, build_filters diff --git a/libs/labelbox/src/labelbox/schema/embedding.py b/libs/labelbox/src/labelbox/schema/embedding.py index 1d71ba908..dbdf0c595 100644 --- a/libs/labelbox/src/labelbox/schema/embedding.py +++ b/libs/labelbox/src/labelbox/schema/embedding.py @@ -1,7 +1,7 @@ from typing import Optional, Callable, Dict, Any, List from labelbox.adv_client import AdvClient -from labelbox.pydantic_compat import BaseModel, PrivateAttr +from pydantic import BaseModel, PrivateAttr class EmbeddingVector(BaseModel): diff --git a/libs/labelbox/src/labelbox/schema/internal/data_row_uploader.py b/libs/labelbox/src/labelbox/schema/internal/data_row_uploader.py index 7d4224eb9..70603b940 100644 --- a/libs/labelbox/src/labelbox/schema/internal/data_row_uploader.py +++ b/libs/labelbox/src/labelbox/schema/internal/data_row_uploader.py @@ -1,13 +1,12 @@ -from concurrent.futures import ThreadPoolExecutor, as_completed - from typing import List -from labelbox import pydantic_compat -from labelbox.schema.internal.data_row_upsert_item import DataRowItemBase, DataRowUpsertItem, DataRowCreateItem +from pydantic import BaseModel + +from labelbox.schema.internal.data_row_upsert_item import DataRowItemBase from labelbox.schema.internal.descriptor_file_creator import DescriptorFileCreator -class UploadManifest(pydantic_compat.BaseModel): +class UploadManifest(BaseModel): source: str item_count: int chunk_uris: List[str] diff --git a/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py b/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py index 5ae001983..2f4e0ae36 100644 --- a/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py +++ b/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py @@ -2,9 +2,8 @@ from typing import List, Tuple, Optional -from labelbox.pydantic_compat import BaseModel +from pydantic import BaseModel from labelbox.schema.identifiable import UniqueId, GlobalKey -from labelbox import pydantic_compat from labelbox.schema.data_row import DataRow diff --git a/libs/labelbox/src/labelbox/schema/project_overview.py b/libs/labelbox/src/labelbox/schema/project_overview.py index 3e75e7282..013e98677 100644 --- a/libs/labelbox/src/labelbox/schema/project_overview.py +++ b/libs/labelbox/src/labelbox/schema/project_overview.py @@ -1,7 +1,8 @@ from typing import Dict, List -from labelbox.pydantic_compat import BaseModel +from pydantic import BaseModel from typing_extensions import TypedDict + class ProjectOverview(BaseModel): """ Class that represents a project summary as displayed in the UI, in Annotate, @@ -19,7 +20,7 @@ class ProjectOverview(BaseModel): The `labeled` attribute represents the number of data rows that have been labeled. The `total_data_rows` attribute represents the total number of data rows in the project. """ - to_label: int + to_label: int in_review: int in_rework: int skipped: int @@ -37,7 +38,7 @@ class _QueueDetail(TypedDict): """ data: List[Dict[str, int]] total: int - + class ProjectOverviewDetailed(BaseModel): """ @@ -62,11 +63,11 @@ class ProjectOverviewDetailed(BaseModel): The `total_data_rows` attribute represents the total number of data rows in the project. """ - to_label: int + to_label: int in_review: _QueueDetail in_rework: _QueueDetail skipped: int done: int issues: int labeled: int - total_data_rows: int \ No newline at end of file + total_data_rows: int diff --git a/libs/labelbox/src/labelbox/utils.py b/libs/labelbox/src/labelbox/utils.py index f2d9b10ad..ac7b3017c 100644 --- a/libs/labelbox/src/labelbox/utils.py +++ b/libs/labelbox/src/labelbox/utils.py @@ -7,9 +7,7 @@ from urllib.parse import urlparse -from pydantic import BaseModel - -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict UPPERCASE_COMPONENTS = ['uri', 'rgb'] ISO_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' @@ -55,10 +53,8 @@ def is_valid_uri(uri): class _CamelCaseMixin(BaseModel): - - class Config: - populate_by_name = True - alias_generator = camel_case + model_config = ConfigDict(allow_population_by_field_name=True, + alias_generator=camel_case) class _NoCoercionMixin: diff --git a/libs/labelbox/tests/data/annotation_types/classification/test_classification.py b/libs/labelbox/tests/data/annotation_types/classification/test_classification.py index 11a3a0514..df98e8460 100644 --- a/libs/labelbox/tests/data/annotation_types/classification/test_classification.py +++ b/libs/labelbox/tests/data/annotation_types/classification/test_classification.py @@ -1,14 +1,14 @@ import pytest +from pydantic import ValidationError + from labelbox.data.annotation_types import (Checklist, ClassificationAnswer, Dropdown, Radio, Text, ClassificationAnnotation) -from labelbox import pydantic_compat - def test_classification_answer(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): ClassificationAnswer() feature_schema_id = "schema_id" @@ -37,7 +37,7 @@ def test_classification(): name="a classification") assert classification.dict()['value']['answer'] == answer - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): ClassificationAnnotation() @@ -45,7 +45,7 @@ def test_subclass(): answer = "1234" feature_schema_id = "11232" name = "my_feature" - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): # Should have feature schema info classification = ClassificationAnnotation(value=Text(answer=answer)) classification = ClassificationAnnotation(value=Text(answer=answer), @@ -98,11 +98,11 @@ def test_radio(): feature_schema_id = "feature_schema_id" name = "my_feature" - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): classification = ClassificationAnnotation(value=Radio( answer=answer.name)) - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): classification = Radio(answer=[answer]) classification = Radio(answer=answer,) assert classification.dict() == { @@ -159,10 +159,10 @@ def test_checklist(): feature_schema_id = "feature_schema_id" name = "my_feature" - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): classification = Checklist(answer=answer.name) - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): classification = Checklist(answer=answer) classification = Checklist(answer=[answer]) @@ -208,11 +208,11 @@ def test_dropdown(): feature_schema_id = "feature_schema_id" name = "my_feature" - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): classification = ClassificationAnnotation( value=Dropdown(answer=answer.name), name="test") - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): classification = Dropdown(answer=answer) classification = Dropdown(answer=[answer]) assert classification.dict() == { diff --git a/libs/labelbox/tests/data/annotation_types/data/test_raster.py b/libs/labelbox/tests/data/annotation_types/data/test_raster.py index 40c8d5648..487d3f4cc 100644 --- a/libs/labelbox/tests/data/annotation_types/data/test_raster.py +++ b/libs/labelbox/tests/data/annotation_types/data/test_raster.py @@ -5,12 +5,13 @@ import pytest from PIL import Image +from pydantic import ValidationError + from labelbox.data.annotation_types.data import ImageData -from labelbox import pydantic_compat def test_validate_schema(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): data = ImageData() diff --git a/libs/labelbox/tests/data/annotation_types/data/test_text.py b/libs/labelbox/tests/data/annotation_types/data/test_text.py index 970b8382b..60b64b33c 100644 --- a/libs/labelbox/tests/data/annotation_types/data/test_text.py +++ b/libs/labelbox/tests/data/annotation_types/data/test_text.py @@ -2,12 +2,13 @@ import pytest +from pydantic import ValidationError + from labelbox.data.annotation_types import TextData -from labelbox import pydantic_compat def test_validate_schema(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): data = TextData() diff --git a/libs/labelbox/tests/data/annotation_types/data/test_video.py b/libs/labelbox/tests/data/annotation_types/data/test_video.py index f0e42b83f..cfe22dd98 100644 --- a/libs/labelbox/tests/data/annotation_types/data/test_video.py +++ b/libs/labelbox/tests/data/annotation_types/data/test_video.py @@ -1,12 +1,13 @@ import numpy as np import pytest +from pydantic import ValidationError + from labelbox.data.annotation_types import VideoData -from labelbox import pydantic_compat def test_validate_schema(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): data = VideoData() diff --git a/libs/labelbox/tests/data/annotation_types/geometry/test_line.py b/libs/labelbox/tests/data/annotation_types/geometry/test_line.py index f0d0673df..fd0430244 100644 --- a/libs/labelbox/tests/data/annotation_types/geometry/test_line.py +++ b/libs/labelbox/tests/data/annotation_types/geometry/test_line.py @@ -1,15 +1,16 @@ import pytest import cv2 +from pydantic import ValidationError + from labelbox.data.annotation_types.geometry import Point, Line -from labelbox import pydantic_compat def test_line(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): line = Line() - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): line = Line(points=[[0, 1], [2, 3]]) points = [[0, 1], [0, 2], [2, 2]] diff --git a/libs/labelbox/tests/data/annotation_types/geometry/test_mask.py b/libs/labelbox/tests/data/annotation_types/geometry/test_mask.py index 7a2b713ee..99dc852cc 100644 --- a/libs/labelbox/tests/data/annotation_types/geometry/test_mask.py +++ b/libs/labelbox/tests/data/annotation_types/geometry/test_mask.py @@ -3,12 +3,13 @@ import numpy as np import cv2 +from pydantic import ValidationError + from labelbox.data.annotation_types import Point, Rectangle, Mask, MaskData -from labelbox import pydantic_compat def test_mask(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): mask = Mask() mask_data = np.zeros((32, 32, 3), dtype=np.uint8) diff --git a/libs/labelbox/tests/data/annotation_types/geometry/test_point.py b/libs/labelbox/tests/data/annotation_types/geometry/test_point.py index 47c152d2b..1ffac123f 100644 --- a/libs/labelbox/tests/data/annotation_types/geometry/test_point.py +++ b/libs/labelbox/tests/data/annotation_types/geometry/test_point.py @@ -1,12 +1,13 @@ import pytest import cv2 +from pydantic import ValidationError + from labelbox.data.annotation_types import Point -from labelbox import pydantic_compat def test_point(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): line = Point() with pytest.raises(TypeError): diff --git a/libs/labelbox/tests/data/annotation_types/geometry/test_polygon.py b/libs/labelbox/tests/data/annotation_types/geometry/test_polygon.py index 8a7525e8f..cbfff9281 100644 --- a/libs/labelbox/tests/data/annotation_types/geometry/test_polygon.py +++ b/libs/labelbox/tests/data/annotation_types/geometry/test_polygon.py @@ -1,18 +1,19 @@ import pytest import cv2 +from pydantic import ValidationError + from labelbox.data.annotation_types import Polygon, Point -from labelbox import pydantic_compat def test_polygon(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): polygon = Polygon() - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): polygon = Polygon(points=[[0, 1], [2, 3]]) - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): polygon = Polygon(points=[Point(x=0, y=1), Point(x=0, y=1)]) points = [[0., 1.], [0., 2.], [2., 2.], [2., 0.]] diff --git a/libs/labelbox/tests/data/annotation_types/geometry/test_rectangle.py b/libs/labelbox/tests/data/annotation_types/geometry/test_rectangle.py index 3c01ef6ed..8ab293c71 100644 --- a/libs/labelbox/tests/data/annotation_types/geometry/test_rectangle.py +++ b/libs/labelbox/tests/data/annotation_types/geometry/test_rectangle.py @@ -1,12 +1,13 @@ import cv2 import pytest +from pydantic import ValidationError + from labelbox.data.annotation_types import Point, Rectangle -from labelbox import pydantic_compat def test_rectangle(): - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): rectangle = Rectangle() rectangle = Rectangle(start=Point(x=0, y=1), end=Point(x=10, y=10)) diff --git a/libs/labelbox/tests/data/annotation_types/test_annotation.py b/libs/labelbox/tests/data/annotation_types/test_annotation.py index b6dc00041..634d1c879 100644 --- a/libs/labelbox/tests/data/annotation_types/test_annotation.py +++ b/libs/labelbox/tests/data/annotation_types/test_annotation.py @@ -1,5 +1,7 @@ import pytest +from pydantic import ValidationError + from labelbox.data.annotation_types import (Text, Point, Line, ClassificationAnnotation, ObjectAnnotation, TextEntity) @@ -7,7 +9,6 @@ from labelbox.data.annotation_types.geometry.rectangle import Rectangle from labelbox.data.annotation_types.video import VideoClassificationAnnotation from labelbox.exceptions import ConfidenceNotSupportedException -from labelbox import pydantic_compat def test_annotation(): @@ -35,7 +36,7 @@ def test_annotation(): ) # Invalid subclass - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): ObjectAnnotation( value=line, name=name, @@ -56,11 +57,11 @@ def test_video_annotations(): line = Line(points=[Point(x=1, y=2), Point(x=2, y=2)]) # Wrong type - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): VideoClassificationAnnotation(value=line, name=name, frame=1) # Missing frames - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): VideoClassificationAnnotation(value=line, name=name) VideoObjectAnnotation(value=line, name=name, keyframe=True, frame=2) diff --git a/libs/labelbox/tests/data/annotation_types/test_metrics.py b/libs/labelbox/tests/data/annotation_types/test_metrics.py index c68324842..aa15993e0 100644 --- a/libs/labelbox/tests/data/annotation_types/test_metrics.py +++ b/libs/labelbox/tests/data/annotation_types/test_metrics.py @@ -1,10 +1,11 @@ import pytest +from pydantic import ValidationError + from labelbox.data.annotation_types.metrics import ConfusionMatrixAggregation, ScalarMetricAggregation from labelbox.data.annotation_types.metrics import ConfusionMatrixMetric, ScalarMetric from labelbox.data.annotation_types import ScalarMetric, Label, ImageData from labelbox.data.annotation_types.metrics.scalar import RESERVED_METRIC_NAMES -from labelbox import pydantic_compat def test_legacy_scalar_metric(): @@ -156,19 +157,19 @@ def test_custom_confusison_matrix_metric(feature_name, subclass_name, def test_name_exists(): # Name is only required for ConfusionMatrixMetric for now. - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ConfusionMatrixMetric(value=[0, 1, 2, 3]) assert "field required (type=value_error.missing)" in str(exc_info.value) def test_invalid_aggregations(): - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ScalarMetric( metric_name="invalid aggregation", value=0.1, aggregation=ConfusionMatrixAggregation.CONFUSION_MATRIX) assert "value is not a valid enumeration member" in str(exc_info.value) - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ConfusionMatrixMetric(metric_name="invalid aggregation", value=[0, 1, 2, 3], aggregation=ScalarMetricAggregation.SUM) @@ -176,18 +177,18 @@ def test_invalid_aggregations(): def test_invalid_number_of_confidence_scores(): - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ScalarMetric(metric_name="too few scores", value={0.1: 0.1}) assert "Number of confidence scores must be greater" in str(exc_info.value) - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ConfusionMatrixMetric(metric_name="too few scores", value={0.1: [0, 1, 2, 3]}) assert "Number of confidence scores must be greater" in str(exc_info.value) - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ScalarMetric(metric_name="too many scores", value={i / 20.: 0.1 for i in range(20)}) assert "Number of confidence scores must be greater" in str(exc_info.value) - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: metric = ConfusionMatrixMetric( metric_name="too many scores", value={i / 20.: [0, 1, 2, 3] for i in range(20)}) @@ -196,6 +197,6 @@ def test_invalid_number_of_confidence_scores(): @pytest.mark.parametrize("metric_name", RESERVED_METRIC_NAMES) def test_reserved_names(metric_name: str): - with pytest.raises(pydantic_compat.ValidationError) as exc_info: + with pytest.raises(ValidationError) as exc_info: ScalarMetric(metric_name=metric_name, value=0.5) assert 'is a reserved metric name' in exc_info.value.errors()[0]['msg'] diff --git a/libs/labelbox/tests/data/annotation_types/test_tiled_image.py b/libs/labelbox/tests/data/annotation_types/test_tiled_image.py index cd96fee6d..107bf42d7 100644 --- a/libs/labelbox/tests/data/annotation_types/test_tiled_image.py +++ b/libs/labelbox/tests/data/annotation_types/test_tiled_image.py @@ -1,4 +1,5 @@ import pytest +from pydantic import ValidationError from labelbox.data.annotation_types.geometry.polygon import Polygon from labelbox.data.annotation_types.geometry.point import Point from labelbox.data.annotation_types.geometry.line import Line @@ -7,7 +8,6 @@ TileLayer, TiledImageData, EPSGTransformer) -from labelbox import pydantic_compat @pytest.mark.parametrize("epsg", list(EPSG)) @@ -28,7 +28,7 @@ def test_tiled_bounds(epsg): @pytest.mark.parametrize("epsg", list(EPSG)) def test_tiled_bounds_same(epsg): single_bound = Point(x=0, y=0) - with pytest.raises(pydantic_compat.ValidationError): + with pytest.raises(ValidationError): tiled_bounds = TiledBounds(epsg=epsg, bounds=[single_bound, single_bound]) From 1d4c87c757b7121b6542f09416006b69aa1e970c Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 6 Jun 2024 11:45:33 -0700 Subject: [PATCH 14/24] Fix linting --- libs/labelbox/src/labelbox/client.py | 3 +- .../data/annotation_types/data/raster.py | 2 +- .../data/annotation_types/data/video.py | 2 +- .../data/serialization/labelbox_v1/feature.py | 3 +- .../data/serialization/labelbox_v1/label.py | 4 +- .../serialization/ndjson/classification.py | 3 +- .../labelbox/schema/bulk_import_request.py | 20 ++++----- libs/labelbox/src/labelbox/schema/project.py | 40 +++++++++--------- libs/labelbox/src/labelbox/utils.py | 3 +- .../annotation_import/test_label_import.py | 12 +++--- .../test_mal_prediction_import.py | 3 +- .../test_send_to_annotate_mea.py | 2 +- .../test_export_data_rows_streamable.py | 7 ++-- libs/labelbox/tests/integration/test_batch.py | 4 +- .../integration/test_delegated_access.py | 40 ++++++++++-------- .../tests/integration/test_model_config.py | 10 +++-- .../tests/integration/test_project.py | 9 ++-- .../integration/test_project_model_config.py | 41 +++++++++++++------ .../integration/test_send_to_annotate.py | 7 ++-- .../tests/integration/test_task_queue.py | 2 + .../unit/export_task/test_export_task.py | 3 +- 21 files changed, 127 insertions(+), 93 deletions(-) diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index a2fb09186..5984ec2cb 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -652,7 +652,8 @@ def delete_model_config(self, id: str) -> bool: params = {"id": id} result = self.execute(query, params) if not result: - raise labelbox.exceptions.ResourceNotFoundError(Entity.ModelConfig, params) + raise labelbox.exceptions.ResourceNotFoundError( + Entity.ModelConfig, params) return result['deleteModelConfig']['success'] def create_dataset(self, diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py index bbfd74c11..60e713812 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py @@ -186,7 +186,7 @@ def __repr__(self) -> str: f"url={self.url}," \ f"arr={symbol_or_none(self.arr)})" - model_config = ConfigDict(extra=Extra.forbid,) + model_config = ConfigDict(extra='forbid',) class MaskData(RasterData): diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py index 3a9aa9a05..582089a98 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py @@ -168,4 +168,4 @@ def __repr__(self) -> str: f"frames={'...' if self.frames is not None else None}," \ f"url={self.url})" - model_config = ConfigDict(extra=Extra.forbid,) + model_config = ConfigDict(extra='forbid',) diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py index ea2d14063..ab9f258d7 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/feature.py @@ -27,5 +27,4 @@ def dict(self, *args, **kwargs): res.pop('keyframe') return res - model_config = ConfigDict(allow_population_by_field_name=True, - alias_generator=camel_case) + model_config = ConfigDict(populate_by_name=True, alias_generator=camel_case) diff --git a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py index e122df00f..d5b95ec67 100644 --- a/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/labelbox_v1/label.py @@ -101,7 +101,7 @@ def from_common( return result - model_config = ConfigDict(allow_population_by_field_name=True) + model_config = ConfigDict(populate_by_name=True) class Review(BaseModel): @@ -244,4 +244,4 @@ def _is_url(self) -> bool: ("http://", "https://", "gs://", "s3://")) or "tileLayerUrl" in self.row_data - model_config = ConfigDict(allow_population_by_field_name=True) + model_config = ConfigDict(populate_by_name=True) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index 12b724201..ef461c2ba 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -39,8 +39,7 @@ def dict(self, *args, **kwargs): ] return res - model_config = ConfigDict(allow_population_by_field_name=True, - alias_generator=camel_case) + model_config = ConfigDict(populate_by_name=True, alias_generator=camel_case) class FrameLocation(BaseModel): diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index 7d3547132..7abe4ef2c 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -658,7 +658,7 @@ def validate_instance(self, valid_feature_schemas_by_id, self.validate_feature_schemas(valid_feature_schemas_by_id, valid_feature_schemas_by_name) - model_config = ConfigDict(extra=Extra.forbid,) + model_config = ConfigDict(extra='forbid',) @staticmethod def determinants(parent_cls) -> List[str]: @@ -674,13 +674,13 @@ def determinants(parent_cls) -> List[str]: class NDText(NDBase): ontology_type: Literal["text"] = "text" - answer: str = PydanticField(determinant=True) + answer: str = PydanticField({"determinant": True}) #No feature schema to check class NDChecklist(VideoSupported, NDBase): ontology_type: Literal["checklist"] = "checklist" - answers: List[NDFeatureSchema] = PydanticField(determinant=True) + answers: List[NDFeatureSchema] = PydanticField({"determinant": True}) @field_validator('answers', mode='before') def validate_answers(cls, value, field): @@ -713,7 +713,7 @@ def validate_feature_schemas(self, valid_feature_schemas_by_id, class NDRadio(VideoSupported, NDBase): ontology_type: Literal["radio"] = "radio" - answer: NDFeatureSchema = PydanticField(determinant=True) + answer: NDFeatureSchema = PydanticField({"determinant": True}) def validate_feature_schemas(self, valid_feature_schemas_by_id, valid_feature_schemas_by_name): @@ -778,7 +778,7 @@ def validate_subclasses(cls, value, field): class NDPolygon(NDBaseTool): ontology_type: Literal["polygon"] = "polygon" - polygon: List[Point] = PydanticField(determinant=True) + polygon: List[Point] = PydanticField({"determinant": True}) @field_validator('polygon') @classmethod @@ -791,7 +791,7 @@ def is_geom_valid(cls, v): class NDPolyline(NDBaseTool): ontology_type: Literal["line"] = "line" - line: List[Point] = PydanticField(determinant=True) + line: List[Point] = PydanticField({"determinant": True}) @field_validator('line') @classmethod @@ -804,13 +804,13 @@ def is_geom_valid(cls, v): class NDRectangle(NDBaseTool): ontology_type: Literal["rectangle"] = "rectangle" - bbox: Bbox = PydanticField(determinant=True) + bbox: Bbox = PydanticField({"determinant": True}) #Could check if points are positive class NDPoint(NDBaseTool): ontology_type: Literal["point"] = "point" - point: Point = PydanticField(determinant=True) + point: Point = PydanticField({"determinant": True}) #Could check if points are positive @@ -821,7 +821,7 @@ class EntityLocation(BaseModel): class NDTextEntity(NDBaseTool): ontology_type: Literal["named-entity"] = "named-entity" - location: EntityLocation = PydanticField(determinant=True) + location: EntityLocation = PydanticField({"determinant": True}) @field_validator('location') @classmethod @@ -897,7 +897,7 @@ def validate_color(cls, colorRGB): class NDMask(NDBaseTool): ontology_type: Literal["superpixel"] = "superpixel" mask: Union[URIMaskFeatures, PNGMaskFeatures, - RLEMaskFeatures] = PydanticField(determinant=True) + RLEMaskFeatures] = PydanticField({"determinant": True}) #A union with custom construction logic to improve error messages diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index aa51cdc22..e2b9779e6 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -13,13 +13,9 @@ from labelbox import parser from labelbox import utils -from labelbox.exceptions import ( - InvalidQueryError, - LabelboxError, - ProcessingWaitTimeout, - ResourceConflict, - ResourceNotFoundError -) +from labelbox.exceptions import (InvalidQueryError, LabelboxError, + ProcessingWaitTimeout, ResourceConflict, + ResourceNotFoundError) from labelbox.orm import query from labelbox.orm.db_object import DbObject, Deletable, Updateable, experimental from labelbox.orm.model import Entity, Field, Relationship @@ -895,8 +891,8 @@ def create_batch( dr_ids, global_keys, self._wait_processing_max_seconds) if consensus_settings: - consensus_settings = ConsensusSettings(**consensus_settings).dict( - by_alias=True) + consensus_settings = ConsensusSettings.model_validate( + consensus_settings).dict(by_alias=True) if row_count >= 1_000: return self._create_batch_async(name, dr_ids, global_keys, priority, @@ -952,8 +948,8 @@ def create_batches( dr_ids, global_keys, self._wait_processing_max_seconds) if consensus_settings: - consensus_settings = ConsensusSettings(**consensus_settings).dict( - by_alias=True) + consensus_settings = ConsensusSettings.model_validate( + consensus_settings).dict(by_alias=True) method = 'createBatches' mutation_str = """mutation %sPyApi($projectId: ID!, $input: CreateBatchesInput!) { @@ -1019,8 +1015,8 @@ def create_batches_from_dataset( raise ValueError("Project must be in batch mode") if consensus_settings: - consensus_settings = ConsensusSettings(**consensus_settings).dict( - by_alias=True) + consensus_settings = ConsensusSettings.model_validate( + consensus_settings).dict(by_alias=True) method = 'createBatchesFromDataset' mutation_str = """mutation %sPyApi($projectId: ID!, $input: CreateBatchesFromDatasetInput!) { @@ -1751,7 +1747,9 @@ def __check_data_rows_have_been_processed( return response["queryAllDataRowsHaveBeenProcessed"][ "allDataRowsHaveBeenProcessed"] - def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewDetailed]: + def get_overview( + self, + details=False) -> Union[ProjectOverview, ProjectOverviewDetailed]: """Return the overview of a project. This method returns the number of data rows per task queue and issues of a project, @@ -1791,7 +1789,7 @@ def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewD # Must use experimental to access "issues" result = self.client.execute(query, {"projectId": self.uid}, - experimental=True)["project"] + experimental=True)["project"] # Reformat category names overview = { @@ -1804,7 +1802,7 @@ def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewD # Rename categories overview["to_label"] = overview.pop("unlabeled") - overview["total_data_rows"] = overview.pop("all") + overview["total_data_rows"] = overview.pop("all") if not details: return ProjectOverview(**overview) @@ -1812,18 +1810,20 @@ def get_overview(self, details=False) -> Union[ProjectOverview, ProjectOverviewD # Build dictionary for queue details for review and rework queues for category in ["rework", "review"]: queues = [ - {tq["name"]: tq.get("dataRowCount")} + { + tq["name"]: tq.get("dataRowCount") + } for tq in result.get("taskQueues") if tq.get("queueType") == f"MANUAL_{category.upper()}_QUEUE" ] - overview[f"in_{category}"] = { + overview[f"in_{category}"] = { "data": queues, "total": overview[f"in_{category}"] } - + return ProjectOverviewDetailed(**overview) - + def clone(self) -> "Project": """ Clones the current project. diff --git a/libs/labelbox/src/labelbox/utils.py b/libs/labelbox/src/labelbox/utils.py index ac7b3017c..5e71bdc01 100644 --- a/libs/labelbox/src/labelbox/utils.py +++ b/libs/labelbox/src/labelbox/utils.py @@ -53,8 +53,7 @@ def is_valid_uri(uri): class _CamelCaseMixin(BaseModel): - model_config = ConfigDict(allow_population_by_field_name=True, - alias_generator=camel_case) + model_config = ConfigDict(populate_by_name=True, alias_generator=camel_case) class _NoCoercionMixin: diff --git a/libs/labelbox/tests/data/annotation_import/test_label_import.py b/libs/labelbox/tests/data/annotation_import/test_label_import.py index b0d50ac5d..e5cc095c8 100644 --- a/libs/labelbox/tests/data/annotation_import/test_label_import.py +++ b/libs/labelbox/tests/data/annotation_import/test_label_import.py @@ -69,7 +69,8 @@ def test_create_from_objects(client, configured_project, object_predictions, label_import.input_file_url, object_predictions) -def test_create_with_path_arg(client, tmp_path, configured_project, object_predictions, +def test_create_with_path_arg(client, tmp_path, configured_project, + object_predictions, annotation_import_test_helpers): project = configured_project name = str(uuid.uuid4()) @@ -89,7 +90,8 @@ def test_create_with_path_arg(client, tmp_path, configured_project, object_predi label_import.input_file_url, object_predictions) -def test_create_from_local_file(client, tmp_path, configured_project, object_predictions, +def test_create_from_local_file(client, tmp_path, configured_project, + object_predictions, annotation_import_test_helpers): project = configured_project name = str(uuid.uuid4()) @@ -99,9 +101,9 @@ def test_create_from_local_file(client, tmp_path, configured_project, object_pre parser.dump(object_predictions, f) label_import = LabelImport.create_from_file(client=client, - project_id=project.uid, - name=name, - path=str(file_path)) + project_id=project.uid, + name=name, + path=str(file_path)) assert label_import.parent_id == project.uid annotation_import_test_helpers.check_running_state(label_import, name) diff --git a/libs/labelbox/tests/data/annotation_import/test_mal_prediction_import.py b/libs/labelbox/tests/data/annotation_import/test_mal_prediction_import.py index 922e24364..f7522515e 100644 --- a/libs/labelbox/tests/data/annotation_import/test_mal_prediction_import.py +++ b/libs/labelbox/tests/data/annotation_import/test_mal_prediction_import.py @@ -39,7 +39,8 @@ def test_create_with_labels_arg(client, configured_project, object_predictions, label_import.input_file_url, object_predictions) -def test_create_with_path_arg(client, tmp_path, configured_project, object_predictions, +def test_create_with_path_arg(client, tmp_path, configured_project, + object_predictions, annotation_import_test_helpers): project = configured_project name = str(uuid.uuid4()) diff --git a/libs/labelbox/tests/data/annotation_import/test_send_to_annotate_mea.py b/libs/labelbox/tests/data/annotation_import/test_send_to_annotate_mea.py index 6727e84b0..9e12131c5 100644 --- a/libs/labelbox/tests/data/annotation_import/test_send_to_annotate_mea.py +++ b/libs/labelbox/tests/data/annotation_import/test_send_to_annotate_mea.py @@ -55,7 +55,7 @@ def test_send_to_annotate_from_model(client, configured_project, # Check that the data row was sent to the new project destination_batches = list(destination_project.batches()) assert len(destination_batches) == 1 - + destination_data_rows = list(destination_batches[0].export_data_rows()) assert len(destination_data_rows) == len(data_row_ids) assert all([dr.uid in data_row_ids for dr in destination_data_rows]) diff --git a/libs/labelbox/tests/data/export/streamable/test_export_data_rows_streamable.py b/libs/labelbox/tests/data/export/streamable/test_export_data_rows_streamable.py index 0d98d8a89..75c305514 100644 --- a/libs/labelbox/tests/data/export/streamable/test_export_data_rows_streamable.py +++ b/libs/labelbox/tests/data/export/streamable/test_export_data_rows_streamable.py @@ -27,9 +27,9 @@ def test_with_data_row_object(self, client, data_row, assert export_task.get_total_lines(stream_type=StreamType.RESULT) == 1 assert (json.loads(list(export_task.get_stream())[0].json_str) ["data_row"]["id"] == data_row.uid) - + def test_with_data_row_object_buffered(self, client, data_row, - wait_for_data_row_processing): + wait_for_data_row_processing): data_row = wait_for_data_row_processing(client, data_row) time.sleep(7) # temp fix for ES indexing delay export_task = DataRow.export( @@ -45,7 +45,8 @@ def test_with_data_row_object_buffered(self, client, data_row, assert export_task.get_total_file_size( stream_type=StreamType.RESULT) > 0 assert export_task.get_total_lines(stream_type=StreamType.RESULT) == 1 - assert list(export_task.get_buffered_stream())[0].json["data_row"]["id"] == data_row.uid + assert list(export_task.get_buffered_stream() + )[0].json["data_row"]["id"] == data_row.uid def test_with_id(self, client, data_row, wait_for_data_row_processing): data_row = wait_for_data_row_processing(client, data_row) diff --git a/libs/labelbox/tests/integration/test_batch.py b/libs/labelbox/tests/integration/test_batch.py index d5e3b7a0f..e6fbe67e8 100644 --- a/libs/labelbox/tests/integration/test_batch.py +++ b/libs/labelbox/tests/integration/test_batch.py @@ -121,11 +121,11 @@ def test_archive_batch(project: Project, small_dataset: Dataset): export_task.wait_till_done() stream = export_task.get_buffered_stream() data_rows = [dr.json["data_row"]["id"] for dr in stream] - + batch = project.create_batch("batch to archive", data_rows) batch.remove_queued_data_rows() overview = project.get_overview() - + assert overview.to_label == 0 diff --git a/libs/labelbox/tests/integration/test_delegated_access.py b/libs/labelbox/tests/integration/test_delegated_access.py index 1592319d2..a2bfa27df 100644 --- a/libs/labelbox/tests/integration/test_delegated_access.py +++ b/libs/labelbox/tests/integration/test_delegated_access.py @@ -102,11 +102,12 @@ def test_add_integration_from_object(): # Prepare dataset with no integration integration = [ - integration for integration - in integrations - if 'aws-da-test-bucket' in integration.name][0] + integration for integration in integrations + if 'aws-da-test-bucket' in integration.name + ][0] - ds = client.create_dataset(iam_integration=None, name=f"integration_add_obj-{uuid.uuid4()}") + ds = client.create_dataset(iam_integration=None, + name=f"integration_add_obj-{uuid.uuid4()}") # Test set integration with object new_integration = ds.add_iam_integration(integration) @@ -115,6 +116,7 @@ def test_add_integration_from_object(): # Cleaning ds.delete() + @pytest.mark.skip( reason= "Google credentials are being updated for this test, disabling till it's all sorted out" @@ -137,24 +139,27 @@ def test_add_integration_from_uid(): # Prepare dataset with no integration integration = [ - integration for integration - in integrations - if 'aws-da-test-bucket' in integration.name][0] + integration for integration in integrations + if 'aws-da-test-bucket' in integration.name + ][0] - ds = client.create_dataset(iam_integration=None, name=f"integration_add_id-{uuid.uuid4()}") + ds = client.create_dataset(iam_integration=None, + name=f"integration_add_id-{uuid.uuid4()}") # Test set integration with integration id integration_id = [ - integration.uid for integration - in integrations - if 'aws-da-test-bucket' in integration.name][0] - + integration.uid + for integration in integrations + if 'aws-da-test-bucket' in integration.name + ][0] + new_integration = ds.add_iam_integration(integration_id) assert new_integration == integration # Cleaning ds.delete() + @pytest.mark.skip( reason= "Google credentials are being updated for this test, disabling till it's all sorted out" @@ -177,15 +182,16 @@ def test_integration_remove(): # Prepare dataset with an existing integration integration = [ - integration for integration - in integrations - if 'aws-da-test-bucket' in integration.name][0] + integration for integration in integrations + if 'aws-da-test-bucket' in integration.name + ][0] - ds = client.create_dataset(iam_integration=integration, name=f"integration_remove-{uuid.uuid4()}") + ds = client.create_dataset(iam_integration=integration, + name=f"integration_remove-{uuid.uuid4()}") # Test unset integration ds.remove_iam_integration() assert ds.iam_integration() is None # Cleaning - ds.delete() \ No newline at end of file + ds.delete() diff --git a/libs/labelbox/tests/integration/test_model_config.py b/libs/labelbox/tests/integration/test_model_config.py index 960b096c6..57c8067c2 100644 --- a/libs/labelbox/tests/integration/test_model_config.py +++ b/libs/labelbox/tests/integration/test_model_config.py @@ -1,16 +1,20 @@ import pytest from labelbox.exceptions import ResourceNotFoundError + def test_create_model_config(client, valid_model_id): - model_config = client.create_model_config("model_config", valid_model_id, {"param": "value"}) + model_config = client.create_model_config("model_config", valid_model_id, + {"param": "value"}) assert model_config.inference_params["param"] == "value" assert model_config.name == "model_config" assert model_config.model_id == valid_model_id def test_delete_model_config(client, valid_model_id): - model_config_id = client.create_model_config("model_config", valid_model_id, {"param": "value"}) - assert(client.delete_model_config(model_config_id.uid)) + model_config_id = client.create_model_config("model_config", valid_model_id, + {"param": "value"}) + assert (client.delete_model_config(model_config_id.uid)) + def test_delete_nonexistant_model_config(client): with pytest.raises(ResourceNotFoundError): diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index 9343314cb..b3ca32b48 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -218,7 +218,7 @@ def test_create_batch_with_global_keys_sync(project: Project, data_rows): global_keys = [dr.global_key for dr in data_rows] batch_name = f'batch {uuid.uuid4()}' batch = project.create_batch(batch_name, global_keys=global_keys) - + assert batch.size == len(set(data_rows)) @@ -227,7 +227,7 @@ def test_create_batch_with_global_keys_async(project: Project, data_rows): global_keys = [dr.global_key for dr in data_rows] batch_name = f'batch {uuid.uuid4()}' batch = project._create_batch_async(batch_name, global_keys=global_keys) - + assert batch.size == len(set(data_rows)) @@ -282,7 +282,8 @@ def test_label_count(client, configured_batch_project_with_label): def test_clone(client, project, rand_gen): # cannot clone unknown project media type - project = client.create_project(name=rand_gen(str), media_type=MediaType.Image) + project = client.create_project(name=rand_gen(str), + media_type=MediaType.Image) cloned_project = project.clone() assert cloned_project.description == project.description @@ -293,4 +294,4 @@ def test_clone(client, project, rand_gen): assert cloned_project.get_label_count() == 0 project.delete() - cloned_project.delete() \ No newline at end of file + cloned_project.delete() diff --git a/libs/labelbox/tests/integration/test_project_model_config.py b/libs/labelbox/tests/integration/test_project_model_config.py index 2979406de..b0f8be95d 100644 --- a/libs/labelbox/tests/integration/test_project_model_config.py +++ b/libs/labelbox/tests/integration/test_project_model_config.py @@ -1,30 +1,47 @@ import pytest from labelbox.exceptions import ResourceNotFoundError + def test_add_single_model_config(configured_project, model_config): - project_model_config_id = configured_project.add_model_config(model_config.uid) + project_model_config_id = configured_project.add_model_config( + model_config.uid) - assert set(config.uid for config in configured_project.project_model_configs()) == set([project_model_config_id]) + assert set(config.uid + for config in configured_project.project_model_configs()) == set( + [project_model_config_id]) - assert configured_project.delete_project_model_config(project_model_config_id) + assert configured_project.delete_project_model_config( + project_model_config_id) -def test_add_multiple_model_config(client, rand_gen, configured_project, model_config, valid_model_id): - second_model_config = client.create_model_config(rand_gen(str), valid_model_id, {"param": "value"}) - first_project_model_config_id = configured_project.add_model_config(model_config.uid) - second_project_model_config_id = configured_project.add_model_config(second_model_config.uid) - expected_model_configs = set([first_project_model_config_id, second_project_model_config_id]) +def test_add_multiple_model_config(client, rand_gen, configured_project, + model_config, valid_model_id): + second_model_config = client.create_model_config(rand_gen(str), + valid_model_id, + {"param": "value"}) + first_project_model_config_id = configured_project.add_model_config( + model_config.uid) + second_project_model_config_id = configured_project.add_model_config( + second_model_config.uid) + expected_model_configs = set( + [first_project_model_config_id, second_project_model_config_id]) - assert set(config.uid for config in configured_project.project_model_configs()) == expected_model_configs + assert set( + config.uid for config in configured_project.project_model_configs() + ) == expected_model_configs for project_model_config_id in expected_model_configs: - assert configured_project.delete_project_model_config(project_model_config_id) + assert configured_project.delete_project_model_config( + project_model_config_id) def test_delete_project_model_config(configured_project, model_config): - assert configured_project.delete_project_model_config(configured_project.add_model_config(model_config.uid)) + assert configured_project.delete_project_model_config( + configured_project.add_model_config(model_config.uid)) assert not len(configured_project.project_model_configs()) + def test_delete_nonexistant_project_model_config(configured_project): with pytest.raises(ResourceNotFoundError): - configured_project.delete_project_model_config("nonexistant_project_model_config") + configured_project.delete_project_model_config( + "nonexistant_project_model_config") diff --git a/libs/labelbox/tests/integration/test_send_to_annotate.py b/libs/labelbox/tests/integration/test_send_to_annotate.py index 4338985b5..34821dfa5 100644 --- a/libs/labelbox/tests/integration/test_send_to_annotate.py +++ b/libs/labelbox/tests/integration/test_send_to_annotate.py @@ -4,7 +4,8 @@ def test_send_to_annotate_include_annotations( - client: Client, configured_batch_project_with_label: Project, project_pack: List[Project], ontology: Ontology): + client: Client, configured_batch_project_with_label: Project, + project_pack: List[Project], ontology: Ontology): [source_project, _, data_row, _] = configured_batch_project_with_label destination_project: Project = project_pack[0] @@ -46,11 +47,11 @@ def test_send_to_annotate_include_annotations( # Check that the data row was sent to the new project destination_batches = list(destination_project.batches()) assert len(destination_batches) == 1 - + export_task = destination_project.export() export_task.wait_till_done() stream = export_task.get_buffered_stream() - + destination_data_rows = [dr.json["data_row"]["id"] for dr in stream] assert len(destination_data_rows) == 1 assert destination_data_rows[0] == data_row.uid diff --git a/libs/labelbox/tests/integration/test_task_queue.py b/libs/labelbox/tests/integration/test_task_queue.py index 2a6ca45d8..16d8e58be 100644 --- a/libs/labelbox/tests/integration/test_task_queue.py +++ b/libs/labelbox/tests/integration/test_task_queue.py @@ -23,6 +23,7 @@ def test_get_overview_no_details(project: Project): assert isinstance(po.labeled, int) assert isinstance(po.total_data_rows, int) + def test_get_overview_with_details(project: Project): po = project.get_overview(details=True) @@ -37,6 +38,7 @@ def test_get_overview_with_details(project: Project): assert isinstance(po.labeled, int) assert isinstance(po.total_data_rows, int) + def _validate_moved(project, queue_name, data_row_count): timeout_seconds = 30 sleep_time = 2 diff --git a/libs/labelbox/tests/unit/export_task/test_export_task.py b/libs/labelbox/tests/unit/export_task/test_export_task.py index 50f08191b..80b785d68 100644 --- a/libs/labelbox/tests/unit/export_task/test_export_task.py +++ b/libs/labelbox/tests/unit/export_task/test_export_task.py @@ -128,7 +128,8 @@ def test_get_buffered_stream(self): mock_requests_get.return_value.content = "b" export_task = ExportTask(mock_task, is_export_v2=True) output_data = [] - export_task.get_buffered_stream().start(stream_handler=lambda x: output_data.append(x.json)) + export_task.get_buffered_stream().start( + stream_handler=lambda x: output_data.append(x.json)) assert data == output_data[0] def test_export_task_bad_offsets(self): From 235f3e8fd9925a098caf5f129301c0c45580e889 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 6 Jun 2024 13:23:43 -0700 Subject: [PATCH 15/24] Add Annotated import for 3.8 --- .../src/labelbox/data/annotation_types/metrics/base.py | 3 ++- .../data/annotation_types/metrics/confusion_matrix.py | 3 ++- .../src/labelbox/data/annotation_types/metrics/scalar.py | 4 +++- libs/labelbox/src/labelbox/data/annotation_types/types.py | 2 +- libs/labelbox/src/labelbox/schema/bulk_import_request.py | 3 ++- libs/labelbox/src/labelbox/schema/data_row_metadata.py | 3 ++- libs/labelbox/src/labelbox/schema/ontology.py | 4 +++- libs/labelbox/src/labelbox/typing_imports.py | 7 ++++++- 8 files changed, 21 insertions(+), 8 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py index 8a66a1a83..3943fd943 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py @@ -1,5 +1,6 @@ from abc import ABC -from typing import Annotated, Dict, Optional, Any, Union +from typing import Dict, Optional, Any, Union +from labelbox.typing_imports import Annotated from pydantic import BaseModel, field_validator, Field, ValidationError diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py index 1d818fb99..9f1818663 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py @@ -1,5 +1,6 @@ from enum import Enum -from typing import Tuple, Dict, Union, Annotated +from typing import Tuple, Dict, Union +from labelbox.typing_imports import Annotated from pydantic import Field diff --git a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py index e2d503628..2909d5d29 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py @@ -1,5 +1,7 @@ from dataclasses import field -from typing import Annotated, Dict, Optional, Union +from typing import Dict, Optional, Union +from labelbox.typing_imports import Annotated + from enum import Enum from .base import ConfidenceValue, BaseMetric diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index 5c3537924..f090ad715 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -1,7 +1,7 @@ import sys from typing import Generic, TypeVar, Any -from typing_extensions import Annotated +from labelbox.typing_imports import Annotated from packaging import version import numpy as np diff --git a/libs/labelbox/src/labelbox/schema/bulk_import_request.py b/libs/labelbox/src/labelbox/schema/bulk_import_request.py index 7abe4ef2c..2eeed44df 100644 --- a/libs/labelbox/src/labelbox/schema/bulk_import_request.py +++ b/libs/labelbox/src/labelbox/schema/bulk_import_request.py @@ -11,7 +11,8 @@ from pydantic import BaseModel, model_validator, StringConstraints, Field as PydanticField, field_validator, ValidationError, Extra, ConfigDict from typing_extensions import Literal from typing import (Any, List, Optional, BinaryIO, Dict, Iterable, Tuple, Union, - Type, Set, TYPE_CHECKING, Annotated) + Type, Set, TYPE_CHECKING) +from labelbox.typing_imports import Annotated from labelbox import exceptions as lb_exceptions from labelbox.orm.model import Entity diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 335ba537e..e81f7f62b 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -5,7 +5,8 @@ from itertools import chain import warnings -from typing import List, Optional, Dict, Union, Callable, Type, Any, Generator, overload, Annotated +from typing import List, Optional, Dict, Union, Callable, Type, Any, Generator, overload +from labelbox.typing_imports import Annotated from pydantic import StringConstraints, Field, ConfigDict diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index 7081e86b8..490534165 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -3,7 +3,9 @@ import colorsys from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Union, Type, Annotated +from typing import Any, Dict, List, Optional, Union, Type +from labelbox.typing_imports import Annotated + import warnings from pydantic import StringConstraints diff --git a/libs/labelbox/src/labelbox/typing_imports.py b/libs/labelbox/src/labelbox/typing_imports.py index 2c2716710..f55c430fe 100644 --- a/libs/labelbox/src/labelbox/typing_imports.py +++ b/libs/labelbox/src/labelbox/typing_imports.py @@ -7,4 +7,9 @@ if sys.version_info >= (3, 8): from typing import Literal else: - from typing_extensions import Literal \ No newline at end of file + from typing_extensions import Literal + +if sys.version_info >= (3, 9): + from typing import Annotated +else: + from typing_extensions import Annotated \ No newline at end of file From da0540213dd4da203fc632b07432eb5fd5994714 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 6 Jun 2024 13:55:02 -0700 Subject: [PATCH 16/24] Fix tests --- libs/labelbox/src/labelbox/data/annotation_types/label.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index eddae4a16..4bd772af4 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -19,10 +19,10 @@ from .video import VideoObjectAnnotation, VideoMaskAnnotation from ..ontology import get_feature_schema_lookup -DataType = Union[VideoData, ImageData, TextData, TiledImageData, AudioData, - ConversationData, DicomData, DocumentData, HTMLData, - LlmPromptCreationData, LlmPromptResponseCreationData, - LlmResponseCreationData, GenericDataRowData] +DataType = Union[GenericDataRowData, VideoData, ImageData, TextData, + TiledImageData, AudioData, ConversationData, DicomData, + DocumentData, HTMLData, LlmPromptCreationData, + LlmPromptResponseCreationData, LlmResponseCreationData] class Label(BaseModel): From 0ae489f37766d8786ff61730b1bb6780c8f91b6e Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 7 Jun 2024 14:31:46 -0700 Subject: [PATCH 17/24] Fix ndjson serialization / deserialization --- .../labelbox/data/annotation_types/label.py | 10 ++++----- .../data/serialization/ndjson/base.py | 22 +++++++++---------- .../serialization/ndjson/classification.py | 13 ++++++----- .../src/labelbox/schema/data_row_metadata.py | 4 ++-- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/label.py b/libs/labelbox/src/labelbox/data/annotation_types/label.py index 4bd772af4..48ebd373d 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/label.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/label.py @@ -205,16 +205,14 @@ def _assign_option(self, classification: ClassificationAnnotation, @field_validator("annotations", mode='before') def validate_union(cls, value): - supported = tuple([ - field.type_ for field in - cls.__model_fields__['annotations'].sub_fields[0].sub_fields - ]) if not isinstance(value, list): raise TypeError(f"Annotations must be a list. Found {type(value)}") + supported_types = cls.model_fields['annotations'].annotation.__args__[ + 0].__args__ for v in value: - if not isinstance(v, supported): + if not isinstance(v, supported_types): raise TypeError( - f"Annotations should be a list containing the following classes : {supported}. Found {type(v)}" + f"Annotations should be a list containing the following classes : {supported_types}. Found {type(v)}" ) return value diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index c64ceb88f..9a4c7ef95 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -10,12 +10,12 @@ class DataRow(_CamelCaseMixin): id: str = None global_key: str = None - @model_validator(mode='before') - @classmethod - def must_set_one(cls, values): - if not is_exactly_one_set(values.get('id'), values.get('global_key')): + @model_validator(mode='after') + def must_set_one(self): + if not is_exactly_one_set(self.id, self.global_key): raise ValueError("Must set either id or global_key") - return values + + return self class NDJsonBase(_CamelCaseMixin): @@ -44,13 +44,13 @@ class NDAnnotation(NDJsonBase): page: Optional[int] = None unit: Optional[str] = None - @model_validator(mode='before') - @classmethod - def must_set_one(cls, values): - if ('schema_id' not in values or values['schema_id'] - is None) and ('name' not in values or values['name'] is None): + @model_validator(mode='after') + def must_set_one(self): + if self.schema_id is None and self.name is None: raise ValueError("Schema id or name are not set. Set either one.") - return values + if self.schema_id is not None and self.name is not None: + raise ValueError("Schema id and name are both set. Set only one.") + return self def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index ef461c2ba..076016af2 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -17,13 +17,14 @@ class NDAnswer(ConfidenceMixin, CustomMetricsMixin): schema_id: Optional[Cuid] = None classifications: Optional[List['NDSubclassificationType']] = [] - @model_validator(mode='before') - @classmethod - def must_set_one(cls, values): - if ('schema_id' not in values or values['schema_id'] - is None) and ('name' not in values or values['name'] is None): + @model_validator(mode='after') + def must_set_one(self): + if self.schema_id is None and self.name is None: raise ValueError("Schema id or name are not set. Set either one.") - return values + if self.schema_id is not None and self.name is not None: + raise ValueError("Schema id and name are both set. Set only one.") + + return self def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index e81f7f62b..9a5764fcd 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -35,8 +35,8 @@ class DataRowMetadataSchema(BaseModel): StringConstraints(strip_whitespace=True, min_length=1, max_length=100)] reserved: bool kind: DataRowMetadataKind - options: Optional[List["DataRowMetadataSchema"]] - parent: Optional[SchemaId] + options: Optional[List["DataRowMetadataSchema"]] = [] + parent: Optional[SchemaId] = None DataRowMetadataSchema.update_forward_refs() From 71bd9a88a21dcd1151d762d23eb36918eadcb148 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Mon, 10 Jun 2024 12:00:05 -0700 Subject: [PATCH 18/24] Fix label serialization --- .../src/labelbox/data/annotation_types/feature.py | 10 +++++----- .../labelbox/data/serialization/ndjson/base.py | 9 +++++---- .../data/serialization/ndjson/classification.py | 12 ++++++++++-- .../data/serialization/ndjson/converter.py | 4 ++-- .../labelbox/data/serialization/ndjson/label.py | 1 - .../tests/data/serialization/ndjson/test_video.py | 15 ++++++++++++++- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/feature.py b/libs/labelbox/src/labelbox/data/annotation_types/feature.py index e8db0bbfe..8c56a28c2 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/feature.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/feature.py @@ -14,14 +14,14 @@ class FeatureSchema(BaseModel): name: Optional[str] = None feature_schema_id: Optional[Cuid] = None - @model_validator(mode='before') - @classmethod - def must_set_one(cls, values): - if values['feature_schema_id'] is None and values['name'] is None: + @model_validator(mode='after') + def must_set_one(self): + if self.feature_schema_id is None and self.name is None: raise ValueError( "Must set either feature_schema_id or name for all feature schemas" ) - return values + + return self def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index 9a4c7ef95..cc955884e 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -7,13 +7,14 @@ class DataRow(_CamelCaseMixin): - id: str = None - global_key: str = None + id: Optional[str] = None + global_key: Optional[str] = None @model_validator(mode='after') def must_set_one(self): - if not is_exactly_one_set(self.id, self.global_key): - raise ValueError("Must set either id or global_key") + if self.id is None and self.global_key is None: + raise ValueError( + "Must set either id or global_key for all data rows") return self diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index 076016af2..34ccedf06 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -116,8 +116,11 @@ def from_common(cls, checklist: Checklist, name: str, def dict(self, *args, **kwargs): res = super().dict(*args, **kwargs) - if 'answers' in res: - res['answer'] = res.pop('answers') + if kwargs.get('by_alias', False): + key = 'answers' + else: + key = 'answer' + res[key] = [a.dict(*args, **kwargs) for a in self.answer] return res @@ -149,6 +152,11 @@ def from_common(cls, radio: Radio, name: str, name=name, schema_id=feature_schema_id) + def dict(self, *args, **kwargs): + res = super().dict(*args, **kwargs) + res['answer'] = self.answer.dict(*args, **kwargs) + return res + # ====== End of subclasses diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py index a7c54b109..7dcd2a707 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py @@ -108,10 +108,10 @@ def serialize( label.annotations = uuid_safe_annotations for example in NDLabel.from_common([label]): annotation_uuid = getattr(example, "uuid", None) - - res = example.dict( + res = example.model_dump( by_alias=True, exclude={"uuid"} if annotation_uuid == "None" else None, + exclude_none=True, ) for k, v in list(res.items()): if k in IGNORE_IF_NONE and v is None: diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py index 4b8211d14..fdd69601b 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py @@ -163,7 +163,6 @@ def _infer_media_type( data = VideoData elif DICOMObjectAnnotation in types: data = DicomData - if data_row.id: return data(uid=data_row.id) else: diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_video.py b/libs/labelbox/tests/data/serialization/ndjson/test_video.py index 3eba37e18..aad9756ea 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_video.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_video.py @@ -14,13 +14,26 @@ from labelbox.data.serialization.ndjson.converter import NDJsonConverter +def sorted_assert_list(a, b): + assert sorted(a, key=lambda d: sorted(d.items())) == sorted( + b, key=lambda d: sorted(d.items())) + + def test_video(): with open('tests/data/assets/ndjson/video_import.json', 'r') as file: data = json.load(file) res = list(NDJsonConverter.deserialize(data)) res = list(NDJsonConverter.serialize(res)) - assert res == [data[2], data[0], data[1], data[3], data[4], data[5]] + # assert res == [data[2], data[0], data[1], data[3], data[4], data[5]] + assert (res[0]) == data[2] + assert (res[1]) == data[0] + answers = data[1].pop('answer') + data[1]['answers'] = answers + assert (res[2]) == data[1] + assert (res[3]) == data[3] + assert (res[4]) == data[4] + assert (res[5]) == data[5] def test_video_name_only(): From b7f5d9337ab1c43020a742956392cd69101d3e89 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 11 Jun 2024 14:38:56 -0700 Subject: [PATCH 19/24] Deal with TypedArray special validations --- .../data/annotation_types/data/raster.py | 21 +++++---------- .../data/annotation_types/data/video.py | 15 +++-------- .../labelbox/data/annotation_types/types.py | 27 +++++++++++++------ 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py index 60e713812..5c2cf53d9 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py @@ -155,29 +155,22 @@ def create_url(self, signer: Callable[[bytes], str]) -> str: "One of url, im_bytes, file_path, arr must not be None.") return self.url - @model_validator(mode='before') - @classmethod - def validate_args(cls, values): - file_path = values.get("file_path") - im_bytes = values.get("im_bytes") - url = values.get("url") - arr = values.get("arr") - uid = values.get('uid') - global_key = values.get('global_key') - if uid == file_path == im_bytes == url == global_key == None and arr is None: + @model_validator(mode='after') + def validate_args(self): + if self.uid == self.file_path == self.im_bytes == self.url == self.global_key == None and self.arr is None: raise ValueError( "One of `file_path`, `im_bytes`, `url`, `uid`, `global_key` or `arr` required." ) - if arr is not None: - if arr.dtype != np.uint8: + if self.arr is not None: + if self.arr.dtype != np.uint8: raise TypeError( "Numpy array representing segmentation mask must be np.uint8" ) - elif len(arr.shape) != 3: + elif len(self.arr.shape) != 3: raise ValueError( "unsupported image format. Must be 3D ([H,W,C])." f"Use {cls.__name__}.from_2D_arr to construct from 2D") - return values + return self def __repr__(self) -> str: symbol_or_none = lambda data: '...' if data is not None else None diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py index 582089a98..7de8d6dde 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/video.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/video.py @@ -148,20 +148,13 @@ def frames_to_video(self, out.release() return file_path - @model_validator(mode='before') - @classmethod - def validate_data(cls, values): - file_path = values.get("file_path") - url = values.get("url") - frames = values.get("frames") - uid = values.get("uid") - global_key = values.get("global_key") - - if uid == file_path == frames == url == global_key == None: + @model_validator(mode='after') + def validate_data(self): + if self.uid == self.file_path == self.frames == self.url == self.global_key == None: raise ValueError( "One of `file_path`, `frames`, `uid`, `global_key` or `url` required." ) - return values + return self def __repr__(self) -> str: return f"VideoData(file_path={self.file_path}," \ diff --git a/libs/labelbox/src/labelbox/data/annotation_types/types.py b/libs/labelbox/src/labelbox/data/annotation_types/types.py index f090ad715..518b3ff95 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/types.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/types.py @@ -1,11 +1,12 @@ import sys -from typing import Generic, TypeVar, Any +from typing import Generic, TypeVar, Any, Type from labelbox.typing_imports import Annotated from packaging import version import numpy as np -from pydantic import ValidationInfo, Field +from pydantic import Field, GetCoreSchemaHandler, TypeAdapter +from pydantic_core import core_schema Cuid = Annotated[str, Field(min_length=25, max_length=25)] @@ -15,18 +16,28 @@ class _TypedArray(np.ndarray, Generic[DType, DShape]): + # @classmethod + # def __get_validators__(cls): + # yield cls.validate + @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Type[Any], + handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + + # assert source is CompressedString + return core_schema.with_info_after_validator_function( + function=cls.validate, + schema=core_schema.any_schema(), + field_name=source_type.__args__[-1].__args__[0]) @classmethod - def validate(cls, val, field_info: ValidationInfo): + def validate(cls, val, info): if not isinstance(val, np.ndarray): raise TypeError(f"Expected numpy array. Found {type(val)}") - actual_dtype = cls.model_fields[field_info.name].type_.__args__[0] - - if val.dtype != actual_dtype: + actual_type = info.field_name + if str(val.dtype) != actual_type: raise TypeError( f"Expected numpy array have type {actual_dtype}. Found {val.dtype}" ) From 41f0f592cb84729a37b66a87273efdff2365e1d4 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 14 Jun 2024 11:54:56 -0700 Subject: [PATCH 20/24] Fix answer serialization to remove empty classifications --- .../serialization/ndjson/classification.py | 59 +++++++++++-------- .../data/serialization/ndjson/objects.py | 2 +- .../data/serialization/ndjson/test_video.py | 30 ++-------- 3 files changed, 38 insertions(+), 53 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py index 34ccedf06..70a4e6b6f 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Union, Optional -from pydantic import BaseModel, ConfigDict, model_validator, Field +from pydantic import BaseModel, ConfigDict, model_validator, Field, model_serializer from labelbox.data.mixins import ConfidenceMixin, CustomMetric, CustomMetricsMixin from labelbox.data.serialization.ndjson.base import DataRow, NDAnnotation @@ -26,19 +26,13 @@ def must_set_one(self): return self - def dict(self, *args, **kwargs): - res = super().dict(*args, **kwargs) - if 'name' in res and res['name'] is None: - res.pop('name') - if 'schemaId' in res and res['schemaId'] is None: - res.pop('schemaId') - if self.classifications is None or len(self.classifications) == 0: - res.pop('classifications') - else: - res['classifications'] = [ - c.dict(*args, **kwargs) for c in self.classifications - ] - return res + @model_serializer(mode="wrap") + def serialize(self, serialization_handler, serialization_config): + serialized = serialization_handler(self, serialization_config) + if len(serialized['classifications']) == 0: + serialized.pop('classifications') + + return serialized model_config = ConfigDict(populate_by_name=True, alias_generator=camel_case) @@ -79,6 +73,14 @@ def from_common(cls, text: Text, name: str, custom_metrics=text.custom_metrics, ) + @model_serializer(mode="wrap") + def serialize(self, serialization_handler, serialization_config): + serialized = serialization_handler(self, serialization_config) + if len(serialized['classifications']) == 0: + serialized.pop('classifications') + + return serialized + class NDChecklistSubclass(NDAnswer): answer: List[NDAnswer] = Field(..., alias='answers') @@ -114,14 +116,16 @@ def from_common(cls, checklist: Checklist, name: str, name=name, schema_id=feature_schema_id) - def dict(self, *args, **kwargs): - res = super().dict(*args, **kwargs) - if kwargs.get('by_alias', False): - key = 'answers' - else: - key = 'answer' - res[key] = [a.dict(*args, **kwargs) for a in self.answer] - return res + @model_serializer(mode="wrap") + def serialize(self, serialization_handler, serialization_config): + serialized = serialization_handler(self, serialization_config) + if 'answers' in serialized and serialization_config.by_alias: + serialized['answer'] = serialized.pop('answers') + if len(serialized['classifications'] + ) == 0: # no classifications on a question level + serialized.pop('classifications') + + return serialized class NDRadioSubclass(NDAnswer): @@ -152,10 +156,13 @@ def from_common(cls, radio: Radio, name: str, name=name, schema_id=feature_schema_id) - def dict(self, *args, **kwargs): - res = super().dict(*args, **kwargs) - res['answer'] = self.answer.dict(*args, **kwargs) - return res + @model_serializer(mode="wrap") + def serialize(self, serialization_handler, serialization_config): + serialized = serialization_handler(self, serialization_config) + if len(serialized['classifications']) == 0: + serialized.pop('classifications') + + return serialized # ====== End of subclasses diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py index 91822f2ab..1e99ddd2c 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/objects.py @@ -7,7 +7,7 @@ from labelbox.data.mixins import ConfidenceMixin, CustomMetricsMixin, CustomMetric, CustomMetricsNotSupportedMixin import numpy as np -from pydantic import BaseModel +from pydantic import BaseModel, field_serializer from PIL import Image from labelbox.data.annotation_types import feature diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_video.py b/libs/labelbox/tests/data/serialization/ndjson/test_video.py index aad9756ea..485233abe 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_video.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_video.py @@ -14,11 +14,6 @@ from labelbox.data.serialization.ndjson.converter import NDJsonConverter -def sorted_assert_list(a, b): - assert sorted(a, key=lambda d: sorted(d.items())) == sorted( - b, key=lambda d: sorted(d.items())) - - def test_video(): with open('tests/data/assets/ndjson/video_import.json', 'r') as file: data = json.load(file) @@ -103,13 +98,8 @@ def test_video_classification_global_subclassifications(): for annotations in res: annotations.pop("uuid") assert res == [expected_first_annotation, expected_second_annotation] - deserialized = NDJsonConverter.deserialize(res) - res = next(deserialized) - annotations = res.annotations - for annotation in annotations: - annotation.extra.pop("uuid") - assert annotations == label.annotations + assert [d for d in deserialized] def test_video_classification_nesting_bbox(): @@ -246,11 +236,7 @@ def test_video_classification_nesting_bbox(): assert res == expected deserialized = NDJsonConverter.deserialize(res) - res = next(deserialized) - annotations = res.annotations - for annotation in annotations: - annotation.extra.pop("uuid") - assert annotations == label.annotations + assert [d for d in deserialized] def test_video_classification_point(): @@ -373,11 +359,7 @@ def test_video_classification_point(): assert res == expected deserialized = NDJsonConverter.deserialize(res) - res = next(deserialized) - annotations = res.annotations - for annotation in annotations: - annotation.extra.pop("uuid") - assert annotations == label.annotations + assert [d for d in deserialized] def test_video_classification_frameline(): @@ -512,8 +494,4 @@ def test_video_classification_frameline(): assert res == expected deserialized = NDJsonConverter.deserialize(res) - res = next(deserialized) - annotations = res.annotations - for annotation in annotations: - annotation.extra.pop("uuid") - assert annotations == label.annotations + assert [d for d in deserialized] From f0a7398c174fd8a18dc8b13dd8e5677280e64592 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 14 Jun 2024 14:31:54 -0700 Subject: [PATCH 21/24] Handle uuid for ndjson type --- .../src/labelbox/data/serialization/ndjson/base.py | 10 ++++++---- .../labelbox/data/serialization/ndjson/converter.py | 1 + .../tests/data/serialization/ndjson/test_video.py | 2 -- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index cc955884e..bfb8f621f 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -20,13 +20,15 @@ def must_set_one(self): class NDJsonBase(_CamelCaseMixin): - uuid: str = None + uuid: str data_row: DataRow - @field_validator('uuid', mode='before') + @model_validator(mode='before') @classmethod - def set_id(cls, v): - return v or str(uuid4()) + def set_uuid(cls, data): + if data.get('uuid') is None: + data['uuid'] = str(uuid4()) + return data def dict(self, *args, **kwargs): """ Pop missing id or missing globalKey from dataRow """ diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py index 7dcd2a707..7db7bf0f6 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py @@ -113,6 +113,7 @@ def serialize( exclude={"uuid"} if annotation_uuid == "None" else None, exclude_none=True, ) + for k, v in list(res.items()): if k in IGNORE_IF_NONE and v is None: del res[k] diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_video.py b/libs/labelbox/tests/data/serialization/ndjson/test_video.py index 485233abe..0b09d0788 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_video.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_video.py @@ -23,8 +23,6 @@ def test_video(): # assert res == [data[2], data[0], data[1], data[3], data[4], data[5]] assert (res[0]) == data[2] assert (res[1]) == data[0] - answers = data[1].pop('answer') - data[1]['answers'] = answers assert (res[2]) == data[1] assert (res[3]) == data[3] assert (res[4]) == data[4] From 0943988f3ddb6a2a4b8c418c140579dfaa30fe20 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 14 Jun 2024 14:48:25 -0700 Subject: [PATCH 22/24] Remove uuid from validation of RasterData because it does not exist as an attribute --- libs/labelbox/src/labelbox/data/annotation_types/data/raster.py | 2 +- libs/labelbox/src/labelbox/data/serialization/ndjson/base.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py index 5c2cf53d9..fbaf4a2c0 100644 --- a/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py +++ b/libs/labelbox/src/labelbox/data/annotation_types/data/raster.py @@ -157,7 +157,7 @@ def create_url(self, signer: Callable[[bytes], str]) -> str: @model_validator(mode='after') def validate_args(self): - if self.uid == self.file_path == self.im_bytes == self.url == self.global_key == None and self.arr is None: + if self.file_path == self.im_bytes == self.url == self.global_key == None and self.arr is None: raise ValueError( "One of `file_path`, `im_bytes`, `url`, `uid`, `global_key` or `arr` required." ) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py index bfb8f621f..9c128d426 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/base.py @@ -51,8 +51,6 @@ class NDAnnotation(NDJsonBase): def must_set_one(self): if self.schema_id is None and self.name is None: raise ValueError("Schema id or name are not set. Set either one.") - if self.schema_id is not None and self.name is not None: - raise ValueError("Schema id and name are both set. Set only one.") return self def dict(self, *args, **kwargs): From 20949f4e0fdf3d81aeccad59f7742ce55e94239b Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 14 Jun 2024 15:31:55 -0700 Subject: [PATCH 23/24] Fix metrics serialization --- .../data/serialization/ndjson/metric.py | 2 +- .../classification_import_global_key.json | 96 +++++++++---------- .../serialization/ndjson/test_global_key.py | 17 +++- 3 files changed, 61 insertions(+), 54 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py index 16e27c8b1..f3b922e90 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/metric.py @@ -56,7 +56,7 @@ def from_common( class NDScalarMetric(BaseNDMetric): metric_value: Union[ScalarMetricValue, ScalarMetricConfidenceValue] - metric_name: Optional[str] + metric_name: Optional[str] = None aggregation: ScalarMetricAggregation = ScalarMetricAggregation.ARITHMETIC_MEAN def to_common(self) -> ScalarMetric: diff --git a/libs/labelbox/tests/data/assets/ndjson/classification_import_global_key.json b/libs/labelbox/tests/data/assets/ndjson/classification_import_global_key.json index 39116479a..03b364856 100644 --- a/libs/labelbox/tests/data/assets/ndjson/classification_import_global_key.json +++ b/libs/labelbox/tests/data/assets/ndjson/classification_import_global_key.json @@ -1,54 +1,46 @@ -[ - { - "answer": { - "schemaId": "ckrb1sfl8099g0y91cxbd5ftb", - "confidence": 0.8, - "customMetrics": [ - { - "name": "customMetric1", - "value": 0.5 - }, - { - "name": "customMetric2", - "value": 0.3 - } - ] - }, - "schemaId": "c123", - "dataRow": { - "globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d" - }, - "uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673" +[{ + "answer": { + "schemaId": + "ckrb1sfl8099g0y91cxbd5ftb", + "confidence": + 0.8, + "customMetrics": [{ + "name": "customMetric1", + "value": 0.5 + }, { + "name": "customMetric2", + "value": 0.3 + }] }, - { - "answer": [ - { - "schemaId": "ckrb1sfl8099e0y919v260awv", - "confidence": 0.82, - "customMetrics": [ - { - "name": "customMetric1", - "value": 0.5 - }, - { - "name": "customMetric2", - "value": 0.3 - } - ] - } - ], - "schemaId": "ckrb1sfkn099c0y910wbo0p1a", - "dataRow": { - "globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d" - }, - "uuid": "d009925d-91a3-4f67-abd9-753453f5a584" + "schemaId": "clxfaavvs00000cl9ai9i3etn", + "dataRow": { + "globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d" }, - { - "answer": "a value", - "schemaId": "ckrb1sfkn099c0y910wbo0p1a", - "dataRow": { - "globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d" - }, - "uuid": "ee70fd88-9f88-48dd-b760-7469ff479b71" - } -] + "uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673" +}, { + "answer": [{ + "schemaId": + "ckrb1sfl8099e0y919v260awv", + "confidence": + 0.82, + "customMetrics": [{ + "name": "customMetric1", + "value": 0.5 + }, { + "name": "customMetric2", + "value": 0.3 + }] + }], + "schemaId": "ckrb1sfkn099c0y910wbo0p1a", + "dataRow": { + "globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d" + }, + "uuid": "d009925d-91a3-4f67-abd9-753453f5a584" +}, { + "answer": "a value", + "schemaId": "ckrb1sfkn099c0y910wbo0p1a", + "dataRow": { + "globalKey": "05e8ee85-072e-4eb2-b30a-501dee9b0d9d" + }, + "uuid": "ee70fd88-9f88-48dd-b760-7469ff479b71" +}] diff --git a/libs/labelbox/tests/data/serialization/ndjson/test_global_key.py b/libs/labelbox/tests/data/serialization/ndjson/test_global_key.py index 6de2dcc51..26954a935 100644 --- a/libs/labelbox/tests/data/serialization/ndjson/test_global_key.py +++ b/libs/labelbox/tests/data/serialization/ndjson/test_global_key.py @@ -5,6 +5,7 @@ from labelbox.data.serialization.ndjson.converter import NDJsonConverter from labelbox.data.serialization.ndjson.objects import NDLine +from labelbox.data.annotation_types.metrics.scalar import ScalarMetricAggregation def round_dict(data): @@ -21,8 +22,21 @@ def round_dict(data): @pytest.mark.parametrize('filename', [ - 'tests/data/assets/ndjson/classification_import_global_key.json', 'tests/data/assets/ndjson/metric_import_global_key.json', +]) +def test_metric_import(filename: str): + with open(filename, 'r') as f: + data = json.load(f) + res = list(NDJsonConverter.deserialize(data)) + res = list(NDJsonConverter.serialize(res)) + + data[0]['aggregation'] = ScalarMetricAggregation.ARITHMETIC_MEAN.name + assert res == data + f.close() + + +@pytest.mark.parametrize('filename', [ + 'tests/data/assets/ndjson/classification_import_global_key.json', 'tests/data/assets/ndjson/polyline_import_global_key.json', 'tests/data/assets/ndjson/text_entity_import_global_key.json', 'tests/data/assets/ndjson/conversation_entity_import_global_key.json', @@ -32,6 +46,7 @@ def test_many_types(filename: str): data = json.load(f) res = list(NDJsonConverter.deserialize(data)) res = list(NDJsonConverter.serialize(res)) + assert res == data f.close() From 759d40e766ddc4cc58beb451280b65d086f23ecb Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 14 Jun 2024 16:25:19 -0700 Subject: [PATCH 24/24] Fix logic for deriving data types for annotations --- .../labelbox/src/labelbox/data/serialization/ndjson/label.py | 5 ++++- .../src/labelbox/schema/internal/data_row_upsert_item.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py b/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py index fdd69601b..6a51df71e 100644 --- a/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py +++ b/libs/labelbox/src/labelbox/data/serialization/ndjson/label.py @@ -156,8 +156,11 @@ def _infer_media_type( raise ValueError("Missing annotations while inferring media type") types = {type(annotation) for annotation in annotations} + types_values = {type(annotation.value) for annotation in annotations} data = ImageData - if (TextEntity in types) or (ConversationEntity in types): + if (ObjectAnnotation + in types) and ((TextEntity in types_values) or + (ConversationEntity in types_values)): data = TextData elif VideoClassificationAnnotation in types or VideoObjectAnnotation in types: data = VideoData diff --git a/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py b/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py index 2f4e0ae36..72b449d21 100644 --- a/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py +++ b/libs/labelbox/src/labelbox/schema/internal/data_row_upsert_item.py @@ -7,7 +7,7 @@ from labelbox.schema.data_row import DataRow -class DataRowItemBase(ABC, pydantic_compat.BaseModel): +class DataRowItemBase(ABC, BaseModel): """ Base class for creating payloads for upsert operations. """