Merge pull request #211 from Labelbox/ms/mea-fix

msokoloff1 · web-flow · commit 6d3bdc5cbfed · 2021-07-28T07:14:21.000-04:00
annotation metric type and demo
diff --git a/examples/model_assisted_labeling/image_mea.ipynb b/examples/model_assisted_labeling/image_mea.ipynb
diff --git a/labelbox/data/annotation_types/__init__.py b/labelbox/data/annotation_types/__init__.py
@@ -26,3 +26,5 @@
 
 from .collection import LabelList
 from .collection import LabelGenerator
+
+from .metrics import ScalarMetric
diff --git a/labelbox/data/annotation_types/data/raster.py b/labelbox/data/annotation_types/data/raster.py
@@ -23,7 +23,7 @@ class RasterData(BaseData):
 
     @classmethod
     def from_2D_arr(cls, arr: TypedArray[Literal['uint8']], **kwargs):
-        if len(arr.shape):
+        if len(arr.shape) != 2:
             raise ValueError(
                 f"Found array with shape {arr.shape}. Expected two dimensions ([W,H])"
             )
@@ -38,7 +38,7 @@ def bytes_to_np(self, image_bytes: bytes) -> np.ndarray:
         Returns:
             numpy array representing the image
         """
-        return np.array(Image.open(BytesIO(image_bytes)))
+        return np.array(Image.open(BytesIO(image_bytes)))[:, :, :3]
 
     def np_to_bytes(self, arr: np.ndarray) -> bytes:
         """
diff --git a/labelbox/data/annotation_types/label.py b/labelbox/data/annotation_types/label.py
@@ -1,5 +1,6 @@
 from collections import defaultdict
-from typing import Any, Callable, Dict, List, Union
+
+from typing import Any, Callable, Dict, List, Union, Optional
 
 from pydantic import BaseModel, validator
 
@@ -9,28 +10,30 @@
 from .classification import ClassificationAnswer
 from .data import VideoData, TextData, RasterData
 from .geometry import Mask
-from .metrics import Metric
+from .metrics import ScalarMetric
+from .types import Cuid
 from .annotation import (ClassificationAnnotation, ObjectAnnotation,
                          VideoClassificationAnnotation, VideoObjectAnnotation)
-from labelbox.data.annotation_types import annotation
 
 
 class Label(BaseModel):
+    uid: Optional[Cuid] = None
     data: Union[VideoData, RasterData, TextData]
     annotations: List[Union[ClassificationAnnotation, ObjectAnnotation,
                             VideoObjectAnnotation,
-                            VideoClassificationAnnotation, Metric]] = []
+                            VideoClassificationAnnotation, ScalarMetric]] = []
     extra: Dict[str, Any] = {}
 
     def object_annotations(self) -> List[ObjectAnnotation]:
-        return self.get_annotations_by_type(ObjectAnnotation)
+        return self._get_annotations_by_type(ObjectAnnotation)
 
     def classification_annotations(self) -> List[ClassificationAnnotation]:
-        return self.get_annotations_by_type(ClassificationAnnotation)
+        return self._get_annotations_by_type(ClassificationAnnotation)
 
-    def get_annotations_by_type(self, annotation_type):
+    def _get_annotations_by_type(self, annotation_type):
         return [
-            annot for annot in self.annotations if isinstance(annotation_type)
+            annot for annot in self.annotations
+            if isinstance(annot, annotation_type)
         ]
 
     def frame_annotations(
diff --git a/labelbox/data/annotation_types/metrics.py b/labelbox/data/annotation_types/metrics.py
@@ -1,6 +1,9 @@
+from typing import Any, Dict
+from labelbox.data.annotation_types.feature import FeatureSchema
 from pydantic import BaseModel
 
 
-class Metric(BaseModel):
+class ScalarMetric(BaseModel):
     """ Class representing metrics """
-    metric_value: float
+    value: float
+    extra: Dict[str, Any] = {}
diff --git a/labelbox/data/serialization/labelbox_v1/converter.py b/labelbox/data/serialization/labelbox_v1/converter.py
@@ -57,6 +57,8 @@ def serialize(
         """
         Converts a labelbox common object to the labelbox json export format
 
+        Note that any metric annotations will not be written since they are not defined in the LBV1 format.
+
         Args:
             labels: Either a LabelList or a LabelGenerator (LabelCollection)
         Returns:
diff --git a/labelbox/data/serialization/labelbox_v1/label.py b/labelbox/data/serialization/labelbox_v1/label.py
@@ -120,12 +120,12 @@ class LBV1Label(BaseModel):
     label: Union[LBV1LabelAnnotations,
                  List[LBV1LabelAnnotationsVideo]] = Field(..., alias='Label')
     data_row_id: str = Field(..., alias="DataRow ID")
-    row_data: str = Field(..., alias="Labeled Data")
+    row_data: str = Field(None, alias="Labeled Data")
+    id: Optional[str] = Field(None, alias='ID')
     external_id: Optional[str] = Field(None, alias="External ID")
 
     created_by: Optional[str] = Extra('Created By')
     project_name: Optional[str] = Extra('Project Name')
-    id: Optional[str] = Extra('ID')
     created_at: Optional[str] = Extra('Created At')
     updated_at: Optional[str] = Extra('Updated At')
     seconds_to_label: Optional[float] = Extra('Seconds to Label')
@@ -149,9 +149,9 @@ def to_common(self) -> Label:
         else:
             annotations = self.label.to_common()
             data = self._infer_media_type()
-            print(data)
 
         return Label(data=data,
+                     uid=self.id,
                      annotations=annotations,
                      extra={
                          field.alias: getattr(self, field_name)
@@ -167,13 +167,8 @@ def from_common(cls, label: Label):
         else:
             label_ = LBV1LabelAnnotations.from_common(label.annotations)
 
-        if label.data.url is None:
-            raise ValueError(
-                "Url attribute required for serializing data objects. "
-                "Use <LabelList,LabelGenerator>.add_url_to_data "
-                "or <LabelList,LabelGenerator>.add_to_dataset")
-
         return LBV1Label(label=label_,
+                         id=label.uid,
                          data_row_id=label.data.uid,
                          row_data=label.data.url,
                          external_id=label.data.external_id,
diff --git a/labelbox/data/serialization/ndjson/classification.py b/labelbox/data/serialization/ndjson/classification.py
@@ -101,7 +101,7 @@ def from_common(cls, text: Text, schema_id: Cuid, extra: Dict[str, Any],
                     data: Union[TextData, RasterData]) -> "NDText":
         return cls(
             answer=text.answer,
-            dataRow={'id': data.uid},
+            data_row={'id': data.uid},
             schema_id=schema_id,
             uuid=extra.get('uuid'),
         )
@@ -116,7 +116,7 @@ def from_common(
         return cls(answer=[
             NDFeature(schema_id=answer.schema_id) for answer in checklist.answer
         ],
-                   dataRow={'id': data.uid},
+                   data_row={'id': data.uid},
                    schema_id=schema_id,
                    uuid=extra.get('uuid'),
                    frames=extra.get('frames'))
@@ -128,7 +128,7 @@ class NDRadio(NDAnnotation, NDRadioSubclass, VideoSupported):
     def from_common(cls, radio: Radio, schema_id: Cuid, extra: Dict[str, Any],
                     data: Union[VideoData, TextData, RasterData]) -> "NDRadio":
         return cls(answer=NDFeature(schema_id=radio.answer.schema_id),
-                   dataRow={'id': data.uid},
+                   data_row={'id': data.uid},
                    schema_id=schema_id,
                    uuid=extra.get('uuid'),
                    frames=extra.get('frames'))
diff --git a/labelbox/data/serialization/ndjson/label.py b/labelbox/data/serialization/ndjson/label.py
@@ -1,4 +1,6 @@
 from itertools import groupby
+from labelbox.data.annotation_types.metrics import ScalarMetric
+
 from operator import itemgetter
 from typing import Dict, Generator, List, Tuple, Union
 from collections import defaultdict
@@ -10,12 +12,13 @@
 from ...annotation_types.data import RasterData, TextData, VideoData
 from ...annotation_types.label import Label
 from ...annotation_types.ner import TextEntity
-from .classification import NDChecklistSubclass, NDClassification, NDRadioSubclass, NDClassificationType
+from .metric import NDMetricAnnotation, NDMetricType
+from .classification import NDChecklistSubclass, NDClassification, NDClassificationType, NDRadioSubclass
 from .objects import NDObject, NDObjectType
 
 
 class NDLabel(BaseModel):
-    annotations: List[Union[NDObjectType, NDClassificationType]]
+    annotations: List[Union[NDObjectType, NDClassificationType, NDMetricType]]
 
     def to_common(self) -> LabelGenerator:
         grouped_annotations = defaultdict(list)
@@ -33,7 +36,8 @@ def from_common(cls,
 
     def _generate_annotations(
         self, grouped_annotations: Dict[str, List[Union[NDObjectType,
-                                                        NDClassificationType]]]
+                                                        NDClassificationType,
+                                                        NDMetricType]]]
     ) -> Generator[Label, None, None]:
         for data_row_id, annotations in grouped_annotations.items():
             annots = []
@@ -42,6 +46,8 @@ def _generate_annotations(
                     annots.append(NDObject.to_common(annotation))
                 elif isinstance(annotation, NDClassificationType.__args__):
                     annots.extend(NDClassification.to_common(annotation))
+                elif isinstance(annotation, NDMetricType):
+                    annots.append(NDMetricAnnotation.to_common(annotation))
                 else:
                     raise TypeError(
                         f"Unsupported annotation. {type(annotation)}")
@@ -99,6 +105,8 @@ def _create_non_video_annotations(cls, label: Label):
                 yield NDClassification.from_common(annotation, label.data)
             elif isinstance(annotation, ObjectAnnotation):
                 yield NDObject.from_common(annotation, label.data)
+            elif isinstance(annotation, ScalarMetric):
+                yield NDMetricAnnotation.from_common(annotation, label.data)
             else:
                 raise TypeError(
                     f"Unable to convert object to MAL format. `{type(annotation.value)}`"
diff --git a/labelbox/data/serialization/ndjson/metric.py b/labelbox/data/serialization/ndjson/metric.py
@@ -0,0 +1,34 @@
+from labelbox.data.annotation_types.data import TextData, RasterData
+from labelbox.data.annotation_types.metrics import ScalarMetric
+from labelbox.data.serialization.ndjson.base import NDJsonBase
+from typing import Union
+
+
+class NDDataRowScalarMetric(NDJsonBase):
+    metric_value: float
+
+    def to_common(self) -> ScalarMetric:
+        return ScalarMetric(value=self.metric_value, extra={'uuid': self.uuid})
+
+    @classmethod
+    def from_common(
+            cls, metric: ScalarMetric,
+            data: Union[TextData, RasterData]) -> "NDDataRowScalarMetric":
+        return NDDataRowScalarMetric(uuid=metric.extra.get('uuid'),
+                                     metric_value=metric.value,
+                                     data_row={'id': data.uid})
+
+
+class NDMetricAnnotation:
+
+    @classmethod
+    def to_common(cls, annotation: NDDataRowScalarMetric) -> ScalarMetric:
+        return annotation.to_common()
+
+    @classmethod
+    def from_common(cls, annotation: ScalarMetric,
+                    data: Union[TextData, RasterData]) -> NDDataRowScalarMetric:
+        return NDDataRowScalarMetric.from_common(annotation, data)
+
+
+NDMetricType = NDDataRowScalarMetric
diff --git a/labelbox/schema/model.py b/labelbox/schema/model.py
@@ -8,15 +8,11 @@ class Model(DbObject):
         can make predictions on new data.
         Attributes:
             name (str)
-            ontology (Relationship): `ToOne` relationship to Ontology
-            model_runs (Relationship): `ToMany` relationship to ModelRun (TODO)
-            slices (Relationship): `ToMany` relationship to Slice (TODO)
+            model_runs (Relationship): `ToMany` relationship to ModelRun
         """
 
     name = Field.String("name")
-
     model_runs = Relationship.ToMany("ModelRun", False)
-    ontology = Relationship.ToOne("Ontology", False)
 
     def create_model_run(self, name):
         """ Creates a model run belonging to this model.
diff --git a/tests/data/annotation_types/test_metrics.py b/tests/data/annotation_types/test_metrics.py
@@ -0,0 +1,29 @@
+from labelbox.data.annotation_types.collection import LabelList
+from labelbox.data.annotation_types import ScalarMetric, Label, RasterData
+
+
+def test_scalar_metric():
+    value = 10
+    metric = ScalarMetric(value=value)
+    assert metric.value == value
+
+    label = Label(data=RasterData(uid="ckrmd9q8g000009mg6vej7hzg"),
+                  annotations=[metric])
+    expected = {
+        'data': {
+            'external_id': None,
+            'uid': 'ckrmd9q8g000009mg6vej7hzg',
+            'im_bytes': None,
+            'file_path': None,
+            'url': None,
+            'arr': None
+        },
+        'annotations': [{
+            'value': 10.0,
+            'extra': {}
+        }],
+        'extra': {},
+        'uid': None
+    }
+    assert label.dict() == expected
+    next(LabelList([label])).dict() == expected
diff --git a/tests/data/assets/ndjson/metric_import.json b/tests/data/assets/ndjson/metric_import.json
@@ -0,0 +1 @@
+[{"uuid" : "a22bbf6e-b2da-4abe-9a11-df84759f7672","dataRow" : {"id": "ckrmdnqj4000007msh9p2a27r"}, "metricValue" : 0.1}]
diff --git a/tests/data/serialization/ndjson/test_metric.py b/tests/data/serialization/ndjson/test_metric.py
@@ -0,0 +1,16 @@
+import json
+from labelbox.data.serialization.labelbox_v1.converter import LBV1Converter
+
+from labelbox.data.serialization.ndjson.converter import NDJsonConverter
+
+
+def test_metric():
+    with open('tests/data/assets/ndjson/metric_import.json', 'r') as file:
+        data = json.load(file)
+
+    label_list = NDJsonConverter.deserialize(data).as_list()
+    reserialized = list(NDJsonConverter.serialize(label_list))
+    assert reserialized == data
+
+    # Just make sure that this doesn't break
+    list(LBV1Converter.serialize(label_list))
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
@@ -6,6 +6,7 @@
 from random import randint
 from string import ascii_letters
 from types import SimpleNamespace
+import time
 import uuid
 import os
 import re
@@ -333,6 +334,7 @@ def configured_project_with_label(client, rand_gen, annotation_submit_fn):
     upload_task = MALPredictionImport.create_from_objects(
         client, project.uid, f'mal-import-{uuid.uuid4()}', predictions)
     upload_task.wait_until_done()
+    time.sleep(1)
     annotation_submit_fn(project.uid, data_row.uid)
     yield project
     dataset.delete()
diff --git a/tests/integration/test_model.py b/tests/integration/test_model.py
@@ -11,12 +11,10 @@ def test_model(client, configured_project, rand_gen):
     data = {"name": rand_gen(str), "ontology_id": ontology.uid}
     model = client.create_model(data["name"], data["ontology_id"])
     assert model.name == data["name"]
-    assert model.ontology().uid == data["ontology_id"]
 
     after = list(client.get_models())
     assert len(after) == len(before) + 1
     assert model in after
 
     model = client.get_model(model.uid)
     assert model.name == data["name"]
-    assert model.ontology().uid == data["ontology_id"]
diff --git a/tests/integration/test_model_run.py b/tests/integration/test_model_run.py
@@ -11,9 +11,7 @@ def test_model_run(client, configured_project_with_label, rand_gen):
     model_run = model.create_model_run(name)
     assert model_run.name == name
     assert model_run.model_id == model.uid
-    # TODO: Uncomment this out once https://github.com/Labelbox/labelbox-python/pull/189 is merged.
-    #assert model_run.created_by_id == client.get_user(
-    #).uid
+    assert model_run.created_by_id == client.get_user().uid
 
     label = list(project.labels())[0]
     model_run.upsert_labels([label.uid])
