Merged

Author: Claire Pajot

nucleus/__init__.py

@@ -1,54 +1,7 @@
 """
 Nucleus Python Library.
 
-Data formats used:
-
-_____________________________________________________________________________________________________
-
-DatasetItem
-
-image_url       |   str     |   The URL containing the image for the given row of data.\n
-reference_id    |   str     |   An optional user-specified identifier to reference this given image.\n
-metadata        |   dict    |   All of column definitions for this item.
-                |           |   The keys should match the user-specified column names,
-                |           |   and the corresponding values will populate the cell under the column.\n
-_____________________________________________________________________________________________________
-
-
-Box2DGeometry:
-
-x               |   float   |   The distance, in pixels, between the left border of the bounding box
-                |           |   and the left border of the image.\n
-y               |   float   |   The distance, in pixels, between the top border of the bounding box
-                |           |   and the top border of the image.\n
-width	        |   float   |   The width in pixels of the annotation.\n
-height	        |   float   |   The height in pixels of the annotation.\n
-
-Box2DAnnotation:
-
-item_id         |   str     |   The internally-controlled item identifier to associate this annotation with.
-                |           |   The reference_id field should be empty if this field is populated.\n
-reference_id    |   str     |   The user-specified reference identifier to associate this annotation with.\n
-                |           |   The item_id field should be empty if this field is populated.
-label	        |   str     |	The label for this annotation (e.g. car, pedestrian, bicycle).\n
-type	        |   str     |   The type of this annotation. It should always be the box string literal.\n
-geometry        |   dict    |   Representation of the bounding box in the Box2DGeometry format.\n
-metadata        |   dict    |   An arbitrary metadata blob for the annotation.\n
-
-_____________________________________________________________________________________________________
-
-Box2DDetection:
-
-item_id         |   str     |   The internally-controlled item identifier to associate this annotation with.
-                |           |   The reference_id field should be empty if this field is populated.\n
-reference_id    |   str     |   The user-specified reference identifier to associate this annotation with.
-                |           |   The item_id field should be empty if this field is populated.\n
-label	        |   str     |	The label for this annotation (e.g. car, pedestrian, bicycle).\n
-type	        |   str     |   The type of this annotation. It should always be the box string literal.\n
-confidence      |   float   |   The optional confidence level of this annotation.
-                |           |   It should be between 0 and 1 (inclusive).\n
-geometry        |   dict    |   Representation of the bounding box in the Box2DGeometry format.\n
-metadata        |   dict    |   An arbitrary metadata blob for the annotation.\n
+For full documentation see: https://dashboard.scale.com/nucleus/docs/api?language=python
 """
 import asyncio
 import json
@@ -83,7 +36,6 @@
     ANNOTATIONS_PROCESSED_KEY,
     AUTOTAGS_KEY,
     DATASET_ID_KEY,
-    DATASET_ITEM_IDS_KEY,
     DEFAULT_NETWORK_TIMEOUT_SEC,
     EMBEDDING_DIMENSION_KEY,
     EMBEDDINGS_URL_KEY,
@@ -246,11 +198,8 @@ def get_dataset_items(self, dataset_id) -> List[DatasetItem]:
             for item in dataset_items:
                 image_url = item.get("original_image_url")
                 metadata = item.get("metadata", None)
-                item_id = item.get("id", None)
                 ref_id = item.get("ref_id", None)
-                dataset_item = DatasetItem(
-                    image_url, ref_id, item_id, metadata
-                )
+                dataset_item = DatasetItem(image_url, ref_id, metadata)
                 constructed_dataset_items.append(dataset_item)
         elif error:
             raise DatasetItemRetrievalError(message=error)
@@ -351,26 +300,19 @@ def delete_dataset(self, dataset_id: str) -> dict:
         return self.make_request({}, f"dataset/{dataset_id}", requests.delete)
 
     @sanitize_string_args
-    def delete_dataset_item(
-        self, dataset_id: str, item_id: str = None, reference_id: str = None
-    ) -> dict:
+    def delete_dataset_item(self, dataset_id: str, reference_id) -> dict:
         """
         Deletes a private dataset based on datasetId.
         Returns an empty payload where response status `200` indicates
         the dataset has been successfully deleted.
         :param payload: { "name": str }
         :return: { "dataset_id": str, "name": str }
         """
-        if item_id:
-            return self.make_request(
-                {}, f"dataset/{dataset_id}/{item_id}", requests.delete
-            )
-        else:  # Assume reference_id is provided
-            return self.make_request(
-                {},
-                f"dataset/{dataset_id}/refloc/{reference_id}",
-                requests.delete,
-            )
+        return self.make_request(
+            {},
+            f"dataset/{dataset_id}/refloc/{reference_id}",
+            requests.delete,
+        )
 
     def populate_dataset(
         self,
@@ -1018,17 +960,13 @@ def create_slice(self, dataset_id: str, payload: dict) -> Slice:
         as a means of identifying items in the dataset.
 
         "name" -- The human-readable name of the slice.
-
-        "dataset_item_ids" -- An optional list of dataset item ids for the items in the slice
-
         "reference_ids" -- An optional list of user-specified identifier for the items in the slice
 
         :param
         dataset_id: id of the dataset
         payload:
         {
             "name": str,
-            "dataset_item_ids": List[str],
             "reference_ids": List[str],
         }
         :return: new Slice object
@@ -1054,14 +992,12 @@ def slice_info(self, slice_id: str) -> dict:
 
         :param
         slice_id: id of the slice
-        id_type: the type of IDs you want in response (either "reference_id" or "dataset_item_id")
-        to identify the DatasetItems
 
         :return:
         {
             "name": str,
             "dataset_id": str,
-            "dataset_item_ids": List[str],
+            "reference_ids": List[str],
         }
         """
         response = self.make_request(
@@ -1113,35 +1049,25 @@ def delete_annotations(
     def append_to_slice(
         self,
         slice_id: str,
-        dataset_item_ids: List[str] = None,
-        reference_ids: List[str] = None,
+        reference_ids: List[str],
     ) -> dict:
         """
         Appends to a slice from items already present in a dataset.
         The caller must exclusively use either datasetItemIds or reference_ids
         as a means of identifying items in the dataset.
 
         :param
-        dataset_item_ids: List[str],
         reference_ids: List[str],
 
         :return:
         {
             "slice_id": str,
         }
         """
-        if dataset_item_ids and reference_ids:
-            raise Exception(
-                "You cannot specify both dataset_item_ids and reference_ids"
-            )
 
-        ids_to_append: Dict[str, Any] = {}
-        if dataset_item_ids:
-            ids_to_append[DATASET_ITEM_IDS_KEY] = dataset_item_ids
-        if reference_ids:
-            ids_to_append[REFERENCE_IDS_KEY] = reference_ids
-
-        response = self.make_request(ids_to_append, f"slice/{slice_id}/append")
+        response = self.make_request(
+            {REFERENCE_IDS_KEY: reference_ids}, f"slice/{slice_id}/append"
+        )
         return response
 
     def list_autotags(self, dataset_id: str) -> List[str]:
@@ -1194,7 +1120,7 @@ def create_custom_index(
 
         :param
         dataset_id: id of dataset that the custom index is being added to.
-        embeddings_urls: list of urls, each of which being a json mapping dataset_item_id -> embedding vector
+        embeddings_urls: list of urls, each of which being a json mapping reference_id -> embedding vector
         embedding_dim: the dimension of the embedding vectors, must be consistent for all embedding vectors in the index.
         """
         return self.make_request(

nucleus/annotation.py

@@ -10,12 +10,10 @@
     BOX_TYPE,
     CATEGORY_TYPE,
     CUBOID_TYPE,
-    DATASET_ITEM_ID_KEY,
     DIMENSIONS_KEY,
     GEOMETRY_KEY,
     HEIGHT_KEY,
     INDEX_KEY,
-    ITEM_ID_KEY,
     LABEL_KEY,
     MASK_TYPE,
     MASK_URL_KEY,
@@ -35,16 +33,7 @@
 
 
 class Annotation:
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
-
-    def _check_ids(self):
-        if self.reference_id and self.item_id:
-            self.item_id = None  # Prefer reference id to item id.
-        if not (self.reference_id or self.item_id):
-            raise Exception(
-                "You must specify either a reference_id or an item_id for an annotation."
-            )
+    reference_id: str
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -95,14 +84,12 @@ def to_payload(self) -> dict:
 class SegmentationAnnotation(Annotation):
     mask_url: str
     annotations: List[Segment]
+    reference_id: str
     annotation_id: Optional[str] = None
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
 
     def __post_init__(self):
         if not self.mask_url:
             raise Exception("You must specify a mask_url.")
-        self._check_ids()
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -114,8 +101,7 @@ def from_json(cls, payload: dict):
                 Segment.from_json(ann)
                 for ann in payload.get(ANNOTATIONS_KEY, [])
             ],
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
         )
 
@@ -126,10 +112,9 @@ def to_payload(self) -> dict:
             ANNOTATIONS_KEY: [ann.to_payload() for ann in self.annotations],
             ANNOTATION_ID_KEY: self.annotation_id,
         }
-        if self.reference_id:
-            payload[REFERENCE_ID_KEY] = self.reference_id
-        else:
-            payload[ITEM_ID_KEY] = self.item_id
+
+        payload[REFERENCE_ID_KEY] = self.reference_id
+
         return payload
 
 
@@ -147,14 +132,14 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
     y: Union[float, int]
     width: Union[float, int]
     height: Union[float, int]
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
+        if self.annotation_id is None:
+            self.annotation_id = f"{self.label}-{self.x}-{self.y}-{self.width}-{self.height}-{self.reference_id}"
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -165,8 +150,7 @@ def from_json(cls, payload: dict):
             y=geometry.get(Y_KEY, 0),
             width=geometry.get(WIDTH_KEY, 0),
             height=geometry.get(HEIGHT_KEY, 0),
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
         )
@@ -218,13 +202,11 @@ def to_payload(self) -> dict:
 class PolygonAnnotation(Annotation):
     label: str
     vertices: List[Point]
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
         if len(self.vertices) > 0:
             if not hasattr(self.vertices[0], X_KEY) or not hasattr(
@@ -248,8 +230,7 @@ def from_json(cls, payload: dict):
             vertices=[
                 Point.from_json(_) for _ in geometry.get(VERTICES_KEY, [])
             ],
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
         )
@@ -274,13 +255,11 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
     position: Point3D
     dimensions: Point3D
     yaw: float
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
 
     @classmethod
@@ -291,8 +270,7 @@ def from_json(cls, payload: dict):
             position=Point3D.from_json(geometry.get(POSITION_KEY, {})),
             dimensions=Point3D.from_json(geometry.get(DIMENSIONS_KEY, {})),
             yaw=geometry.get(YAW_KEY, 0),
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
         )
@@ -307,8 +285,7 @@ def to_payload(self) -> dict:
                 YAW_KEY: self.yaw,
             },
         }
-        if self.reference_id:
-            payload[REFERENCE_ID_KEY] = self.reference_id
+        payload[REFERENCE_ID_KEY] = self.reference_id
         if self.annotation_id:
             payload[ANNOTATION_ID_KEY] = self.annotation_id
         if self.metadata:
@@ -321,21 +298,18 @@ def to_payload(self) -> dict:
 class CategoryAnnotation(Annotation):
     label: str
     taxonomy_name: str
-    reference_id: Optional[str] = None
-    item_id: Optional[str] = None
+    reference_id: str
     metadata: Optional[Dict] = None
 
     def __post_init__(self):
-        self._check_ids()
         self.metadata = self.metadata if self.metadata else {}
 
     @classmethod
     def from_json(cls, payload: dict):
         return cls(
             label=payload.get(LABEL_KEY, 0),
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
-            reference_id=payload.get(REFERENCE_ID_KEY, None),
-            item_id=payload.get(DATASET_ITEM_ID_KEY, None),
+            reference_id=payload[REFERENCE_ID_KEY],
             metadata=payload.get(METADATA_KEY, {}),
         )