Merge remote-tracking branch 'origin/master' into vinjai/autocurate

Author: Vinjai Vale

nucleus/constants.py

@@ -86,6 +86,7 @@
 TYPE_KEY = "type"
 UPDATED_ITEMS = "updated_items"
 UPDATE_KEY = "update"
+UPLOAD_TO_SCALE_KEY = "upload_to_scale"
 URL_KEY = "url"
 VERTICES_KEY = "vertices"
 WIDTH_KEY = "width"

nucleus/dataset_item.py

@@ -10,6 +10,7 @@
     IMAGE_URL_KEY,
     METADATA_KEY,
     ORIGINAL_IMAGE_URL_KEY,
+    UPLOAD_TO_SCALE_KEY,
     REFERENCE_ID_KEY,
     TYPE_KEY,
     URL_KEY,
@@ -92,12 +93,17 @@ class DatasetItem:  # pylint: disable=R0902
     reference_id: Optional[str] = None
     metadata: Optional[dict] = None
     pointcloud_location: Optional[str] = None
+    upload_to_scale: Optional[bool] = True
 
     def __post_init__(self):
         assert self.reference_id is not None, "reference_id is required."
         assert bool(self.image_location) != bool(
             self.pointcloud_location
         ), "Must specify exactly one of the image_location, pointcloud_location parameters"
+        if self.pointcloud_location and not self.upload_to_scale:
+            raise NotImplementedError(
+                "Skipping upload to Scale is not currently implemented for pointclouds."
+            )
         self.local = (
             is_local_path(self.image_location) if self.image_location else None
         )
@@ -133,6 +139,7 @@ def from_json(cls, payload: dict, is_scene=False):
             image_location=image_url,
             reference_id=payload.get(REFERENCE_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            upload_to_scale=payload.get(UPLOAD_TO_SCALE_KEY, None),
         )
 
     def local_file_exists(self):
@@ -158,6 +165,7 @@ def to_payload(self, is_scene=False) -> dict:
                 self.image_location
             ), "Must specify image_location for DatasetItems not in a LidarScene"
             payload[IMAGE_URL_KEY] = self.image_location
+            payload[UPLOAD_TO_SCALE_KEY] = self.upload_to_scale
 
         return payload
 

tests/test_dataset.py

@@ -1,5 +1,7 @@
 import copy
 import math
+from nucleus.model import Model
+from nucleus.prediction import BoxPrediction
 import os
 
 import pytest
@@ -176,12 +178,17 @@ def check_is_expected_response(response):
 
     # Plain image upload
     ds_items_plain = []
-    for url in TEST_IMG_URLS:
+    for i, url in enumerate(TEST_IMG_URLS):
+        # Upload just the first item in privacy mode
+        upload_to_scale = i == 0
         ds_items_plain.append(
             DatasetItem(
-                image_location=url, reference_id=url.split("/")[-1] + "_plain"
+                image_location=url,
+                upload_to_scale=upload_to_scale,
+                reference_id=url.split("/")[-1] + "_plain",
             )
         )
+
     response = dataset.append(ds_items_plain)
     check_is_expected_response(response)
 
@@ -289,8 +296,8 @@ def test_dataset_append_async_with_1_bad_url(dataset: Dataset):
             "started_image_processing": f"Dataset: {dataset.id}, Job: {job.job_id}",
         },
         "job_progress": "1.00",
-        "completed_steps": 1,
-        "total_steps": 1,
+        "completed_steps": 4,
+        "total_steps": 4,
     }
     # The error is fairly detailed and subject to change. What's important is we surface which URLs failed.
     assert (