nits

Author: Drew Kaul

nucleus/__init__.py

@@ -135,7 +135,7 @@
 )
 from .slice import Slice
 from .upload_response import UploadResponse
-from .scene import Scene, LidarScene
+from .scene import Scene, LidarScene, Frame, SceneDatasetItem, CameraParams
 
 # pylint: disable=E1101
 # TODO: refactor to reduce this file to under 1000 lines.

nucleus/annotation.py

@@ -310,15 +310,6 @@ def to_payload(self) -> dict:
         }
 
 
-def check_all_frame_paths_remote(frames: List[str]):
-    for frame_url in frames:
-        if is_local_path(frame_url):
-            raise ValueError(
-                f"All paths must be remote, but {frame_url} is either "
-                "local, or a remote URL type that is not supported."
-            )
-
-
 def check_all_mask_paths_remote(
     annotations: Sequence[Union[Annotation]],
 ):

nucleus/dataset.py

@@ -13,7 +13,6 @@
 from .annotation import (
     Annotation,
     check_all_mask_paths_remote,
-    check_all_frame_paths_remote,
 )
 from .constants import (
     DATASET_ITEM_IDS_KEY,
@@ -23,13 +22,10 @@
     DATASET_SLICES_KEY,
     DEFAULT_ANNOTATION_UPDATE_MODE,
     EXPORTED_ROWS,
-    FRAMES_KEY,
     NAME_KEY,
     REFERENCE_IDS_KEY,
     REQUEST_ID_KEY,
-    SCENES_KEY,
     UPDATE_KEY,
-    URL_KEY,
 )
 from .dataset_item import (
     DatasetItem,
@@ -43,6 +39,7 @@
 )
 
 WARN_FOR_LARGE_UPLOAD = 50000
+WARN_FOR_LARGE_SCENES_UPLOAD = 5
 
 
 class Dataset:
@@ -268,62 +265,46 @@ def append_scenes(
         scenes: List[LidarScene],
         update: Optional[bool] = False,
         asynchronous: Optional[bool] = False,
-    ) -> Union[dict, AsyncJob]:
-        """TODO: Add updated docstring here"""
-        for scene in scenes:
-            scene.validate()
-
-        if asynchronous:
-            check_all_scene_paths_remote(scenes)
-            request_id = serialize_and_write_to_presigned_url(
-                scenes, self.id, self._client
-            )
-            response = self._client.make_request(
-                payload={REQUEST_ID_KEY: request_id, UPDATE_KEY: update},
-                route=f"{self.id}/upload_scenes?async=1",
-            )
-            return AsyncJob.from_json(response, self._client)
-
-        payload = construct_append_scenes_payload(scenes, update)
-        response = self._client.make_request(
-            payload=payload,
-            route=f"{self.id}/upload_scenes",
-        )
-        return response
-
-    def upload_scenes(
-        self,
-        payload: dict,
-        update: Optional[bool] = False,
-        asynchronous: bool = False,
     ) -> Union[dict, AsyncJob]:
         """
-        Uploads scenes with given frames to the dataset
+        Appends scenes with given frames (containing pointclouds and optional images) to the dataset
 
         Parameters:
-        :param payload: dictionary containing scenes to be uploaded
+        :param scenes: scenes to upload
         :param update: if True, overwrite scene on collision
-        :param aynchronous: if True, return a job object representing asynchronous ingestion job
+        :param asynchronous: if True, return a job object representing asynchronous ingestion job
         :return:
         {
             'dataset_id': str,
             'new_scenes': int,
+            'ignored_scenes': int,
+            'scenes_errored': int,
+            'errors': List[str],
         }
         """
+        for scene in scenes:
+            scene.validate()
+
+        if len(scenes) > WARN_FOR_LARGE_SCENES_UPLOAD and not asynchronous:
+            print(
+                "Tip: for large uploads, get faster performance by importing your data "
+                "into Nucleus directly from a cloud storage provider. See "
+                "https://dashboard.scale.com/nucleus/docs/api?language=python#guide-for-large-ingestions"
+                " for details."
+            )
+
         if asynchronous:
-            for scene in payload[SCENES_KEY]:
-                for frame in scene[FRAMES_KEY]:
-                    check_all_frame_paths_remote(frame[URL_KEY])
+            check_all_scene_paths_remote(scenes)
             request_id = serialize_and_write_to_presigned_url(
-                [payload], self.id, self._client
+                scenes, self.id, self._client
             )
             response = self._client.make_request(
                 payload={REQUEST_ID_KEY: request_id, UPDATE_KEY: update},
                 route=f"{self.id}/upload_scenes?async=1",
             )
             return AsyncJob.from_json(response, self._client)
 
-        # TODO: create client method for sync scene upload
+        payload = construct_append_scenes_payload(scenes, update)
         response = self._client.make_request(
             payload=payload,
             route=f"{self.id}/upload_scenes",

nucleus/scene.py

@@ -130,8 +130,8 @@ def to_json(self) -> str:
 
 @dataclass
 class Frame:
-    index: Union[int, None] = None
     items: Dict[str, SceneDatasetItem] = field(default_factory=dict)
+    index: Union[int, None] = None
 
     def __post_init__(self):
         for key, value in self.items.items():
@@ -197,7 +197,7 @@ def validate(self):
 
     def add_item(self, index: int, sensor_name: str, item: SceneDatasetItem):
         if index not in self.frames_dict:
-            new_frame = Frame(index, {sensor_name: item})
+            new_frame = Frame(index=index, items={sensor_name: item})
             self.frames_dict[index] = new_frame
         else:
             self.frames_dict[index].items[sensor_name] = item
@@ -254,7 +254,6 @@ def to_json(self) -> str:
 
 @dataclass
 class LidarScene(Scene):
-    # TODO: call validate in scene upload
     def validate(self):
         super().validate()
         lidar_sources = flatten(

nucleus/utils.py

@@ -122,7 +122,7 @@ def convert_export_payload(api_payload):
 
 
 def serialize_and_write(
-    upload_units: Sequence[Union[DatasetItem, Annotation, Dict, LidarScene]],
+    upload_units: Sequence[Union[DatasetItem, Annotation, LidarScene]],
     file_pointer,
 ):
     for unit in upload_units:
@@ -157,7 +157,7 @@ def upload_to_presigned_url(presigned_url: str, file_pointer: IO):
 
 
 def serialize_and_write_to_presigned_url(
-    upload_units: Sequence[Union[DatasetItem, Annotation, Dict, LidarScene]],
+    upload_units: Sequence[Union[DatasetItem, Annotation, LidarScene]],
     dataset_id: str,
     client,
 ):

tests/test_scene.py

@@ -1,5 +1,10 @@
-from nucleus.constants import SCENES_KEY, UPDATE_KEY
 import pytest
+from nucleus.constants import SCENES_KEY, UPDATE_KEY
+
+from nucleus import (
+    CuboidAnnotation,
+    LidarScene,
+)
 
 from .helpers import (
     TEST_DATASET_3D_NAME,
@@ -8,11 +13,6 @@
     assert_cuboid_annotation_matches_dict,
 )
 
-from nucleus import (
-    CuboidAnnotation,
-    LidarScene,
-)
-
 
 @pytest.fixture()
 def dataset(CLIENT):