retries, batch size max, require async for scenes

Author: Ubuntu

nucleus/__init__.py

@@ -323,7 +323,7 @@ def populate_dataset(
         self,
         dataset_id: str,
         dataset_items: List[DatasetItem],
-        batch_size: int = 30,
+        batch_size: int = 20,
         update: bool = False,
     ):
         """
@@ -494,6 +494,9 @@ async def _make_files_request(
         files,
         route: str,
         session: aiohttp.ClientSession,
+        retry_attempt=0,
+        max_retries=3,
+        sleep_intervals=(1, 3, 9),
     ):
         """
         Makes an async post request with files to a Nucleus endpoint.
@@ -518,6 +521,7 @@ async def _make_files_request(
             )
 
         for sleep_time in RetryStrategy.sleep_times + [-1]:
+
             async with session.post(
                 endpoint,
                 data=form,
@@ -541,15 +545,27 @@ async def _make_files_request(
                     continue
 
                 if not response.ok:
-                    self.handle_bad_response(
-                        endpoint,
-                        session.post,
-                        aiohttp_response=(
-                            response.status,
-                            response.reason,
-                            data,
-                        ),
-                    )
+                    if retry_attempt < max_retries:
+                        time.sleep(sleep_intervals[retry_attempt])
+                        retry_attempt += 1
+                        return self._make_files_request(
+                            files,
+                            route,
+                            session,
+                            retry_attempt,
+                            max_retries,
+                            sleep_intervals,
+                        )
+                    else:
+                        self.handle_bad_response(
+                            endpoint,
+                            session.post,
+                            aiohttp_response=(
+                                response.status,
+                                response.reason,
+                                data,
+                            ),
+                        )
 
                 return data
 

nucleus/dataset.py

@@ -261,6 +261,9 @@ def append(
             'ignored_items': int,
         }
         """
+        assert (
+            batch_size is None or batch_size < 30
+        ), "Please specify a batch size smaller than 30 to avoid timeouts."
         dataset_items = [
             item for item in items if isinstance(item, DatasetItem)
         ]
@@ -270,6 +273,9 @@ def append(
                 "You must append either DatasetItems or Scenes to the dataset."
             )
         if scenes:
+            assert (
+                asynchronous
+            ), "In order to avoid timeouts, you must set asynchronous=True when uploading scenes."
             return self.append_scenes(scenes, update, asynchronous)
 
         check_for_duplicate_reference_ids(dataset_items)