bugfix: add an arbitrary limit to fetch all datasets, but avoid pagination code

Author: bpblanken

v03_pipeline/lib/misc/terra_data_repository.py

@@ -31,7 +31,7 @@ def _tdr_request(resource: str) -> dict:
 
 
 def _get_dataset_ids() -> list[str]:
-    res_body = _tdr_request('datasets')
+    res_body = _tdr_request('datasets?limit=50000')  # Arbitrary large number :/
     items = res_body['items']
     for item in items:
         if not any(x['cloudResource'] == BIGQUERY_RESOURCE for x in item['storage']):

v03_pipeline/lib/misc/terra_data_repository_test.py

@@ -137,7 +137,7 @@ class TerraDataRepositoryTest(unittest.TestCase):
     @responses.activate
     def test_get_dataset_ids(self, _: Mock) -> None:
         responses.get(
-            os.path.join(TDR_ROOT_URL, 'datasets'),
+            os.path.join(TDR_ROOT_URL, 'datasets?limit=50000'),
             body=json.dumps(
                 {
                     'total': 3,