Add support for downloading waymo from mlcommons checkpoint (#235)

* add support for downloading mlcommons checkpoint

Author: anandhu-eng

script/get-dataset-waymo/customize.py

@@ -11,12 +11,19 @@ def preprocess(i):
     if os_info['platform'] == "windows":
         return {'return': 1, 'error': 'Script not supported in windows yet!'}
 
-    if env.get('MLC_DATASET_WAYMO_PATH', '') == '':
-        return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \\`--waymo_path\\`as automatic download of this dataset is not supported yet.'}
-
-    if not os.path.exists(env['MLC_DATASET_WAYMO_PATH']):
-        return {
-            'return': 1, 'error': f"Path {env['MLC_DATASET_WAYMO_PATH']} does not exists!"}
+    if env.get('MLC_DATASET_WAYMO_PATH', '') != '':
+        if not os.path.exists(env['MLC_DATASET_WAYMO_PATH']):
+            return {
+                'return': 1, 'error': f"Path {env['MLC_DATASET_WAYMO_PATH']} does not exists!"}
+    else:
+        env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes"
+        if env['MLC_DOWNLOAD_SRC'] == "mlcommons":
+            i['run_script_input']['script_name'] = 'run-rclone'
+            if env.get('MLC_OUTDIRNAME', '') != '':
+                env['MLC_DATASET_WAYMO_PATH'] = env['MLC_OUTDIRNAME']
+            else:
+                env['MLC_DATASET_WAYMO_PATH'] = os.path.join(
+                    os.getcwd(), "kitti_format")
 
     return {'return': 0}
 

script/get-dataset-waymo/meta.yaml

@@ -17,3 +17,18 @@ variations:
     group: dataset-format
     env:
       MLC_DATASET_WAYMO_FORMAT: kitti
+  mlc:
+    group: download-src
+    default: true
+    prehook_deps:
+      - tags: get,rclone
+        enable_if_env:
+          MLC_TMP_REQUIRE_DOWNLOAD:
+          - yes
+      - tags: get,rclone-config,_waymo
+        force_cache: true
+        enable_if_env:
+          MLC_TMP_REQUIRE_DOWNLOAD:
+          - yes
+    env:
+      MLC_DOWNLOAD_SRC: mlcommons

script/get-dataset-waymo/run-rclone.sh

@@ -0,0 +1,4 @@
+cmd="rclone sync mlc-waymo:waymo_preprocessed_dataset/kitti_format ${MLC_DATASET_WAYMO_PATH} -P"
+echo $cmd
+eval $cmd
+test $? -eq 0 || exit $?

script/get-rclone-config/meta.yaml

@@ -19,3 +19,7 @@ variations:
     env:
       MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-llama3-1 drive config_is_local=false scope=drive.readonly root_folder_id=12K-2yvmr1ZSZ7SLrhidCbWc0BriN98am'
       MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-llama3-1:'
+  waymo:
+    env:
+      MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-waymo drive config_is_local=false scope=drive.readonly root_folder_id=1xbfnaUurFeXliFFl1i1gj48eRU2NDiH5'
+      MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-waymo:'