diff --git a/script/get-dataset-waymo/customize.py b/script/get-dataset-waymo/customize.py index 273feef06..cb625f443 100644 --- a/script/get-dataset-waymo/customize.py +++ b/script/get-dataset-waymo/customize.py @@ -11,12 +11,19 @@ def preprocess(i): if os_info['platform'] == "windows": return {'return': 1, 'error': 'Script not supported in windows yet!'} - if env.get('MLC_DATASET_WAYMO_PATH', '') == '': - return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \\`--waymo_path\\`as automatic download of this dataset is not supported yet.'} - - if not os.path.exists(env['MLC_DATASET_WAYMO_PATH']): - return { - 'return': 1, 'error': f"Path {env['MLC_DATASET_WAYMO_PATH']} does not exists!"} + if env.get('MLC_DATASET_WAYMO_PATH', '') != '': + if not os.path.exists(env['MLC_DATASET_WAYMO_PATH']): + return { + 'return': 1, 'error': f"Path {env['MLC_DATASET_WAYMO_PATH']} does not exists!"} + else: + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + if env['MLC_DOWNLOAD_SRC'] == "mlcommons": + i['run_script_input']['script_name'] = 'run-rclone' + if env.get('MLC_OUTDIRNAME', '') != '': + env['MLC_DATASET_WAYMO_PATH'] = env['MLC_OUTDIRNAME'] + else: + env['MLC_DATASET_WAYMO_PATH'] = os.path.join( + os.getcwd(), "kitti_format") return {'return': 0} diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index bfbba995f..63bbf2472 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -17,3 +17,18 @@ variations: group: dataset-format env: MLC_DATASET_WAYMO_FORMAT: kitti + mlc: + group: download-src + default: true + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_waymo + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_DOWNLOAD_SRC: mlcommons diff --git a/script/get-dataset-waymo/run-rclone.sh b/script/get-dataset-waymo/run-rclone.sh new file mode 100644 index 000000000..caca3ff98 --- /dev/null +++ b/script/get-dataset-waymo/run-rclone.sh @@ -0,0 +1,4 @@ +cmd="rclone sync mlc-waymo:waymo_preprocessed_dataset/kitti_format ${MLC_DATASET_WAYMO_PATH} -P" +echo $cmd +eval $cmd +test $? -eq 0 || exit $? \ No newline at end of file diff --git a/script/get-rclone-config/meta.yaml b/script/get-rclone-config/meta.yaml index 8cc949d25..f2f3332f0 100644 --- a/script/get-rclone-config/meta.yaml +++ b/script/get-rclone-config/meta.yaml @@ -19,3 +19,7 @@ variations: env: MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-llama3-1 drive config_is_local=false scope=drive.readonly root_folder_id=12K-2yvmr1ZSZ7SLrhidCbWc0BriN98am' MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-llama3-1:' + waymo: + env: + MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-waymo drive config_is_local=false scope=drive.readonly root_folder_id=1xbfnaUurFeXliFFl1i1gj48eRU2NDiH5' + MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-waymo:'