From d94ab453fa2a20fcd8074a5579f403e046754e0d Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 27 Jan 2025 16:45:06 +0530 Subject: [PATCH 1/2] add dataset and model download scripts automotive ref --- script/get-dataset-waymo/COPYRIGHT.md | 9 ++++++ script/get-dataset-waymo/customize.py | 27 +++++++++++++++++ script/get-dataset-waymo/meta.yaml | 19 ++++++++++++ script/get-dataset-waymo/run.sh | 8 +++++ script/get-ml-model-pointpillars/COPYRIGHT.md | 9 ++++++ script/get-ml-model-pointpillars/customize.py | 30 +++++++++++++++++++ script/get-ml-model-pointpillars/meta.yaml | 26 ++++++++++++++++ script/get-ml-model-pointpillars/run.sh | 8 +++++ .../COPYRIGHT.md | 9 ++++++ .../customize.py | 30 +++++++++++++++++++ .../get-ml-model-resnet50-deeplab/meta.yaml | 27 +++++++++++++++++ script/get-ml-model-resnet50-deeplab/run.sh | 8 +++++ 12 files changed, 210 insertions(+) create mode 100644 script/get-dataset-waymo/COPYRIGHT.md create mode 100644 script/get-dataset-waymo/customize.py create mode 100644 script/get-dataset-waymo/meta.yaml create mode 100644 script/get-dataset-waymo/run.sh create mode 100644 script/get-ml-model-pointpillars/COPYRIGHT.md create mode 100644 script/get-ml-model-pointpillars/customize.py create mode 100644 script/get-ml-model-pointpillars/meta.yaml create mode 100644 script/get-ml-model-pointpillars/run.sh create mode 100644 script/get-ml-model-resnet50-deeplab/COPYRIGHT.md create mode 100644 script/get-ml-model-resnet50-deeplab/customize.py create mode 100644 script/get-ml-model-resnet50-deeplab/meta.yaml create mode 100644 script/get-ml-model-resnet50-deeplab/run.sh diff --git a/script/get-dataset-waymo/COPYRIGHT.md b/script/get-dataset-waymo/COPYRIGHT.md new file mode 100644 index 000000000..a059b0c49 --- /dev/null +++ b/script/get-dataset-waymo/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2024-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-dataset-waymo/customize.py b/script/get-dataset-waymo/customize.py new file mode 100644 index 000000000..6b6b021ff --- /dev/null +++ b/script/get-dataset-waymo/customize.py @@ -0,0 +1,27 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_DATASET_WAYMO_PATH', '') == '': + return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \`--waymo_path\`as automatic download of this dataset is not supported yet.'} + + if not os.path.exists(env['MLC_ML_MODEL_POINT_PILLARS_PATH']): + return {'return': 1, 'error': f"Path {env['MLC_ML_MODEL_POINT_PILLARS_PATH']} does not exists!"} + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + return {'return': 0} diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml new file mode 100644 index 000000000..bfbba995f --- /dev/null +++ b/script/get-dataset-waymo/meta.yaml @@ -0,0 +1,19 @@ +alias: get-dataset-waymo +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- dataset +- waymo +uid: 21b269c753b64437 +new_env_keys: + - MLC_DATASET_WAYMO_PATH +input_mapping: + waymo_path: MLC_DATASET_WAYMO_PATH +variations: + kitti_format: + default: true + group: dataset-format + env: + MLC_DATASET_WAYMO_FORMAT: kitti diff --git a/script/get-dataset-waymo/run.sh b/script/get-dataset-waymo/run.sh new file mode 100644 index 000000000..3197bb8ad --- /dev/null +++ b/script/get-dataset-waymo/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency diff --git a/script/get-ml-model-pointpillars/COPYRIGHT.md b/script/get-ml-model-pointpillars/COPYRIGHT.md new file mode 100644 index 000000000..a059b0c49 --- /dev/null +++ b/script/get-ml-model-pointpillars/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2024-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-ml-model-pointpillars/customize.py b/script/get-ml-model-pointpillars/customize.py new file mode 100644 index 000000000..5a0cfa5db --- /dev/null +++ b/script/get-ml-model-pointpillars/customize.py @@ -0,0 +1,30 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_ML_MODEL_POINT_PILLARS_PATH', '') == '': + return {'return': 1, 'error': 'Please provide path to pointpillars model using tag \`--pp_path\`as automatic download of this model is not supported yet.'} + + if os.path.isdir(env['MLC_ML_MODEL_POINT_PILLARS_PATH']): + if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": + env['MLC_ML_MODEL_POINT_PILLARS_PATH'] = os.path.join(env['MLC_ML_MODEL_POINT_PILLARS_PATH'], "pp.onnx") + else: + env['MLC_ML_MODEL_POINT_PILLARS_PATH'] = os.path.join(env['MLC_ML_MODEL_POINT_PILLARS_PATH'], "pp_ep36.pth") + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + return {'return': 0} diff --git a/script/get-ml-model-pointpillars/meta.yaml b/script/get-ml-model-pointpillars/meta.yaml new file mode 100644 index 000000000..18470e4c0 --- /dev/null +++ b/script/get-ml-model-pointpillars/meta.yaml @@ -0,0 +1,26 @@ +alias: get-ml-model-pointpillars +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- ml-model +- ml +- model +- pointpillars +uid: 3562621a8994411d +new_env_keys: + - MLC_ML_MODEL_POINT_PILLARS_PATH +input_mapping: + pp_path: MLC_ML_MODEL_POINT_PILLARS_PATH +variations: + gpu: + default: true + group: device + env: + MLC_ML_MODEL_PP_FORMAT: pth + cpu: + group: device + env: + MLC_ML_MODEL_PP_FORMAT: onnx + diff --git a/script/get-ml-model-pointpillars/run.sh b/script/get-ml-model-pointpillars/run.sh new file mode 100644 index 000000000..3197bb8ad --- /dev/null +++ b/script/get-ml-model-pointpillars/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency diff --git a/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md b/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md new file mode 100644 index 000000000..a059b0c49 --- /dev/null +++ b/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2024-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-ml-model-resnet50-deeplab/customize.py b/script/get-ml-model-resnet50-deeplab/customize.py new file mode 100644 index 000000000..02b6c3dde --- /dev/null +++ b/script/get-ml-model-resnet50-deeplab/customize.py @@ -0,0 +1,30 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': + return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \`--waymo_path\`as automatic download of this dataset is not supported yet.'} + + if os.path.isdir(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH']): + if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'], "deeplabv3+.onnx") + else: + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'], "best_deeplabv3plus_resnet50_waymo_os16.pth") + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + return {'return': 0} diff --git a/script/get-ml-model-resnet50-deeplab/meta.yaml b/script/get-ml-model-resnet50-deeplab/meta.yaml new file mode 100644 index 000000000..f731b5d9c --- /dev/null +++ b/script/get-ml-model-resnet50-deeplab/meta.yaml @@ -0,0 +1,27 @@ +alias: get-dataset-waymo +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- ml-model +- ml +- model +- resnet50-deeplab +- resnet50 +- deeplab +uid: 93097b691a6a4fce +new_env_keys: + - MLC_ML_MODEL_DPLAB_RESNET50_PATH +input_mapping: + waymo_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH +variations: + gpu: + default: true + group: device + env: + MLC_ML_MODEL_DPLAB_RESNET50_FORMAT: pth + cpu: + group: device + env: + MLC_ML_MODEL_DPLAB_RESNET50_FORMAT: onnx diff --git a/script/get-ml-model-resnet50-deeplab/run.sh b/script/get-ml-model-resnet50-deeplab/run.sh new file mode 100644 index 000000000..3197bb8ad --- /dev/null +++ b/script/get-ml-model-resnet50-deeplab/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency From ca162ac19904f48bbb3301b614b2d456a15fcaa9 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 27 Jan 2025 16:46:20 +0530 Subject: [PATCH 2/2] update copyrights and code clean --- script/get-dataset-waymo/COPYRIGHT.md | 2 +- script/get-dataset-waymo/customize.py | 9 +++++---- script/get-ml-model-pointpillars/COPYRIGHT.md | 2 +- script/get-ml-model-pointpillars/customize.py | 8 +++++--- script/get-ml-model-resnet50-deeplab/COPYRIGHT.md | 2 +- script/get-ml-model-resnet50-deeplab/customize.py | 11 +++++++---- script/get-ml-model-resnet50-deeplab/meta.yaml | 4 ++-- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/script/get-dataset-waymo/COPYRIGHT.md b/script/get-dataset-waymo/COPYRIGHT.md index a059b0c49..d2ceead84 100644 --- a/script/get-dataset-waymo/COPYRIGHT.md +++ b/script/get-dataset-waymo/COPYRIGHT.md @@ -1,6 +1,6 @@ # Copyright Notice -© 2024-2025 MLCommons. All Rights Reserved. +© 2025-2026 MLCommons. All Rights Reserved. This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: diff --git a/script/get-dataset-waymo/customize.py b/script/get-dataset-waymo/customize.py index 6b6b021ff..273feef06 100644 --- a/script/get-dataset-waymo/customize.py +++ b/script/get-dataset-waymo/customize.py @@ -12,11 +12,12 @@ def preprocess(i): return {'return': 1, 'error': 'Script not supported in windows yet!'} if env.get('MLC_DATASET_WAYMO_PATH', '') == '': - return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \`--waymo_path\`as automatic download of this dataset is not supported yet.'} + return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \\`--waymo_path\\`as automatic download of this dataset is not supported yet.'} + + if not os.path.exists(env['MLC_DATASET_WAYMO_PATH']): + return { + 'return': 1, 'error': f"Path {env['MLC_DATASET_WAYMO_PATH']} does not exists!"} - if not os.path.exists(env['MLC_ML_MODEL_POINT_PILLARS_PATH']): - return {'return': 1, 'error': f"Path {env['MLC_ML_MODEL_POINT_PILLARS_PATH']} does not exists!"} - return {'return': 0} diff --git a/script/get-ml-model-pointpillars/COPYRIGHT.md b/script/get-ml-model-pointpillars/COPYRIGHT.md index a059b0c49..d2ceead84 100644 --- a/script/get-ml-model-pointpillars/COPYRIGHT.md +++ b/script/get-ml-model-pointpillars/COPYRIGHT.md @@ -1,6 +1,6 @@ # Copyright Notice -© 2024-2025 MLCommons. All Rights Reserved. +© 2025-2026 MLCommons. All Rights Reserved. This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: diff --git a/script/get-ml-model-pointpillars/customize.py b/script/get-ml-model-pointpillars/customize.py index 5a0cfa5db..b6685c889 100644 --- a/script/get-ml-model-pointpillars/customize.py +++ b/script/get-ml-model-pointpillars/customize.py @@ -12,13 +12,15 @@ def preprocess(i): return {'return': 1, 'error': 'Script not supported in windows yet!'} if env.get('MLC_ML_MODEL_POINT_PILLARS_PATH', '') == '': - return {'return': 1, 'error': 'Please provide path to pointpillars model using tag \`--pp_path\`as automatic download of this model is not supported yet.'} + return {'return': 1, 'error': 'Please provide path to pointpillars model using tag \\`--pp_path\\`as automatic download of this model is not supported yet.'} if os.path.isdir(env['MLC_ML_MODEL_POINT_PILLARS_PATH']): if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": - env['MLC_ML_MODEL_POINT_PILLARS_PATH'] = os.path.join(env['MLC_ML_MODEL_POINT_PILLARS_PATH'], "pp.onnx") + env['MLC_ML_MODEL_POINT_PILLARS_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PILLARS_PATH'], "pp.onnx") else: - env['MLC_ML_MODEL_POINT_PILLARS_PATH'] = os.path.join(env['MLC_ML_MODEL_POINT_PILLARS_PATH'], "pp_ep36.pth") + env['MLC_ML_MODEL_POINT_PILLARS_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PILLARS_PATH'], "pp_ep36.pth") return {'return': 0} diff --git a/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md b/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md index a059b0c49..d2ceead84 100644 --- a/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md +++ b/script/get-ml-model-resnet50-deeplab/COPYRIGHT.md @@ -1,6 +1,6 @@ # Copyright Notice -© 2024-2025 MLCommons. All Rights Reserved. +© 2025-2026 MLCommons. All Rights Reserved. This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: diff --git a/script/get-ml-model-resnet50-deeplab/customize.py b/script/get-ml-model-resnet50-deeplab/customize.py index 02b6c3dde..0df3b1c3f 100644 --- a/script/get-ml-model-resnet50-deeplab/customize.py +++ b/script/get-ml-model-resnet50-deeplab/customize.py @@ -12,14 +12,17 @@ def preprocess(i): return {'return': 1, 'error': 'Script not supported in windows yet!'} if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': - return {'return': 1, 'error': 'Please provide path to kitti dataset using tag \`--waymo_path\`as automatic download of this dataset is not supported yet.'} + return {'return': 1, 'error': 'Please provide path to deeplab resnet 50 model using tag \\`--dp_resnet50_path\\`as automatic download of this dataset is not supported yet.'} if os.path.isdir(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH']): if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'], "deeplabv3+.onnx") + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'], "deeplabv3+.onnx") else: - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'], "best_deeplabv3plus_resnet50_waymo_os16.pth") - + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'], + "best_deeplabv3plus_resnet50_waymo_os16.pth") + return {'return': 0} diff --git a/script/get-ml-model-resnet50-deeplab/meta.yaml b/script/get-ml-model-resnet50-deeplab/meta.yaml index f731b5d9c..c8c8b84e1 100644 --- a/script/get-ml-model-resnet50-deeplab/meta.yaml +++ b/script/get-ml-model-resnet50-deeplab/meta.yaml @@ -1,4 +1,4 @@ -alias: get-dataset-waymo +alias: get-dataset-deeplab-resnet50 automation_alias: script automation_uid: 5b4e0237da074764 cache: true @@ -14,7 +14,7 @@ uid: 93097b691a6a4fce new_env_keys: - MLC_ML_MODEL_DPLAB_RESNET50_PATH input_mapping: - waymo_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH + dp_resnet50_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH variations: gpu: default: true