diff --git a/script/app-mlperf-automotive-mlcommons-python/customize.py b/script/app-mlperf-automotive-mlcommons-python/customize.py index 4ea2ce0f1..688a52dc9 100644 --- a/script/app-mlperf-automotive-mlcommons-python/customize.py +++ b/script/app-mlperf-automotive-mlcommons-python/customize.py @@ -251,7 +251,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, print(env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']) cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend onnx --dataset nuscenes --nuscenes-root {os.path.dirname(env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/"))} --dataset-path {env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH']} --checkpoint {env['MLC_ML_MODEL_BEVFORMER_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" print(cmd) - elif env['MLC_MODEL'] in ['ssd-resnet50']: + elif env['MLC_MODEL'] in ['ssd']: run_dir = env['MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH'] env['RUN_DIR'] = run_dir @@ -265,7 +265,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_SSD_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" - elif env['MLC_MODEL'] in ['deeplab_v3+']: + elif env['MLC_MODEL'] in ['deeplabv3plus']: run_dir = env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH'] env['RUN_DIR'] = run_dir @@ -275,7 +275,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, backend = "onnx" if env.get( 'MLC_MLPERF_BACKEND') == "onnxruntime" else env.get('MLC_MLPERF_BACKEND') - cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" + cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" ########################################################################## diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index dd288d46e..cf95e283b 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -339,6 +339,10 @@ variations: add_deps_recursive: ml-model-bevformer: tags: _onnx + ml-model-ssd: + tags: _onnx + ml-model-deeplabv3-plus: + tags: _onnx onnxruntime,cpu: env: @@ -362,7 +366,7 @@ variations: tags: raw,_pytorch ml-model-bevformer: tags: _pytorch - ml-model-ssd-resnet50: + ml-model-ssd: tags: _pytorch ml-model-deeplabv3-plus: tags: _pytorch @@ -375,7 +379,7 @@ variations: pytorch: env: MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html - torchvision: + ml-engine-torchvision: env: MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html torchaudio: @@ -457,7 +461,7 @@ variations: pytorch: version_max: "2.5.1" version_max_usable: "2.5.1" - torchvision: + ml-engine-torchvision: version_max: "0.20.1" version_max_usable": "0.20.1" torchaudio: @@ -473,26 +477,26 @@ variations: - tags: get,generic-python-lib,_package.pyquaternion - tags: get,generic-python-lib,_package.tqdm - tags: get,generic-python-lib,_package.nuscenes-devkit - - tags: get,preprocessed,dataset,nuscenes + - tags: get,preprocessed,dataset,nuscenes,_mlc,_validation skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - preprocessed-dataset-mlcommons-nuscenes - - tags: get,ml-model,bevformer + - tags: get,ml-model,bevformer,_mlc,_rclone skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - ml-model-bevformer - ssd-resnet50: + ssd: group: models add_deps_recursive: pytorch: version_max: "2.3.1" version_max_usable: "2.3.1" - torchvision: + ml-engine-torchvision: version_max: "0.18.1" version_max_usable": "0.18.1" torchaudio: @@ -502,32 +506,35 @@ variations: - tags: get,generic-python-lib,_package.Cython - tags: get,generic-python-lib,_package.scikit-image - tags: get,generic-python-lib,_package.faster-coco-eval + - tags: get,generic-python-lib,_torchvision + names: + - ml-engine-torchvision - tags: get,generic-python-lib,_package.torchinfo - tags: get,generic-python-lib,_package.torchmetrics - tags: get,generic-sys-util,_libgl1-mesa-glx - tags: get,generic-python-lib,_package.onnx - tags: get,generic-python-lib,_package.onnxruntime - tags: get,generic-python-lib,_package.tqdm - - tags: get,preprocessed,dataset,cognata,_mlc + - tags: get,preprocessed,dataset,cognata,_mlc,_2d_object_det,_validation skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - - preprocessed-dataset-mlcommons-cognata-ssd-resnet50 + - preprocessed-dataset-mlcommons-cognata-ssd - tags: get,ml-model,ssd,resnet50,_mlc,_rclone skip_if_any_env: MLC_RUN_STATE_DOCKER: - "yes" names: - - ml-model-ssd-resnet50 + - ml-model-ssd - deeplab-v3+: + deeplabv3plus: group: models add_deps_recursive: pytorch: version_max: "2.3.1" version_max_usable: "2.3.1" - torchvision: + ml-engine-torchvision: version_max: "0.18.1" version_max_usable": "0.18.1" torchaudio: @@ -537,6 +544,9 @@ variations: - tags: get,generic-python-lib,_package.Cython - tags: get,generic-python-lib,_package.scikit-image - tags: get,generic-python-lib,_package.scikit-learn + - tags: get,generic-python-lib,_torchvision + names: + - ml-engine-torchvision - tags: get,generic-python-lib,_package.torchinfo - tags: get,generic-python-lib,_package.torchmetrics - tags: get,generic-sys-util,_libgl1-mesa-glx @@ -544,13 +554,13 @@ variations: - tags: get,generic-python-lib,_package.onnxruntime - tags: get,generic-python-lib,_package.tqdm - tags: get,generic-python-lib,_package.ijson - - tags: get,preprocessed,dataset,cognata,_mlc,_segmentation + - tags: get,preprocessed,dataset,cognata,_mlc,_segmentation,_validation skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - preprocessed-dataset-mlcommons-cognata-deeplabv3-plus - - tags: get,ml-model,deeplabv3-plus + - tags: get,ml-model,deeplabv3-plus,_mlc,_rclone skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" @@ -598,4 +608,4 @@ variations: batch_size.#: group: batch-size env: - MLC_MLPERF_LOADGEN_MAX_BATCHSIZE: "#" \ No newline at end of file + MLC_MLPERF_LOADGEN_MAX_BATCHSIZE: "#" diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 101a7b851..db81b5313 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -119,13 +119,15 @@ post_deps: docker: mlc_repo: anandhu-eng@mlperf-automations - mlc_repo_branch: automotive + mlc_repo_branch: automotive2 use_host_group_id: True use_host_user_id: True real_run: false user: mlcuser interactive: True mlc_repos_off: 'mlc pull repo mlcommons@cm4abtf --branch=poc' + pre_run_cmds: + - mlc pull repo deps: - tags: get,abtf,scratch,space mounts: @@ -188,16 +190,16 @@ variations: tags: _onnxruntime ml-model-bevformer: tags: _onnx - ml-model-ssd-resnet50: + ml-model-ssd: tags: _onnx - ml-model-deeplab-v3+: + ml-model-deeplabv3plus: tags: _onnx onnx_dynamic: base: - onnxruntime add_deps_recursive: - ml-model-deeplab-v3+: + ml-model-deeplabv3plus: tags: _onnx_dynamic onnxruntime,cpu: @@ -221,9 +223,9 @@ variations: tags: _pytorch ml-model-bevformer: tags: _pytorch - ml-model-ssd-resnet50: + ml-model-ssd: tags: _pytorch - ml-model-deeplab-v3+: + ml-model-deeplabv3plus: tags: _pytorch @@ -263,11 +265,11 @@ variations: MLC_MODEL: bevformer docker: deps: - - tags: get,preprocessed,dataset,nuscenes + - tags: get,preprocessed,dataset,nuscenes,_mlc,_validation enable_if_env: MLC_USE_DATASET_FROM_HOST: - "yes" - - tags: get,ml-model,bevformer + - tags: get,ml-model,bevformer,_mlc,_rclone enable_if_env: MLC_USE_MODEL_FROM_HOST: - "yes" @@ -292,16 +294,16 @@ variations: - nuscenes-accuracy-script tags: run,accuracy,mlperf,_nuscenes - ssd-resnet50: + ssd: group: models default_env: MLC_USE_DATASET_FROM_HOST: yes env: - MLC_MODEL: ssd-resnet50 + MLC_MODEL: ssd docker: deps: - - tags: get,preprocessed,dataset,cognata + - tags: get,preprocessed,dataset,cognata,_mlc,_2d_object_det,_validation enable_if_env: MLC_USE_DATASET_FROM_HOST: - "yes" @@ -310,13 +312,13 @@ variations: MLC_USE_MODEL_FROM_HOST: - "yes" names: - - ml-model-ssd-resnet50 + - ml-model-ssd mounts: - "${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}:${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}" - "${{ MLC_ML_MODEL_SSD_PATH }}:${{ MLC_ML_MODEL_SSD_PATH }}" add_deps_recursive: abtf-inference-implementation: - tags: _ssd-resnet50 + tags: _ssd posthook_deps: - enable_if_env: MLC_MLPERF_LOADGEN_MODE: @@ -329,31 +331,31 @@ variations: - cognata-ssd-accuracy-script tags: run,accuracy,mlperf,_cognata_ssd - deeplab-v3+: + deeplabv3plus: group: models default_env: MLC_USE_DATASET_FROM_HOST: yes env: - MLC_MODEL: deeplab-v3+ + MLC_MODEL: deeplabv3plus docker: deps: - - tags: get,preprocessed,dataset,cognata,_segmentation + - tags: get,preprocessed,dataset,cognata,_mlc,_segmentation,_validation enable_if_env: MLC_USE_DATASET_FROM_HOST: - "yes" - - tags: get,ml-model,ssd,resnet50,_mlc,_rclone + - tags: get,ml-model,deeplabv3-plus,_mlc,_rclone enable_if_env: MLC_USE_MODEL_FROM_HOST: - "yes" names: - - ml-model-deeplab-v3+ + - ml-model-deeplabv3plus mounts: - "${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}:${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}" - "${{ MLC_ML_MODEL_DEEPLABV3_PLUS_PATH }}:${{ MLC_ML_MODEL_DEEPLABV3_PLUS_PATH }}" add_deps_recursive: abtf-inference-implementation: - tags: _deeplab-v3+ + tags: _deeplabv3plus posthook_deps: - enable_if_env: MLC_MLPERF_LOADGEN_MODE: @@ -460,4 +462,4 @@ variations: MLC_MLPERF_LOADGEN_MAX_BATCHSIZE: '#' add_deps_recursive: abtf-inference-implementation: - tags: _batch_size.# \ No newline at end of file + tags: _batch_size.# diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index b8f248337..e0e5ed1d3 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -177,7 +177,7 @@ variations: new_env_keys: - MLC_LIBGLX_VERSION state: - libgl: # tbd: complete for other flavours of linux + libgl1-mesa-glx: # tbd: complete for other flavours of linux apt: libgl1-mesa-glx brew: '' dnf: '' diff --git a/script/get-preprocessed-dataset-cognata/customize.py b/script/get-preprocessed-dataset-cognata/customize.py index 1e4f0beba..c77e3bb17 100644 --- a/script/get-preprocessed-dataset-cognata/customize.py +++ b/script/get-preprocessed-dataset-cognata/customize.py @@ -8,7 +8,8 @@ def preprocess(i): env = i['env'] - if env.get('MLC_NUSCENES_DATASET_TYPE', '') == "prebuilt": + if env.get('MLC_NUSCENES_DATASET_TYPE', '') == "prebuilt" and env.get( + 'MLC_PREPROCESSED_DATASET_COGNATA_PATH', '') == '': env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" return {'return': 0} diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index 458f740fa..b2d628611 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -13,6 +13,20 @@ deps: - inference-src - accuracy-check-src tags: get,mlcommons,inference,src + skip_if_env: + MLC_DATASET: + - nuscenes + - cognata_deeplab + - cognata_ssd +- names: + - automotive-src + - accuracy-check-src + tags: get,mlcommons,automotive,src + enable_if_env: + MLC_DATASET: + - nuscenes + - cognata_deeplab + - cognata_ssd input_mapping: rerun: MLC_RERUN result_dir: MLC_MLPERF_ACCURACY_RESULTS_DIR @@ -297,13 +311,13 @@ variations: MLC_RUN_STATE_DOCKER: - "yes" names: - - preprocessed-dataset-mlcommons-cognata-ssd-resnet50 + - preprocessed-dataset-mlcommons-cognata-ssd - tags: get,ml-model,ssd,resnet50,_mlc,_rclone skip_if_any_env: MLC_RUN_STATE_DOCKER: - "yes" names: - - ml-model-ssd-resnet50 + - ml-model-ssd env: MLC_DATASET: cognata_ssd group: dataset