Changes for automotive 0.5

script/app-mlperf-automotive-mlcommons-python/customize.py

@@ -251,7 +251,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
         print(env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'])
         cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend onnx --dataset nuscenes --nuscenes-root {os.path.dirname(env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/"))} --dataset-path {env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH']} --checkpoint {env['MLC_ML_MODEL_BEVFORMER_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
         print(cmd)
-    elif env['MLC_MODEL'] in ['ssd-resnet50']:
+    elif env['MLC_MODEL'] in ['ssd']:
         run_dir = env['MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH']
 
         env['RUN_DIR'] = run_dir
@@ -265,7 +265,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
 
         cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_SSD_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
 
-    elif env['MLC_MODEL'] in ['deeplab_v3+']:
+    elif env['MLC_MODEL'] in ['deeplabv3plus']:
         run_dir = env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH']
 
         env['RUN_DIR'] = run_dir
@@ -275,7 +275,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
         backend = "onnx" if env.get(
             'MLC_MLPERF_BACKEND') == "onnxruntime" else env.get('MLC_MLPERF_BACKEND')
 
-        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
+        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
 
     ##########################################################################
 

script/app-mlperf-automotive-mlcommons-python/meta.yaml

@@ -201,6 +201,10 @@ deps:
       - tvm-pytorch
       MLC_MLPERF_DEVICE:
       - cpu
+    enable_if_env:
+      MLC_MODEL:
+      - deeplabv3plus
+      - ssd
 
   ## Torchvision (CUDA)
   - tags: get,generic-python-lib,_torchvision_cuda
@@ -362,7 +366,7 @@ variations:
         tags: raw,_pytorch
       ml-model-bevformer:
         tags: _pytorch
-      ml-model-ssd-resnet50:
+      ml-model-ssd:
         tags: _pytorch
       ml-model-deeplabv3-plus:
         tags: _pytorch
@@ -375,7 +379,7 @@ variations:
       pytorch:
         env:
           MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
-      torchvision:
+      ml-engine-torchvision:
         env:
           MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
       torchaudio:
@@ -457,7 +461,7 @@ variations:
       pytorch:
         version_max: "2.5.1"
         version_max_usable: "2.5.1"
-      torchvision:
+      ml-engine-torchvision:
         version_max: "0.20.1"
         version_max_usable": "0.20.1"
       torchaudio:
@@ -486,13 +490,13 @@ variations:
         names:
           - ml-model-bevformer
 
-  ssd-resnet50:
+  ssd:
     group: models
     add_deps_recursive:
       pytorch:
         version_max: "2.3.1"
         version_max_usable: "2.3.1"
-      torchvision:
+      ml-engine-torchvision:
         version_max: "0.18.1"
         version_max_usable": "0.18.1"
       torchaudio:
@@ -513,21 +517,21 @@ variations:
           MLC_RUN_STATE_DOCKER:
             - "yes"
         names:
-          - preprocessed-dataset-mlcommons-cognata-ssd-resnet50
+          - preprocessed-dataset-mlcommons-cognata-ssd
       - tags: get,ml-model,ssd,resnet50,_mlc,_rclone
         skip_if_any_env:
           MLC_RUN_STATE_DOCKER:
             - "yes"
         names:
-          - ml-model-ssd-resnet50
+          - ml-model-ssd
 
-  deeplab-v3+:
+  deeplabv3plus:
     group: models
     add_deps_recursive:
       pytorch:
         version_max: "2.3.1"
         version_max_usable: "2.3.1"
-      torchvision:
+      ml-engine-torchvision:
         version_max: "0.18.1"
         version_max_usable": "0.18.1"
       torchaudio:

script/app-mlperf-automotive/meta.yaml

@@ -119,13 +119,15 @@ post_deps:
 
 docker:
   mlc_repo: anandhu-eng@mlperf-automations
-  mlc_repo_branch: automotive
+  mlc_repo_branch: automotive2
   use_host_group_id: True
   use_host_user_id: True
   real_run: false
   user: mlcuser
   interactive: True
   mlc_repos_off: 'mlc pull repo mlcommons@cm4abtf --branch=poc'
+  pre_run_cmds:
+    - mlc pull repo
   deps:
     - tags: get,abtf,scratch,space
   mounts:
@@ -188,16 +190,16 @@ variations:
         tags: _onnxruntime
       ml-model-bevformer:
         tags: _onnx
-      ml-model-ssd-resnet50:
+      ml-model-ssd:
         tags: _onnx
-      ml-model-deeplab-v3+:
+      ml-model-deeplabv3plus:
         tags: _onnx
 
   onnx_dynamic:
     base:
       - onnxruntime
     add_deps_recursive:
-      ml-model-deeplab-v3+:
+      ml-model-deeplabv3plus:
         tags: _onnx_dynamic
 
   onnxruntime,cpu:
@@ -221,9 +223,9 @@ variations:
         tags: _pytorch
       ml-model-bevformer:
         tags: _pytorch
-      ml-model-ssd-resnet50:
+      ml-model-ssd:
         tags: _pytorch
-      ml-model-deeplab-v3+:
+      ml-model-deeplabv3plus:
         tags: _pytorch
 
 
@@ -292,16 +294,16 @@ variations:
       - nuscenes-accuracy-script
       tags: run,accuracy,mlperf,_nuscenes
 
-  ssd-resnet50:
+  ssd:
     group:
       models
     default_env:
       MLC_USE_DATASET_FROM_HOST: yes
     env:
-      MLC_MODEL: ssd-resnet50
+      MLC_MODEL: ssd
     docker:
       deps:
-        - tags: get,preprocessed,dataset,cognata
+        - tags: get,preprocessed,dataset,cognata,_2d_obj_det
           enable_if_env:
             MLC_USE_DATASET_FROM_HOST:
               - "yes"
@@ -310,13 +312,13 @@ variations:
             MLC_USE_MODEL_FROM_HOST:
               - "yes"
           names:
-            - ml-model-ssd-resnet50
+            - ml-model-ssd
       mounts:
         - "${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}:${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}"
         - "${{ MLC_ML_MODEL_SSD_PATH }}:${{ MLC_ML_MODEL_SSD_PATH }}"
     add_deps_recursive:
       abtf-inference-implementation:
-        tags: _ssd-resnet50
+        tags: _ssd
     posthook_deps:
     - enable_if_env:
         MLC_MLPERF_LOADGEN_MODE:
@@ -329,31 +331,31 @@ variations:
       - cognata-ssd-accuracy-script
       tags: run,accuracy,mlperf,_cognata_ssd
 
-  deeplab-v3+:
+  deeplabv3plus:
     group:
       models
     default_env:
       MLC_USE_DATASET_FROM_HOST: yes
     env:
-      MLC_MODEL: deeplab-v3+
+      MLC_MODEL: deeplabv3plus
     docker:
       deps:
         - tags: get,preprocessed,dataset,cognata,_segmentation
           enable_if_env:
             MLC_USE_DATASET_FROM_HOST:
               - "yes"
-        - tags: get,ml-model,ssd,resnet50,_mlc,_rclone
+        - tags:  get,ml-model,deeplabv3-plus
           enable_if_env:
             MLC_USE_MODEL_FROM_HOST:
               - "yes"
           names:
-            - ml-model-deeplab-v3+
+            - ml-model-deeplabv3plus
       mounts:
         - "${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}:${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}"
         - "${{ MLC_ML_MODEL_DEEPLABV3_PLUS_PATH }}:${{ MLC_ML_MODEL_DEEPLABV3_PLUS_PATH }}"
     add_deps_recursive:
       abtf-inference-implementation:
-        tags: _deeplab-v3+
+        tags: _deeplabv3plus
     posthook_deps:
     - enable_if_env:
         MLC_MLPERF_LOADGEN_MODE:

script/get-generic-sys-util/meta.yaml

@@ -162,7 +162,7 @@ variations:
     new_env_keys:
     - MLC_LIBGLX_VERSION
     state:
-      libgl: # tbd: complete for other flavours of linux
+      libgl1-mesa-glx: # tbd: complete for other flavours of linux
         apt: libgl1-mesa-glx
         brew: ''
         dnf: ''

script/get-preprocessed-dataset-cognata/customize.py

@@ -8,7 +8,8 @@ def preprocess(i):
 
     env = i['env']
 
-    if env.get('MLC_NUSCENES_DATASET_TYPE', '') == "prebuilt":
+    if env.get('MLC_NUSCENES_DATASET_TYPE', '') == "prebuilt" and env.get(
+            'MLC_PREPROCESSED_DATASET_COGNATA_PATH', '') == '':
         env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes"
 
     return {'return': 0}

script/process-mlperf-accuracy/meta.yaml

@@ -13,6 +13,20 @@ deps:
   - inference-src
   - accuracy-check-src
   tags: get,mlcommons,inference,src
+  skip_if_env:
+    MLC_DATASET:
+      - nuscenes
+      - cognata_deeplab
+      - cognata_ssd
+- names:
+  - automotive-src
+  - accuracy-check-src
+  tags: get,mlcommons,automotive,src
+  enable_if_env:
+    MLC_DATASET:
+      - nuscenes
+      - cognata_deeplab
+      - cognata_ssd
 input_mapping:
   rerun: MLC_RERUN
   result_dir: MLC_MLPERF_ACCURACY_RESULTS_DIR
@@ -297,13 +311,13 @@ variations:
           MLC_RUN_STATE_DOCKER:
             - "yes"
         names:
-          - preprocessed-dataset-mlcommons-cognata-ssd-resnet50
+          - preprocessed-dataset-mlcommons-cognata-ssd
       - tags: get,ml-model,ssd,resnet50,_mlc,_rclone
         skip_if_any_env:
           MLC_RUN_STATE_DOCKER:
             - "yes"
         names:
-          - ml-model-ssd-resnet50
+          - ml-model-ssd
     env:
       MLC_DATASET: cognata_ssd
     group: dataset