Fix for early stopping and command generation

script/app-mlperf-automotive-mlcommons-python/customize.py

@@ -198,6 +198,8 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
 
     q = '"' if os_info['platform'] == 'windows' else "'"
 
+    device = env['MLC_MLPERF_DEVICE']
+
     ##########################################################################
     # Grigori added for ABTF demo
 
@@ -249,7 +251,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
             "bevformer",
             "bevformer_tiny.py")
         print(env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'])
-        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend onnx --dataset nuscenes --nuscenes-root {os.path.dirname(env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/"))} --dataset-path {env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH']} --checkpoint {env['MLC_ML_MODEL_BEVFORMER_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
+        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend onnx --dataset nuscenes --device {"cuda" if device == "gpu" else "cpu"}  --nuscenes-root {os.path.dirname(env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/"))} --dataset-path {env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH']} --checkpoint {env['MLC_ML_MODEL_BEVFORMER_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
         print(cmd)
     elif env['MLC_MODEL'] in ['ssd']:
         run_dir = env['MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH']
@@ -263,7 +265,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
 
         config_path = "baseline_8MP_ss_scales_fm1_5x5_all"
 
-        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_SSD_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
+        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --cognata-root-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_SSD_PATH']} --config {config_path} --device {"cuda" if device == "gpu" else "cpu"} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
 
     elif env['MLC_MODEL'] in ['deeplabv3plus']:
         run_dir = env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH']
@@ -275,7 +277,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options,
         backend = "onnx" if env.get(
             'MLC_MLPERF_BACKEND') == "onnxruntime" else env.get('MLC_MLPERF_BACKEND')
 
-        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
+        cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --device {"cuda" if device == "gpu" else "cpu"} --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}"""
 
     ##########################################################################
 

script/app-mlperf-automotive-mlcommons-python/meta.yaml

@@ -60,6 +60,7 @@ input_mapping:
   target_latency: MLC_MLPERF_LOADGEN_TARGET_LATENCY
   offline_target_qps: MLC_MLPERF_LOADGEN_OFFLINE_TARGET_QPS
   server_target_qps: MLC_MLPERF_LOADGEN_SERVER_TARGET_QPS
+  constantstream_target_qps: MLC_MLPERF_LOADGEN_CONSTANTSTREAM_TARGET_QPS
   singlestream_target_latency: MLC_MLPERF_LOADGEN_SINGLESTREAM_TARGET_LATENCY
   multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY
   output: MLC_MLPERF_OUTPUT_DIR
@@ -457,6 +458,9 @@ variations:
 
   bevformer:
     group: models
+    default_env:
+      MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE: "99.9"
+      MLC_MLPERF_DEFAULT_MAX_QUERY_COUNT: 6636
     add_deps_recursive:
       pytorch:
         version_max: "2.5.1"
@@ -492,6 +496,9 @@ variations:
 
   ssd:
     group: models
+    default_env:
+      MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE: "99.9"
+      MLC_MLPERF_DEFAULT_MAX_QUERY_COUNT: 6636
     add_deps_recursive:
       pytorch:
         version_max: "2.3.1"
@@ -515,7 +522,7 @@ variations:
       - tags: get,generic-python-lib,_package.onnx
       - tags: get,generic-python-lib,_package.onnxruntime
       - tags: get,generic-python-lib,_package.tqdm
-      - tags: get,preprocessed,dataset,cognata,_mlc,_2d_object_det,_validation
+      - tags: get,preprocessed,dataset,cognata,_mlc,_2d_obj_det,_validation
         skip_if_env:
           MLC_RUN_STATE_DOCKER:
             - "yes"
@@ -530,6 +537,9 @@ variations:
 
   deeplabv3plus:
     group: models
+    default_env:
+      MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE: "99.9"
+      MLC_MLPERF_DEFAULT_MAX_QUERY_COUNT: 6636
     add_deps_recursive:
       pytorch:
         version_max: "2.3.1"
@@ -601,6 +611,9 @@ variations:
   server:
     env:
       MLC_MLPERF_LOADGEN_SCENARIO: Server
+  constantstream:
+    env:
+      MLC_MLPERF_LOADGEN_SCENARIO: ConstantStream
 
   mvp_demo:
     env:

script/app-mlperf-automotive/customize.py

@@ -68,7 +68,7 @@ def postprocess(i):
 
     mlperf_log = MLPerfLog(os.path.join(output_dir, "mlperf_log_detail.txt"))
     if mode == "performance":
-        if scenario in ["Offline", "Server"]:
+        if scenario in ["Offline", "Server", "ConstantStream"]:
             metric = "target_qps"
             result = mlperf_log['result_mean_latency_ns'] / 1000000
         elif scenario.endswith("Stream"):

script/app-mlperf-automotive/meta.yaml

@@ -59,6 +59,7 @@ input_mapping:
   target_latency: MLC_MLPERF_LOADGEN_TARGET_LATENCY
   offline_target_qps: MLC_MLPERF_LOADGEN_OFFLINE_TARGET_QPS
   server_target_qps: MLC_MLPERF_LOADGEN_SERVER_TARGET_QPS
+  constantstream_target_qps: MLC_MLPERF_LOADGEN_CONSTANTSTREAM_TARGET_QPS
   singlestream_target_latency: MLC_MLPERF_LOADGEN_SINGLESTREAM_TARGET_LATENCY
   multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY
   output: MLC_MLPERF_OUTPUT_DIR
@@ -303,7 +304,7 @@ variations:
       MLC_MODEL: ssd
     docker:
       deps:
-        - tags: get,preprocessed,dataset,cognata,_mlc,_2d_object_det,_validation
+        - tags: get,preprocessed,dataset,cognata,_mlc,_2d_obj_det,_validation
           enable_if_env:
             MLC_USE_DATASET_FROM_HOST:
               - "yes"
@@ -381,6 +382,9 @@ variations:
       abtf-inference-implementation:
         tags: _cpu
 
+  gpu:
+    alias: cuda
+
   cuda:
     group: device
     env:
@@ -413,12 +417,14 @@ variations:
     add_deps_recursive:
       abtf-inference-implementation:
         tags: _offline
+
   multistream:
     env:
       MLC_MLPERF_LOADGEN_SCENARIO: MultiStream
     add_deps_recursive:
       abtf-inference-implementation:
         tags: _multistream
+
   singlestream:
     group: loadgen-scenario
     default: true
@@ -427,6 +433,14 @@ variations:
     add_deps_recursive:
       abtf-inference-implementation:
         tags: _singlestream
+
+  constantstream:
+    env:
+      MLC_MLPERF_LOADGEN_SCENARIO: ConstantStream
+    add_deps_recursive:
+      abtf-inference-implementation:
+        tags: _constantstream
+
   server:
     env:
       MLC_MLPERF_LOADGEN_SCENARIO: Server

script/app-mlperf-inference-amd/meta.yaml

@@ -91,12 +91,12 @@ deps:
     - inference-src
 
   # Download MLPerf inference loadgen
-  - tags: get,mlcommons,inference,loadgen
+  - tags: get,mlcommons,inference,loadgen,_inference
     names:
     - inference-loadgen
 
   # Creates user conf for given SUT
-  - tags: generate,user-conf,mlperf,inference
+  - tags: generate,user-conf,mlperf,inference,_inference
     names:
     - user-conf-generator
 

script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml

@@ -28,7 +28,7 @@ deps:
   tags: get,cuda
 - names:
   - loadgen
-  tags: get,loadgen
+  tags: get,loadgen,_inference
 - names:
   - inference-src
   tags: get,mlcommons,inference,src
@@ -121,7 +121,7 @@ post_deps:
 prehook_deps:
 - names:
   - user-conf-generator
-  tags: generate,user-conf,mlperf,inference
+  tags: generate,user-conf,mlperf,inference,_inference
 - enable_if_env:
     MLC_MLPERF_SKIP_RUN:
     - 'no'

script/app-mlperf-inference-dummy/meta.yaml

@@ -91,12 +91,12 @@ deps:
     - inference-src
 
   # Download MLPerf inference loadgen
-  - tags: get,mlcommons,inference,loadgen
+  - tags: get,mlcommons,inference,loadgen,_inference
     names:
     - inference-loadgen
 
   # Creates user conf for given SUT
-  - tags: generate,user-conf,mlperf,inference
+  - tags: generate,user-conf,mlperf,inference,_inference
     names:
     - user-conf-generator
 

script/app-mlperf-inference-intel/meta.yaml

@@ -392,7 +392,7 @@ variations:
       - tags: get,mlcommons,inference,src
         names:
         - inference-src
-      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build
+      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build,_inference
         names:
         - inference-loadgen
 
@@ -510,7 +510,7 @@ variations:
       - tags: get,mlcommons,inference,src
         names:
         - inference-src
-      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build
+      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build,_inference
         names:
         - inference-loadgen
 
@@ -627,7 +627,7 @@ variations:
       - tags: get,mlcommons,inference,src
         names:
         - inference-src
-      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build
+      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build,_inference
         names:
         - inference-loadgen
 
@@ -699,7 +699,7 @@ variations:
       - tags: get,mlcommons,inference,src
         names:
         - inference-src
-      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build
+      - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build,_inference
         names:
         - inference-loadgen
 
@@ -924,7 +924,7 @@ variations:
       - tags: get,mlcommons,inference,src
         names:
         - inference-src
-      - tags: get,mlcommons,inference,loadgen,_custom-python
+      - tags: get,mlcommons,inference,loadgen,_custom-python,_inference
         names:
         - inference-loadgen
       - tags: get,ml-model,dlrm,_pytorch
@@ -1074,7 +1074,7 @@ variations:
         - inference-src
 
       # Creates user conf for given SUT
-      - tags: generate,user-conf,mlperf,inference
+      - tags: generate,user-conf,mlperf,inference,_inference
         names:
         - user-conf-generator
       - tags: get,generic-sys-util,_rsync

script/app-mlperf-inference-mlcommons-cpp/meta.yaml

@@ -71,7 +71,7 @@ deps:
   # Install MLPerf inference dependencies
 
   # Install MLPerf loadgen
-  - tags: get,loadgen
+  - tags: get,loadgen,_inference
     names:
     - loadgen
 
@@ -129,7 +129,7 @@ deps:
     tags: get,ml-model,retinanet,_onnx,_fp32
 
   # Creates user conf for given SUT
-  - tags: generate,user-conf,mlperf,inference
+  - tags: generate,user-conf,mlperf,inference,_inference
     names:
     - user-conf-generator
 

script/app-mlperf-inference-mlcommons-python/meta.yaml

@@ -684,15 +684,15 @@ deps:
   # Install MLPerf inference dependencies
 
   # Creates user conf for given SUT
-  - tags: generate,user-conf,mlperf,inference
+  - tags: generate,user-conf,mlperf,inference,_inference
     names:
       - user-conf-generator
     skip_if_env:
       MLC_RUN_STATE_DOCKER:
         - "yes"
 
   # Install MLPerf loadgen
-  - tags: get,loadgen
+  - tags: get,loadgen,_inference
     names:
       - loadgen
       - mlperf-inference-loadgen

script/app-mlperf-inference-qualcomm/meta.yaml

@@ -111,12 +111,12 @@ deps:
     - inference-src
 
   # Download MLPerf inference loadgen
-  - tags: get,mlcommons,inference,loadgen
+  - tags: get,mlcommons,inference,loadgen,_inference
     names:
     - inference-loadgen
 
   # Creates user conf for given SUT
-  - tags: generate,user-conf,mlperf,inference
+  - tags: generate,user-conf,mlperf,inference,_inference
     names:
     - user-conf-generator
 

script/app-mlperf-inference-redhat/meta.yaml

@@ -91,12 +91,12 @@ deps:
     - inference-src
 
   # Download MLPerf inference loadgen
-  - tags: get,mlcommons,inference,loadgen
+  - tags: get,mlcommons,inference,loadgen,_inference
     names:
     - inference-loadgen
 
   # Creates user conf for given SUT
-  - tags: generate,user-conf,mlperf,inference
+  - tags: generate,user-conf,mlperf,inference,_inference
     names:
     - user-conf-generator
 

script/generate-mlperf-inference-submission/meta.yaml

@@ -13,12 +13,8 @@ deps:
   - python
   - python3
   tags: get,python3
-- names:
-  - inference-src
-  tags: mlcommons,inference,src
 - tags: get,sut,system-description
 - tags: install,pip-package,for-mlc-python,_package.tabulate
-- tags: get,mlperf,inference,utils
 - names:
   - get-mlperf-results-dir
   skip_if_env:
@@ -109,6 +105,20 @@ post_deps:
   - mlperf-inference-submission-checker
   - submission-checker
   tags: submission,inference,checker,mlc
+variations:
+  inference:
+    default: true
+    deps:
+      - names:
+        - inference-src
+        tags: mlcommons,inference,src
+      - tags: get,mlperf,inference,utils
+  automotive:
+    deps:
+      - names:
+        - automotive-src
+        tags: mlcommons,automotive,src
+      - tags: get,mlperf,automotive,utils
 tags:
 - generate
 - submission

script/generate-mlperf-inference-user-conf/customize.py

@@ -100,7 +100,7 @@ def preprocess(i):
     query_count = None
 
     value = None
-    if scenario in ['Offline', 'Server']:
+    if scenario in ['Offline', 'Server', 'ConstantStream']:
         metric = "target_qps"
         tolerance = 1.01
         # value = env.get('MLC_MLPERF_LOADGEN_SERVER_TARGET_QPS') if scenario == "Server" else env.get('MLC_MLPERF_LOADGEN_OFFLINE_TARGET_QPS')
@@ -343,7 +343,7 @@ def preprocess(i):
             ".sample_concatenate_permutation = 0" + "\n"
         max_duration_fast_s = int(env.get('MLC_MLPERF_MAX_DURATION_FAST', 120))
         max_duration_fast = str(max_duration_fast_s * 1000)  # in milliseconds
-        if scenario == "Server":
+        if scenario == "Server" or scenario == "ConstantStream":
             user_conf += ml_model_name + "." + scenario + \
                 f".max_duration = {max_duration_fast}" + "\n"
             target_qps = conf['target_qps']
@@ -368,9 +368,13 @@ def preprocess(i):
                 user_conf += ml_model_name + "." + scenario + ".min_duration = " + \
                     env['MLC_MLPERF_INFERENCE_MIN_DURATION'] + " \n"
             elif not is_false(env.get('MLC_MLPERF_USE_MAX_DURATION', 'yes')) and not is_true(env.get(
-                    'MLC_MLPERF_MODEL_EQUAL_ISSUE_MODE', 'no')):
+                    'MLC_MLPERF_MODEL_EQUAL_ISSUE_MODE', 'no')) and env.get('MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE') != '99.9':
                 user_conf += ml_model_name + "." + scenario + \
                     f".max_duration = {max_duration_valid}" + "\n"
+            if scenario in ["SingleStream", "ConstantStream"] and env.get('MLC_MLPERF_SINGLESTREAM_TARGET_LATENCY_PERCENTILE') == '99.9' and env.get(
+                    'MLC_MLPERF_DEFAULT_MAX_QUERY_COUNT', '') != '' and env.get('MLC_MLPERF_TARGET_LATENCY', '') == '':
+                user_conf += ml_model_name + "." + scenario + \
+                    f".max_query_count = {env.get('MLC_MLPERF_DEFAULT_MAX_QUERY_COUNT')}" + "\n"
             if scenario == "MultiStream":
                 user_conf += ml_model_name + "." + scenario + ".min_query_count = " + \
                     env.get(