diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 86932d896..a2f3a5fbe 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -1,3 +1,4 @@ +# This workflow will run configured tests for any updated MLC script name: Test script on modified meta on: @@ -10,52 +11,35 @@ jobs: get_modified_files: runs-on: ubuntu-latest outputs: - processed_files: ${{ steps.filter-modified-files.outputs.processed_files }} + processed_files: ${{ steps.modified-files.outputs.processed_files }} steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - - name: Install dependencies - run: pip install pyyaml - - - name: Fetch base branch - run: | - git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - - name: Get list of changed files - id: modified-files - run: | - git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt - files=$(paste -sd, changed_files.txt) - echo "files=$files" >> $GITHUB_OUTPUT - - - name: Filter changed files - id: filter-modified-files - env: - FILES: ${{ steps.modified-files.outputs.files }} - run: | - processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) - echo "processed_files<> $GITHUB_OUTPUT - echo "$processed" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Debug processed_files output - run: | - echo "Processed files output:" - echo "${{ steps.filter-modified-files.outputs.processed_files }}" + - name: 'Checkout' + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + pip install pyyaml + + - name: Get changed files + id: modified-files + run: | + git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} + git fetch upstream + changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only | paste -sd, -) + echo "$changed_files" | python3 .github/scripts/list_modified_files.py process_modified_files: - needs: get_modified_files runs-on: ubuntu-latest - if: needs.get_modified_files.outputs.processed_files != '[]' + needs: get_modified_files + if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != '' strategy: fail-fast: false matrix: @@ -69,7 +53,8 @@ jobs: - name: Process meta.yaml file run: | - echo "Processing ${{ matrix.file_info.file }} (run #${{ matrix.file_info.num_run }})" + echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}" + pip install mlcflow mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - mlc test script ${{ matrix.file_info.uid }} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet + mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet diff --git a/.github/workflows/test-mlperf-automotive.yml b/.github/workflows/test-mlperf-automotive.yml new file mode 100644 index 000000000..a03560dc1 --- /dev/null +++ b/.github/workflows/test-mlperf-automotive.yml @@ -0,0 +1,43 @@ +name: Test MLPerf automotive + +on: + pull_request_target: + branches: [ "main", "dev" ] + paths: + - 'script/**meta.yaml' + +jobs: + fetch-secret: + runs-on: ubuntu-latest + outputs: + gdrive_secret: ${{ steps.op-load-secret.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }} + steps: + - name: Load secret from 1Password + id: op-load-secret + uses: 1password/load-secrets-action@v2 + with: + export-env: false + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential + + run-mlperf: + runs-on: ubuntu-latest + needs: + - fetch-secret + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Set RCLONE Service account env var from secret + shell: bash + run: | + echo "::add-mask::${{ needs.fetch-secret.outputs.gdrive_secret }}" + echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV + - name: Run MLPerf + run: | + pip install mlcflow + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + mlc test script app,mlperf,automotive diff --git a/script/app-mlperf-automotive-mlcommons-python/customize.py b/script/app-mlperf-automotive-mlcommons-python/customize.py index 16805cc35..4ea2ce0f1 100644 --- a/script/app-mlperf-automotive-mlcommons-python/customize.py +++ b/script/app-mlperf-automotive-mlcommons-python/customize.py @@ -1,248 +1,293 @@ -from mlc import utils -from utils import is_true -import os -import json -import shutil -import subprocess - - -def preprocess(i): - - os_info = i['os_info'] - env = i['env'] - state = i['state'] - script_path = i['run_script_input']['path'] - - if is_true(env.get('MLC_MLPERF_SKIP_RUN', '')): - return {'return': 0} - - if is_true(env.get('MLC_RUN_DOCKER_CONTAINER', '')): - return {'return': 0} - - if is_true(env.get('MLC_MLPERF_POWER', '')): - power = "yes" - else: - power = "no" - - rerun = True if env.get("MLC_RERUN", "") != '' else False - - if 'MLC_MLPERF_LOADGEN_SCENARIO' not in env: - env['MLC_MLPERF_LOADGEN_SCENARIO'] = "Offline" - - if 'MLC_MLPERF_LOADGEN_MODE' not in env: - env['MLC_MLPERF_LOADGEN_MODE'] = "accuracy" - - if 'MLC_MODEL' not in env: - return { - 'return': 1, 'error': "Please select a variation specifying the model to run"} - - # if env['MLC_MODEL'] == "resnet50": - # cmd = "cp " + os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") + " " + os.path.join(env['MLC_DATASET_PATH'], - # "val_map.txt") - # ret = os.system(cmd) - - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] = " " + \ - env.get('MLC_MLPERF_LOADGEN_EXTRA_OPTIONS', '') + " " - - if 'MLC_MLPERF_LOADGEN_QPS' not in env: - env['MLC_MLPERF_LOADGEN_QPS_OPT'] = "" - else: - env['MLC_MLPERF_LOADGEN_QPS_OPT'] = " --qps " + \ - env['MLC_MLPERF_LOADGEN_QPS'] - - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += env['MLC_MLPERF_LOADGEN_QPS_OPT'] - - if 'MLC_NUM_THREADS' not in env: - if 'MLC_MINIMIZE_THREADS' in env: - env['MLC_NUM_THREADS'] = str(int(env['MLC_HOST_CPU_TOTAL_CORES']) // - (int(env.get('MLC_HOST_CPU_SOCKETS', '1')) * int(env.get('MLC_HOST_CPU_TOTAL_CORES', '1')))) - else: - env['MLC_NUM_THREADS'] = env.get('MLC_HOST_CPU_TOTAL_CORES', '1') - - if env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '') != '' and not env.get( - 'MLC_MLPERF_MODEL_SKIP_BATCHING', False): - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --max-batchsize " + \ - str(env['MLC_MLPERF_LOADGEN_MAX_BATCHSIZE']) - - if env.get('MLC_MLPERF_LOADGEN_BATCH_SIZE', '') != '': - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --batch-size " + \ - str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE']) - - if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get( - 'MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and env.get('MLC_MLPERF_RUN_STYLE', '') != "valid": - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \ - env['MLC_MLPERF_LOADGEN_QUERY_COUNT'] - - print("Using MLCommons Inference source from '" + - env['MLC_MLPERF_INFERENCE_SOURCE'] + "'") - - if 'MLC_MLPERF_CONF' not in env: - env['MLC_MLPERF_CONF'] = os.path.join( - env['MLC_MLPERF_INFERENCE_SOURCE'], "mlperf.conf") - - x = "" if os_info['platform'] == 'windows' else "'" - - env['MODEL_DIR'] = env.get('MLC_ML_MODEL_PATH') - if not env['MODEL_DIR']: - env['MODEL_DIR'] = os.path.dirname( - env.get( - 'MLC_MLPERF_CUSTOM_MODEL_PATH', - env.get('MLC_ML_MODEL_FILE_WITH_PATH'))) - - RUN_CMD = "" - - scenario = env['MLC_MLPERF_LOADGEN_SCENARIO'] - scenario_extra_options = '' - - NUM_THREADS = env['MLC_NUM_THREADS'] - if int(NUM_THREADS) > 2 and env['MLC_MLPERF_DEVICE'] == "gpu": - NUM_THREADS = "2" # Don't use more than 2 threads when run on GPU - - if env['MLC_MODEL'] in ['resnet50', 'retinanet', 'stable-diffusion-xl']: - scenario_extra_options += " --threads " + NUM_THREADS - - ml_model_name = env['MLC_MODEL'] - if 'MLC_MLPERF_USER_CONF' in env: - user_conf_path = env['MLC_MLPERF_USER_CONF'] - x = "" if os_info['platform'] == 'windows' else "'" - scenario_extra_options += " --user_conf " + x + user_conf_path + x - - mode = env['MLC_MLPERF_LOADGEN_MODE'] - mode_extra_options = "" - - # Grigori blocked for ABTF to preprocess data set on the fly for now - # we can later move it to a separate script to preprocess data set - -# if 'MLC_DATASET_PREPROCESSED_PATH' in env and env['MLC_MODEL'] in [ 'resnet50', 'retinanet' ]: -# #dataset_options = " --use_preprocessed_dataset --preprocessed_dir "+env['MLC_DATASET_PREPROCESSED_PATH'] -# if env.get('MLC_MLPERF_LAST_RELEASE') not in [ "v2.0", "v2.1" ]: -# dataset_options = " --use_preprocessed_dataset --cache_dir "+env['MLC_DATASET_PREPROCESSED_PATH'] -# else: -# dataset_options = "" -# if env['MLC_MODEL'] == "retinanet": -# dataset_options += " --dataset-list "+ env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] -# elif env['MLC_MODEL'] == "resnet50": -# dataset_options += " --dataset-list "+ os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") -# env['DATA_DIR'] = env.get('MLC_DATASET_PREPROCESSED_PATH') -# else: -# if 'MLC_DATASET_PREPROCESSED_PATH' in env: -# env['DATA_DIR'] = env.get('MLC_DATASET_PREPROCESSED_PATH') -# else: -# env['DATA_DIR'] = env.get('MLC_DATASET_PATH') -# dataset_options = '' - - # Grigori added for ABTF -# dataset_path = env.get('MLC_DATASET_PATH') -# env['DATA_DIR'] = dataset_path - -# dataset_options = " --dataset-list " + env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] -# dataset_options += " --cache_dir " + os.path.join(script_path, 'preprocessed-dataset') - - dataset_options = '' - - if env.get('MLC_MLPERF_EXTRA_DATASET_ARGS', '') != '': - dataset_options += " " + env['MLC_MLPERF_EXTRA_DATASET_ARGS'] - - if mode == "accuracy": - mode_extra_options += " --accuracy" - env['MLC_OUTPUT_PREDICTIONS_PATH'] = os.path.join( - env['MLC_DATASET_MLCOMMONS_COGNATA_PATH'], - env['MLC_DATASET_MLCOMMONS_COGNATA_SERIAL_NUMBERS'], - 'Cognata_Camera_01_8M_png', - 'output') - - elif mode == "performance": - pass - - elif mode == "compliance": - - audit_full_path = env['MLC_MLPERF_INFERENCE_AUDIT_PATH'] - mode_extra_options = " --audit '" + audit_full_path + "'" - - if env.get('MLC_MLPERF_OUTPUT_DIR', '') == '': - env['MLC_MLPERF_OUTPUT_DIR'] = os.getcwd() - - mlperf_implementation = env.get('MLC_MLPERF_IMPLEMENTATION', 'reference') - - # Generate CMD - - # Grigori updated for ABTF demo -# cmd, run_dir = get_run_cmd(os_info, env, scenario_extra_options, mode_extra_options, dataset_options, mlperf_implementation) - cmd, run_dir = get_run_cmd_reference( - os_info, env, scenario_extra_options, mode_extra_options, dataset_options, script_path) - - if env.get('MLC_NETWORK_LOADGEN', '') == "lon": - - run_cmd = i['state']['mlperf_inference_run_cmd'] - env['MLC_SSH_RUN_COMMANDS'] = [] - env['MLC_SSH_RUN_COMMANDS'].append( - run_cmd.replace( - "--network=lon", - "--network=sut") + " &") - - env['MLC_MLPERF_RUN_CMD'] = cmd - env['MLC_RUN_DIR'] = run_dir - env['MLC_RUN_CMD'] = cmd - env['CK_PROGRAM_TMP_DIR'] = env.get('MLC_ML_MODEL_PATH') # for tvm - - if env.get('MLC_HOST_PLATFORM_FLAVOR', '') == "arm64": - env['MLC_HOST_PLATFORM_FLAVOR'] = "aarch64" - - if not env.get('MLC_COGNATA_ACCURACY_DUMP_FILE'): - env['MLC_COGNATA_ACCURACY_DUMP_FILE'] = os.path.join( - env['OUTPUT_DIR'], "accuracy.txt") - - return {'return': 0} - - -def get_run_cmd_reference(os_info, env, scenario_extra_options, - mode_extra_options, dataset_options, script_path=None): - - q = '"' if os_info['platform'] == 'windows' else "'" - - ########################################################################## - # Grigori added for ABTF demo - - if env['MLC_MODEL'] in ['retinanet']: - - run_dir = os.path.join(script_path, 'ref') - - env['RUN_DIR'] = run_dir - - env['OUTPUT_DIR'] = env['MLC_MLPERF_OUTPUT_DIR'] - - cognata_dataset_path = env['MLC_DATASET_MLCOMMONS_COGNATA_PATH'] -# cognata_dataset_path = env['MLC_DATASET_PATH'] # Using open images -# dataset for some tests - - path_to_model = env.get( - 'MLC_MLPERF_CUSTOM_MODEL_PATH', - env.get( - 'MLC_ML_MODEL_FILE_WITH_PATH', - env.get('MLC_ML_MODEL_CODE_WITH_PATH'))) - env['MODEL_FILE'] = path_to_model - - cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " " + os.path.join(run_dir, "python", "main.py") + " --profile " + env['MLC_MODEL'] + "-" + env['MLC_MLPERF_BACKEND'] + \ - " --model=" + q + path_to_model + q + \ - " --dataset=" + env["MLC_MLPERF_VISION_DATASET_OPTION"] + \ - " --dataset-path=" + q + cognata_dataset_path + q + \ - " --cache_dir=" + q + os.path.join(script_path, 'tmp-preprocessed-dataset') + q + \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \ - " --output " + q + env['OUTPUT_DIR'] + q + " " + \ - env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + dataset_options - - ########################################################################## - - return cmd, run_dir - - -def postprocess(i): - - env = i['env'] - - state = i['state'] - - inp = i['input'] - - return {'return': 0} +from mlc import utils +from utils import is_true +import os +import json +import shutil +import subprocess + + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + state = i['state'] + script_path = i['run_script_input']['path'] + + logger = i['automation'].logger + + if is_true(env.get('MLC_MLPERF_SKIP_RUN', '')): + return {'return': 0} + + if is_true(env.get('MLC_RUN_DOCKER_CONTAINER', '')): + return {'return': 0} + + if is_true(env.get('MLC_MLPERF_POWER', '')): + power = "yes" + else: + power = "no" + + rerun = True if env.get("MLC_RERUN", "") != '' else False + + if 'MLC_MLPERF_LOADGEN_SCENARIO' not in env: + env['MLC_MLPERF_LOADGEN_SCENARIO'] = "Offline" + + if 'MLC_MLPERF_LOADGEN_MODE' not in env: + env['MLC_MLPERF_LOADGEN_MODE'] = "accuracy" + + if 'MLC_MODEL' not in env: + return { + 'return': 1, 'error': "Please select a variation specifying the model to run"} + + # if env['MLC_MODEL'] == "resnet50": + # cmd = "cp " + os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") + " " + os.path.join(env['MLC_DATASET_PATH'], + # "val_map.txt") + # ret = os.system(cmd) + + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] = " " + \ + env.get('MLC_MLPERF_LOADGEN_EXTRA_OPTIONS', '') + " " + + if 'MLC_MLPERF_LOADGEN_QPS' not in env: + env['MLC_MLPERF_LOADGEN_QPS_OPT'] = "" + else: + env['MLC_MLPERF_LOADGEN_QPS_OPT'] = " --qps " + \ + env['MLC_MLPERF_LOADGEN_QPS'] + + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += env['MLC_MLPERF_LOADGEN_QPS_OPT'] + + if 'MLC_NUM_THREADS' not in env: + if 'MLC_MINIMIZE_THREADS' in env: + env['MLC_NUM_THREADS'] = str(int(env['MLC_HOST_CPU_TOTAL_CORES']) // + (int(env.get('MLC_HOST_CPU_SOCKETS', '1')) * int(env.get('MLC_HOST_CPU_TOTAL_CORES', '1')))) + else: + env['MLC_NUM_THREADS'] = env.get('MLC_HOST_CPU_TOTAL_CORES', '1') + + if env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '') != '' and not env.get( + 'MLC_MLPERF_MODEL_SKIP_BATCHING', False): + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --max-batchsize " + \ + str(env['MLC_MLPERF_LOADGEN_MAX_BATCHSIZE']) + + if env.get('MLC_MLPERF_LOADGEN_BATCH_SIZE', '') != '': + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --batch-size " + \ + str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE']) + + if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get( + 'MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and env.get('MLC_MLPERF_RUN_STYLE', '') != "valid": + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \ + env['MLC_MLPERF_LOADGEN_QUERY_COUNT'] + + print("Using MLCommons Inference source from '" + + env['MLC_MLPERF_INFERENCE_SOURCE'] + "'") + + env['MODEL_DIR'] = env.get('MLC_ML_MODEL_PATH') + if not env['MODEL_DIR']: + env['MODEL_DIR'] = os.path.dirname( + env.get( + 'MLC_MLPERF_CUSTOM_MODEL_PATH', + env.get('MLC_ML_MODEL_FILE_WITH_PATH'))) + + RUN_CMD = "" + + scenario = env['MLC_MLPERF_LOADGEN_SCENARIO'] + scenario_extra_options = '' + + NUM_THREADS = env['MLC_NUM_THREADS'] + if int(NUM_THREADS) > 2 and env['MLC_MLPERF_DEVICE'] == "gpu": + NUM_THREADS = "2" # Don't use more than 2 threads when run on GPU + + if env['MLC_MODEL'] in ['resnet50', 'retinanet', 'stable-diffusion-xl']: + scenario_extra_options += " --threads " + NUM_THREADS + + ml_model_name = env['MLC_MODEL'] + if 'MLC_MLPERF_USER_CONF' in env: + user_conf_path = env['MLC_MLPERF_USER_CONF'] + x = "" if os_info['platform'] == 'windows' else "'" + scenario_extra_options += " --user_conf " + x + user_conf_path + x + + mode = env['MLC_MLPERF_LOADGEN_MODE'] + mode_extra_options = "" + + # Grigori blocked for ABTF to preprocess data set on the fly for now + # we can later move it to a separate script to preprocess data set + +# if 'MLC_DATASET_PREPROCESSED_PATH' in env and env['MLC_MODEL'] in [ 'resnet50', 'retinanet' ]: +# #dataset_options = " --use_preprocessed_dataset --preprocessed_dir "+env['MLC_DATASET_PREPROCESSED_PATH'] +# if env.get('MLC_MLPERF_LAST_RELEASE') not in [ "v2.0", "v2.1" ]: +# dataset_options = " --use_preprocessed_dataset --cache_dir "+env['MLC_DATASET_PREPROCESSED_PATH'] +# else: +# dataset_options = "" +# if env['MLC_MODEL'] == "retinanet": +# dataset_options += " --dataset-list "+ env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] +# elif env['MLC_MODEL'] == "resnet50": +# dataset_options += " --dataset-list "+ os.path.join(env['MLC_DATASET_AUX_PATH'], "val.txt") +# env['DATA_DIR'] = env.get('MLC_DATASET_PREPROCESSED_PATH') +# else: +# if 'MLC_DATASET_PREPROCESSED_PATH' in env: +# env['DATA_DIR'] = env.get('MLC_DATASET_PREPROCESSED_PATH') +# else: +# env['DATA_DIR'] = env.get('MLC_DATASET_PATH') +# dataset_options = '' + + # Grigori added for ABTF +# dataset_path = env.get('MLC_DATASET_PATH') +# env['DATA_DIR'] = dataset_path + +# dataset_options = " --dataset-list " + env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] +# dataset_options += " --cache_dir " + os.path.join(script_path, 'preprocessed-dataset') + + dataset_options = '' + + if env.get('MLC_MLPERF_EXTRA_DATASET_ARGS', '') != '': + dataset_options += " " + env['MLC_MLPERF_EXTRA_DATASET_ARGS'] + + if mode == "accuracy": + mode_extra_options += " --accuracy" + if env.get('MLC_MODEL', '') == "retinanet": + env['MLC_OUTPUT_PREDICTIONS_PATH'] = os.path.join( + env['MLC_DATASET_MLCOMMONS_COGNATA_PATH'], + env['MLC_DATASET_MLCOMMONS_COGNATA_SERIAL_NUMBERS'], + 'Cognata_Camera_01_8M_png', + 'output') + + elif mode == "performance": + pass + + elif mode == "compliance": + + audit_full_path = env['MLC_MLPERF_INFERENCE_AUDIT_PATH'] + mode_extra_options = " --audit '" + audit_full_path + "'" + + if env.get('MLC_MLPERF_OUTPUT_DIR', '') == '': + env['MLC_MLPERF_OUTPUT_DIR'] = os.getcwd() + + mlperf_implementation = env.get('MLC_MLPERF_IMPLEMENTATION', 'reference') + + # Generate CMD + + # Grigori updated for ABTF demo +# cmd, run_dir = get_run_cmd(os_info, env, scenario_extra_options, mode_extra_options, dataset_options, mlperf_implementation) + cmd, run_dir = get_run_cmd_reference( + os_info, env, scenario_extra_options, mode_extra_options, dataset_options, logger, script_path) + + if env.get('MLC_NETWORK_LOADGEN', '') == "lon": + + run_cmd = i['state']['mlperf_inference_run_cmd'] + env['MLC_SSH_RUN_COMMANDS'] = [] + env['MLC_SSH_RUN_COMMANDS'].append( + run_cmd.replace( + "--network=lon", + "--network=sut") + " &") + + env['MLC_MLPERF_RUN_CMD'] = cmd + env['MLC_RUN_DIR'] = run_dir + env['MLC_RUN_CMD'] = cmd + env['CK_PROGRAM_TMP_DIR'] = env.get('MLC_ML_MODEL_PATH') # for tvm + + if env.get('MLC_HOST_PLATFORM_FLAVOR', '') == "arm64": + env['MLC_HOST_PLATFORM_FLAVOR'] = "aarch64" + + if env.get('MLC_MODEL', '') == "retinanet": + if not env.get('MLC_COGNATA_ACCURACY_DUMP_FILE'): + env['MLC_COGNATA_ACCURACY_DUMP_FILE'] = os.path.join( + env['OUTPUT_DIR'], "accuracy.txt") + + return {'return': 0} + + +def get_run_cmd_reference(os_info, env, scenario_extra_options, + mode_extra_options, dataset_options, logger, script_path=None): + + q = '"' if os_info['platform'] == 'windows' else "'" + + ########################################################################## + # Grigori added for ABTF demo + + if env['MLC_MODEL'] in ['retinanet']: + + run_dir = os.path.join(script_path, 'ref') + + env['RUN_DIR'] = run_dir + + env['OUTPUT_DIR'] = env['MLC_MLPERF_OUTPUT_DIR'] + + cognata_dataset_path = env['MLC_DATASET_MLCOMMONS_COGNATA_PATH'] +# cognata_dataset_path = env['MLC_DATASET_PATH'] # Using open images +# dataset for some tests + + path_to_model = env.get( + 'MLC_MLPERF_CUSTOM_MODEL_PATH', + env.get( + 'MLC_ML_MODEL_FILE_WITH_PATH', + env.get('MLC_ML_MODEL_CODE_WITH_PATH'))) + env['MODEL_FILE'] = path_to_model + + cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " " + os.path.join(run_dir, "python", "main.py") + " --profile " + env['MLC_MODEL'] + "-" + env['MLC_MLPERF_BACKEND'] + \ + " --model=" + q + path_to_model + q + \ + " --dataset=" + env["MLC_MLPERF_VISION_DATASET_OPTION"] + \ + " --dataset-path=" + q + cognata_dataset_path + q + \ + " --cache_dir=" + q + os.path.join(script_path, 'tmp-preprocessed-dataset') + q + \ + " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \ + " --output " + q + env['OUTPUT_DIR'] + q + " " + \ + env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ + scenario_extra_options + mode_extra_options + dataset_options + + elif env['MLC_MODEL'] in ['bevformer']: + run_dir = env['MLC_MLPERF_INFERENCE_BEVFORMER_PATH'] + + env['RUN_DIR'] = run_dir + + env['OUTPUT_DIR'] = env['MLC_MLPERF_OUTPUT_DIR'] + + if env['MLC_MLPERF_BACKEND'] != "onnxruntime": + logger.warning( + "Unsupported backend {MLC_MLPERF_BACKEND}, defaulting to onnx") + env['MLC_MLPERF_BACKEND'] = "onnx" + + config_path = os.path.join( + run_dir, + "projects", + "configs", + "bevformer", + "bevformer_tiny.py") + print(env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']) + cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend onnx --dataset nuscenes --nuscenes-root {os.path.dirname(env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/"))} --dataset-path {env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH']} --checkpoint {env['MLC_ML_MODEL_BEVFORMER_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" + print(cmd) + elif env['MLC_MODEL'] in ['ssd-resnet50']: + run_dir = env['MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH'] + + env['RUN_DIR'] = run_dir + + env['OUTPUT_DIR'] = env['MLC_MLPERF_OUTPUT_DIR'] + + backend = "onnx" if env.get( + 'MLC_MLPERF_BACKEND') == "onnxruntime" else env.get('MLC_MLPERF_BACKEND') + + config_path = "baseline_8MP_ss_scales_fm1_5x5_all" + + cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_SSD_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" + + elif env['MLC_MODEL'] in ['deeplab_v3+']: + run_dir = env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH'] + + env['RUN_DIR'] = run_dir + + env['OUTPUT_DIR'] = env['MLC_MLPERF_OUTPUT_DIR'] + + backend = "onnx" if env.get( + 'MLC_MLPERF_BACKEND') == "onnxruntime" else env.get('MLC_MLPERF_BACKEND') + + cmd = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(run_dir, "main.py")} --output {env['OUTPUT_DIR']} --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} --backend {backend} --dataset cognata --dataset-path {env['MLC_PREPROCESSED_DATASET_COGNATA_PATH']} --checkpoint {env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH']} --config {config_path} {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} {scenario_extra_options} {mode_extra_options} {dataset_options}""" + + ########################################################################## + + return cmd, run_dir + + +def postprocess(i): + + env = i['env'] + + state = i['state'] + + inp = i['input'] + + return {'return': 0} diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index e5567ac27..dd288d46e 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -9,10 +9,13 @@ category: "Modular MLPerf inference benchmark pipeline for ABTF model" # User-friendly tags to find this CM script tags: -- demo +- automotive +- mlcommons +- reference - run-mlperf-inference - object-detection - abtf-model +- demo # Default environment @@ -174,26 +177,19 @@ deps: names: - ml-engine-pytorch - pytorch - enable_if_env: - MLC_MLPERF_BACKEND: - - pytorch - - tvm-pytorch + skip_if_env: MLC_MLPERF_DEVICE: - - cpu - - rocm + - gpu ## Pytorch (CUDA) - tags: get,generic-python-lib,_torch_cuda names: - ml-engine-pytorch - pytorch - enable_if_env: - MLC_MLPERF_BACKEND: - - pytorch - - tvm-pytorch - - ray + skip_if_env: MLC_MLPERF_DEVICE: - - gpu + - cpu + - rocm ## Torchvision (CPU) - tags: get,generic-python-lib,_torchvision @@ -259,7 +255,7 @@ deps: # Creates user conf for given SUT - - tags: generate,user-conf,mlperf,inference + - tags: generate,user-conf,mlperf,inference,_automotive names: - user-conf-generator @@ -273,7 +269,7 @@ deps: - loadgen - mlperf-inference-loadgen - - tags: get,loadgen + - tags: get,loadgen,_automotive enable_if_any_env: MLC_MLPERF_LOADGEN_BUILD_FROM_SRC: - "on" @@ -340,6 +336,9 @@ variations: group: framework env: MLC_MLPERF_BACKEND: onnxruntime + add_deps_recursive: + ml-model-bevformer: + tags: _onnx onnxruntime,cpu: env: @@ -361,10 +360,27 @@ variations: tags: _NCHW ml-model: tags: raw,_pytorch + ml-model-bevformer: + tags: _pytorch + ml-model-ssd-resnet50: + tags: _pytorch + ml-model-deeplabv3-plus: + tags: _pytorch env: MLC_MLPERF_BACKEND: pytorch MLC_MLPERF_BACKEND_VERSION: <<>> - + + pytorch,cpu: + add_deps_recursive: + pytorch: + env: + MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html + torchvision: + env: + MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html + torchaudio: + env: + MLC_GENERIC_PYTHON_PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu/torch_stable.html @@ -383,6 +399,9 @@ variations: abtf-demo-model: group: models + add_deps_recursive: + automotive-src: + tags: _sha.ee526dc63d9ca2636000343c5d2d16132145719e deps: - tags: get,generic-python-lib,_opencv-python - tags: get,generic-python-lib,_numpy @@ -399,13 +418,15 @@ variations: - tags: get,ml-model,abtf-ssd-pytorch,_abtf-mvp names: - ml-model-abtf - env: MLC_MODEL: retinanet abtf-poc-model: group: models default: true + add_deps_recursive: + automotive-src: + tags: _sha.ee526dc63d9ca2636000343c5d2d16132145719e deps: - tags: get,generic-python-lib,_opencv-python - tags: get,generic-python-lib,_numpy @@ -427,10 +448,115 @@ variations: - tags: get,ml-model,abtf-ssd-pytorch,_abtf-poc names: - ml-model-abtf - env: MLC_MODEL: retinanet + bevformer: + group: models + add_deps_recursive: + pytorch: + version_max: "2.5.1" + version_max_usable: "2.5.1" + torchvision: + version_max: "0.20.1" + version_max_usable": "0.20.1" + torchaudio: + version_max: "2.5.1" + version_max_usable": "2.5.1" + deps: + - tags: get,generic-python-lib,_package.opencv-python + - tags: get,generic-python-lib,_package.numpy + version_max: "1.26.4" + version_max_usable: "1.26.4" + - tags: get,generic-python-lib,_package.onnx + - tags: get,generic-python-lib,_package.pillow + - tags: get,generic-python-lib,_package.pyquaternion + - tags: get,generic-python-lib,_package.tqdm + - tags: get,generic-python-lib,_package.nuscenes-devkit + - tags: get,preprocessed,dataset,nuscenes + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-mlcommons-nuscenes + - tags: get,ml-model,bevformer + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-bevformer + + ssd-resnet50: + group: models + add_deps_recursive: + pytorch: + version_max: "2.3.1" + version_max_usable: "2.3.1" + torchvision: + version_max: "0.18.1" + version_max_usable": "0.18.1" + torchaudio: + version_max: "2.3.1" + version_max_usable": "2.3.1" + deps: + - tags: get,generic-python-lib,_package.Cython + - tags: get,generic-python-lib,_package.scikit-image + - tags: get,generic-python-lib,_package.faster-coco-eval + - tags: get,generic-python-lib,_package.torchinfo + - tags: get,generic-python-lib,_package.torchmetrics + - tags: get,generic-sys-util,_libgl1-mesa-glx + - tags: get,generic-python-lib,_package.onnx + - tags: get,generic-python-lib,_package.onnxruntime + - tags: get,generic-python-lib,_package.tqdm + - tags: get,preprocessed,dataset,cognata,_mlc + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-mlcommons-cognata-ssd-resnet50 + - tags: get,ml-model,ssd,resnet50,_mlc,_rclone + skip_if_any_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-ssd-resnet50 + + deeplab-v3+: + group: models + add_deps_recursive: + pytorch: + version_max: "2.3.1" + version_max_usable: "2.3.1" + torchvision: + version_max: "0.18.1" + version_max_usable": "0.18.1" + torchaudio: + version_max: "2.3.1" + version_max_usable": "2.3.1" + deps: + - tags: get,generic-python-lib,_package.Cython + - tags: get,generic-python-lib,_package.scikit-image + - tags: get,generic-python-lib,_package.scikit-learn + - tags: get,generic-python-lib,_package.torchinfo + - tags: get,generic-python-lib,_package.torchmetrics + - tags: get,generic-sys-util,_libgl1-mesa-glx + - tags: get,generic-python-lib,_package.onnx + - tags: get,generic-python-lib,_package.onnxruntime + - tags: get,generic-python-lib,_package.tqdm + - tags: get,generic-python-lib,_package.ijson + - tags: get,preprocessed,dataset,cognata,_mlc,_segmentation + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-mlcommons-cognata-deeplabv3-plus + - tags: get,ml-model,deeplabv3-plus + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-deeplabv3-plus + # Target devices cpu: group: device @@ -460,10 +586,16 @@ variations: singlestream: env: MLC_MLPERF_LOADGEN_SCENARIO: SingleStream - MLC_MLPERF_LOADGEN_MAX_BATCHSIZE: 1 + default_variations: + batch-size: batch_size.1 server: env: MLC_MLPERF_LOADGEN_SCENARIO: Server mvp_demo: env: + + batch_size.#: + group: batch-size + env: + MLC_MLPERF_LOADGEN_MAX_BATCHSIZE: "#" \ No newline at end of file diff --git a/script/app-mlperf-automotive/customize.py b/script/app-mlperf-automotive/customize.py index 1333e0719..b7d8598c5 100644 --- a/script/app-mlperf-automotive/customize.py +++ b/script/app-mlperf-automotive/customize.py @@ -1,103 +1,421 @@ -from mlc import utils -import os -import json -import shutil -import subprocess -import mlperf_utils -from log_parser import MLPerfLog - - -def preprocess(i): - - os_info = i['os_info'] - env = i['env'] - state = i['state'] - script_path = i['run_script_input']['path'] - - if 'cmd' in i['input']: - state['mlperf_inference_run_cmd'] = "mlcr " + \ - " ".join(i['input']['cmd']) - - state['mlperf-inference-implementation'] = {} - - run_state = i['run_script_input']['run_state'] - state['mlperf-inference-implementation']['script_id'] = run_state['script_id'] + \ - ":" + ",".join(run_state['script_variation_tags']) - - return {'return': 0} - - -def postprocess(i): - - env = i['env'] - state = i['state'] - - inp = i['input'] - os_info = i['os_info'] - - xsep = '^' if os_info['platform'] == 'windows' else '\\' - - env['CMD'] = '' - - # if env.get('MLC_MLPERF_USER_CONF', '') == '': - # return {'return': 0} - - output_dir = env['MLC_MLPERF_OUTPUT_DIR'] - mode = env['MLC_MLPERF_LOADGEN_MODE'] - - model = env['MLC_MODEL'] - model_full_name = env.get('MLC_ML_MODEL_FULL_NAME', model) - - scenario = env['MLC_MLPERF_LOADGEN_SCENARIO'] - - if not os.path.exists(output_dir) or not os.path.exists( - os.path.join(output_dir, "mlperf_log_summary.txt")): - # No output, fake_run? - return {'return': 0} - - mlperf_log = MLPerfLog(os.path.join(output_dir, "mlperf_log_detail.txt")) - if mode == "performance": - result = mlperf_log['result_mean_latency_ns'] / 1000000 - elif mode == "accuracy": - if not env.get( - 'MLC_COGNATA_ACCURACY_DUMP_FILE'): # can happen while reusing old runs - env['MLC_COGNATA_ACCURACY_DUMP_FILE'] = os.path.join( - output_dir, "accuracy.txt") - acc = "" - if os.path.exists(env['MLC_COGNATA_ACCURACY_DUMP_FILE']): - with open(env['MLC_COGNATA_ACCURACY_DUMP_FILE'], "r") as f: - acc = f.readline() - result = acc - else: - return {'return': 1, 'error': f"Unknown mode {mode}"} - - valid = {'performance': True, 'accuracy': True} # its POC - power_result = None # No power measurement in POC - - # result, valid, power_result = mlperf_utils.get_result_from_log(env['MLC_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode) - - if not state.get('mlc-mlperf-inference-results'): - state['mlc-mlperf-inference-results'] = {} - if not state.get('mlc-mlperf-inference-results-last'): - state['mlc-mlperf-inference-results-last'] = {} - if not state['mlc-mlperf-inference-results'].get( - state['MLC_SUT_CONFIG_NAME']): - state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME']] = {} - if not state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] - ].get(model): - state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME']][model] = {} - if not state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] - ][model].get(scenario): - state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] - ][model][scenario] = {} - - state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] - ][model][scenario][mode] = result - state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] - ][model][scenario][mode + '_valid'] = valid.get(mode, False) - - state['mlc-mlperf-inference-results-last'][mode] = result - state['mlc-mlperf-inference-results-last'][mode + - '_valid'] = valid.get(mode, False) - - return {'return': 0} +from mlc import utils +import os +import json +import shutil +import subprocess +import mlperf_utils +from log_parser import MLPerfLog +from utils import * +import copy +import platform +import sys + + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + state = i['state'] + script_path = i['run_script_input']['path'] + + if 'cmd' in i['input']: + state['mlperf_inference_run_cmd'] = "mlcr " + \ + " ".join(i['input']['cmd']) + + state['abtf-inference-implementation'] = {} + + run_state = i['run_script_input']['run_state'] + state['abtf-inference-implementation']['script_id'] = run_state['script_id'] + \ + ":" + ",".join(run_state['script_variation_tags']) + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + inp = i['input'] + os_info = i['os_info'] + + xsep = '^' if os_info['platform'] == 'windows' else '\\' + q = '"' if os_info['platform'] == 'windows' else "'" + + logger = i['automation'].logger + + env['CMD'] = '' + + # if env.get('MLC_MLPERF_USER_CONF', '') == '': + # return {'return': 0} + + output_dir = env['MLC_MLPERF_OUTPUT_DIR'] + mode = env['MLC_MLPERF_LOADGEN_MODE'] + + mlc = i['automation'].action_object + + result_sut_folder_path = env['MLC_MLPERF_INFERENCE_RESULTS_SUT_PATH'] + + model = env['MLC_MODEL'] + model_full_name = env.get('MLC_ML_MODEL_FULL_NAME', model) + + scenario = env['MLC_MLPERF_LOADGEN_SCENARIO'] + + if not os.path.exists(output_dir) or not os.path.exists( + os.path.join(output_dir, "mlperf_log_summary.txt")): + # No output, fake_run? + return {'return': 0} + + mlperf_log = MLPerfLog(os.path.join(output_dir, "mlperf_log_detail.txt")) + if mode == "performance": + if scenario in ["Offline", "Server"]: + metric = "target_qps" + result = mlperf_log['result_mean_latency_ns'] / 1000000 + elif scenario.endswith("Stream"): + metric = "target_latency" + result = mlperf_log['result_mean_latency_ns'] + else: + return {'return': 1, + 'error': 'Unsupported scenario: {}'.format(scenario)} + import yaml + sut_name = state['MLC_SUT_CONFIG_NAME'] + sut_config = state['MLC_SUT_CONFIG'][sut_name] + sut_config_path = state['MLC_SUT_CONFIG_PATH'][sut_name] + if scenario not in sut_config[model_full_name]: + sut_config[model_full_name][scenario] = {} + sut_config[model_full_name][scenario][metric] = result + + print( + f"SUT: {sut_name}, model: {model_full_name}, scenario: {scenario}, {metric} (mean value) updated as {result}") + with open(sut_config_path, "w") as f: + yaml.dump(sut_config, f) + logger.info(f"New config stored in {sut_config_path}") + elif mode == "accuracy": + acc = "" + if env.get('MLC_MLPERF_INFERENCE_VERSION', '') == "mvp-demo" and env.get( + 'MLC_MLPERF_INFERENCE_VERSION') == "poc-demo": + if not env.get( + 'MLC_COGNATA_ACCURACY_DUMP_FILE'): # can happen while reusing old runs + env['MLC_COGNATA_ACCURACY_DUMP_FILE'] = os.path.join( + output_dir, "accuracy.txt") + if os.path.exists(env['MLC_COGNATA_ACCURACY_DUMP_FILE']): + with open(env['MLC_COGNATA_ACCURACY_DUMP_FILE'], "r") as f: + acc = f.readline() + result = acc + else: + return {'return': 1, 'error': f"Unknown mode {mode}"} + + valid = {'performance': True, 'accuracy': True} # its POC + power_result = None # No power measurement in POC + + # result, valid, power_result = mlperf_utils.get_result_from_log(env['MLC_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode) + + if not state.get('mlc-mlperf-inference-results'): + state['mlc-mlperf-inference-results'] = {} + if not state.get('mlc-mlperf-inference-results-last'): + state['mlc-mlperf-inference-results-last'] = {} + if not state['mlc-mlperf-inference-results'].get( + state['MLC_SUT_CONFIG_NAME']): + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME']] = {} + if not state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ].get(model): + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME']][model] = {} + if not state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ][model].get(scenario): + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ][model][scenario] = {} + + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ][model][scenario][mode] = result + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ][model][scenario][mode + '_valid'] = valid.get(mode, False) + + state['mlc-mlperf-inference-results-last'][mode] = result + state['mlc-mlperf-inference-results-last'][mode + + '_valid'] = valid.get(mode, False) + + if mode in ["performance", "accuracy"] and env.get( + 'MLC_MLPERF_INFERENCE_VERSION', '') not in ["", "mvp-demo", "poc-demo"]: + # if measurements file exist read it + if os.path.exists("measurements.json"): + with open("measurements.json", "r") as file: + measurements = json.load(file) # Load JSON data from the file + else: + measurements = {} + measurements['starting_weights_filename'] = env.get( + 'MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME', env.get( + 'MLC_ML_MODEL_FILE', measurements.get( + 'starting_weights_filename', 'TBD'))) + measurements['retraining'] = env.get( + 'MLC_ML_MODEL_RETRAINING', measurements.get( + 'retraining', 'no')) + measurements['input_data_types'] = env.get( + 'MLC_ML_MODEL_INPUTS_DATA_TYPE', measurements.get( + 'input_data_types', 'fp32')) + measurements['weight_data_types'] = env.get( + 'MLC_ML_MODEL_WEIGHTS_DATA_TYPE', measurements.get( + 'weight_data_types', 'fp32')) + measurements['weight_transformations'] = env.get( + 'MLC_ML_MODEL_WEIGHT_TRANSFORMATIONS', measurements.get( + 'weight_transformations', 'none')) + + os.chdir(output_dir) + + if not os.path.exists("mlperf_log_summary.txt"): + return {'return': 0} + + mlperf_log_summary = '' + if os.path.isfile("mlperf_log_summary.txt"): + with open("mlperf_log_summary.txt", "r") as fp: + mlperf_log_summary = fp.read() + + if mlperf_log_summary != '': + state['app_mlperf_inference_log_summary'] = {} + for x in mlperf_log_summary.split('\n'): + y = x.split(': ') + if len(y) == 2: + state['app_mlperf_inference_log_summary'][y[0].strip().lower() + ] = y[1].strip() + + if not is_false(env.get("MLC_MLPERF_PRINT_SUMMARY", "")): + logger.info("\n") + logger.info(mlperf_log_summary) + + with open("measurements.json", "w") as fp: + json.dump(measurements, fp, indent=2) + + mlc_sut_info = {} + mlc_sut_info['system_name'] = state['MLC_SUT_META']['system_name'] + mlc_sut_info['implementation'] = env['MLC_MLPERF_IMPLEMENTATION'] + mlc_sut_info['device'] = env['MLC_MLPERF_DEVICE'] + mlc_sut_info['framework'] = state['MLC_SUT_META']['framework'] + mlc_sut_info['run_config'] = env['MLC_MLPERF_INFERENCE_SUT_RUN_CONFIG'] + with open(os.path.join(result_sut_folder_path, "mlc-sut-info.json"), "w") as fp: + json.dump(mlc_sut_info, fp, indent=2) + + system_meta = state['MLC_SUT_META'] + with open("system_meta.json", "w") as fp: + json.dump(system_meta, fp, indent=2) + + # map the custom model for inference result to the official model + # if custom model name is not set, the official model name will be + # mapped to itself + official_model_name = model + if "efficientnet" in official_model_name or "mobilenet" in official_model_name: + official_model_name = "resnet" + model_mapping = {model_full_name: official_model_name} + with open("model_mapping.json", "w") as fp: + json.dump(model_mapping, fp, indent=2) + + # Add to the state + state['app_mlperf_inference_measurements'] = copy.deepcopy( + measurements) + + if os.path.exists(env['MLC_MLPERF_CONF']): + shutil.copy(env['MLC_MLPERF_CONF'], 'mlperf.conf') + + if os.path.exists(env['MLC_MLPERF_USER_CONF']): + shutil.copy(env['MLC_MLPERF_USER_CONF'], 'user.conf') + + result, valid, power_result = mlperf_utils.get_result_from_log( + env['MLC_MLPERF_LAST_RELEASE'], model, scenario, output_dir, mode, env.get('MLC_MLPERF_INFERENCE_SOURCE_VERSION')) + + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ][model][scenario][mode] = result + state['mlc-mlperf-inference-results'][state['MLC_SUT_CONFIG_NAME'] + ][model][scenario][mode + '_valid'] = valid.get(mode, False) + + state['mlc-mlperf-inference-results-last'][mode] = result + state['mlc-mlperf-inference-results-last'][mode + + '_valid'] = valid.get(mode, False) + + # Power not included in v0.5, code should be added in future + + # Record basic host info + host_info = { + "os_version": platform.platform(), + "cpu_version": platform.processor(), + "python_version": sys.version, + } + try: + import importlib.metadata + mlc_version = importlib.metadata.version("mlc") + host_info["mlc_version"] = mlc_version + except Exception as e: + error = format(e) + mlc_version = "unknown" + + x = '' + if env.get('MLC_HOST_OS_FLAVOR', '') != '': + x += env['MLC_HOST_OS_FLAVOR'] + if env.get('MLC_HOST_OS_VERSION', '') != '': + x += ' ' + env['MLC_HOST_OS_VERSION'] + if x != '': + host_info['os_version_sys'] = x + + if env.get('MLC_HOST_SYSTEM_NAME', '') != '': + host_info['system_name'] = env['MLC_HOST_SYSTEM_NAME'] + + # Check CM automation repository + repo_name = 'mlcommons@mlperf-automations' + repo_hash = '' + r = mlc.access({'action': 'find', 'automation': 'repo', + 'item': 'mlcommons@mlperf-automations,9e97bb72b0474657'}) + if r['return'] == 0 and len(r['list']) == 1: + repo_path = r['list'][0].path + if os.path.isdir(repo_path): + repo_name = os.path.basename(repo_path) + + # Check dev + # if repo_name == 'cm4mlops': repo_name = 'mlcommons@cm4mlops' + + r = utils.run_system_cmd({ + 'path': repo_path, + 'cmd': 'git rev-parse HEAD'}) + if r['return'] == 0: + repo_hash = r['output'] + + host_info['mlc_repo_name'] = repo_name + host_info['mlc_repo_git_hash'] = repo_hash + + with open("mlc-host-info.json", "w") as fp: + fp.write(json.dumps(host_info, indent=2) + '\n') + + # Prepare README + if "cmd" in inp: + cmd = "mlc run script \\\n\t" + " \\\n\t".join(inp['cmd']) + xcmd = "mlc run script " + xsep + "\n\t" + \ + (" " + xsep + "\n\t").join(inp['cmd']) + else: + cmd = "" + xcmd = "" + + readme_init = "*Check [MLC MLPerf docs](https://docs.mlcommons.org/automotive) for more details.*\n\n" + + readme_body = "## Host platform\n\n* OS version: {}\n* CPU version: {}\n* Python version: {}\n* MLC version: {}\n\n".format(platform.platform(), + platform.processor(), sys.version, mlc_version) + + x = repo_name + if repo_hash != '': + x += ' --checkout=' + str(repo_hash) + + readme_body += "## MLC Run Command\n\nSee [MLC installation guide](https://docs.mlcommons.org/mlcflow/install/).\n\n" + \ + "```bash\npip install -U mlcflow\n\nmlc rm cache -f\n\nmlc pull repo {}\n\n{}\n```".format( + x, xcmd) + + readme_body += "\n*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf,\n" + \ + " you should simply reload {} without checkout and clean MLC cache as follows:*\n\n".format(repo_name) + \ + "```bash\nmlc rm repo {}\nmlc pull repo {}\nmlc rm cache -f\n\n```".format( + repo_name, repo_name) + + extra_readme_init = '' + extra_readme_body = '' + if env.get('MLC_MLPERF_README', '') == "yes": + extra_readme_body += "\n## Dependent MLPerf Automation scripts\n\n" + + script_tags = inp['tags'] + script_adr = inp.get('adr', {}) + + mlc_input = {'action': 'run', + 'automation': 'script', + 'tags': script_tags, + 'adr': script_adr, + 'print_deps': True, + 'env': env, + 'quiet': True, + 'silent': True, + 'fake_run': True + } + r = mlc.access(mlc_input) + if r['return'] > 0: + return r + + print_deps = r['new_state']['print_deps'] + count = 1 + for dep in print_deps: + extra_readme_body += "\n\n" + str(count) + ". `" + dep + "`\n" + count = count + 1 + + if state.get( + 'abtfabtf-inference-implementation') and state['abtfabtf-inference-implementation'].get('print_deps'): + + extra_readme_body += "\n## Dependent automation scripts for the MLPerf Automotive Implementation\n" + + print_deps = state['abtfabtf-inference-implementation']['print_deps'] + count = 1 + for dep in print_deps: + extra_readme_body += "\n\n" + \ + str(count) + ". `" + dep + "`\n" + count = count + 1 + + readme = readme_init + readme_body + extra_readme = extra_readme_init + extra_readme_body + + with open("README.md", "w") as fp: + fp.write(readme) + if extra_readme: + with open("README-extra.md", "w") as fp: + fp.write(extra_readme) + + if state.get( + 'abtf-inference-implementation') and state['abtf-inference-implementation'].get('version_info'): + env['MLC_MLPERF_RUN_JSON_VERSION_INFO_FILE'] = os.path.join( + output_dir, "mlc-version-info.json") + env['MLC_MLPERF_RUN_DEPS_GRAPH'] = os.path.join( + output_dir, "mlc-deps.png") + env['MLC_MLPERF_RUN_DEPS_MERMAID'] = os.path.join( + output_dir, "mlc-deps.mmd") + with open(os.path.join(output_dir, "mlc-version-info.json"), "w") as f: + f.write( + json.dumps( + state['abtf-inference-implementation']['version_info'], + indent=2)) + + if env.get('MLC_DUMP_SYSTEM_INFO', True): + dump_script_output( + "detect,os", + env, + state, + 'new_env', + os.path.join( + output_dir, + "os_info.json"), mlc) + dump_script_output( + "detect,cpu", + env, + state, + 'new_env', + os.path.join( + output_dir, + "cpu_info.json"), mlc) + env['MLC_DUMP_RAW_PIP_FREEZE_FILE_PATH'] = os.path.join( + env['MLC_MLPERF_OUTPUT_DIR'], "pip_freeze.raw") + dump_script_output( + "dump,pip,freeze", + env, + state, + 'new_state', + os.path.join( + output_dir, + "pip_freeze.json"), mlc) + + return {'return': 0} + + +def dump_script_output(script_tags, env, state, output_key, dump_file, mlc): + + mlc_input = {'action': 'run', + 'automation': 'script', + 'tags': script_tags, + 'env': env, + 'state': state, + 'quiet': True, + 'silent': True, + } + r = mlc.access(mlc_input) + if r['return'] > 0: + return r + with open(dump_file, "w") as f: + f.write(json.dumps(r[output_key], indent=2)) + + return {'return': 0} diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 896008c39..101a7b851 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -4,14 +4,16 @@ uid: f7488ce376484fd2 automation_alias: script automation_uid: 5b4e0237da074764 -category: "Modular MLPerf inference benchmark pipeline for ABTF model" +category: "Modular MLPerf automotive benchmark pipeline for ABTF models" # User-friendly tags to find this CM script tags: - app - app-mlperf-inference +- app-mlperf-inference-automotive - mlperf-inference +- mlperf-inference-automotive - abtf-inference predeps: no @@ -90,26 +92,53 @@ deps: # Use mlc inside scripts #- tags: get,generic-python-lib,_package.mlcflow - - tags: get,mlperf,inference,utils + - tags: get,mlperf,automotive,utils + +posthook_deps: + - tags: get,mlperf,sut,description #populate system meta information like framework + - tags: get,platform,details + enable_if_env: + MLC_GET_PLATFORM_DETAILS: + - yes + skip_if_env: + MLC_MLPERF_LOADGEN_MODE: + - accuracy + env: + MLC_PLATFORM_DETAILS_FILE_PATH: '<<>>/system_info.txt' + +post_deps: + - tags: draw,graph,from-json + enable_if_env: + MLC_MLPERF_RUN_JSON_VERSION_INFO_FILE: + - on + env: + MLC_JSON_INPUT_FILE: <<>> + MLC_OUTPUT_IMAGE_PATH: <<>> + MLC_OUTPUT_MERMAID_PATH: <<>> docker: - mlc_repo: gateoverflow@mlperf-automations - mlc_repo_branch: dev + mlc_repo: anandhu-eng@mlperf-automations + mlc_repo_branch: automotive use_host_group_id: True use_host_user_id: True real_run: false + user: mlcuser interactive: True mlc_repos_off: 'mlc pull repo mlcommons@cm4abtf --branch=poc' deps: - tags: get,abtf,scratch,space mounts: - "${{ MLC_ABTF_SCRATCH_PATH_DATASETS }}:${{ MLC_ABTF_SCRATCH_PATH_DATASETS }}" + - "${{ MLC_ML_MODEL_FILE_WITH_PATH }}:${{ MLC_ML_MODEL_FILE_WITH_PATH }}" # Variations to customize dependencies variations: + reference: + alias: mlcommons-python + # Implementation mlcommons-python: group: implementation @@ -121,7 +150,7 @@ variations: - names: - python-reference-abtf-inference - abtf-inference-implementation - tags: run-mlperf-inference,demo,abtf-model + tags: run-mlperf-inference,abtf-model skip_if_env: MLC_SKIP_RUN: - yes @@ -157,7 +186,19 @@ variations: add_deps_recursive: abtf-inference-implementation: tags: _onnxruntime - + ml-model-bevformer: + tags: _onnx + ml-model-ssd-resnet50: + tags: _onnx + ml-model-deeplab-v3+: + tags: _onnx + + onnx_dynamic: + base: + - onnxruntime + add_deps_recursive: + ml-model-deeplab-v3+: + tags: _onnx_dynamic onnxruntime,cpu: env: @@ -178,6 +219,12 @@ variations: add_deps_recursive: abtf-inference-implementation: tags: _pytorch + ml-model-bevformer: + tags: _pytorch + ml-model-ssd-resnet50: + tags: _pytorch + ml-model-deeplab-v3+: + tags: _pytorch abtf-demo-model: @@ -204,10 +251,120 @@ variations: enable_if_env: MLC_DATASET_MLCOMMONS_COGNATA_DOWNLOAD_IN_HOST: - yes - mounts: - "${{ MLC_DATASET_MLCOMMONS_COGNATA_PATH }}:${{ MLC_DATASET_MLCOMMONS_COGNATA_PATH }}" - + + bevformer: + group: + models + default_env: + MLC_USE_DATASET_FROM_HOST: yes + env: + MLC_MODEL: bevformer + docker: + deps: + - tags: get,preprocessed,dataset,nuscenes + enable_if_env: + MLC_USE_DATASET_FROM_HOST: + - "yes" + - tags: get,ml-model,bevformer + enable_if_env: + MLC_USE_MODEL_FROM_HOST: + - "yes" + names: + - ml-model-bevformer + mounts: + - "${{ MLC_PREPROCESSED_DATASET_NUSCENES_PATH }}:${{ MLC_PREPROCESSED_DATASET_NUSCENES_PATH }}" + - "${{ MLC_ML_MODEL_BEVFORMER_PATH }}:${{ MLC_ML_MODEL_BEVFORMER_PATH }}" + - "${{ MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH }}:${{ MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH }}" + add_deps_recursive: + abtf-inference-implementation: + tags: _bevformer + posthook_deps: + - enable_if_env: + MLC_MLPERF_LOADGEN_MODE: + - accuracy + - all + MLC_MLPERF_ACCURACY_RESULTS_DIR: + - 'on' + names: + - mlperf-accuracy-script + - nuscenes-accuracy-script + tags: run,accuracy,mlperf,_nuscenes + + ssd-resnet50: + group: + models + default_env: + MLC_USE_DATASET_FROM_HOST: yes + env: + MLC_MODEL: ssd-resnet50 + docker: + deps: + - tags: get,preprocessed,dataset,cognata + enable_if_env: + MLC_USE_DATASET_FROM_HOST: + - "yes" + - tags: get,ml-model,ssd,resnet50,_mlc,_rclone + enable_if_env: + MLC_USE_MODEL_FROM_HOST: + - "yes" + names: + - ml-model-ssd-resnet50 + mounts: + - "${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}:${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}" + - "${{ MLC_ML_MODEL_SSD_PATH }}:${{ MLC_ML_MODEL_SSD_PATH }}" + add_deps_recursive: + abtf-inference-implementation: + tags: _ssd-resnet50 + posthook_deps: + - enable_if_env: + MLC_MLPERF_LOADGEN_MODE: + - accuracy + - all + MLC_MLPERF_ACCURACY_RESULTS_DIR: + - 'on' + names: + - mlperf-accuracy-script + - cognata-ssd-accuracy-script + tags: run,accuracy,mlperf,_cognata_ssd + + deeplab-v3+: + group: + models + default_env: + MLC_USE_DATASET_FROM_HOST: yes + env: + MLC_MODEL: deeplab-v3+ + docker: + deps: + - tags: get,preprocessed,dataset,cognata,_segmentation + enable_if_env: + MLC_USE_DATASET_FROM_HOST: + - "yes" + - tags: get,ml-model,ssd,resnet50,_mlc,_rclone + enable_if_env: + MLC_USE_MODEL_FROM_HOST: + - "yes" + names: + - ml-model-deeplab-v3+ + mounts: + - "${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}:${{ MLC_PREPROCESSED_DATASET_COGNATA_PATH }}" + - "${{ MLC_ML_MODEL_DEEPLABV3_PLUS_PATH }}:${{ MLC_ML_MODEL_DEEPLABV3_PLUS_PATH }}" + add_deps_recursive: + abtf-inference-implementation: + tags: _deeplab-v3+ + posthook_deps: + - enable_if_env: + MLC_MLPERF_LOADGEN_MODE: + - accuracy + - all + MLC_MLPERF_ACCURACY_RESULTS_DIR: + - 'on' + names: + - mlperf-accuracy-script + - cognata-deeplab-accuracy-script + tags: run,accuracy,mlperf,_cognata_deeplab # Target devices cpu: @@ -234,8 +391,18 @@ variations: docker: all_gpus: 'yes' base_image: nvcr.io/nvidia/pytorch:24.03-py3 + os_version: 22.04 + + v0.5: {} + + mvp-demo: {} + poc-demo: {} + v0.5,mlcommons-python,cpu: + docker: + base_image: ubuntu:22.04 + os_version: 22.04 # Loadgen scenarios offline: @@ -286,3 +453,11 @@ variations: MLC_DATASET_MLCOMMONS_COGNATA_GROUP_NAMES: Cognata_Camera_01_8M MLC_ABTF_ML_MODEL_TRAINING_FORCE_COGNATA_LABELS: 'yes' MLC_ABTF_ML_MODEL_SKIP_WARMUP: 'yes' + + batch_size.#: + group: batch_size + env: + MLC_MLPERF_LOADGEN_MAX_BATCHSIZE: '#' + add_deps_recursive: + abtf-inference-implementation: + tags: _batch_size.# \ No newline at end of file diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 9f3776c2d..b240e4164 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -228,7 +228,7 @@ def preprocess(i): f.write('RUN ' + env['MLC_DOCKER_EXTRA_SYS_DEPS'] + EOL) if env['MLC_DOCKER_OS'] == "ubuntu": - if int(env['MLC_DOCKER_OS_VERSION'].split('.')[0]) >= 23: + if int(str(env['MLC_DOCKER_OS_VERSION']).split('.')[0]) >= 23: if "--break-system-packages" not in env.get( 'MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''): env['MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS'] = " --break-system-packages" diff --git a/script/generate-mlperf-inference-user-conf/meta.yaml b/script/generate-mlperf-inference-user-conf/meta.yaml index fbba97b33..4288bd443 100644 --- a/script/generate-mlperf-inference-user-conf/meta.yaml +++ b/script/generate-mlperf-inference-user-conf/meta.yaml @@ -84,13 +84,20 @@ deps: OUTPUT_BASE_DIR: - "on" - ######################################################################## - # Install MLPerf inference dependencies - - # Download MLPerf inference source - - tags: get,mlcommons,inference,src - names: - - inference-src - # Get SUT configs (System Under Test) - tags: get,sut,configs + +variations: + inference: + default: true + group: benchmark_wg + deps: + - tags: get,mlcommons,inference,src + names: + - inference-src + automotive: + group: benchmark_wg + deps: + - tags: get,mlcommons,automotive,src + names: + - automotive-src \ No newline at end of file diff --git a/script/get-dataset-cognata-mlcommons/customize.py b/script/get-dataset-cognata-mlcommons/customize.py index be725599d..64406d44e 100644 --- a/script/get-dataset-cognata-mlcommons/customize.py +++ b/script/get-dataset-cognata-mlcommons/customize.py @@ -8,6 +8,10 @@ def preprocess(i): env = i['env'] + if env.get('MLC_COGNATA_DATASET_TYPE', '') == "release": + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + return {'return': 0} + mlc_cache_dataset_path = env.get( 'MLC_CUSTOM_CACHE_ENTRY_DATASET_MLCOMMONS_COGNATA_PATH', '').strip() @@ -61,6 +65,9 @@ def postprocess(i): logger = automation.logger + if env.get('MLC_COGNATA_DATASET_TYPE', '') == "release": + return {'return': 0} + cur_dir = os.getcwd() quiet = is_true(env.get('MLC_QUIET', False)) diff --git a/script/get-dataset-cognata-mlcommons/meta.yaml b/script/get-dataset-cognata-mlcommons/meta.yaml index 309b6ba90..ecb10799a 100644 --- a/script/get-dataset-cognata-mlcommons/meta.yaml +++ b/script/get-dataset-cognata-mlcommons/meta.yaml @@ -48,9 +48,11 @@ deps: - custom-cache-entry-mlcommons-cognata-dataset tags: create,custom,cache,entry extra_cache_tags: dataset,cognata,mlcommons-cognata - skip_if_env: + skip_if_any_env: MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: - 'off' + MLC_DOWNLOAD_SRC: + - mlcommons env_key: DATASET_MLCOMMONS_COGNATA # this script will prepare env MLC_CUSTOM_CACHE_ENTRY_{env_key}_PATH @@ -58,9 +60,11 @@ deps: prehook_deps: - names: - gdrive-downloader-cognata - skip_if_env: + skip_if_any_env: MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: - 'on' + MLC_DOWNLOAD_SRC: + - mlcommons enable_if_env: MLC_DATASET_MLCOMMONS_COGNATA_IMPORTED: - 'no' @@ -76,9 +80,11 @@ prehook_deps: - names: - rclone-downloader-cognata - skip_if_env: + skip_if_any_env: MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: - 'on' + MLC_DOWNLOAD_SRC: + - mlcommons enable_if_env: MLC_DATASET_MLCOMMONS_COGNATA_IMPORTED: - 'no' @@ -101,27 +107,33 @@ prehook_deps: - python - python3 tags: get,python3 - skip_if_env: + skip_if_any_env: MLC_DATASET_MLCOMMONS_COGNATA_IMPORTED: - 'yes' + MLC_DOWNLOAD_SRC: + - mlcommons enable_if_env: MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: - 'on' # Python package to read/write Excel files - tags: get,generic-python-lib,_package.openpyxl - skip_if_env: + skip_if_any_env: MLC_DATASET_MLCOMMONS_COGNATA_IMPORTED: - 'yes' + MLC_DOWNLOAD_SRC: + - mlcommons enable_if_env: MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: - 'on' # Tool to download large files - tags: get,aria2 - skip_if_env: + skip_if_any_env: MLC_DATASET_MLCOMMONS_COGNATA_IMPORTED: - 'yes' + MLC_DOWNLOAD_SRC: + - mlcommons enable_if_env: MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: - 'on' @@ -143,6 +155,11 @@ variations: MLC_DATASET_MLCOMMONS_COGNATA_GROUP_NAMES: "Cognata_Camera_01_8M" MLC_DATASET_MLCOMMONS_COGNATA_FILE_NAMES: "" + release: + group: dataset-type + env: + MLC_COGNATA_DATASET_TYPE: "release" + rclone: group: download-tool default: true @@ -153,6 +170,57 @@ variations: group: download-tool env: MLC_DATASET_MLCOMMONS_COGNATA_DOWNLOAD_TOOL: gdrive + + mlc: + group: download-src + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_MLCOMMONS_COGNATA_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_MLCOMMONS_COGNATA_PATH + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/unprocessed' + extra_cache_tags: dataset,cognata,release + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons + + rclone,mlc: + add_deps_recursive: + dae: + tags: _rclone + env: + MLC_RCLONE_COPY_USING: sync + + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + new_env_keys: - MLC_DATASET_MLCOMMONS_COGNATA* diff --git a/script/get-dataset-cognata-mlcommons/run.sh b/script/get-dataset-cognata-mlcommons/run.sh new file mode 100644 index 000000000..5563a95da --- /dev/null +++ b/script/get-dataset-cognata-mlcommons/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency + +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" == "true" && "$MLC_COGNATA_DATASET_TYPE" == "release" ]]; then + cd "${MLC_DATASET_MLCOMMONS_COGNATA_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd - || exit +fi \ No newline at end of file diff --git a/script/get-dataset-nuscenes/COPYRIGHT.md b/script/get-dataset-nuscenes/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-dataset-nuscenes/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-dataset-nuscenes/customize.py b/script/get-dataset-nuscenes/customize.py new file mode 100644 index 000000000..d4abad774 --- /dev/null +++ b/script/get-dataset-nuscenes/customize.py @@ -0,0 +1,23 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + return {'return': 0} diff --git a/script/get-dataset-nuscenes/meta.yaml b/script/get-dataset-nuscenes/meta.yaml new file mode 100644 index 000000000..e12787bdf --- /dev/null +++ b/script/get-dataset-nuscenes/meta.yaml @@ -0,0 +1,57 @@ +alias: get-dataset-nuscenes +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- dataset +- nuscenes +uid: ec2a0842c9a644f5 +new_env_keys: + - MLC_DATASET_NUSCENES_PATH +variations: + mlc: + group: download-src + default: true + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_waymo + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_NUSCENES_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_NUSCENES_PATH + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset + extra_cache_tags: nuscenes,dataset + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + default: true + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run diff --git a/script/get-dataset-nuscenes/run.sh b/script/get-dataset-nuscenes/run.sh new file mode 100644 index 000000000..abe5a17c7 --- /dev/null +++ b/script/get-dataset-nuscenes/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} + +#To export any variable +#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out + +#${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency + +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "true" ]]; then + cd "${MLC_DATASET_NUSCENES_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd "${MLC_DATASET_NUSCENES_PATH}/nuscenes" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd - || exit +fi \ No newline at end of file diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index eac6ecdb1..f058b3127 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -153,6 +153,20 @@ variations: brew: '' dnf: mesa-libGL yum: mesa-libGL + libgl1-mesa-glx: + env: + MLC_SYS_UTIL_NAME: libgl1-mesa-glx # tbd: regular expression for version as well as whether its installed? + MLC_SYS_UTIL_CHECK_CMD: 'ldconfig -p | grep -i libGLX_mesa.so.*' + default_env: + MLC_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE: yes + new_env_keys: + - MLC_LIBGLX_VERSION + state: + libgl: # tbd: complete for other flavours of linux + apt: libgl1-mesa-glx + brew: '' + dnf: '' + yum: '' libsm6: env: MLC_SYS_UTIL_NAME: libsm6 # tbd: regular expression for version as well as whether its installed? diff --git a/script/get-git-repo/meta.yaml b/script/get-git-repo/meta.yaml index efdf3bf63..468468bb5 100644 --- a/script/get-git-repo/meta.yaml +++ b/script/get-git-repo/meta.yaml @@ -79,7 +79,7 @@ variations: git-history: full-history env: MLC_GIT_SHA: '#' - group: checkout + group: post-checkout short-history: default: true env: diff --git a/script/get-ml-model-abtf-ssd-pytorch/customize.py b/script/get-ml-model-abtf-ssd-pytorch/customize.py index a4fa7f16c..e63e92fff 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/customize.py +++ b/script/get-ml-model-abtf-ssd-pytorch/customize.py @@ -26,6 +26,9 @@ def preprocess(i): 'error': 'ML model {} is not found'.format(ml_model)} env['MLC_ML_MODEL_FILE_WITH_PATH'] = ml_model + # handles download from mlcommons gdrive + elif env.get('MLC_DOWNLOAD_SRC', '') == "mlcommons" and env.get('MLC_ML_MODEL_SSD_PATH', '') == '': + env['MLC_TMP_REQUIRE_DOWNLOAD'] = 'yes' return {'return': 0} @@ -35,7 +38,13 @@ def postprocess(i): env = i['env'] if env.get('MLC_ML_MODEL_FILE_WITH_PATH', '') == '': - env['MLC_ML_MODEL_FILE_WITH_PATH'] = 'model-weights-skipped' + if env.get('MLC_ML_MODEL_SSD_PATH', '') == '': + env['MLC_ML_MODEL_FILE_WITH_PATH'] = 'model-weights-skipped' + else: + if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): + env['MLC_ML_MODEL_SSD_PATH'] = os.path.join( + env['MLC_ML_MODEL_SSD_PATH'], env['MLC_ML_MODEL_FILENAME']) + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_SSD_PATH'] env['MLC_ML_MODEL_FILE'] = os.path.basename( env['MLC_ML_MODEL_FILE_WITH_PATH']) diff --git a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml index b9f70ebc3..017acec37 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml @@ -17,6 +17,8 @@ tags: - get - ml-model - abtf-ssd-pytorch +- ssd +- resnet50 - cmc @@ -38,9 +40,11 @@ deps: names: - abtf-ssd-pytorch-git-repo - abtf-ml-model-code-git-repo - skip_if_env: + skip_if_any_env: MLC_SKIP_MODEL_CODE_DOWNLOAD: - 'yes' + MLC_DOWNLOAD_SRC: + - 'mlcommons' env: MLC_GIT_AUTH: 'yes' MLC_GIT_CHECKOUT_PATH_ENV_NAME: MLC_ABTF_SSD_PYTORCH @@ -62,9 +66,11 @@ deps: names: - abtf-ml-model-weights - abtf-ml-model-weights-download - skip_if_env: + skip_if_any_env: MLC_SKIP_MODEL_WEIGHTS_DOWNLOAD: - 'yes' + MLC_DOWNLOAD_SRC: + - 'mlcommons' update_tags_from_env_with_prefix: _url.: - MLC_ML_MODEL_URL @@ -77,6 +83,7 @@ new_env_keys: print_env_at_the_end: MLC_ML_MODEL_FILE_WITH_PATH: Path to the ML model weights MLC_ML_MODEL_CODE_WITH_PATH: Path to the ML model code + MLC_ML_MODEL_SSD_PATH: Path to ssd resnet50 model variations: @@ -172,3 +179,81 @@ variations: adr: abtf-ml-model-weights-download: tags: _gdown + + onnx: + group: model-format + default: true + env: + MLC_MODEL_FORMAT: onnx + + onnx,mlc: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_ssd/ssd_resnet50.onnx + MLC_ML_MODEL_FILENAME: ssd_resnet50.onnx + + pytorch: + group: model-format + env: + MLC_MODEL_FORMAT: pth + + pytorch,mlc: + env: + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_ssd/baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth + MLC_ML_MODEL_FILENAME: baseline_8MP_ss_scales_fm1_5x5_all_ep60.pth + + rclone,mlc: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + env: + MLC_RCLONE_COPY_USING: sync + + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + + mlc: + group: download-src + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_SSD_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_MLC_MODEL_SSD_PATH + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + extra_cache_tags: ml,model,ssd,resnet50 + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons +tests: + run_inputs: + - variations_list: + - onnx,rclone,mlc,dry-run + - pytorch,rclone,mlc,dry-run \ No newline at end of file diff --git a/script/get-ml-model-bevformer/COPYRIGHT.md b/script/get-ml-model-bevformer/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-ml-model-bevformer/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-ml-model-bevformer/customize.py b/script/get-ml-model-bevformer/customize.py new file mode 100644 index 000000000..9afc37740 --- /dev/null +++ b/script/get-ml-model-bevformer/customize.py @@ -0,0 +1,31 @@ +from mlc import utils +from utils import is_true +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_ML_MODEL_BEVFORMER_PATH', '') == '': + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): + env['MLC_ML_MODEL_BEVFORMER_PATH'] = os.path.join( + env['MLC_ML_MODEL_BEVFORMER_PATH'], env['MLC_ML_MODEL_FILENAME']) + + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_BEVFORMER_PATH'] + + return {'return': 0} diff --git a/script/get-ml-model-bevformer/meta.yaml b/script/get-ml-model-bevformer/meta.yaml new file mode 100644 index 000000000..e4c156030 --- /dev/null +++ b/script/get-ml-model-bevformer/meta.yaml @@ -0,0 +1,81 @@ +alias: get-ml-model-bevformer +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- ml-model +- bevformer +uid: 438a053f666443bd +new_env_keys: + - MLC_ML_MODEL_BEVFORMER_PATH + - MLC_ML_MODEL_FILE_WITH_PATH +print_env_at_the_end: + MLC_ML_MODEL_BEVFORMER_PATH: BevFormer checkpoint path +variations: + onnx: + group: model-format + default: true + env: + MLC_MODEL_FORMAT: onnx + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_bevformer/bevformer_tiny.onnx + MLC_ML_MODEL_FILENAME: bevformer_tiny.onnx + pytorch: + group: model-format + env: + MLC_MODEL_FORMAT: pth + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_bevformer/bevformer_tiny_epoch_24.pth + MLC_ML_MODEL_FILENAME: bevformer_tiny_epoch_24.onnx + mlc: + group: download-src + default: true + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.mlc-nuscenes + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_BEVFORMER_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_BEVFORMER_PATH + MLC_DOWNLOAD_URL: 'mlc-nuscenes:<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + extra_cache_tags: ml,model,bevformer + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + default: true + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + run_inputs: + - variations_list: + - onnx,rclone,mlc,dry-run + - pytorch,rclone,mlc,dry-run \ No newline at end of file diff --git a/script/get-ml-model-bevformer/run.sh b/script/get-ml-model-bevformer/run.sh new file mode 100644 index 000000000..a9bf588e2 --- /dev/null +++ b/script/get-ml-model-bevformer/run.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/script/get-ml-model-deeplabv3_plus/COPYRIGHT.md b/script/get-ml-model-deeplabv3_plus/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-ml-model-deeplabv3_plus/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-ml-model-deeplabv3_plus/customize.py b/script/get-ml-model-deeplabv3_plus/customize.py new file mode 100644 index 000000000..2fff179b1 --- /dev/null +++ b/script/get-ml-model-deeplabv3_plus/customize.py @@ -0,0 +1,30 @@ +from mlc import utils +import os +from utils import is_true + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if os_info['platform'] == "windows": + return {'return': 1, 'error': 'Script not supported in windows yet!'} + + if env.get('MLC_ML_MODEL_DEEPLABV3_PLUS_PATH', '') == '': + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): + env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH'] = os.path.join( + env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH'], env['MLC_ML_MODEL_FILENAME']) + env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_DEEPLABV3_PLUS_PATH'] + + return {'return': 0} diff --git a/script/get-ml-model-deeplabv3_plus/meta.yaml b/script/get-ml-model-deeplabv3_plus/meta.yaml new file mode 100644 index 000000000..66215eb09 --- /dev/null +++ b/script/get-ml-model-deeplabv3_plus/meta.yaml @@ -0,0 +1,91 @@ +alias: get-ml-model-deeplabv3-plus +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- ml-model +- deeplab +- v3-plus +- deeplabv3-plus +uid: cfb2d53b9dbc4dc0 +new_env_keys: + - MLC_ML_MODEL_DEEPLABV3_PLUS_PATH + - MLC_ML_MODEL_FILE_WITH_PATH +print_env_at_the_end: + MLC_ML_MODEL_DEEPLABV3_PLUS_PATH: DeepLabV3+ checkpoint path +variations: + onnx: + group: model-format + default: true + env: + MLC_MODEL_FORMAT: onnx + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_8mp.onnx + MLC_ML_MODEL_FILENAME: deeplabv3+_8mp.onnx + dynamic: {} + onnx,dynamic: + group: model-format + env: + MLC_MODEL_FORMAT: onnx + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/deeplabv3+_dynamic.onnx + MLC_ML_MODEL_FILENAME: deeplabv3+_dynamic.onnx + pytorch: + group: model-format + env: + MLC_MODEL_FORMAT: pth + MLC_MODEL_RCLONE_FILEPATH: model_checkpoint_deeplab/latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth + MLC_ML_MODEL_FILENAME: latest_deeplabv3plus_resnet50_cognata_os16_it100000.pth + mlc: + group: download-src + default: true + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPLABV3_PLUS_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_DEEPLABV3_PLUS_PATH + MLC_DOWNLOAD_URL: 'mlc-cognata:mlc_cognata_dataset/<<>>' + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + extra_cache_tags: ml,model,deeplabv3,plus + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + default: true + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + run_inputs: + - variations_list: + - onnx,rclone,mlc,dry-run + - onnx_dynamic,rclone,mlc,dry-run + - pytorch,rclone,mlc,dry-run \ No newline at end of file diff --git a/script/get-ml-model-deeplabv3_plus/run.sh b/script/get-ml-model-deeplabv3_plus/run.sh new file mode 100644 index 000000000..05a7907cf --- /dev/null +++ b/script/get-ml-model-deeplabv3_plus/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash + diff --git a/script/get-mlperf-automotive-src/COPYRIGHT.md b/script/get-mlperf-automotive-src/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-mlperf-automotive-src/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-mlperf-automotive-src/customize.py b/script/get-mlperf-automotive-src/customize.py new file mode 100644 index 000000000..69b336134 --- /dev/null +++ b/script/get-mlperf-automotive-src/customize.py @@ -0,0 +1,154 @@ +from mlc import utils +from utils import is_true +import os +import shutil + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + meta = i['meta'] + + script_path = i['run_script_input']['path'] + + if env.get('MLC_GIT_CHECKOUT', '') == '' and env.get( + 'MLC_GIT_URL', '') == '' and env.get('MLC_VERSION', '') == '': + # if custom checkout and url parameters are not set and MLC_VERSION is + # not specified + env['MLC_VERSION'] = "master" + env["MLC_GIT_CHECKOUT"] = "master" + env["MLC_GIT_URL"] = "https://github.com/mlcommons/mlperf_automotive" + elif env.get('MLC_GIT_CHECKOUT', '') != '' and env.get('MLC_TMP_GIT_CHECKOUT', '') != '' and env.get('MLC_GIT_CHECKOUT', '') != env.get('MLC_TMP_GIT_CHECKOUT', ''): + # if checkout branch is assigned inside version and custom branch is + # also specified + return { + "return": 1, "error": "Conflicting branches between version assigned and user specified."} + elif env.get('MLC_GIT_URL', '') != '' and env.get('MLC_TMP_GIT_URL', '') != '' and env.get('MLC_GIT_URL', '') != env.get('MLC_TMP_GIT_URL', ''): + # if GIT URL is assigned inside version and custom branch is also + # specified + return { + "return": 1, "error": "Conflicting URL's between version assigned and user specified."} + + if env.get('MLC_VERSION', '') == '': + env['MLC_VERSION'] = "custom" + + # check whether branch and url is specified, + # if not try to assign the values specified in version parameters, + # if version parameters does not have the value to a parameter, set the + # default one + if env.get('MLC_GIT_CHECKOUT', '') == '' and env.get( + 'MLC_GIT_CHECKOUT_TAG', '') == '': + if env.get('MLC_TMP_GIT_CHECKOUT', '') != '': + env["MLC_GIT_CHECKOUT"] = env["MLC_TMP_GIT_CHECKOUT"] + else: + env["MLC_GIT_CHECKOUT"] = "master" + + if env.get('MLC_GIT_URL', '') == '': + if env.get('MLC_TMP_GIT_URL', '') != '': + env["MLC_GIT_URL"] = env["MLC_TMP_GIT_URL"] + else: + env["MLC_GIT_URL"] = "https://github.com/mlcommons/mlperf_automotive" + + if env.get("MLC_MLPERF_LAST_RELEASE", '') == '': + env["MLC_MLPERF_LAST_RELEASE"] = "v0.5" + + if 'MLC_GIT_DEPTH' not in env: + env['MLC_GIT_DEPTH'] = '' + + if 'MLC_GIT_RECURSE_SUBMODULES' not in env: + env['MLC_GIT_RECURSE_SUBMODULES'] = '' + submodules = [] + possible_submodules = { + "pybind": "third_party/pybind", + } + for submodule in possible_submodules: + env_name = submodule.upper().replace("-", "_") + if is_true(env.get("MLC_SUBMODULE_" + env_name)): + submodules.append(possible_submodules[submodule]) + + env['MLC_GIT_SUBMODULES'] = ",".join(submodules) + + if env.get('MLC_GIT_PATCH_FILENAME', '') != '': + patch_file_name = env['MLC_GIT_PATCH_FILENAME'] + env['MLC_GIT_PATCH_FILEPATHS'] = os.path.join( + script_path, 'patch', patch_file_name) + + need_version = env.get('MLC_VERSION', '') + versions = meta['versions'] + + if need_version != '' and not need_version in versions: + env['MLC_GIT_CHECKOUT'] = need_version + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + automotive_root = env['MLC_MLPERF_INFERENCE_SOURCE'] + env['MLC_MLPERF_INFERENCE_BEVFORMER_PATH'] = os.path.join( + automotive_root, 'automotive', 'camera-3d-detection') + env['MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH'] = os.path.join( + automotive_root, 'automotive', '2d-object-detection') + env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH'] = os.path.join( + automotive_root, 'automotive', 'semantic-segmentation') + + env['MLC_GET_DEPENDENT_CACHED_PATH'] = automotive_root + +# 20221024: we save and restore env in the main script and can clean env here for determinism +# if '+PYTHONPATH' not in env: env['+PYTHONPATH'] = [] + env['+PYTHONPATH'] = [] + + if os.path.exists(os.path.join(automotive_root, "loadgen", "VERSION.txt")): + with open(os.path.join(automotive_root, "loadgen", "VERSION.txt")) as f: + version_info = f.read().strip() + env['MLC_MLPERF_INFERENCE_SOURCE_VERSION'] = version_info + + if is_true(env.get('MLC_GET_MLPERF_IMPLEMENTATION_ONLY', '')): + return {'return': 0} + + env['MLC_MLPERF_INFERENCE_CONF_PATH'] = os.path.join( + automotive_root, 'mlperf.conf') + env['+PYTHONPATH'].append( + os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], + 'tools', + 'submission')) + + # To be uncommented after Pablo's PR is merged: https://github.com/mlcommons/mlperf_automotive/pull/14 + # valid_models = get_valid_models( + # env['MLC_MLPERF_LAST_RELEASE'], + # env['MLC_MLPERF_INFERENCE_SOURCE']) + + # state['MLC_MLPERF_AUTOMOTIVE_MODELS'] = valid_models + + if env.get('MLC_GIT_REPO_CURRENT_HASH', '') != '': + env['MLC_VERSION'] += "-git-" + env['MLC_GIT_REPO_CURRENT_HASH'] + + return {'return': 0, 'version': env['MLC_VERSION']} + + +def get_valid_models(mlperf_version, mlperf_path): + + import sys + + submission_checker_dir = os.path.join(mlperf_path, "tools", "submission") + + sys.path.append(submission_checker_dir) + + if not os.path.exists(os.path.join( + submission_checker_dir, "submission_checker.py")): + shutil.copy(os.path.join(submission_checker_dir, "submission-checker.py"), os.path.join(submission_checker_dir, + "submission_checker.py")) + + import submission_checker as checker + + config = checker.MODEL_CONFIG + + valid_models = config[mlperf_version]["models"] + + return valid_models diff --git a/script/get-mlperf-automotive-src/meta.yaml b/script/get-mlperf-automotive-src/meta.yaml new file mode 100644 index 000000000..9fa26d5ca --- /dev/null +++ b/script/get-mlperf-automotive-src/meta.yaml @@ -0,0 +1,103 @@ +alias: get-mlperf-automotive-src +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: MLPerf benchmark support +default_env: + MLC_GIT_CHECKOUT_FOLDER: automotive + MLC_GIT_DEPTH: --depth 4 + MLC_GIT_PATCH: 'no' + MLC_GIT_RECURSE_SUBMODULES: '' +default_version: master +deps: +- tags: detect,os +- names: + - python + - python3 + tags: get,python3 +new_env_keys: +- MLC_MLPERF_INFERENCE_BEVFORMER_PATH +- MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH +- MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH +- MLC_MLPERF_LAST_RELEASE +- MLC_MLPERF_INFERENCE_SOURCE +- MLC_MLPERF_INFERENCE_SOURCE_VERSION +- +PYTHONPATH +prehook_deps: +- env: + MLC_GIT_CHECKOUT_PATH_ENV_NAME: MLC_MLPERF_INFERENCE_SOURCE + extra_cache_tags: automotive,src + force_env_keys: + - MLC_GIT_* + names: + - automotive-git-repo + tags: get,git,repo + update_tags_from_env_with_prefix: + _branch.: + - MLC_GIT_CHECKOUT + _repo.: + - MLC_GIT_URL + _sha.: + - MLC_GIT_SHA + _submodules.: + - MLC_GIT_SUBMODULES +print_env_at_the_end: + MLC_MLPERF_INFERENCE_SOURCE: Path to MLPerf automotive benchmark source +tags: +- get +- src +- source +- automotive +- automotive-src +- automotive-source +- mlperf +- mlcommons +uid: c3842e6e35d947ef +variations: + branch.#: + default_version: custom + env: + MLC_GIT_CHECKOUT: '#' + group: checkout + full-history: + env: + MLC_GIT_DEPTH: '' + group: git-history + no-recurse-submodules: + env: + MLC_GIT_RECURSE_SUBMODULES: '' + patch: + ad: + automotive-git-repo: + tags: _patch + env: + MLC_GIT_PATCH: 'yes' + pybind: + env: + MLC_SUBMODULE_PYBIND: 'yes' + recurse-submodules: + env: + MLC_GIT_RECURSE_SUBMODULES: ' --recurse-submodules' + repo.#: + env: + MLC_GIT_URL: '#' + sha.#: + env: + MLC_GIT_SHA: '#' + group: checkout + short-history: + default: true + env: + MLC_GIT_DEPTH: --depth 10 + group: git-history + submodules.#: + env: + MLC_GIT_SUBMODULES: '#' +versions: + custom: + env: + MLC_MLPERF_LAST_RELEASE: v0.5 + master: + env: + MLC_MLPERF_LAST_RELEASE: v0.5 + MLC_TMP_GIT_CHECKOUT: master diff --git a/script/get-mlperf-automotive-utils/COPYRIGHT.md b/script/get-mlperf-automotive-utils/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-mlperf-automotive-utils/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-mlperf-automotive-utils/customize.py b/script/get-mlperf-automotive-utils/customize.py new file mode 100644 index 000000000..7ae5d02bd --- /dev/null +++ b/script/get-mlperf-automotive-utils/customize.py @@ -0,0 +1,36 @@ +from mlc import utils +from utils import is_true +import os +import sys + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = is_true(env.get('MLC_QUIET', False)) + + utils_path = env['MLC_TMP_CURRENT_SCRIPT_PATH'] + + env['+PYTHONPATH'] = [utils_path] + + submission_checker_dir = os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], "tools", "submission") + + sys.path.append(submission_checker_dir) + sys.path.append(utils_path) + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + + return {'return': 0} diff --git a/script/get-mlperf-automotive-utils/meta.yaml b/script/get-mlperf-automotive-utils/meta.yaml new file mode 100644 index 000000000..bdd5c667b --- /dev/null +++ b/script/get-mlperf-automotive-utils/meta.yaml @@ -0,0 +1,25 @@ +alias: get-mlperf-automotive-utils +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: false +tags: +- get +- mlperf +- automotive +- util +- utils +- functions +uid: c20cfade1c184f83 +deps: + - tags: get,mlperf,automotive,src + names: + - automotive-src +new_env_keys: + - '+PYTHONPATH' + - MLC_MLPERF_INFERENCE_BEVFORMER_PATH + - MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH + - MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH + - MLC_MLPERF_LAST_RELEASE + - MLC_MLPERF_INFERENCE_SOURCE + - MLC_MLPERF_INFERENCE_VERSION + - MLC_MLPERF_INFERENCE_SOURCE_VERSION \ No newline at end of file diff --git a/script/get-mlperf-automotive-utils/mlperf_utils.py b/script/get-mlperf-automotive-utils/mlperf_utils.py new file mode 100644 index 000000000..f7441cedd --- /dev/null +++ b/script/get-mlperf-automotive-utils/mlperf_utils.py @@ -0,0 +1,353 @@ +import sys +import os +import submission_checker as checker +from log_parser import MLPerfLog + + +def get_result_from_log(version, model, scenario, + result_path, mode, automotive_src_version=None): + + config = checker.Config( + version, + None, + ignore_uncommited=False, + skip_power_check=False, + ) + mlperf_model = config.get_mlperf_model(model) + # scenario = checker.SCENARIO_MAPPING[scenario] + + result = '' + power_result = None + valid = {} + if mode == "performance": + # has_power = os.path.exists(os.path.join(result_path, "..", "power")) + version_tuple = None + if automotive_src_version: + version_tuple = tuple(map(int, automotive_src_version.split('.'))) + + if version_tuple and version_tuple >= (4, 1, 22): + result_ = checker.get_performance_metric( + config, mlperf_model, result_path, scenario) + else: + result_ = checker.get_performance_metric( + config, mlperf_model, result_path, scenario, None, None, has_power) + mlperf_log = MLPerfLog( + os.path.join( + result_path, + "mlperf_log_detail.txt")) + if ( + "result_validity" not in mlperf_log.get_keys() + or mlperf_log["result_validity"] != "VALID" + ): + valid['performance'] = False + else: + valid['performance'] = True + + if "stream" in scenario.lower(): + result = result_ / 1000000 # convert to milliseconds + else: + result = result_ + result = str(round(result, 3)) + + # if has_power: + # power_valid, power_metric, scenario, avg_power_efficiency = checker.get_power_metric( + # config, scenario, result_path, True, result_) + # power_result = f"{round(power_metric,3)},{round(avg_power_efficiency,3)}" + # valid['power'] = power_valid + + elif mode == "accuracy" and os.path.exists(os.path.join(result_path, 'accuracy.txt')): + + acc_valid, acc_results, acc_targets, acc_limits = get_accuracy_metric( + config, mlperf_model, result_path) + valid['accuracy'] = acc_valid + + if len(acc_results) == 1: + for acc in acc_results: + result = str(round(float(acc_results[acc]), 5)) + else: + result = '(' + result_list = [] + for i, acc in enumerate(acc_results): + result_list.append(str(round(float(acc_results[acc]), 5))) + result += ", ".join(result_list) + ")" + + return result, valid, power_result + + +def get_accuracy_metric(config, model, path): + + import re + is_valid = False + all_accuracy_valid = True + acc = None + result_acc = None + target = config.get_accuracy_target(model) + acc_upper_limit = config.get_accuracy_upper_limit(model) + patterns = [] + acc_targets = [] + acc_limits = [None] * (len(target) // 2) + up_patterns = [None] * (len(target) // 2) + acc_types = [] + + if acc_upper_limit is not None: + acc_limit_check = True + + for ii in range(0, len(target), 2): + acc_type1, tmp = target[ii:ii + 2] + for i in range(0, len(acc_upper_limit), 2): + acc_type, acc_target = acc_upper_limit[i:i + 2] + if acc_type != acc_type1: + continue + acc_limits[ii // 2] = acc_target + up_patterns[ii // 2] = checker.ACC_PATTERN[acc_type] + + for i in range(0, len(target), 2): + acc_type, acc_target = target[i:i + 2] + acc_types.append(acc_type) + patterns.append(checker.ACC_PATTERN[acc_type]) + acc_targets.append(acc_target) + + acc_seen = [False for _ in acc_targets] + acc_results = {} + with open(os.path.join(path, "accuracy.txt"), "r", encoding="utf-8") as f: + for line in f: + for i, (pattern, acc_target, acc_type) in enumerate( + zip(patterns, acc_targets, acc_types)): + m = re.match(pattern, line) + if m: + acc = m.group(1) + + acc_results[acc_type] = acc + + if acc is not None and float(acc) >= acc_target: + all_accuracy_valid &= True + acc_seen[i] = True + elif acc is not None: + all_accuracy_valid = False + # log.warning("%s accuracy not met: expected=%f, found=%s", path, acc_target, acc) + if i == 0 and acc: + result_acc = acc + acc = None + if acc_upper_limit is not None: + for i, (pattern, acc_limit) in enumerate( + zip(up_patterns, acc_limits)): + if not pattern: + continue + m = re.match(pattern, line) + if m: + acc = m.group(1) + if acc is not None and acc_upper_limit is not None and float( + acc) > acc_limit: + acc_limit_check = False + # log.warning("%s accuracy not met: upper limit=%f, found=%s", path, acc_limit, acc) + acc = None + if all(acc_seen): + break + is_valid = all_accuracy_valid & all(acc_seen) + if acc_upper_limit is not None: + is_valid &= acc_limit_check + + return is_valid, acc_results, acc_targets, acc_limits + + +def get_result_string(version, model, scenario, result_path, has_power, sub_res, + division="open", system_json=None, model_precision="fp32", automotive_src_version=None): + + config = checker.Config( + version, + None, + ignore_uncommited=False, + skip_power_check=False, + ) + mlperf_model = config.get_mlperf_model(model) + performance_path = os.path.join(result_path, "performance", "run_1") + accuracy_path = os.path.join(result_path, "accuracy") + scenario = checker.SCENARIO_MAPPING[scenario.lower()] + + fname = os.path.join(performance_path, "mlperf_log_detail.txt") + mlperf_log = MLPerfLog(fname) + effective_scenario = mlperf_log["effective_scenario"] + inferred = False + result = {} + + version_tuple = None + if automotive_src_version: + version_tuple = tuple(map(int, automotive_src_version.split('.'))) + + if version_tuple and version_tuple >= (4, 1, 22): + performance_result = checker.get_performance_metric( + config, mlperf_model, performance_path, scenario) + else: + performance_result = checker.get_performance_metric( + config, mlperf_model, performance_path, scenario, None, None) + if "stream" in scenario.lower(): + performance_result_ = performance_result / 1000000 # convert to milliseconds + else: + performance_result_ = performance_result + result['performance'] = round(performance_result_, 3) + + if scenario != effective_scenario: + inferred, inferred_result = checker.get_inferred_result( + scenario, effective_scenario, performance_result, mlperf_log, config, False) + + # if has_power: + # is_valid, power_metric, scenario, avg_power_efficiency = checker.get_power_metric( + # config, scenario, performance_path, True, performance_result) + # if "stream" in scenario.lower(): + # power_metric_unit = "milliJoules" + # else: + # power_metric_unit = "Watts" + # power_result_string = f"`Power consumed`: `{round(power_metric, 3)} {power_metric_unit}`, `Power efficiency`: `{round(avg_power_efficiency * 1000, 3)} samples per Joule`" + + # power_result = round(power_metric, 3) + # power_efficiency_result = round(avg_power_efficiency, 3) + # result['power'] = power_result + # result['power_efficiency'] = power_efficiency_result + + # compliance_list = ["TEST01", "TEST04", "TEST06"] + # if division == "closed": + # for test in compliance_list: + # test_path = os.path.join(result_path, test) + # if os.path.exists( + # test_path): # We dont consider missing test folders now - submission checker will do that + # # test_pass = checker.check_compliance_dir(test_path, mlperf_model, scenario, config, "closed", system_json, sub_res) + # test_pass = checker.check_compliance_perf_dir( + # test_path) if test != "TEST06" else True + # if test_pass and test in ["TEST01", "TEST06"]: + # # test_pass = checker.check_compliance_acc_dir(test_path, mlperf_model, config) + # pass # accuracy truncation script is done after submission generation. We assume here that it'll pass + # if test_pass: + # result[test] = "passed" + # else: + # result[test] = "failed" + + acc_valid, acc_results, acc_targets, acc_limits = get_accuracy_metric( + config, mlperf_model, accuracy_path) + + result_field = checker.RESULT_FIELD[effective_scenario] + + performance_result_string = f"`{result_field}`: `{performance_result}`\n" + if inferred: + inferred_result_field = checker.RESULT_FIELD[scenario] + performance_result_string += f"Inferred result: `{inferred_result_field}`: `{inferred_result}` \n" + + accuracy_result_string = '' + accuracy_results = [] + for i, acc in enumerate(acc_results): + accuracy_results.append(str(round(float(acc_results[acc]), 5))) + accuracy_result_string += f"`{acc}`: `{round(float(acc_results[acc]), 5)}`" + if not acc_limits or not acc_limits[i]: + accuracy_result_string += f", Required accuracy for closed division `>= {round(acc_targets[i], 5)}`" + else: + accuracy_result_string += f", Required accuracy for closed division `>= {round(acc_targets[i], 5)}` and `<= {round(acc_limits[i], 5)}`" + accuracy_result_string += "\n" + + if len(accuracy_results) == 1: + accuracy_result = accuracy_results[0] + else: + accuracy_result = "(" + ", ".join(accuracy_results) + ")" + result['accuracy'] = accuracy_result + + result_string = f"\n\n## Results\n" + result_string += f"\nPlatform: {sub_res}\n" + result_string += f"\nModel Precision: {model_precision}\n" + result_string += "\n### Accuracy Results \n" + accuracy_result_string + result_string += "\n### Performance Results \n" + performance_result_string + # if has_power: + # result_string += "\n### Power Results \n" + power_result_string + + return result_string, result + + +def get_result_table(results): + + headers = [ + "Model", + "Scenario", + "Accuracy", + "Throughput", + "Latency (in ms)", + "Power Efficiency (in samples/J)", + "TEST01", + "TEST04"] + table = [] + for model in results: + for scenario in results[model]: + row = [] + row.append(model) + row.append(scenario) + if results[model][scenario].get('accuracy'): + val = str(results[model][scenario]['accuracy']) + if not results[model][scenario].get('accuracy_valid', True): + val = "X " + val + row.append(val) + else: + row.append("-") + + if results[model][scenario].get('performance'): + + if "stream" in scenario.lower(): + if float(results[model][scenario]['performance']) == 0: + row.append("-") + elif scenario.lower() == "singlestream": + val_qps = str( + round( + 1000 / + float( + results[model][scenario]['performance']), + 3)) + if not results[model][scenario].get( + 'performance_valid', True): # we explicitly mark invalid results + val_qps = "X " + val_qps + row.append(val_qps) + elif scenario.lower() == "multistream": + val_qps = str( + round( + 8000 / + float( + results[model][scenario]['performance']), + 3)) + if not results[model][scenario].get( + 'performance_valid', True): + val_qps = "X " + val_qps + row.append(val_qps) + val = str(results[model][scenario]['performance']) + if not results[model][scenario].get( + 'performance_valid', True): + val = "X " + val + row.append(val) + else: + val = str(results[model][scenario]['performance']) + if not results[model][scenario].get( + 'performance_valid', True): + val = "X " + val + row.append(val) + row.append("-") + + # val1 = results[model][scenario].get('TEST01') + # val2 = results[model][scenario].get('TEST05') + # val3 = results[model][scenario].get('TEST04') + + # if results[model][scenario].get('power','') != '': + # row.append(results[model][scenario]['power']) + # if results[model][scenario].get('power_efficiency', '') != '': + # val = str(results[model][scenario]['power_efficiency']) + # if not results[model][scenario].get('power_valid', True): + # val = "X " + val + # row.append(val) + # elif val1 or val3: # Don't output unless there are any further column data + # row.append(None) + + # if val1: + # row.append(val1) + # if val3: + # row.append(val3) + + # else: + # if val3: + # row.append("missing") + # row.append(val3) + + table.append(row) + + return table, headers diff --git a/script/get-mlperf-inference-loadgen/meta.yaml b/script/get-mlperf-inference-loadgen/meta.yaml index ad59163fd..ed75db192 100644 --- a/script/get-mlperf-inference-loadgen/meta.yaml +++ b/script/get-mlperf-inference-loadgen/meta.yaml @@ -24,10 +24,21 @@ deps: - MLC_GIT_CHECKOUT names: - inference-src-loadgen - skip_if_env: + skip_if_any_env: MLC_MLPERF_INFERENCE_LOADGEN_DOWNLOAD: - 'YES' + MLC_INFERENCE_AUTOMOTIVE_REPO: + - 'YES' tags: get,mlcommons,inference,src +- force_env_keys: + - MLC_GIT_URL + - MLC_GIT_CHECKOUT + names: + - automotive-src + enable_if_env: + MLC_INFERENCE_AUTOMOTIVE_REPO: + - 'YES' + tags: get,mlcommons,automotive,src - enable_if_env: MLC_MLPERF_INFERENCE_LOADGEN_DOWNLOAD: - 'YES' @@ -49,7 +60,7 @@ deps: - enable_if_env: MLC_HOST_OS_TYPE: - windows - skip_if_env: + skip_if_any_env: MLC_TMP_MLPERF_INFERENCE_LOADGEN_INSTALL_FROM_PIP: - 'yes' names: @@ -155,7 +166,9 @@ variations: '+ CXXFLAGS': - '-Werror' - '-Wno-unused-parameter' - + automotive: + env: + MLC_INFERENCE_AUTOMOTIVE_REPO: 'YES' versions: custom: add_deps: diff --git a/script/get-preprocessed-dataset-cognata/COPYRIGHT.md b/script/get-preprocessed-dataset-cognata/COPYRIGHT.md new file mode 100644 index 000000000..2d6a2775e --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2023-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-preprocessed-dataset-cognata/README.md b/script/get-preprocessed-dataset-cognata/README.md new file mode 100644 index 000000000..fee3d0ae4 --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/README.md @@ -0,0 +1 @@ +Please see [https://docs.mlcommons.org/cm4mlops/scripts/AI-ML-datasets/get-preprocessed-dataset-kits19](https://docs.mlcommons.org/cm4mlops/scripts/AI-ML-datasets/get-preprocessed-dataset-kits19) for the documentation of this CM script. diff --git a/script/get-preprocessed-dataset-cognata/customize.py b/script/get-preprocessed-dataset-cognata/customize.py new file mode 100644 index 000000000..1e4f0beba --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/customize.py @@ -0,0 +1,24 @@ +from mlc import utils +import os +import shutil +from utils import is_true + + +def preprocess(i): + + env = i['env'] + + if env.get('MLC_NUSCENES_DATASET_TYPE', '') == "prebuilt": + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + + return {'return': 0} + + +def postprocess(i): + env = i['env'] + if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): + env['MLC_PREPROCESSED_DATASET_COGNATA_PATH'] = os.path.join( + env['MLC_PREPROCESSED_DATASET_COGNATA_PATH'], + env['MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME']) + + return {'return': 0} diff --git a/script/get-preprocessed-dataset-cognata/meta.yaml b/script/get-preprocessed-dataset-cognata/meta.yaml new file mode 100644 index 000000000..c8bac0417 --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/meta.yaml @@ -0,0 +1,114 @@ +alias: get-preprocessed-dataset-cognata +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: AI/ML datasets +default_env: + MLC_DATASET: cognata +new_env_keys: +- MLC_PREPROCESSED_DATASET_* +tags: +- get +- dataset +- cognata +- preprocessed +uid: 29b3a984ff444de9 +print_env_at_the_end: + MLC_PREPROCESSED_DATASET_COGNATA_PATH: Preprocessed Cognata dataset path +variations: + validation: + default: true + group: dataset-type + env: + MLC_DATASET_COGNATA_TYPE: validation + calibration: + group: dataset-type + env: + MLC_DATASET_COGNATA_TYPE: calibration + 2d_obj_det: + default: true + group: task + env: + MLC_DATASET_COGNATA_TASK: 2d_object_detection + segmentation: + group: task + env: + MLC_DATASET_COGNATA_TASK: segmentation + validation,2d_obj_det: + env: + MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAMEE: val_2d + MLC_DATASET_COGNATA_TAR_FILENAME: val_2d.tar.gz + MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + calibration,2d_obj_det: + env: + MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_2d + MLC_DATASET_COGNATA_TAR_FILENAME: calib_2d.tar.gz + MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + validation,segmentation: + env: + MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_seg + MLC_DATASET_COGNATA_TAR_FILENAME: val_seg.tar.gz + MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + calibration,segmentation: + env: + MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_seg + MLC_DATASET_COGNATA_TAR_FILENAME: calib_seg.tar.gz + MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + prebuilt: + default: true + group: dataset-src + env: + MLC_NUSCENES_DATASET_TYPE: prebuilt + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + mlc: + group: download-src + default: true + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.mlc-cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + extra_cache_tags: nuscenes,dataset + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + default: true + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + run_inputs: + - variations_list: + - validation,prebuilt,2d_obj_det,rclone,mlc,dry-run + - calibration,prebuilt,2d_obj_det,rclone,mlc,dry-run + - validation,prebuilt,segmentation,rclone,mlc,dry-run + - calibration,prebuilt,segmentation,rclone,mlc,dry-run \ No newline at end of file diff --git a/script/get-preprocessed-dataset-cognata/run.sh b/script/get-preprocessed-dataset-cognata/run.sh new file mode 100644 index 000000000..0c141b8e6 --- /dev/null +++ b/script/get-preprocessed-dataset-cognata/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then + cd "${MLC_PREPROCESSED_DATASET_COGNATA_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd - || exit +fi \ No newline at end of file diff --git a/script/get-preprocessed-dataset-nuscenes/COPYRIGHT.md b/script/get-preprocessed-dataset-nuscenes/COPYRIGHT.md new file mode 100644 index 000000000..2d6a2775e --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2023-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-preprocessed-dataset-nuscenes/README.md b/script/get-preprocessed-dataset-nuscenes/README.md new file mode 100644 index 000000000..fee3d0ae4 --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/README.md @@ -0,0 +1 @@ +Please see [https://docs.mlcommons.org/cm4mlops/scripts/AI-ML-datasets/get-preprocessed-dataset-kits19](https://docs.mlcommons.org/cm4mlops/scripts/AI-ML-datasets/get-preprocessed-dataset-kits19) for the documentation of this CM script. diff --git a/script/get-preprocessed-dataset-nuscenes/customize.py b/script/get-preprocessed-dataset-nuscenes/customize.py new file mode 100644 index 000000000..5236b5c78 --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/customize.py @@ -0,0 +1,36 @@ +from mlc import utils +import os +import shutil +from utils import is_true + + +def preprocess(i): + + env = i['env'] + + if env.get('MLC_NUSCENES_DATASET_TYPE', '') == "prebuilt" and env.get( + 'MLC_PREPROCESSED_DATASET_NUSCENES_PATH', '') == '': + env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + + return {'return': 0} + + +def postprocess(i): + env = i['env'] + + if is_true(env.get('MLC_TMP_REQUIRE_DOWNLOAD', '')): + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'] = os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'], + env['MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME']) + if env.get( + 'MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH', '') != '': + shutil.copy( + os.path.join( + env['MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH'], + env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME']), + os.path.join( + os.path.dirname( + env['MLC_PREPROCESSED_DATASET_NUSCENES_PATH'].rstrip("/")), + env['MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME'])) + + return {'return': 0} diff --git a/script/get-preprocessed-dataset-nuscenes/meta.yaml b/script/get-preprocessed-dataset-nuscenes/meta.yaml new file mode 100644 index 000000000..6436574a6 --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/meta.yaml @@ -0,0 +1,132 @@ +alias: get-preprocessed-dataset-nuscenes +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: AI/ML datasets +default_env: + MLC_DATASET: nuscenes +new_env_keys: +- MLC_PREPROCESSED_DATASET_* +tags: +- get +- dataset +- nuscenes +- preprocessed +uid: 0e403a2861984a4e +print_env_at_the_end: + MLC_PREPROCESSED_DATASET_NUSCENES_PATH: Preprocessed Nuscenes dataset path + MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH: Path containing minimum files for accuracy checker +variations: + validation: + default: true + group: dataset-type + env: + MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME: val_3d + MLC_DATASET_NUSCENES_TAR_FILENAME: val_3d.tar.gz + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> + calibration: + group: dataset-type + env: + MLC_DATASET_NUSCENES_EXTRACTED_FOLDER_NAME: calib_3d + MLC_DATASET_NUSCENES_TAR_FILENAME: calib_3d.tar.gz + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/preprocessed/<<>> + prebuilt: + default: true + group: dataset-src + env: + MLC_NUSCENES_DATASET_TYPE: prebuilt + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_PATH + MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' + mlc: + group: download-src + default: true + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.mlc-nuscenes + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + extra_cache_tags: nuscenes,dataset + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + env: + MLC_DOWNLOAD_SRC: mlcommons + mlc,validation: + env: + MLC_DATASET_NUSCENES_SCENE_PICKLE_FILENAME: scene_lengths.pkl + MLC_DATASET_NUSCENES_ACC_CHECKER_DEP_FILES_TAR_NAME: nuscenes_min.tar.gz + MLC_DATASET_NUSCENES_ACC_REQ_FILE_EXTRACTED_FOLDER_NAME: nuscenes + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + extra_cache_tags: nuscenes,dataset,scene_lengths + force_cache: true + names: + - dae_sl + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_SCENE_LENGTHS_PATH + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/scene_lengths.pkl + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + extra_cache_tags: nuscenes,dataset,accuracy_checker + force_cache: true + names: + - dae_ac + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH + MLC_DOWNLOAD_URL: mlc-nuscenes:nuscenes_dataset/nuscenes_min.tar.gz + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + dae_sl: + tags: _rclone + dae_ac: + tags: _rclone + default: true + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run +tests: + run_inputs: + - variations_list: + - validation,prebuilt,rclone,mlc,dry-run + - calibration,prebuilt,rclone,mlc,dry-run \ No newline at end of file diff --git a/script/get-preprocessed-dataset-nuscenes/run.sh b/script/get-preprocessed-dataset-nuscenes/run.sh new file mode 100644 index 000000000..16337c44d --- /dev/null +++ b/script/get-preprocessed-dataset-nuscenes/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then + cd "${MLC_PREPROCESSED_DATASET_NUSCENES_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd "${MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH}" || exit + for f in *.tar.gz; do + tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + done + cd - || exit +fi \ No newline at end of file diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 4ef3fbd43..afb359ce7 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -1,271 +1,283 @@ -from mlc import utils -import os - - -def preprocess(i): - - os_info = i['os_info'] - - xsep = ';' if os_info['platform'] == 'windows' else ':' - - env = i['env'] - logger = i['automation'].logger - - results_dir = env.get("MLC_MLPERF_ACCURACY_RESULTS_DIR", "") - - if results_dir == "": - logger.error("Please set MLC_MLPERF_ACCURACY_RESULTS_DIR") - return {'return': -1} - - # In fact, we expect only 1 command line here - run_cmds = [] - - if env.get('MLC_MAX_EXAMPLES', '') != '' and env.get( - 'MLC_MLPERF_RUN_STYLE', '') != 'valid': - max_examples_string = " --max_examples " + env['MLC_MAX_EXAMPLES'] - else: - max_examples_string = "" - - results_dir_split = results_dir.split(xsep) - dataset = env['MLC_DATASET'] - regenerate_accuracy_file = env.get( - 'MLC_MLPERF_REGENERATE_ACCURACY_FILE', env.get( - 'MLC_RERUN', False)) - - for result_dir in results_dir_split: - - out_file = os.path.join(result_dir, 'accuracy.txt') - - if os.path.exists(out_file) and ( - os.stat(out_file).st_size != 0) and not regenerate_accuracy_file: - continue - - if dataset == "openimages": - if env.get('MLC_DATASET_PATH_ROOT', '') != '': - dataset_dir = env['MLC_DATASET_PATH_ROOT'] - if 'DATASET_ANNOTATIONS_FILE_PATH' in env: - del (env['DATASET_ANNOTATIONS_FILE_PATH']) - else: - env['DATASET_ANNOTATIONS_FILE_PATH'] = env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] - dataset_dir = os.getcwd() # not used, just to keep the script happy - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " " + "'" + os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", - "accuracy-openimages.py") + "'" + " --mlperf-accuracy-file " + "'" + os.path.join(result_dir, - "mlperf_log_accuracy.json") + "'" + " --openimages-dir " + "'" + dataset_dir + "'" + " --verbose > " + "'" + \ - out_file + "'" - - elif dataset == "imagenet": - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", - "accuracy-imagenet.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, - "mlperf_log_accuracy.json") + "' --imagenet-val-file '" + os.path.join(env['MLC_DATASET_AUX_PATH'], - "val.txt") + "' --dtype " + env.get('MLC_ACCURACY_DTYPE', "float32") + " > '" + out_file + "'" - - elif dataset == "squad": - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_BERT_PATH'], - "accuracy-squad.py") + "' --val_data '" + env['MLC_DATASET_SQUAD_VAL_PATH'] + \ - "' --log_file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --vocab_file '" + env['MLC_ML_MODEL_BERT_VOCAB_FILE_WITH_PATH'] + \ - "' --out_file '" + os.path.join(result_dir, 'predictions.json') + \ - "' --features_cache_file '" + os.path.join(env['MLC_MLPERF_INFERENCE_BERT_PATH'], 'eval_features.pickle') + \ - "' --output_dtype " + env['MLC_ACCURACY_DTYPE'] + env.get( - 'MLC_OUTPUT_TRANSPOSED', '') + max_examples_string + " > '" + out_file + "'" - - elif dataset == "cnndm": - if env.get('MLC_MLPERF_IMPLEMENTATION', '') == 'intel': - accuracy_checker_file = env['MLC_MLPERF_INFERENCE_INTEL_GPTJ_ACCURACY_FILE_WITH_PATH'] - env['+PYTHONPATH'] = [os.path.dirname(env['MLC_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_FILE_WITH_PATH'])] + [ - os.path.dirname(env['MLC_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_ITEM_FILE_WITH_PATH'])] + env['+PYTHONPATH'] - suffix_string = " --model-name-or-path '" + \ - env['GPTJ_CHECKPOINT_PATH'] + "'" - else: - accuracy_checker_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "gpt-j", - "evaluation.py") - suffix_string = " --dtype " + \ - env.get('MLC_ACCURACY_DTYPE', "float32") - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --dataset-file '" + \ - env['MLC_DATASET_EVAL_PATH'] + "'" + \ - suffix_string + " > '" + out_file + "'" - - elif dataset == "openorca": - accuracy_checker_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "llama2-70b", - "evaluate-accuracy.py") - if env.get('MLC_VLLM_SERVER_MODEL_NAME', '') == '': - checkpoint_path = env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] - else: - checkpoint_path = env['MLC_VLLM_SERVER_MODEL_NAME'] - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + checkpoint_path + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --dataset-file '" + env['MLC_DATASET_PREPROCESSED_PATH'] + "'" + " --dtype " + env.get( - 'MLC_ACCURACY_DTYPE', "int32") + " > '" + out_file + "'" - - elif dataset == "openorca-gsm8k-mbxp-combined": - accuracy_checker_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b", - "evaluate-accuracy.py") - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['MIXTRAL_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --dataset-file '" + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] + "'" + \ - " --dtype " + env.get('MLC_ACCURACY_DTYPE', - "float32") + " > '" + out_file + "'" - - elif dataset == "coco2014": - env['+PYTHONPATH'] = [ - os.path.join( - env['MLC_MLPERF_INFERENCE_SOURCE'], - "text_to_image", - "tools"), - os.path.join( - env['MLC_MLPERF_INFERENCE_SOURCE'], - "text_to_image", - "tools", - "fid")] - extra_options = "" - - if env.get('MLC_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += ( - f""" --statistics-path '{ - env['MLC_SDXL_STATISTICS_FILE_PATH']}'""" - ) - - if env.get('MLC_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += ( - f""" --compliance-images-path '{ - env['MLC_SDXL_COMPLIANCE_IMAGES_PATH']}' """ - ) - else: - extra_options += f""" --compliance-images-path '{ - os.path.join( - result_dir, "images")}' """ - - if env.get('MLC_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += ( - f" --ids-path '{env['MLC_COCO2014_SAMPLE_ID_PATH']}' " - ) - - if env.get('MLC_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += ( - f" --device '{env['MLC_SDXL_ACCURACY_RUN_DEVICE']}' " - ) - - # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", - "accuracy_coco.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --caption-path '" + os.path.join( - env['MLC_MLPERF_INFERENCE_SOURCE'], - "text_to_image", - "coco2014", - "captions", - "captions_source.tsv") + "'" + extra_options + " > '" + out_file + "'" - - elif dataset == "kits19": - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_3DUNET_PATH'], - "accuracy_kits.py") + \ - "' --preprocessed_data_dir '" + env['MLC_DATASET_PREPROCESSED_PATH'] +\ - "' --postprocessed_data_dir '" + result_dir +\ - "' --log_file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ - "' --output_dtype " + \ - env['MLC_ACCURACY_DTYPE'] + " > '" + out_file + "'" - - elif dataset == "librispeech": - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_RNNT_PATH'], - "accuracy_eval.py") + \ - "' --dataset_dir '" + os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], "..") +\ - "' --manifest '" + env['MLC_DATASET_PREPROCESSED_JSON'] +\ - "' --log_dir '" + result_dir + \ - "' --output_dtype " + \ - env['MLC_ACCURACY_DTYPE'] + " > '" + out_file + "'" - - elif dataset == "terabyte": - extra_options = "" - if env.get('MLC_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += ( - f""" --aggregation-trace-file '{ - env['MLC_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' """ - ) - if env.get('MLC_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += ( - f""" --day-23-file '{ - env['MLC_DLRM_V2_DAY23_FILE_PATH']}' """ - ) - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", - "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, - "mlperf_log_accuracy.json") + "'" + extra_options + \ - " --dtype " + env.get('MLC_ACCURACY_DTYPE', - "float32") + " > '" + out_file + "'" - - elif dataset == "igbh": - if env.get('MLC_DATASET_IGBH_SIZE', '') == '': - if env.get('MLC_MLPERF_SUBMISSION_GENERATION_STYLE', - '') == "full": - env['MLC_DATASET_IGBH_SIZE'] = "full" - else: - env['MLC_DATASET_IGBH_SIZE'] = "tiny" - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "graph", "R-GAT", "tools", "accuracy_igbh.py") + "' --mlperf-accuracy-file '" + os.path.join( - result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['MLC_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['MLC_DATASET_IGBH_SIZE'] + "' --output-file '" + out_file + "'" - - elif dataset == "dataset_llama3": - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "llama3.1-405b", "evaluate-accuracy.py") + "' --checkpoint-path '" + env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join( - result_dir, "mlperf_log_accuracy.json") + "' --dtype '" + env['MLC_ACCURACY_DTYPE'] + "' --dataset-file '" + env['MLC_DATASET_LLAMA3_PATH'] + "' > '" + out_file + "'" - - elif dataset == "waymo": - CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "automotive", "3d-object-detection", "accuracy_waymo.py") + "' --mlperf-accuracy-file '" + os.path.join( - result_dir, "mlperf_log_accuracy.json") + "' --waymo-dir '" + env['MLC_DATASET_WAYMO_PATH'] + "' > '" + out_file + "'" - - else: - return {'return': 1, 'error': 'Unsupported dataset'} - - run_cmds.append(CMD) - - if os_info['platform'] == 'windows': - env['MLC_RUN_CMDS'] = ( - '\n'.join(run_cmds)).replace( - "'", - '"').replace( - '>', - '^>') - else: - env['MLC_RUN_CMDS'] = "??".join(run_cmds) - - return {'return': 0} - - -def postprocess(i): - - os_info = i['os_info'] - env = i['env'] - state = i['state'] - logger = i['automation'].logger - xsep = ';' if os_info['platform'] == 'windows' else ':' - - results_dir = env.get("MLC_MLPERF_ACCURACY_RESULTS_DIR", "") - - results_dir_split = results_dir.split(xsep) - - for result_dir in results_dir_split: - accuracy_file = os.path.join(result_dir, "accuracy.txt") - - if os.path.exists(accuracy_file): - logger.info('') - logger.info('Accuracy file: {}'.format(accuracy_file)) - logger.info('') - - x = '' - with open(accuracy_file, "r") as fp: - x = fp.read() - - if x != '': - logger.info(f"{x}") - - # Trying to extract accuracy dict - for y in x.split('\n'): - if y.startswith('{') and y.endswith('}'): - - import json - - try: - z = json.loads(y) - state['app_mlperf_inference_accuracy'] = z - - break - except ValueError as e: - pass - - logger.info('') - return {'return': 0} +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + xsep = ';' if os_info['platform'] == 'windows' else ':' + + env = i['env'] + logger = i['automation'].logger + + results_dir = env.get("MLC_MLPERF_ACCURACY_RESULTS_DIR", "") + + if results_dir == "": + logger.error("Please set MLC_MLPERF_ACCURACY_RESULTS_DIR") + return {'return': -1} + + # In fact, we expect only 1 command line here + run_cmds = [] + + if env.get('MLC_MAX_EXAMPLES', '') != '' and env.get( + 'MLC_MLPERF_RUN_STYLE', '') != 'valid': + max_examples_string = " --max_examples " + env['MLC_MAX_EXAMPLES'] + else: + max_examples_string = "" + + results_dir_split = results_dir.split(xsep) + dataset = env['MLC_DATASET'] + regenerate_accuracy_file = env.get( + 'MLC_MLPERF_REGENERATE_ACCURACY_FILE', env.get( + 'MLC_RERUN', False)) + + for result_dir in results_dir_split: + + out_file = os.path.join(result_dir, 'accuracy.txt') + + if os.path.exists(out_file) and ( + os.stat(out_file).st_size != 0) and not regenerate_accuracy_file: + continue + + if dataset == "openimages": + if env.get('MLC_DATASET_PATH_ROOT', '') != '': + dataset_dir = env['MLC_DATASET_PATH_ROOT'] + if 'DATASET_ANNOTATIONS_FILE_PATH' in env: + del (env['DATASET_ANNOTATIONS_FILE_PATH']) + else: + env['DATASET_ANNOTATIONS_FILE_PATH'] = env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] + dataset_dir = os.getcwd() # not used, just to keep the script happy + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " " + "'" + os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", + "accuracy-openimages.py") + "'" + " --mlperf-accuracy-file " + "'" + os.path.join(result_dir, + "mlperf_log_accuracy.json") + "'" + " --openimages-dir " + "'" + dataset_dir + "'" + " --verbose > " + "'" + \ + out_file + "'" + + elif dataset == "imagenet": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", + "accuracy-imagenet.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, + "mlperf_log_accuracy.json") + "' --imagenet-val-file '" + os.path.join(env['MLC_DATASET_AUX_PATH'], + "val.txt") + "' --dtype " + env.get('MLC_ACCURACY_DTYPE', "float32") + " > '" + out_file + "'" + + elif dataset == "squad": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_BERT_PATH'], + "accuracy-squad.py") + "' --val_data '" + env['MLC_DATASET_SQUAD_VAL_PATH'] + \ + "' --log_file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --vocab_file '" + env['MLC_ML_MODEL_BERT_VOCAB_FILE_WITH_PATH'] + \ + "' --out_file '" + os.path.join(result_dir, 'predictions.json') + \ + "' --features_cache_file '" + os.path.join(env['MLC_MLPERF_INFERENCE_BERT_PATH'], 'eval_features.pickle') + \ + "' --output_dtype " + env['MLC_ACCURACY_DTYPE'] + env.get( + 'MLC_OUTPUT_TRANSPOSED', '') + max_examples_string + " > '" + out_file + "'" + + elif dataset == "cnndm": + if env.get('MLC_MLPERF_IMPLEMENTATION', '') == 'intel': + accuracy_checker_file = env['MLC_MLPERF_INFERENCE_INTEL_GPTJ_ACCURACY_FILE_WITH_PATH'] + env['+PYTHONPATH'] = [os.path.dirname(env['MLC_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_FILE_WITH_PATH'])] + [ + os.path.dirname(env['MLC_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_ITEM_FILE_WITH_PATH'])] + env['+PYTHONPATH'] + suffix_string = " --model-name-or-path '" + \ + env['GPTJ_CHECKPOINT_PATH'] + "'" + else: + accuracy_checker_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "gpt-j", + "evaluation.py") + suffix_string = " --dtype " + \ + env.get('MLC_ACCURACY_DTYPE', "float32") + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --dataset-file '" + \ + env['MLC_DATASET_EVAL_PATH'] + "'" + \ + suffix_string + " > '" + out_file + "'" + + elif dataset == "openorca": + accuracy_checker_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "llama2-70b", + "evaluate-accuracy.py") + if env.get('MLC_VLLM_SERVER_MODEL_NAME', '') == '': + checkpoint_path = env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] + else: + checkpoint_path = env['MLC_VLLM_SERVER_MODEL_NAME'] + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + checkpoint_path + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --dataset-file '" + env['MLC_DATASET_PREPROCESSED_PATH'] + "'" + " --dtype " + env.get( + 'MLC_ACCURACY_DTYPE', "int32") + " > '" + out_file + "'" + + elif dataset == "openorca-gsm8k-mbxp-combined": + accuracy_checker_file = os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b", + "evaluate-accuracy.py") + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['MIXTRAL_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --dataset-file '" + env['MLC_DATASET_MIXTRAL_PREPROCESSED_PATH'] + "'" + \ + " --dtype " + env.get('MLC_ACCURACY_DTYPE', + "float32") + " > '" + out_file + "'" + + elif dataset == "coco2014": + env['+PYTHONPATH'] = [ + os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], + "text_to_image", + "tools"), + os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], + "text_to_image", + "tools", + "fid")] + extra_options = "" + + if env.get('MLC_SDXL_STATISTICS_FILE_PATH', '') != '': + extra_options += ( + f""" --statistics-path '{ + env['MLC_SDXL_STATISTICS_FILE_PATH']}'""" + ) + + if env.get('MLC_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': + extra_options += ( + f""" --compliance-images-path '{ + env['MLC_SDXL_COMPLIANCE_IMAGES_PATH']}' """ + ) + else: + extra_options += f""" --compliance-images-path '{ + os.path.join( + result_dir, "images")}' """ + + if env.get('MLC_COCO2014_SAMPLE_ID_PATH', '') != '': + extra_options += ( + f" --ids-path '{env['MLC_COCO2014_SAMPLE_ID_PATH']}' " + ) + + if env.get('MLC_SDXL_ACCURACY_RUN_DEVICE', '') != '': + extra_options += ( + f" --device '{env['MLC_SDXL_ACCURACY_RUN_DEVICE']}' " + ) + + # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['MLC_DATASET_ANNOTATIONS_FILE_PATH'] + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", + "accuracy_coco.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --caption-path '" + os.path.join( + env['MLC_MLPERF_INFERENCE_SOURCE'], + "text_to_image", + "coco2014", + "captions", + "captions_source.tsv") + "'" + extra_options + " > '" + out_file + "'" + + elif dataset == "kits19": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_3DUNET_PATH'], + "accuracy_kits.py") + \ + "' --preprocessed_data_dir '" + env['MLC_DATASET_PREPROCESSED_PATH'] +\ + "' --postprocessed_data_dir '" + result_dir +\ + "' --log_file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --output_dtype " + \ + env['MLC_ACCURACY_DTYPE'] + " > '" + out_file + "'" + + elif dataset == "librispeech": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_RNNT_PATH'], + "accuracy_eval.py") + \ + "' --dataset_dir '" + os.path.join(env['MLC_DATASET_PREPROCESSED_PATH'], "..") +\ + "' --manifest '" + env['MLC_DATASET_PREPROCESSED_JSON'] +\ + "' --log_dir '" + result_dir + \ + "' --output_dtype " + \ + env['MLC_ACCURACY_DTYPE'] + " > '" + out_file + "'" + + elif dataset == "terabyte": + extra_options = "" + if env.get('MLC_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': + extra_options += ( + f""" --aggregation-trace-file '{ + env['MLC_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' """ + ) + if env.get('MLC_DLRM_V2_DAY23_FILE_PATH', '') != '': + extra_options += ( + f""" --day-23-file '{ + env['MLC_DLRM_V2_DAY23_FILE_PATH']}' """ + ) + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", + "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, + "mlperf_log_accuracy.json") + "'" + extra_options + \ + " --dtype " + env.get('MLC_ACCURACY_DTYPE', + "float32") + " > '" + out_file + "'" + + elif dataset == "igbh": + if env.get('MLC_DATASET_IGBH_SIZE', '') == '': + if env.get('MLC_MLPERF_SUBMISSION_GENERATION_STYLE', + '') == "full": + env['MLC_DATASET_IGBH_SIZE'] = "full" + else: + env['MLC_DATASET_IGBH_SIZE'] = "tiny" + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "graph", "R-GAT", "tools", "accuracy_igbh.py") + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['MLC_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['MLC_DATASET_IGBH_SIZE'] + "' --output-file '" + out_file + "'" + + elif dataset == "dataset_llama3": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "llama3.1-405b", "evaluate-accuracy.py") + "' --checkpoint-path '" + env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --dtype '" + env['MLC_ACCURACY_DTYPE'] + "' --dataset-file '" + env['MLC_DATASET_LLAMA3_PATH'] + "' > '" + out_file + "'" + + elif dataset == "waymo": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "automotive", "3d-object-detection", "accuracy_waymo.py") + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --waymo-dir '" + env['MLC_DATASET_WAYMO_PATH'] + "' > '" + out_file + "'" + + elif dataset == "nuscenes": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_BEVFORMER_PATH'], "accuracy_nuscenes_cpu.py") + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --nuscenes-dir '" + env['MLC_PREPROCESSED_DATASET_NUSCENES_ACC_CHECKER_MIN_FILES_PATH'] + "' --config '" + os.path.join(env['MLC_MLPERF_INFERENCE_BEVFORMER_PATH'], "projects" + "configs" + "bevformer" + "bevformer_tiny.py") + "' > '" + out_file + "'" + + elif dataset == "cognata_ssd": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SSD_RESNET50_PATH'], "accuracy_cognata.py") + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['MLC_PREPROCESSED_DATASET_COGNATA_PATH'] + "' --dataset cognata --config '" + "baseline_8MP_ss_scales_fm1_5x5_all" + "' > '" + out_file + "'" + + elif dataset == "cognata_deeplab": + CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_DEEPLABV3PLUS_PATH'], "accuracy_cognata.py") + "' --mlperf-accuracy-file '" + os.path.join( + result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['MLC_PREPROCESSED_DATASET_COGNATA_PATH'] + "' > '" + out_file + "'" + + else: + return {'return': 1, 'error': 'Unsupported dataset'} + + run_cmds.append(CMD) + + if os_info['platform'] == 'windows': + env['MLC_RUN_CMDS'] = ( + '\n'.join(run_cmds)).replace( + "'", + '"').replace( + '>', + '^>') + else: + env['MLC_RUN_CMDS'] = "??".join(run_cmds) + + return {'return': 0} + + +def postprocess(i): + + os_info = i['os_info'] + env = i['env'] + state = i['state'] + logger = i['automation'].logger + xsep = ';' if os_info['platform'] == 'windows' else ':' + + results_dir = env.get("MLC_MLPERF_ACCURACY_RESULTS_DIR", "") + + results_dir_split = results_dir.split(xsep) + + for result_dir in results_dir_split: + accuracy_file = os.path.join(result_dir, "accuracy.txt") + + if os.path.exists(accuracy_file): + logger.info('') + logger.info('Accuracy file: {}'.format(accuracy_file)) + logger.info('') + + x = '' + with open(accuracy_file, "r") as fp: + x = fp.read() + + if x != '': + logger.info(f"{x}") + + # Trying to extract accuracy dict + for y in x.split('\n'): + if y.startswith('{') and y.endswith('}'): + + import json + + try: + z = json.loads(y) + state['app_mlperf_inference_accuracy'] = z + + break + except ValueError as e: + pass + + logger.info('') + return {'return': 0} diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index cd14ef67a..458f740fa 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -273,3 +273,54 @@ variations: env: MLC_DATASET: waymo group: dataset + nuscenes: + deps: + - tags: get,preprocessed,dataset,nuscenes + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-mlcommons-nuscenes + - tags: get,ml-model,bevformer + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-bevformer + env: + MLC_DATASET: nuscenes + group: dataset + cognata_ssd: + deps: + - tags: get,preprocessed,dataset,cognata,_mlc + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-mlcommons-cognata-ssd-resnet50 + - tags: get,ml-model,ssd,resnet50,_mlc,_rclone + skip_if_any_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-ssd-resnet50 + env: + MLC_DATASET: cognata_ssd + group: dataset + cognata_deeplab: + deps: + - tags: get,preprocessed,dataset,cognata,_mlc,_segmentation + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - preprocessed-dataset-mlcommons-cognata-deeplabv3-plus + - tags: get,ml-model,deeplabv3-plus + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + names: + - ml-model-deeplabv3-plus + env: + MLC_DATASET: cognata_deeplab + group: dataset diff --git a/script/run-mlperf-automotive-app/customize.py b/script/run-mlperf-automotive-app/customize.py index c3dda9c7a..10983fc24 100644 --- a/script/run-mlperf-automotive-app/customize.py +++ b/script/run-mlperf-automotive-app/customize.py @@ -116,7 +116,8 @@ def preprocess(i): test_list = [] - variation_implementation = "_" + \ + variation_benchmark_version = "_" + env["MLC_MLPERF_INFERENCE_VERSION"] + variation_implementation = ",_" + \ env.get("MLC_MLPERF_IMPLEMENTATION", "reference") variation_model = ",_" + env["MLC_MLPERF_MODEL"] variation_backend = ",_" + \ @@ -135,7 +136,7 @@ def preprocess(i): else: variation_quantization_string = "" - tags = "app,abtf-inference," + variation_implementation + variation_model + variation_backend + variation_device + \ + tags = "app,abtf-inference," + variation_benchmark_version + variation_implementation + variation_model + variation_backend + variation_device + \ variation_run_style + variation_reproducibility + \ variation_quantization_string + power_variation verbose = inp.get('v', False) @@ -151,37 +152,37 @@ def preprocess(i): add_deps_recursive[key] = adr_from_meta[key] if env.get('MLC_MLPERF_LOADGEN_MAX_BATCHSIZE', '') != '': - if not add_deps_recursive.get('mlperf-inference-implementation', {}): - add_deps_recursive['mlperf-inference-implementation'] = {} - if add_deps_recursive['mlperf-inference-implementation'].get( + if not add_deps_recursive.get('abtf-inference-implementation', {}): + add_deps_recursive['abtf-inference-implementation'] = {} + if add_deps_recursive['abtf-inference-implementation'].get( 'tags', '') == '': - add_deps_recursive['mlperf-inference-implementation']['tags'] = '' + add_deps_recursive['abtf-inference-implementation']['tags'] = '' else: - add_deps_recursive['mlperf-inference-implementation']['tags'] += ',' - add_deps_recursive['mlperf-inference-implementation']['tags'] += "_batch_size." + \ + add_deps_recursive['abtf-inference-implementation']['tags'] += ',' + add_deps_recursive['abtf-inference-implementation']['tags'] += "_batch_size." + \ env['MLC_MLPERF_LOADGEN_MAX_BATCHSIZE'] if env.get('MLC_MLPERF_INFERENCE_SUT_VARIATION', '') != '': - if not add_deps_recursive.get('mlperf-inference-implementation', {}): - add_deps_recursive['mlperf-inference-implementation'] = {} - if add_deps_recursive['mlperf-inference-implementation'].get( + if not add_deps_recursive.get('abtf-inference-implementation', {}): + add_deps_recursive['abtf-inference-implementation'] = {} + if add_deps_recursive['abtf-inference-implementation'].get( 'tags', '') == '': - add_deps_recursive['mlperf-inference-implementation']['tags'] = '' + add_deps_recursive['abtf-inference-implementation']['tags'] = '' else: - add_deps_recursive['mlperf-inference-implementation']['tags'] += ',' - add_deps_recursive['mlperf-inference-implementation']['tags'] += "_" + \ + add_deps_recursive['abtf-inference-implementation']['tags'] += ',' + add_deps_recursive['abtf-inference-implementation']['tags'] += "_" + \ env['MLC_MLPERF_INFERENCE_SUT_VARIATION'] if env.get('MLC_NETWORK_LOADGEN', '') != '': - if not add_deps_recursive.get('mlperf-inference-implementation', {}): - add_deps_recursive['mlperf-inference-implementation'] = {} + if not add_deps_recursive.get('abtf-inference-implementation', {}): + add_deps_recursive['abtf-inference-implementation'] = {} network_variation_tag = f"_network-{env['MLC_NETWORK_LOADGEN']}" - if add_deps_recursive['mlperf-inference-implementation'].get( + if add_deps_recursive['abtf-inference-implementation'].get( 'tags', '') == '': - add_deps_recursive['mlperf-inference-implementation']['tags'] = '' + add_deps_recursive['abtf-inference-implementation']['tags'] = '' else: - add_deps_recursive['mlperf-inference-implementation']['tags'] += ',' - add_deps_recursive['mlperf-inference-implementation']['tags'] += network_variation_tag + add_deps_recursive['abtf-inference-implementation']['tags'] += ',' + add_deps_recursive['abtf-inference-implementation']['tags'] += network_variation_tag if env.get('MLC_OUTPUT_FOLDER_NAME', '') == '': env['MLC_OUTPUT_FOLDER_NAME'] = env['MLC_MLPERF_RUN_STYLE'] + "_results" @@ -290,9 +291,9 @@ def preprocess(i): if state.get("mlc-mlperf-inference-results"): # print(state["mlc-mlperf-inference-results"]) for sut in state["mlc-mlperf-inference-results"]: # only one sut will be there - # Better to do this in a stand alone CM script with proper deps but + # Better to do this in a stand alone MLC script with proper deps but # currently we manage this by modifying the sys path of the python - # executing CM + # executing MLC import mlperf_utils # noqa logger.info(f"{sut}") diff --git a/script/run-mlperf-automotive-app/meta.yaml b/script/run-mlperf-automotive-app/meta.yaml index c70ac9e97..74b9e34c9 100644 --- a/script/run-mlperf-automotive-app/meta.yaml +++ b/script/run-mlperf-automotive-app/meta.yaml @@ -43,6 +43,7 @@ input_mapping: save_console_log: MLC_SAVE_CONSOLE_LOG execution_mode: MLC_MLPERF_RUN_STYLE find_performance: MLC_MLPERF_FIND_PERFORMANCE_MODE + framework: MLC_MLPERF_BACKEND gh_token: MLC_GH_TOKEN gpu_name: MLC_NVIDIA_GPU_NAME hw_name: MLC_HW_NAME @@ -105,8 +106,8 @@ deps: skip_if_env: MLC_MLPERF_USE_DOCKER: [ on ] - names: - - inference-src - tags: get,mlcommons,inference,src + - automotive-src + tags: get,mlcommons,automotive,src skip_if_env: MLC_MLPERF_USE_DOCKER: [ on ] - tags: get,sut,description @@ -121,7 +122,7 @@ deps: skip_if_env: OUTPUT_BASE_DIR: [ on ] - tags: install,pip-package,for-mlc-python,_package.tabulate -- tags: get,mlperf,inference,utils +- tags: get,mlperf,automotive,utils skip_if_env: MLC_MLPERF_USE_DOCKER: [ on ] @@ -214,6 +215,11 @@ variations: compiler: tags: gcc group: benchmark-version + + v0.5: + group: benchmark-version + env: + MLC_MLPERF_INFERENCE_VERSION: v0.5 performance-and-accuracy: default: true