From 4897ededd5fe085127035a09da0a74c752f6a745 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 13 Dec 2024 07:21:09 +0000 Subject: [PATCH 01/17] [Automated Commit] Format Codebase --- script/get-docker/customize.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/script/get-docker/customize.py b/script/get-docker/customize.py index 30ccf627d..fbf0a5bbf 100644 --- a/script/get-docker/customize.py +++ b/script/get-docker/customize.py @@ -52,9 +52,8 @@ def detect_version(i): if "podman" in r['string'].lower(): tool = "podman" - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return': 0, 'version': version, "tool":tool} + return {'return': 0, 'version': version, "tool": tool} def postprocess(i): @@ -76,7 +75,7 @@ def postprocess(i): env['CM_DOCKER_CACHE_TAGS'] = 'version-' + version env['CM_DOCKER_VERSION'] = version - + env['CM_CONTAINER_TOOL'] = tool return {'return': 0, 'version': version} From 51d4fdf78a0cad333fc7e7dd25176369b8d7ccf1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:07:47 +0000 Subject: [PATCH 02/17] Fixes for rgat submission generation --- script/get-ml-model-rgat/_cm.yaml | 1 + script/process-mlperf-accuracy/customize.py | 24 ++++++++++++++------- script/run-mlperf-inference-app/_cm.yaml | 2 ++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/script/get-ml-model-rgat/_cm.yaml b/script/get-ml-model-rgat/_cm.yaml index d7615acd2..c4ecc56e0 100644 --- a/script/get-ml-model-rgat/_cm.yaml +++ b/script/get-ml-model-rgat/_cm.yaml @@ -63,4 +63,5 @@ variations: group: download-tool rclone,fp32: env: + CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-rclone CM_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/R-GAT/RGAT.pt diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index f1d8b7874..254c51772 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -118,18 +118,24 @@ def preprocess(i): extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += f" --statistics-path '{env['CM_SDXL_STATISTICS_FILE_PATH']}' " + extra_options += f" --statistics-path '{ + env['CM_SDXL_STATISTICS_FILE_PATH']}' " if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += f" --compliance-images-path '{env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " + extra_options += f" --compliance-images-path '{ + env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " else: - extra_options += f""" --compliance-images-path '{os.path.join(result_dir, "images")}' """ + extra_options += f""" --compliance-images-path '{ + os.path.join( + result_dir, "images")}' """ if env.get('CM_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += f" --ids-path '{env['CM_COCO2014_SAMPLE_ID_PATH']}' " + extra_options += f" --ids-path '{ + env['CM_COCO2014_SAMPLE_ID_PATH']}' " if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " + extra_options += f" --device '{ + env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", @@ -162,9 +168,11 @@ def preprocess(i): elif dataset == "terabyte": extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += f" --aggregation-trace-file '{env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + extra_options += f" --aggregation-trace-file '{ + env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += f" --day-23-file '{env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + extra_options += f" --day-23-file '{ + env['CM_DLRM_V2_DAY23_FILE_PATH']}' " CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + "'" + extra_options + \ @@ -179,7 +187,7 @@ def preprocess(i): else: env['CM_DATASET_IGBH_SIZE'] = "tiny" CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "graph", "R-GAT", "tools", "accuracy_igbh.py") + "' --mlperf-accuracy-file '" + os.path.join( - result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['CM_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['CM_DATASET_IGBH_SIZE'] + "' > '" + out_file + "'" + result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['CM_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['CM_DATASET_IGBH_SIZE'] + "' --output-file '" + out_file + "'" else: return {'return': 1, 'error': 'Unsupported dataset'} diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 689aaabc3..12b57a6f7 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -344,6 +344,7 @@ variations: env: CM_MLPERF_INFERENCE_VERSION: '4.1' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.1_default + CM_MLPERF_SUBMISSION_CHECKER_VERSION: v4.1 adr: get-mlperf-inference-results-dir: tags: _version.r4_1 @@ -357,6 +358,7 @@ variations: env: CM_MLPERF_INFERENCE_VERSION: '5.0-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r5.0-dev_default + CM_MLPERF_SUBMISSION_CHECKER_VERSION: v5.0 group: benchmark-version adr: get-mlperf-inference-results-dir: From 7ecb64ae71ad276d7e30614229d0efbd57b23b82 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 13 Dec 2024 09:08:05 +0000 Subject: [PATCH 03/17] [Automated Commit] Format Codebase --- script/process-mlperf-accuracy/customize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 254c51772..579b1eedd 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -118,11 +118,11 @@ def preprocess(i): extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += f" --statistics-path '{ + extra_options += f" - -statistics - path '{ env['CM_SDXL_STATISTICS_FILE_PATH']}' " if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += f" --compliance-images-path '{ + extra_options += f" - -compliance - images - path '{ env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " else: extra_options += f""" --compliance-images-path '{ @@ -130,11 +130,11 @@ def preprocess(i): result_dir, "images")}' """ if env.get('CM_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += f" --ids-path '{ + extra_options += f" - -ids - path '{ env['CM_COCO2014_SAMPLE_ID_PATH']}' " if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += f" --device '{ + extra_options += f" - -device '{ env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] @@ -168,10 +168,10 @@ def preprocess(i): elif dataset == "terabyte": extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += f" --aggregation-trace-file '{ + extra_options += f" - -aggregation - trace - file '{ env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += f" --day-23-file '{ + extra_options += f" - -day - 23 - file '{ env['CM_DLRM_V2_DAY23_FILE_PATH']}' " CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, From 77da71d08c6ba14460029f741d4bf8ce304811da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:19:34 +0000 Subject: [PATCH 04/17] Fix issue with autoformat --- script/process-mlperf-accuracy/customize.py | 34 +++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 579b1eedd..0926f332f 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -118,24 +118,30 @@ def preprocess(i): extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += f" - -statistics - path '{ - env['CM_SDXL_STATISTICS_FILE_PATH']}' " + extra_options += ( + f""" --statistics-path '{ + env['CM_SDXL_STATISTICS_FILE_PATH']}'""" + ) if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += f" - -compliance - images - path '{ - env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " + extra_options += ( + f""" --compliance-images-path '{ + env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' """ + ) else: extra_options += f""" --compliance-images-path '{ os.path.join( result_dir, "images")}' """ if env.get('CM_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += f" - -ids - path '{ - env['CM_COCO2014_SAMPLE_ID_PATH']}' " + extra_options += ( + f" --ids-path '{env['CM_COCO2014_SAMPLE_ID_PATH']}' " + ) if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += f" - -device '{ - env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " + extra_options += ( + f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " + ) # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", @@ -168,11 +174,15 @@ def preprocess(i): elif dataset == "terabyte": extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += f" - -aggregation - trace - file '{ - env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + extra_options += ( + f" --aggregation-trace-file '{ + env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + ) if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += f" - -day - 23 - file '{ - env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + extra_options += ( + f" --day-23-file '{ + env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + ) CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + "'" + extra_options + \ From c6ba03982103a5ded10b4cef5ebf4d361bfb9ec0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 13 Dec 2024 09:19:54 +0000 Subject: [PATCH 05/17] [Automated Commit] Format Codebase --- script/process-mlperf-accuracy/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 0926f332f..698227d86 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -175,12 +175,12 @@ def preprocess(i): extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': extra_options += ( - f" --aggregation-trace-file '{ + f" - -aggregation - trace - file '{ env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " ) if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': extra_options += ( - f" --day-23-file '{ + f" - -day - 23 - file '{ env['CM_DLRM_V2_DAY23_FILE_PATH']}' " ) CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", From 85b975a6d4f1831b4d5b9966931df5f5ebe1a0a2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:22:55 +0000 Subject: [PATCH 06/17] Fix issue with autoformat --- script/process-mlperf-accuracy/customize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 698227d86..21569fd57 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -175,13 +175,13 @@ def preprocess(i): extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': extra_options += ( - f" - -aggregation - trace - file '{ - env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + f""" --aggregation-trace-file '{ + env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' """ ) if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': extra_options += ( - f" - -day - 23 - file '{ - env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + f""" --day-23-file '{ + env['CM_DLRM_V2_DAY23_FILE_PATH']}' """ ) CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, From 541b27c6e09e33afa2bc5e3c8cdb5e88e3ff9a85 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:24:18 +0000 Subject: [PATCH 07/17] Update test-mlperf-inference-rgat.yml --- .github/workflows/test-mlperf-inference-rgat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index de5b0fbb6..9f2f5a307 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -31,7 +31,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 - name: Push Results if: github.repository_owner == 'gateoverflow' env: From deeaa9df4ff237640305085e36e158b116dc8cfb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 11:09:56 +0000 Subject: [PATCH 08/17] Update test-mlperf-inference-rgat.yml --- .github/workflows/test-mlperf-inference-rgat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index 9f2f5a307..478e666c6 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -31,7 +31,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.fix_submission_generation --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 - name: Push Results if: github.repository_owner == 'gateoverflow' env: From 18cacd453896cb131637e784864d84dc8f1a0513 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 12:17:13 +0000 Subject: [PATCH 09/17] Update _cm.yaml --- script/get-dataset-mlperf-inference-igbh/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-dataset-mlperf-inference-igbh/_cm.yaml b/script/get-dataset-mlperf-inference-igbh/_cm.yaml index 796d5674e..eac7179b7 100644 --- a/script/get-dataset-mlperf-inference-igbh/_cm.yaml +++ b/script/get-dataset-mlperf-inference-igbh/_cm.yaml @@ -28,6 +28,7 @@ deps: - tags: get,generic-python-lib,_package.colorama - tags: get,generic-python-lib,_package.tqdm + prehook_deps: #paper - env: From c1dd2078b1a2200be5ccba7bef0426b5cc57bf4d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 14:08:57 +0000 Subject: [PATCH 10/17] Make r5.0-dev the default version for mlperf-inference --- script/run-mlperf-inference-app/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 12b57a6f7..8fe9b88d1 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -327,7 +327,6 @@ variations: tags: _version.r4_0-dev r4.1-dev: - default: true env: CM_MLPERF_INFERENCE_VERSION: '4.1-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.1-dev_default @@ -355,6 +354,7 @@ variations: group: benchmark-version r5.0-dev: + default: true env: CM_MLPERF_INFERENCE_VERSION: '5.0-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r5.0-dev_default From ed22615425428a6659e10a12e3a96fac8e28a901 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 14:26:26 +0000 Subject: [PATCH 11/17] Update _cm.yaml --- script/app-image-classification-onnx-py/_cm.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/app-image-classification-onnx-py/_cm.yaml b/script/app-image-classification-onnx-py/_cm.yaml index 740a8a18a..e53b91ec2 100644 --- a/script/app-image-classification-onnx-py/_cm.yaml +++ b/script/app-image-classification-onnx-py/_cm.yaml @@ -22,7 +22,6 @@ default_env: deps: - tags: detect,os -#- tags: get,sys-utils-cm - names: - python - python3 From f0326b235843f7f77ba39a1eb0fdf5b129a2be16 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 14:41:17 +0000 Subject: [PATCH 12/17] Update CM_MLPERF_LAST_RELEASE for get-mlperf-inference-src --- script/get-mlperf-inference-src/_cm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-mlperf-inference-src/_cm.yaml b/script/get-mlperf-inference-src/_cm.yaml index c100e32e8..b8bd39092 100644 --- a/script/get-mlperf-inference-src/_cm.yaml +++ b/script/get-mlperf-inference-src/_cm.yaml @@ -134,10 +134,10 @@ variations: versions: custom: env: - CM_MLPERF_LAST_RELEASE: v4.1 + CM_MLPERF_LAST_RELEASE: v5.0 deepsparse: env: - CM_MLPERF_LAST_RELEASE: v4.1 + CM_MLPERF_LAST_RELEASE: v5.0 CM_TMP_GIT_CHECKOUT: deepsparse CM_TMP_GIT_URL: https://github.com/neuralmagic/inference main: From 6c4d3a4e536ead50363f9a75d3d26c7200aa9bc0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 14:42:07 +0000 Subject: [PATCH 13/17] Update test-mlperf-inference-rgat.yml --- .github/workflows/test-mlperf-inference-rgat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index 87a9314ca..1902eff54 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -1,7 +1,7 @@ name: MLPerf inference rgat on: - pull_request_target: + pull_request: branches: [ "main", "dev" ] paths: - '.github/workflows/test-mlperf-inference-rgat.yml' From 5ee6846a8cd73ce58a48fa21c4551c5c8ca66023 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 15:00:16 +0000 Subject: [PATCH 14/17] Pre-create results and measurements dirs for mlperf-inference submission generation --- .../customize.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 040e7cb09..8d4010a18 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -160,11 +160,15 @@ def generate_submission(env, state, inp, submission_division): print('* MLPerf inference submitter: {}'.format(submitter)) if env.get('CM_MLPERF_SUT_SW_NOTES_EXTRA', '') != '': - sw_notes = f"{system_meta_tmp['sw_notes']} {env['CM_MLPERF_SUT_SW_NOTES_EXTRA']}" + sw_notes = f"{ + system_meta_tmp['sw_notes']} { + env['CM_MLPERF_SUT_SW_NOTES_EXTRA']}" system_meta_tmp['sw_notes'] = sw_notes if env.get('CM_MLPERF_SUT_HW_NOTES_EXTRA', '') != '': - hw_notes = f"{system_meta_tmp['hw_notes']} {env['CM_MLPERF_SUT_HW_NOTES_EXTRA']}" + hw_notes = f"{ + system_meta_tmp['hw_notes']} { + env['CM_MLPERF_SUT_HW_NOTES_EXTRA']}" system_meta_tmp['hw_notes'] = hw_notes path_submission = os.path.join(path_submission_division, submitter) @@ -297,6 +301,10 @@ def generate_submission(env, state, inp, submission_division): system_path = os.path.join(path_submission, "systems") submission_system_path = system_path + if not os.path.isdir(submission_path): + os.makedirs(submission_path) + if not os.path.isdir(measurement_path): + os.makedirs(measurement_path) if not os.path.isdir(submission_system_path): os.makedirs(submission_system_path) system_file = os.path.join(submission_system_path, sub_res + ".json") From 06b31af62ba4f80e6ac0a23001cbc992b4b7d549 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 15:30:08 +0000 Subject: [PATCH 15/17] Pre-create results and measurements dirs for mlperf-inference submission generation --- script/generate-mlperf-inference-submission/customize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 8d4010a18..a27086237 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -160,15 +160,15 @@ def generate_submission(env, state, inp, submission_division): print('* MLPerf inference submitter: {}'.format(submitter)) if env.get('CM_MLPERF_SUT_SW_NOTES_EXTRA', '') != '': - sw_notes = f"{ + sw_notes = f"""{ system_meta_tmp['sw_notes']} { - env['CM_MLPERF_SUT_SW_NOTES_EXTRA']}" + env['CM_MLPERF_SUT_SW_NOTES_EXTRA']}""" system_meta_tmp['sw_notes'] = sw_notes if env.get('CM_MLPERF_SUT_HW_NOTES_EXTRA', '') != '': - hw_notes = f"{ + hw_notes = f"""{ system_meta_tmp['hw_notes']} { - env['CM_MLPERF_SUT_HW_NOTES_EXTRA']}" + env['CM_MLPERF_SUT_HW_NOTES_EXTRA']}""" system_meta_tmp['hw_notes'] = hw_notes path_submission = os.path.join(path_submission_division, submitter) From 4b9a98881dd06a9e60e999dec74f45616982d74c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 15:31:46 +0000 Subject: [PATCH 16/17] Use master branch of inference-src for rgat gh action --- .github/workflows/test-mlperf-inference-rgat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index 1902eff54..03941023e 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -31,7 +31,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.fix_submission_generation --adr.mlperf-implementation.tags=_branch.fix_submission_generation --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 - name: Push Results if: github.repository_owner == 'gateoverflow' env: From 2861fa9e6f7faf33a24bf6975df97e011359b17c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 14 Dec 2024 01:20:03 +0000 Subject: [PATCH 17/17] pytorch version_max change to 2.4.0 from 2.4.1 --- script/app-mlperf-inference-mlcommons-python/_cm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 89646244b..45401431f 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -1239,8 +1239,8 @@ variations: CM_MODEL: rgat add_deps_recursive: pytorch: - version_max: "2.4.1" - version_max_usable: "2.4.1" + version_max: "2.4.0" + version_max_usable: "2.4.0" deps: - tags: get,generic-python-lib,_package.colorama - tags: get,generic-python-lib,_package.tqdm