From 4897ededd5fe085127035a09da0a74c752f6a745 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 13 Dec 2024 07:21:09 +0000 Subject: [PATCH 01/10] [Automated Commit] Format Codebase --- script/get-docker/customize.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/script/get-docker/customize.py b/script/get-docker/customize.py index 30ccf627d..fbf0a5bbf 100644 --- a/script/get-docker/customize.py +++ b/script/get-docker/customize.py @@ -52,9 +52,8 @@ def detect_version(i): if "podman" in r['string'].lower(): tool = "podman" - print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) - return {'return': 0, 'version': version, "tool":tool} + return {'return': 0, 'version': version, "tool": tool} def postprocess(i): @@ -76,7 +75,7 @@ def postprocess(i): env['CM_DOCKER_CACHE_TAGS'] = 'version-' + version env['CM_DOCKER_VERSION'] = version - + env['CM_CONTAINER_TOOL'] = tool return {'return': 0, 'version': version} From 51d4fdf78a0cad333fc7e7dd25176369b8d7ccf1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:07:47 +0000 Subject: [PATCH 02/10] Fixes for rgat submission generation --- script/get-ml-model-rgat/_cm.yaml | 1 + script/process-mlperf-accuracy/customize.py | 24 ++++++++++++++------- script/run-mlperf-inference-app/_cm.yaml | 2 ++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/script/get-ml-model-rgat/_cm.yaml b/script/get-ml-model-rgat/_cm.yaml index d7615acd2..c4ecc56e0 100644 --- a/script/get-ml-model-rgat/_cm.yaml +++ b/script/get-ml-model-rgat/_cm.yaml @@ -63,4 +63,5 @@ variations: group: download-tool rclone,fp32: env: + CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/graph/R-GAT#download-model-using-rclone CM_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/R-GAT/RGAT.pt diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index f1d8b7874..254c51772 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -118,18 +118,24 @@ def preprocess(i): extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += f" --statistics-path '{env['CM_SDXL_STATISTICS_FILE_PATH']}' " + extra_options += f" --statistics-path '{ + env['CM_SDXL_STATISTICS_FILE_PATH']}' " if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += f" --compliance-images-path '{env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " + extra_options += f" --compliance-images-path '{ + env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " else: - extra_options += f""" --compliance-images-path '{os.path.join(result_dir, "images")}' """ + extra_options += f""" --compliance-images-path '{ + os.path.join( + result_dir, "images")}' """ if env.get('CM_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += f" --ids-path '{env['CM_COCO2014_SAMPLE_ID_PATH']}' " + extra_options += f" --ids-path '{ + env['CM_COCO2014_SAMPLE_ID_PATH']}' " if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " + extra_options += f" --device '{ + env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", @@ -162,9 +168,11 @@ def preprocess(i): elif dataset == "terabyte": extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += f" --aggregation-trace-file '{env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + extra_options += f" --aggregation-trace-file '{ + env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += f" --day-23-file '{env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + extra_options += f" --day-23-file '{ + env['CM_DLRM_V2_DAY23_FILE_PATH']}' " CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + "'" + extra_options + \ @@ -179,7 +187,7 @@ def preprocess(i): else: env['CM_DATASET_IGBH_SIZE'] = "tiny" CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "graph", "R-GAT", "tools", "accuracy_igbh.py") + "' --mlperf-accuracy-file '" + os.path.join( - result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['CM_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['CM_DATASET_IGBH_SIZE'] + "' > '" + out_file + "'" + result_dir, "mlperf_log_accuracy.json") + "' --dataset-path '" + env['CM_DATASET_IGBH_PATH'] + "' --dataset-size '" + env['CM_DATASET_IGBH_SIZE'] + "' --output-file '" + out_file + "'" else: return {'return': 1, 'error': 'Unsupported dataset'} diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 689aaabc3..12b57a6f7 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -344,6 +344,7 @@ variations: env: CM_MLPERF_INFERENCE_VERSION: '4.1' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.1_default + CM_MLPERF_SUBMISSION_CHECKER_VERSION: v4.1 adr: get-mlperf-inference-results-dir: tags: _version.r4_1 @@ -357,6 +358,7 @@ variations: env: CM_MLPERF_INFERENCE_VERSION: '5.0-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r5.0-dev_default + CM_MLPERF_SUBMISSION_CHECKER_VERSION: v5.0 group: benchmark-version adr: get-mlperf-inference-results-dir: From 7ecb64ae71ad276d7e30614229d0efbd57b23b82 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 13 Dec 2024 09:08:05 +0000 Subject: [PATCH 03/10] [Automated Commit] Format Codebase --- script/process-mlperf-accuracy/customize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 254c51772..579b1eedd 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -118,11 +118,11 @@ def preprocess(i): extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += f" --statistics-path '{ + extra_options += f" - -statistics - path '{ env['CM_SDXL_STATISTICS_FILE_PATH']}' " if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += f" --compliance-images-path '{ + extra_options += f" - -compliance - images - path '{ env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " else: extra_options += f""" --compliance-images-path '{ @@ -130,11 +130,11 @@ def preprocess(i): result_dir, "images")}' """ if env.get('CM_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += f" --ids-path '{ + extra_options += f" - -ids - path '{ env['CM_COCO2014_SAMPLE_ID_PATH']}' " if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += f" --device '{ + extra_options += f" - -device '{ env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] @@ -168,10 +168,10 @@ def preprocess(i): elif dataset == "terabyte": extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += f" --aggregation-trace-file '{ + extra_options += f" - -aggregation - trace - file '{ env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += f" --day-23-file '{ + extra_options += f" - -day - 23 - file '{ env['CM_DLRM_V2_DAY23_FILE_PATH']}' " CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, From 77da71d08c6ba14460029f741d4bf8ce304811da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:19:34 +0000 Subject: [PATCH 04/10] Fix issue with autoformat --- script/process-mlperf-accuracy/customize.py | 34 +++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 579b1eedd..0926f332f 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -118,24 +118,30 @@ def preprocess(i): extra_options = "" if env.get('CM_SDXL_STATISTICS_FILE_PATH', '') != '': - extra_options += f" - -statistics - path '{ - env['CM_SDXL_STATISTICS_FILE_PATH']}' " + extra_options += ( + f""" --statistics-path '{ + env['CM_SDXL_STATISTICS_FILE_PATH']}'""" + ) if env.get('CM_SDXL_COMPLIANCE_IMAGES_PATH', '') != '': - extra_options += f" - -compliance - images - path '{ - env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' " + extra_options += ( + f""" --compliance-images-path '{ + env['CM_SDXL_COMPLIANCE_IMAGES_PATH']}' """ + ) else: extra_options += f""" --compliance-images-path '{ os.path.join( result_dir, "images")}' """ if env.get('CM_COCO2014_SAMPLE_ID_PATH', '') != '': - extra_options += f" - -ids - path '{ - env['CM_COCO2014_SAMPLE_ID_PATH']}' " + extra_options += ( + f" --ids-path '{env['CM_COCO2014_SAMPLE_ID_PATH']}' " + ) if env.get('CM_SDXL_ACCURACY_RUN_DEVICE', '') != '': - extra_options += f" - -device '{ - env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " + extra_options += ( + f" --device '{env['CM_SDXL_ACCURACY_RUN_DEVICE']}' " + ) # env['DATASET_ANNOTATIONS_FILE_PATH'] = env['CM_DATASET_ANNOTATIONS_FILE_PATH'] CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "text_to_image", "tools", @@ -168,11 +174,15 @@ def preprocess(i): elif dataset == "terabyte": extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': - extra_options += f" - -aggregation - trace - file '{ - env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + extra_options += ( + f" --aggregation-trace-file '{ + env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + ) if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': - extra_options += f" - -day - 23 - file '{ - env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + extra_options += ( + f" --day-23-file '{ + env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + ) CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + "'" + extra_options + \ From c6ba03982103a5ded10b4cef5ebf4d361bfb9ec0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 13 Dec 2024 09:19:54 +0000 Subject: [PATCH 05/10] [Automated Commit] Format Codebase --- script/process-mlperf-accuracy/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 0926f332f..698227d86 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -175,12 +175,12 @@ def preprocess(i): extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': extra_options += ( - f" --aggregation-trace-file '{ + f" - -aggregation - trace - file '{ env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " ) if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': extra_options += ( - f" --day-23-file '{ + f" - -day - 23 - file '{ env['CM_DLRM_V2_DAY23_FILE_PATH']}' " ) CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", From 85b975a6d4f1831b4d5b9966931df5f5ebe1a0a2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:22:55 +0000 Subject: [PATCH 06/10] Fix issue with autoformat --- script/process-mlperf-accuracy/customize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 698227d86..21569fd57 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -175,13 +175,13 @@ def preprocess(i): extra_options = "" if env.get('CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH', '') != '': extra_options += ( - f" - -aggregation - trace - file '{ - env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' " + f""" --aggregation-trace-file '{ + env['CM_DLRM_V2_AGGREGATION_TRACE_FILE_PATH']}' """ ) if env.get('CM_DLRM_V2_DAY23_FILE_PATH', '') != '': extra_options += ( - f" - -day - 23 - file '{ - env['CM_DLRM_V2_DAY23_FILE_PATH']}' " + f""" --day-23-file '{ + env['CM_DLRM_V2_DAY23_FILE_PATH']}' """ ) CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['CM_MLPERF_INFERENCE_DLRM_V2_PATH'], "pytorch", "tools", "accuracy-dlrm.py") + "' --mlperf-accuracy-file '" + os.path.join(result_dir, From 541b27c6e09e33afa2bc5e3c8cdb5e88e3ff9a85 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 09:24:18 +0000 Subject: [PATCH 07/10] Update test-mlperf-inference-rgat.yml --- .github/workflows/test-mlperf-inference-rgat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index de5b0fbb6..9f2f5a307 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -31,7 +31,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 - name: Push Results if: github.repository_owner == 'gateoverflow' env: From deeaa9df4ff237640305085e36e158b116dc8cfb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 11:09:56 +0000 Subject: [PATCH 08/10] Update test-mlperf-inference-rgat.yml --- .github/workflows/test-mlperf-inference-rgat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index 9f2f5a307..478e666c6 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -31,7 +31,7 @@ jobs: cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 + cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.fix_submission_generation --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 - name: Push Results if: github.repository_owner == 'gateoverflow' env: From 18cacd453896cb131637e784864d84dc8f1a0513 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 12:17:13 +0000 Subject: [PATCH 09/10] Update _cm.yaml --- script/get-dataset-mlperf-inference-igbh/_cm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-dataset-mlperf-inference-igbh/_cm.yaml b/script/get-dataset-mlperf-inference-igbh/_cm.yaml index 796d5674e..eac7179b7 100644 --- a/script/get-dataset-mlperf-inference-igbh/_cm.yaml +++ b/script/get-dataset-mlperf-inference-igbh/_cm.yaml @@ -28,6 +28,7 @@ deps: - tags: get,generic-python-lib,_package.colorama - tags: get,generic-python-lib,_package.tqdm + prehook_deps: #paper - env: From c1dd2078b1a2200be5ccba7bef0426b5cc57bf4d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 13 Dec 2024 14:08:57 +0000 Subject: [PATCH 10/10] Make r5.0-dev the default version for mlperf-inference --- script/run-mlperf-inference-app/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 12b57a6f7..8fe9b88d1 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -327,7 +327,6 @@ variations: tags: _version.r4_0-dev r4.1-dev: - default: true env: CM_MLPERF_INFERENCE_VERSION: '4.1-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r4.1-dev_default @@ -355,6 +354,7 @@ variations: group: benchmark-version r5.0-dev: + default: true env: CM_MLPERF_INFERENCE_VERSION: '5.0-dev' CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS: r5.0-dev_default