From b555ada5c8cdd46a8114b8fa4270ad62c3747273 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 19:18:29 +0000 Subject: [PATCH 01/24] Updated results summary [skip ci] --- README.md | 54 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 28 +++++++++++----------- summary.json | 30 ++++++++++++------------ summary.xlsx | Bin 13800 -> 13790 bytes summary_results.json | 30 ++++++++++++------------ 6 files changed, 72 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index a77e71b6a..8f465ba38 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,47 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 02:59:44,734 submission_checker1.py:3259 INFO] Results=51, NoResults=0, Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3266 INFO] --- -[2025-01-26 02:59:44,734 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3259 INFO] Results=51, NoResults=0, Power Results=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3266 INFO] --- +[2025-01-26 19:18:24,650 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3272 INFO] Open Results=49, Open Power Results=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3272 INFO] Open Results=49, Open Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3282 INFO] --- -[2025-01-26 02:59:44,734 submission_checker1.py:3284 INFO] Systems=39, Power Systems=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 02:59:44,735 submission_checker1.py:3293 INFO] Open Systems=39, Open Power Systems=0 -[2025-01-26 02:59:44,735 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 02:59:44,735 submission_checker1.py:3303 INFO] --- -[2025-01-26 02:59:44,735 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 02:59:45,832 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 19:18:24,650 submission_checker1.py:3282 INFO] --- +[2025-01-26 19:18:24,650 submission_checker1.py:3284 INFO] Systems=39, Power Systems=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3293 INFO] Open Systems=39, Open Power Systems=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 19:18:24,650 submission_checker1.py:3303 INFO] --- +[2025-01-26 19:18:24,650 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 19:18:25,692 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.6996 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.441643 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4042 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.197574 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.401784 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.3606 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.3711 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.69575 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.5726 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.406436 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0518 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.13282 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -54,19 +54,19 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 30 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 32 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 33 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2716 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 33 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2312 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 34 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43976 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8984 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 35 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426066 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0912 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426567 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.16895 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4a8e7b555..84fa17bc2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -490 +491 diff --git a/summary.csv b/summary.csv index 7f6de3767..10987d83c 100644 --- a/summary.csv +++ b/summary.csv @@ -1,25 +1,25 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.6996,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.441643,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4042,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.197574,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.401784,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.3606,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.3711,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.69575,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.5726,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.406436,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412174,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0518,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.13282,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45726,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -32,19 +32,19 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2716,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2312,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43976,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8984,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426066,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0912,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426567,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9898,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.16895,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 484e33c7f..88c9decea 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.6996, + "Result":1.73704, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -99,7 +99,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.441643, + "Result":0.401784, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.4042, + "Result":22.3606, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.197574, + "Result":0.3711, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.69575, + "Result":7.5726, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406436, + "Result":0.412174, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0518, + "Result":20.948, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.13282, + "Result":9.45726, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -999,8 +999,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2716, - "Accuracy":"acc: 75.000", + "Result":10.2312, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1059,7 +1059,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43976, + "Result":0.436753, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1089,7 +1089,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8984, + "Result":23.0772, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1209,7 +1209,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426066, + "Result":0.426567, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,7 +1239,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0912, + "Result":22.9898, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1359,7 +1359,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.23584, + "Result":6.16895, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 14274c0b1623c50ea98cac264e816b41a85497d8..adf60913fb6ccf304ee04f03ee34931c074c92bf 100644 GIT binary patch delta 6593 zcmZu$X*iVc+a7DO?;4VfA8~}i3F#vE0 zfYnjpN6zL4SUt@C>Vz!tnX1a<%Z9RTtX7Ym7&X4Toaci* zZIH|L89BLDI)xy)FMo%AeMuyPHrXcf#rVtwX=_(-+6CkFQW}oGN3&D(7jGO!DZS0} z8X3EnrgwkXeYxAcAa#{^{Nwaa-dRod&7iat){UEE*sq12ZF3sm+(Z5>sZ8u-lGmgG`K(g>0 zB3XuNbh>#dKYlsk?3%j$5ILs(&3&NGKj{F;o4NNS)N7Qhc5hZ*-04jGWfER7gnw>M zX>|7?_oJ=7;;E1)P{!}zxqj%DclU*3edQ@h_L;{c<`6j!v?Zd1C&XY5 zYuF%2j!tX7^w}UeyA2=sWqf;zU%ZGAC^yX6*rB|0I!8!kgKSex@K>Xq>R1mYDk zoBhJi*^b1cBQY(Hk4-%`u6oU{_=hXCQi{5M`)YQ_aSGM4CZbY5g>iEH*{4fnz`0{k z^Xm^d6Prcq4dGF-Zdlaho+~Fu?-mM9m@|_r>6z0MY| zp8uMiDV(jdG-wbOr3p;@dk><1@Y>qs@}@E_*g5%RDR`;0+x|@R;f~VdS@q+Gk6}#% zgQ%~cs`rRDQ$7Te@?4hA@|S$n$@Y7Ja%zG%4W>ZaK16f-sk`@D+fNjLGD#!I!sYeV z3JwMPB#3jNuVT_(>$CwgVE{VPAHOYtMXu6EmBkm75~N zzPa~39J^*NcPN!yVmb`-e!ov$a@#cx6 zjd`QwiLpRAmG6IQ1Rmyhd?+IoUnsCO?V%ZD`^j10*~Zai(tDR_FnbY*TukE?I`X+> zxx4i0DeyfO4lV$Lb_fjfK=p0Hujum9M_J)n5uaE!`saI~DxE2TnWlH6!RRHx8@*X< z#Dt~IoVcAFj}Z5wJi660b|@C{(b9C=FzNg3plr#K9{j@fXi%adw-LMq#i1h?T7;UV z()*W`rRimH%m)*0Bv&G63a6qI;#G)UI!kDie6{dlwiicm9g%)R+&o)SNISI(*x6NBwWGHT>c= zKBN05^Kow!rt`ZG6nYh-0CgDyJwEyEz{xMLDSz6~#L^(78##zv&KS~7)Fmf-rMKIT z3kxr0ctzV|sNCe9woCHWFa4m|-Vh)1ri->;UdS+|iVy|PXE4L-aA1)wnf!G6KU=+T z(C>aj>Xm6`YU(|u{PlR<$XwOsXbo|H6F0firN^7B7lrG=H0>DXZ8UJnwk<+*uru1; zAONt2f3Ojfl{NnmE@7$7M`Z=I%OfB{Dpxy(A;q?oU%J0m?Bw}m5MK_>a27x!)*RU*9b3*=_@!mJ`yR$}cRe^E?u7$} z3SmT#>S5*-Xhn&VsM$CFr9U~O2g!+TFKy}%43@x`(uHG6>5d+l+?uge6S^bm(&N&; z#!L+V#8R5^({vKlDQ*K?&>{8L@JO$N&1r{>74$h~-R9zJf^n;y?d`uOl?)sU^ng&QWuEu2ORI_#j?_w0zWWxf8y<4`S@_&X$_ zfy_sIfA7W1ga(8(W{iA@>b(Dq(H-d%6Q4C=bVULEbyx>WpyZR>5iw8SjqH!jR{@OX@jIicwzaaJ()IRh?i>kdKxEvM+4x zq8;c6V`aP&emj@XwL((A#jr1+g~=iKx!17W&JT%IlZwZ}Z+}(9?VDd@G7{i%t~0pz z{^6K!1%U(u0Yz0eU$Ad~mdDs~*u$4`E2PsL(6AEmzj* z10l{Rmg#&+8xb2&VZ0dOy z)g0H2^QsJMB$m0IE7?I&(wwb>qNKU})wq#mK8lj%CbH8f+pOkO7)))nEA@M!$Fx; zR{ha0Y_e_=B(GLNi+%UH)vdfKN`mr*d1FniIAJzM!tgf;pFW|nHMy2ZDGFlMZ6@2< z;NG$$XkaP7lZEKk?-y4%`o+mEow`UEsw%;N;*KPcy<~$k(SZ&==FRCRK|Xh=2>G*k2Xe`{oJG4*b;(MIub;OFgPqLLi_wP74lWC`-^H$OA+} zLze!8HJJz@=$444E_AI4UKR{aCz^RBYG1yEJe=q-HY8M%aFhb$b_aEJ~g0~ zmY4}biAKP20?-Vc0yKl?5&XLlQIIEbZoR>fTLCtUc4{V?B^lxV7tGkWfTAQ+)0Cnl zv>z#eivAB=*8joHbKE$$`U-~>%aydcNf8+IEHteGBXNv^w?a7I@K^C%<7IJkHU*^| z8-dI~7)$+^aB!SQWHuZ^VQROFuYvQv$2FWpYSf}X6Si$}%1uoH@}P$E(%v(VdNkJD&b$K ztX%0iqyV^9S`ZC2M-whkWVe&!yig3wQgyr}c~Mv3%eTe{T@_KL;*~Z*GvBR~AT|x! zwW6XRUdA#w9EK`Ops_{&cP70JmD;Pw>c@EXmzna?%iDRlru!l2!&@9d0LlfN1 z`FYZ-4oi6lU2V|=nN~Rd7<7=&J87mexcNn#H33!}jy7bFIJkp$pckj?$wUbRU9?q} zwby$IO6UHmL>}+BwKgs?%WhQofGa0I3@wgH#qdhz~g)m={h9F^#uS85! zqJwz45`}Hr6(tJWw0}C_iEvUG7oO3B&?Xtq@*95MW8VX72fsr_gf;p_y$fcQ1DP^Up>O@NFSW~)UF0`^Q+J@2?rWmz?J^a}0G_)fR!OSWZ!gwoc2`&nrH;9wk!cjl=vplDb(p7+B_!PW` zI(nmOh~>t$^E*(F0w}DqDk^+6gNNFx8_go(ZY6Y3Q-2j9mVlDWsX`n8j50! zug;#E?F@$*RwXzS2kc-z$7`)fycc0;yw8OZLr8x=OX*meVF9qrIOc(cYTGQ z4QdKAOur@>?<5SS`}OkvkQwEbOibo9tVYGd{CLvXt=2tx@XQuQ^rMX63}`LN4j}33 zHoTJXz$Vrn-a_rb=_CGvc&)!ry)E$89iJA3GC{z>qp7`&0n+M~5QpX@n>G&7<~`RW zXKqeW(HF?O987vvP>}G_m@bquU1a#Fzpo}jOL(H~v)s62-6$%~qTBswDU*c&sUHU} zYD8qM6^Cp6`eZ5ExL-07IFjT3!YQs~2*e#vFuRDeB*4-XD9O09FH1?rZLfN6Y7JLo zj#@xOYhDR^9>~{d-456O9u}c$tRZqWlLtwSkUC8MC3@)mPrR|%DxJThJgI82n_JpwJ(zw)THua{n_H435W?h%Q#Ej$O{9t*gvO=;sSJ=X zDl?I!5H`-T<&L#IC(P)QBCjo;PApIGg)>oA@FeNC6g^1|7;-&G{@CD?UU-QRgLjQ{ zf|tJNlQ2f>L%_Celo9*Yl@dz9yfU4qGIcGXFwSm;m$Jh}&Kb``vJ&;9B{)(5)^-fC zvmC=~LK<&oE%9qXNC)1eT|EW6w5SeP*uw(v>eautXLpbo>8}mu_e1eDEVs$^HJb&w z<4&EuXzj86rBX>Pk1uTzos_pg73e8@z}dobpxk=B-Qv1q5c#N;I5JN8(4Z!cX3-Ck zk!hIT*wj$3@Nr)uhf0V!*RymbyRort+~`npPxeXL+1NZljvG9DLrgmT;iYu)P_u%# z{(Fn~&uSsP_M3te1>e8B0cK}b>J`h@T`4P#&LL&LV_7!uAm_A+&;KMf?u0wG_34x0 zPxU*ic^4b^X6o%5$;Z_v3QHq_Z(f=Occ~8^cL!QVWx3C%-fkTGsoVp7cmB;~YpDB- z%EE+(mKy+Q>=P=WqGN0K*_y@%2+JbL#*&_rdWMpU@1()bQ`bIE#zyq=jbkp7|76gB z33s`uZx>jpw&rQo{<6Czc$R%l?j=6Yz%h~pQVSN&q<6ejhQ*F||9!XehMn&T|8Zz$ z19qHgu~sNw`?)HgA=Hd)v}uG|PxIWKPZZUfXp}Hn?o%Ohx%VOW5n#b(kB4X}!B;26W9U>E95z-h z?RtbrhTD>pKJ6$yo-e?gIkBt`&{<2pklsZVV@ICTYLAD6x5cuoCtmy$EG`9O4v>4L z#OwPh>s*3Hy~QJ-WWv$LKroWmhkWy}tV#xk=AJ! z*hy@R4lR)QbMwsR($nZ3$Wgr7l)V^R13uy~tfyKz$09}L77T4UU#IXEJ*74-`w53m z=-l?6G&BI-$P4A=pRk{W7&W8|V9Pz_Fva%Fs|BTHjpE5sLGsc~kC2(h%YPq9d);!^ zAC~PSg(_H%O1lU)wn$fBe;?CQHWtICBoPC?NBDmHeOQD@+TCibwyX})mYh5Jv-{63 zx9jUY2-~HN@7f@yme70C#y)g1*b~&vvhU_j@<|VK1Z%(g3CriqLg1x*`(n)Z>W2qL zD}y&HT03flt%`Z|7ekSj`*(*RM=J z)mWHb5H4-%`>WHR=slke2&Q_=GXFV8I+X8y2Md+6-PoHuiUyPS*+; z4phtiHMTr^XQ<=n5%M{ETC3(vgKVhFL}Q*3Y&FZL+R}fzb)*^ZjY##dvMxSuVdK-h z8+W;cxG2WZLA}qAQ#W`2+ac#$jE=Li_xEQ*e}4|neesBx*krpw@uXXs8F0+QpLgZ3)92J&t)a=sk}f_XB9Cjtt$t)&INI zUtCk9>JTfMpxu8N?Z!0}KpknO^PfZBq6zAKr)W2(p#Z8#X`OnyHRKyW2j$dizv0eD z-j)Ezj^z;B9Ry_S>*v4Hw-YnV)}vHJ)ciKXXQU4EJ;OreD-UltFfqcUX)0{gw@g^u zw|IZ9v0kq3Z1J1X{`JUVMv7yk2Bk|ou&qONJy6_(Q@ihXBr%`x0TF)5w7lNW`Bfk^ zcTC*#(d`=8Azmvk8ENL7kR4`c$8a-fCq~~(xmnSXg$F0b5W`!;p^qM8R;wUHe$*EY zu479OX$*K0edR@9i1$XwrmooWT;LzHy00fWxsLpq#Bi%oE2y(s;p1dY-|cE5-z2}d zj5|X8t&$vNQg$w{9u{}x90Qa25(`Qd1`6!yj<+u!@pOtIkVp~t#+WW?TN=wQP229O zCq&I{+t2h>yXEOA-iOVmU&;=*)ao7!9v-qDAI9sENh~Z7p#PPtRX=^cq^DlnU61?H zW@CW1ndU7seW>X8SPVwgo&23y{M-L0x2=24v`&*RjdD$VpOp1yTI+v=N|(vhB9eRN zsUlxbLmPgm*(gsjY3bln2WrZ zv;fxXoWmC-exsVS>jVL@w|sBJuwKC?&nGEITb~AZ7rCfrFn7eajW?>a9=SdLRJYj> zEbnmZNo<#0{Z@BX0nIpKQwJ*uD*bA+yD#^Y$s+&RNp1xF<*@HhVvpGxndE>p z7ytX;1S_pzPt}5rR{#n8duP%C04`tvfO3)kcSXmTV?Qd~U~#$z07L}JgoQs0^bW&X zDsp4Niu}|A64)IH9;~e*yTHFjPyqlI)BphGpUi((bWANaT2YIN3frY9LfXBiB#OO{ZU>}!&>QAuM9gDLwG$!=^RSz=^J zV(hXcvQr|Fx(yV*oW?eFV)%N%OXWVW>Hv`Our{c?|T`jb1%{bJp%wuaZ@E#8cd{KYf4 z?fUumvlUazS_J--REN>bTaG-aJt{t%J;Q{`l>O}F@(Az1RB_UZB)&bK4pxaA^_8ac zIBMziyy|gO61L)|DtP#e`1&XLn~T!$pPsvAKfy6km#pzR*64 zQf;}kn!a)^E&TqxGgpsuZu0HK-OesGr#WW?IhOYyCHx21f$JnmovKHuB=`u1QqO3(- zbHkvi_YI*B%N7h_^GWSc@DJ7q6R6AZJwp_KCp9}b@^TNBEAlWE)jPj^s~1aXhk)hq z!u61#q%$FK&q4s=?7q(zHVL^bk4qMN$)6xuw*22Pt5U*Ge=>dWBz$$MM=rhn{>LQl z#60Tp=?dC8vh7}g(;K@NtZuscbW`kqHqApfZk6GcCj!uO&6kcw=K9TkIiDV`5ofpV z?k=t#FpLx{3#$qshNU$X>BvEMqMT0Dq(6ox)ZRWiSq@WBA*~Ws9PYP&;2!)+B$oZ~ zB&eR+VfZmm`7G|J4F5ccx4Cmyb9{^At>vmC2=cFN@qhsqD#Bc{Sqn=(VD}?kyLO zqwse5ysI7j7#o&MrTve0x58xea+6FqJ13qT)n8F{9>Yxg<)AOg9JVR5@1jZ7&soP z4`f`(q;(3pEu@&+Y$P>80JvA7DyB)Fw70p6xXKOH?6+^KkO1h^+iX1{0A)7zG( zTq9eKO@Hmry%x{8!$T4~`qHke0}vcUAKU6L86dkB{NjLUM!E^y#%!5PNi!<9ZUgZY4My7X>9_o5ZF3`9Y`e)ZR{O& zE;|u?1h@k@JCBa7zi14FLP_v@o-1=DtPGG=81*uaIW>_7Hu<}sNQJH`(|X1U2p*)p zSqlHdQL|hRx-sBx0Q%(q7~t42x2C!5?GL}h0G;r4Le&o_`-ezKW5DXy!hAXqyVdDQ zxXbs{V6nY8W{@#bOX_y2?WQnZnMteaAE--piY<~hTyI_JPNZSH0HQFRAJNj`z&nc{ zJ&LjVBJqtKRfZsEDJ0p{$lrQ$Sp3Ni4K)-^w`nX%7XxdFk^p%y>JS&Y^`z(!b|#v& z{lEORJhm_|(ys#7k{+1W+x^a!h!n`A+uJoDB(%~*OC%GoEtFLAzjnyzVX*YfhFLN~ zc}hz;Q9Aul5+D(158^Y=MPG_`0aXgYb0<3(9chC>jcy^51Q_dy&?pNH#K!)#fD9am;}eaE8zo2u}QdqoEGLD?=pRqbr*R|I zUE`4HYJ)aI#0iTe;z}%kdBV1kP>9tU-?yWxFtd-`M@H+$32e{UJVK01iw?bec)c4( zF)iDwDJJ}G#dmP%0IXXEehI_?$pUVG1_4=aTIiQ%f-an?R|~XDN*3<`5Il2?xz)(c z3({vd>7QBI(BsJor$v!B5s1b*H1WmrNodiwCi@;t!&jmy}R%khb%6zUx>yF_5bRt_UAE!QwAG592Z+CVs@ms5A|b^Uyu&K~=_vZ@oDY#Kw#~R2aLejRavWZ_>1V^ zR0MzF0RE~T>#s4&GGS@MJ7MWaaNCzvPCWPqevP7Rv){gAa4*eU(bugB^T=f7=0#!# zYQO;Y0wzOgKPBpAVLVWY6q9OF5Zx;wl%siy9$bjj{I}1kjJb#sKB?R)1*zONmltbK z>RL-*j;Ou$;bKDnbrHs-by|!GLwYkgbZ$sa#P4(;I1r1CUUH)<=7q+4EqK*5ao%5rIP=m` z*?O=S%gH^It}6%L1`h(tDdnj!Lr}_NUEmP%XGx&Ep29or7{t_VV9BmyZ3sa%xkQWc zkN=jNjF8N70#)3mzBcDCS}H2*TkI|H3Rf15F2!QpGTQYPAP|(ImbeR#RwgsO#es@m z6;<&@dP3uVG;1MT*a0XHj5t8c^x~2P2~|^k5%l;JJMGuj@QAW)I=qD6d%yPq)$3Ca z;NjS(A@OK^o^|##kPFWuJAwHo#hH(!1h((Q+X9bnUUYs?gMarC*y!f_`oUYk-%L7T zDE|HXfia3x62)DRz?Q|mVB!4A?Val9zihp|p56Wd(1qw;M7ok-%&7 zx)}fUx?Q8L$f#IpiKPfdM#;mh7zW5QAd-^KXUjpTuF_$+3vG;Xfl-Mz{2!iqe4#^1 z2z4ellP(i|y^0<3?7YR;n#7@d1L_zWZ_ncAGD+rc?17Jp6_$*`p*-#-#+0JpjxnYb zT_haO7PD?ep$;H>uzShI7s6fNu#C}gz1!3RihvO`WzC#97}O!*IHiRl^D1&}KU5cv zWZ6c>H}=3rUlo*$!u|ppQUHBo3@Lz;;WKPq8y6{nG2^RXukYA@fVWtFVA1quVY=#2 zM}on=>&-cvG;}}$YU0v%#NNXoEQ;=NhIImpA|C1jH5g4xeNu0xJFEsq-NIX7(2HS2 z--xDdqf>x-lBWEruL0W*(3fq|c3!q<6C&>j^>kEjr8x3%MHU4UsA66Y zLnsrD)g%x*rN5jBZT9JgKq0Y;y$}f#FUeb{Hu0};N3=n0;^1j96C<{@S(+v(aQQx~ zwSt+YI0d!d5(!36cj&~76U`{pq%r7)v1|m>t zRSXLuZ-TUq`;?$EeA>1Dtv|MoyM%h_@@P%lTJ!elXPd|Uz7i-UZ@aImBgq2R^_}%_ z0Vr4fCXssVBKO&C=CaTRIrf9+FwUiQ7QkRm>sny_^;Uo8u9Ht`GBS$4mO0S4_4dCweg$8 zkq7y#<h4RlCJ$ueQ4PcpiW@$6Ee7@L<1C(|a|(#I;w)MpYjc+d`*|(U+obsbA#; z)@YKj*R*a#7)={B||T_2A9$`h_B|J(;)sBwR^Rm^wLXD`bB>U zt5Q7ne_2m6|7AVp#wJRo6m1HwjPX4#fl^yi_UR@W*7;aPFR&BdCGPS#bs`}TDpS1> z)!^=cafAvB8jUFp1QGi6`K^tsGSCBmiC;zO03-&{=63oI2k2-D?F&qLB>=A|yqQ%0 z@!q@7#MG@K!vHL=0;PIODkbR{Mvo2QjYPP0Dw1i+y#`hB)|LDv?1pfhKbXmGo$H@! zxG1U_$hV*=IbMd^01XeO^)Ospl+Nq7S-j7*zmevIcE{dulf);!E05|uwj(V+e8qvG zHOUHH`b{*i6tHIJgX8QeaOIq7AWNP?P6D&JA>CL4EK{=-P%dOGkt$|`oFwI`{|QH^j_LYmDwaG;9McJT#pV)%?y}a%Rw0SU z)*R1<^OVX%(0ceOaU&x=DcJeG^OM-r0SurV>mFnDi$(}uzHg`3NM*iGPLT4{`}Frb zRc0YVZ`T?{_gudb6k|{D3ZdRO{LS1}U7UtU)F;Mkv$rxkZi14f(yTH#l5Np#p_Ey8 zl^R7kK3vV}qJDmi61E4*9>nW&p@cn2Yc`8g&*JrC#KFk`%F#)7j(ucJP#x1WXGvJA zMkMMHq1mr&HJyrDl?Q*W{5-4g>{R=>S{5#P<%;td9eK%M;@a@+ggxQC4W??HWY84x zNO5Uekq0M( zRm#$pqzRuM8PVD&kq57wa)vMlji6^TCbXS`4 zPZT^)_k2vwEoUn^tvchm8XN-L4`}%&&UUIW*@C}T%U+O*L-PwbE++F5YqC>Jhy6R8 zOg%Ph6UeD%AY#e zbowobEM|I2BvUpM3d_v?%!RAoy6En!-TsxyhZK8qIv@w@4ZcWqJKxgF(X_zk(85*V z+_o2--H`OeD|i&D>JP#H_5z8J(&G;dcYE;s#892}KF10_a`ELOaEl<8%fg|i_DVta zVrK8ogC`^rz3}QCIkq?NizD;sq*Rqqhf`wAr9o>Ay@T%>y)kH=?#e#k*Kk*sr7AvUHbbEB@s`)wM*CaPo5vLGm5;?MAzH7L43 zPwHo^>a>*z8#zAg7ZP4^x!gQY*;NN;@j+ufaC~Bd5GN;cEpm~;>}4`@#uo!|j|V}~ z_eS0!_sCWv}$JZ1x4Mb!u5d{ zu9sDZ*=*^h3I51Rnl&`Gk&~%-F|IMBCsCP0>v{5U9$Pf8n4P0KKC4>cFQbS;jy=Q# zB1ufuBV0y?25H=p)_1>n!={U9HhCT*JwhRT}>7$jC(M z;(MAcuA2eWp(`Kop%?1j@Ub_Wk2i%VzMl$`W$E)%`-D`wm1XVsE4!|N+b-&i>}h(} zSd{lL+S|)DMcy@0J_hWLjOY-vdE0Y@*$td&R}Qr-XLtRYrNq)KZV=kXt7uXs<%Phl z^bjj&Vp{MY5*huK4l*Qtm|76mhQxMzn~PaDk8Fgk+yUCB_QI7JO9wm~3#Vk=r>;5u z&5cgkax@Fd?t$i6pZWA>`3&4|9*(P2aB)xl2Rc}a{>&Df=6qSJxpIlctai8NnCM+a)8ZgM-1zD#ID2pE6n^Iwhr1!w z*LVCW+zpDA^+o)uI#0(;nVF|mF`*nLL(l0i*1r6~*Kl=gY%fIQtXfV6o69p`?eZuPgiKIwrkAVzRR_}#(Jr;~^xMuxIh_*Ks$g`VQyTrD-@R4QJ1>cON+tJGWwr{>&SOa$dy8ZI3 zS<|g(H@R7+^%3KsN8pg8u`qV1uQA-ui-ur_8>$%O{=}R5Or}U6H|IHrR&V*&aZkL| z1Mh^BM6Kal>?OmJ4>Y5?qiJj7F`pN(lfFvLQ{!?rrFm^bknA9g?~u*aUh9a$C#S*nj)y$X<03wKre zl3eO~KwLc_y(8YO<*78_N#AJPuF)8nYV(`@_$P-svvtXZuPE_sXsrSxjA-7Yy-S{& zeG|XYt0F#EIrEkLW)hzF%Rb>!ZsN{)gT3f2dRpvGA3`L(OGXf5+9L~!-Bs6Hj$DG3 za@rb705{@|&>R*)4O@QR;PZZ^lJLmSB_;t5)E zLys$r_X&1HHnHG*2~)o6LX?1VW7Q7hZF(}-e3jD8w*YS1)Zo}G_U*TqX1dabw?kCi*WPU`L8MG|H|!*g#3bxf70WL?4i zIhqVwHaVviGt)gYQKydk{JNm{%86-EAk(r+qq0-MuA6*YlO=}n5lbbZLbcl|$kM=E>( diff --git a/summary_results.json b/summary_results.json index d90b92267..d47197c50 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/Community/code", "Unique ID (e.g. for Audit)": "edge/open/Community/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableCommunity", - "Performance_Result": 1.6996, + "Performance_Result": 1.73704, "Performance_Units": "Samples/s" }, { @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2716, + "Performance_Result": 10.2312, "Performance_Units": "Samples/s" }, { @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.13282, + "Performance_Result": 9.45726, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.69575, + "Performance_Result": 7.5726, "Performance_Units": "Samples/s" }, { @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.4042, + "Performance_Result": 22.3606, "Performance_Units": "Samples/s" }, { @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.441643, + "Performance_Result": 0.401784, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.197574, + "Performance_Result": 0.3711, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0912, + "Performance_Result": 22.9898, "Performance_Units": "Samples/s" }, { @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.426066, + "Performance_Result": 0.426567, "Performance_Units": "Samples/s" }, { @@ -1112,7 +1112,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.23584, + "Performance_Result": 6.16895, "Performance_Units": "Samples/s" }, { @@ -1220,7 +1220,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8984, + "Performance_Result": 23.0772, "Performance_Units": "Samples/s" }, { @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.43976, + "Performance_Result": 0.436753, "Performance_Units": "Samples/s" }, { @@ -1364,7 +1364,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.406436, + "Performance_Result": 0.412174, "Performance_Units": "Samples/s" }, { @@ -1436,7 +1436,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0518, + "Performance_Result": 20.948, "Performance_Units": "Samples/s" }, { From 0c50fb4e39f5b9f3713f3f77f38511b8ae2c93df Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 21:24:27 +0000 Subject: [PATCH 02/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13790 -> 13790 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8f465ba38..31619ecb4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 19:18:24,650 submission_checker1.py:3259 INFO] Results=51, NoResults=0, Power Results=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3266 INFO] --- -[2025-01-26 19:18:24,650 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3259 INFO] Results=51, NoResults=0, Power Results=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3266 INFO] --- +[2025-01-26 21:24:22,618 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3272 INFO] Open Results=49, Open Power Results=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3272 INFO] Open Results=49, Open Power Results=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3282 INFO] --- -[2025-01-26 19:18:24,650 submission_checker1.py:3284 INFO] Systems=39, Power Systems=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3293 INFO] Open Systems=39, Open Power Systems=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 19:18:24,650 submission_checker1.py:3303 INFO] --- -[2025-01-26 19:18:24,650 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 19:18:25,692 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 21:24:22,618 submission_checker1.py:3282 INFO] --- +[2025-01-26 21:24:22,618 submission_checker1.py:3284 INFO] Systems=39, Power Systems=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3293 INFO] Open Systems=39, Open Power Systems=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 21:24:22,618 submission_checker1.py:3303 INFO] --- +[2025-01-26 21:24:22,618 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 21:24:23,790 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -54,7 +54,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 30 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 32 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 33 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2312 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 33 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3445 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 34 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 84fa17bc2..a786da04d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -491 +492 diff --git a/summary.csv b/summary.csv index 10987d83c..5c6085f6c 100644 --- a/summary.csv +++ b/summary.csv @@ -32,7 +32,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2312,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3445,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 88c9decea..8f2b018da 100644 --- a/summary.json +++ b/summary.json @@ -999,8 +999,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2312, - "Accuracy":"acc: 75.800", + "Result":10.3445, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index adf60913fb6ccf304ee04f03ee34931c074c92bf..605ba43bb4924581c1ac930d1dda3adada8a7354 100644 GIT binary patch delta 1839 zcmV+~2hjN5Yu;?6+U?DMoS=f+Dm`V|nN=GPWSz@rAWyLjHc` z012_&gn*f+RE{pEPpiAtSjRu+afD_(6JeU1G>C~Ch$ns;gvsQj@o;}`v>HfAFA2OT zP54RUg^R|I&L77@NTik~;-v8;<@Ct1g#W~2PnhYHCu(GzX0a#LzuCkR(~Nt6fq3Fv z#*sxacC6S7lLqqBJdr02hqw)tC*glN?=BG1B8|?m2s_8JbDah}I<{oz*eZYpIPiJm zWno&zACgdtGT7%Lk7OCmxJbB$4tS>AQATga<0wpc8U8EhS;&Q@F%Gf?^ zbfe2N^o^Sn6Dh}`(>w^}_wvktZW^W8PyV8j<)b*1Qn&ha=7o_r3Zqc!i(Z7}aT&c) zzSeBM%Tk{Uk!t%-|I`?LpGQ3Nq(0fxCJ)lFeD*TUt2e3CjX2{{iP7M$GPsO2O>AI{ zNf{43>8Y?$j zk3AJdLqAQ%;Y3rnaCOArwk*#82i}&J2R_)O1(MPced~Wmk33C^zt65gS-bhDdGqhf zv&&O-!Ozpo=cs@Gh!}3;^0bxC5?&s)i_qMi7pF$9Yj}jJcN5QRKxQHC#M0+U@dS9L{CtMMi7xjx^KLU&?qr z*B;anR~8QyrFrK0T23qj#|Cf);7tJ6&TNUh0N&aVutil}3s-xDT^G|})N}$P<71ww zs?A>mcKV`{&wm=usL4!X2;Zm77R+p?-zi#xHYRfqL>t0+?o2ZHr$ zO{|uR*Jr4A_fHqOu45yI8QnDF2)=O^&OwXd-Z0wSII0}GZ=Iv2ZML;Kw^O-fdbzAR zX6S6XSC@CgLHD+Q-#bD?>25U{!^Cmj7IWGNonM}R%6Y1E)zA@gnaMCPaBD1TQK?OKqc4Pbq85>}{FqOYPrvT>Cog?H%;(3yb!p z8yZ+LR6IbG+s}$AA5Y}d37dB*kK~E6EljxfYO?xFgqDY}@hwC1)!HfQzF4(^Xwe34 zSPPMtrW6UL**`|h^94q9=A9l#oc-Teil-V>{f%WOYwS9#VJ6mJd|sa{TG+(;YsTtP z#-=81w$>hH9Mf?e_x(+Vs#s}Z2ce7J&G4#sqYy{?g1Tw!?cGE-;+!nZE93F$-RqKe zb1g(HA!O%&vsx!%2n1(VW@wYYFCc$v!!Qtp?|llRyDeL3LlL$EZON&DLQ`6D6VfIT zwSJJ)I)3}gPU0lsQ;(~e{dRUyxj3jM`32{t(G@DP93`N+5mN6e^s!#1k0|khX~7z! zp+X&aw5ab&ZYeho-kh=Ef&`yLkoJ_@3hhI%6l2f#pqS61Lq~CIoMIuCZijy@132X;8wTadsbuIZOj^(4 z=stKk8r!zb+Sy=@`3wBDdVL>OO{MNn#R1i&;FLRH!MM7_*F`*B!+co96K^Gmr!MLf zZ>L3$eISw;fF9$XmCxDp%X%btHMuP$TiGYd*n)V+7uJ#q`TLav zB*bzP0%o34Il7!at?pK19sigoF`BWQM_GE(B06p%mIhfErIVA^!~MDDwGbD68v1dT zvXj;e5*aZ;E5$Z*Rv^0<;Xb86JN-G^NGQyIrBq*{=}F_ zVuNDr7>OUHEfi!$Do$Dsaa*WJqyGxlT_D6Gt(}OorNquHbm#xD1iky2w3Xp zQC7tt(n#dbC`7H9cS_M(s#qa+eSwfc1CN3lPO;z+2AUbtXM6}^$Z zR&2h@vw(4)Df>_VR2YMR#Vq%QI@wny53;d%_H)InHl=*Bz45!`rpwb%hK}iv#U|7Z9gjB{QL6k z@)TXL^DGY->fb*i3%7A~+Dc~yFO&R;Po>{ZS~81ynH{apX(2M1-7G|-7xX3!b+6IP z)TwUksJgwZAyT0hM&ogRQPvu5uB3Dv`-x9A7mu`dyE>RebCLV8=FQ!aVtV#V8n5Qs zgFNEe;vu87$o)XciDlrJ0B!-e4dBX|EpZpXy$u0dR@Jp|HAmQWF%3ty6KXjdvs_kf z_8PF$7nyv)({M(0OD7r+0%~a^pEF~NGk_UoF#3TgSH05(?{wOKtUPv7$r5AZnKn(f zDMohY8a6F#kPIDwmd8F6)jN zI-Bm*<=t@5z3umZju4T$J=?M{aa`B4oHjz|muIU{YTCNfwy|Ymk6=<$q58KE7d>)> z9HPtPCJ4O(^az=*-geyf61b8;TQ_CjhS2@(&)&^oc-QL>2dCF}SG{sIRvg@^nP*y5 z#|TY$;{B2_v?$6_1jZ$tjONodRM}7t9U<4&Jq!#vI(C(Rnwm2arG8E0H)6S{$>qsZ z{96MrOEv{?U_MbSUE)fxE^%Og0$fo9xS|MfMG@eNBES_z6n8FhKraRKQa~>S^in`C z1@uxtF9q~cKraRKQb4Z>=rsYoCZN{@^qPQP6VPh{dQCvD3FtKey%wO?0`yvdUJKA` z0eUS!uLbCTwE(>qpx09LvT(u{w`*Cw`EB2jX zb8j-K_CZ(+3PLRds8l=eOAfs67VYQ`9IL*!Y1=>Udsy2m5!0-QLj=t5KmjvsC}0vl z$oP88oy5UnbQnyhuW4JRY%ZJCC}DeMtU~s&K7@>aO=X9eb5>RePi^smMr1d(HCYW6 zq$UV~7OW9Qiq#A{sKk20MkQ{p4~HYOYkq(pkZf;d>Tl~qCi>oN`-(egVfl=%`O4A= zW3xVOzP2>N*sREAtIHSI8YnMcVmr7tF)oRSTc@khC|MHKB>5MKo})C(o>v9>@Co-7 zz9wRSe>s1$`2r&<^G=T=&i-#Kk-rA##qdSh{zQ4&(87m%k5W48y46k}O5^`?4 zi#(rU1%U|`7(M_~I}*_eDfo!AKdv1Bf@5ewfes#%jNpqmfK&c%!=PL_l?=UwyVkQf zy7yj=#;)tKZZcS7euBSNukXXEsnmy4aX@t`IOPslFs?4~brBEOG9Om)#2X3XsgL@d zx6>lWJ`hO^K#y_H%ID Date: Sun, 26 Jan 2025 21:27:09 +0000 Subject: [PATCH 03/24] Updated results summary [skip ci] --- README.md | 107 +++++++++++++++++++++--------------------- dbversion | 2 +- summary.csv | 3 +- summary.json | 108 +++++++++++++++++++++++++++---------------- summary.xlsx | Bin 13790 -> 13904 bytes summary_results.json | 82 +++++++++++++++++++++++--------- 6 files changed, 185 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index 31619ecb4..8182fb559 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 21:24:22,618 submission_checker1.py:3259 INFO] Results=51, NoResults=0, Power Results=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3266 INFO] --- -[2025-01-26 21:24:22,618 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3266 INFO] --- +[2025-01-26 21:27:03,671 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3272 INFO] Open Results=49, Open Power Results=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3282 INFO] --- -[2025-01-26 21:24:22,618 submission_checker1.py:3284 INFO] Systems=39, Power Systems=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3293 INFO] Open Systems=39, Open Power Systems=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 21:24:22,618 submission_checker1.py:3303 INFO] --- -[2025-01-26 21:24:22,618 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 21:24:23,790 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 21:27:03,671 submission_checker1.py:3282 INFO] --- +[2025-01-26 21:27:03,671 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 21:27:03,671 submission_checker1.py:3303 INFO] --- +[2025-01-26 21:27:03,671 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 21:27:04,923 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.401784 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.413475 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.3606 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.3711 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,42 +33,43 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.5726 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 25 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | -| 26 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 32 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 33 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3445 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426567 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.16895 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 50 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71562 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 20 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 26 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3445 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426567 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 45 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.16895 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 50 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 51 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file diff --git a/dbversion b/dbversion index a786da04d..6cd3ae2db 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -492 +493 diff --git a/summary.csv b/summary.csv index 5c6085f6c..186d6846c 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.401784,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.413475,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.3606,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.3711,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,6 +11,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.5726,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71562,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 8f2b018da..b14ff30e0 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.401784, + "Result":0.413475, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -360,6 +360,36 @@ "weight_data_types":"fp32" }, "12":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":1.71562, + "Accuracy":"acc: 80.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +779,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +809,7 @@ "Units":"Samples\/s", "weight_data_types":"int8" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +839,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -899,7 +929,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,7 +959,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "31":{ + "32":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -959,7 +989,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "32":{ + "33":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -989,7 +1019,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "33":{ + "34":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1019,7 +1049,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "34":{ + "35":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1049,7 +1079,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "35":{ + "36":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1079,7 +1109,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "36":{ + "37":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1109,7 +1139,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "37":{ + "38":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1139,7 +1169,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "38":{ + "39":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1169,7 +1199,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "39":{ + "40":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1199,7 +1229,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "40":{ + "41":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1229,7 +1259,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "41":{ + "42":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1259,7 +1289,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "42":{ + "43":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1289,7 +1319,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "43":{ + "44":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1319,7 +1349,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "44":{ + "45":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1349,7 +1379,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "45":{ + "46":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1379,7 +1409,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "46":{ + "47":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1409,7 +1439,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "47":{ + "48":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1439,7 +1469,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "48":{ + "49":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1469,7 +1499,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "49":{ + "50":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", @@ -1499,7 +1529,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "50":{ + "51":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", diff --git a/summary.xlsx b/summary.xlsx index 605ba43bb4924581c1ac930d1dda3adada8a7354..92dceba79e97da734411c236c1b84d5f2ba6c1ef 100644 GIT binary patch delta 6704 zcmZvBXHXOE`*o-yy$DD*A)zDEn?yQ<-b<*4-g^@;sFcu@ByR<_>+*WndA271~o#&As zqJDC_Z53o-&)D(q!AqM(wbaAZuUFd#CvW!}vp;K%4;q-Z7Q2>YQ@&2>nMICwSp#~YdZ<*J~NxffY%%#!^nEY~f$w-*~436&G>yNvL9 zV5~RvK3~u6=T-4VNa##~!>6Px|4EOi`6=TGk7ws4u!HIkSZnRn^}#`Rqq}*uxBGjY z1I(Q*@{d`UcURp?k|Tn`BIShswB;K*S>$g?6;|`lz4xB+-tnz(yL@OCm23IXSEOS9 zzQQFk@^pCkhk+zqb2crylf4gDBQ9cambIEZJ#HwLgj>fKfAiUE8g_gDb+~*b&Yf{? zBFpjfYV~=!hYU0|KDpWDM<3o@P9#&Y9e>mI3+wovgC){rnyjw9`b&rdbJgs;wJyH0 z!M?>a|GJ^Kjmdw;41$&L7Wp)H*mvwu_{A(&NR|H=V*$uM2A?CK|MJ zN9cm^50s;gtc1czqQN z3O{?IzW959ijkqdC<_cax_~C4eugQ75IZY3aflsm4C-r#Qp!#7OiM7RFHl_u{1jIR zMx1Qcesu8TiYj5m;y( zjYt1k@snMxB*CX)i9&lePmMx1?pHjOo2u8IQ{T!j=|C55b)M|tcIpqZ-aRR_q$-Yh z15)4;{c`(W3;B3JWR(3?m~CC?P~q(){NSE@pvk8pX`6tz0xQWsPnQ$++&gWNSD&8U z>#MDNm(ASk`C`((u3~1c>3G?><;0{eZ)I7+b-vSn#5?0<*-KTMC&|OLz5RPr!4juY z7sjn-Q)4#MjX%4C10@uEu4Xt{lCRdH<}X}2qE)}(kyGLofy?l8HJi&3@w#ctpC*=C z#TnV)!2I`2PwgMCC8~8MFhIU5Av>~hi<3tw>ZJMau_#hH-bOi|3v~GFkH0jxa9IVv z&PzK`oMs)#N;z01Jj3}ynls~`<}~j${ZEU`!o1&Suslb(&`%xHmws|eN{YYeGI@f9 zN((Ye?W3&q+@3VoK5I9h!n4~7`H<|^I0+Gjcj^j-&y+GlGMHxB3k3Tmk-fa4_P-); z8V4?q^-?duK#Ycr=x_w-uAWjjBCKnCJi0JAE4-Zh?%)FCxh#6tluurnQ6n(M1TC*4 z%ca4bg5WZ}t|DGrTxrACM0-Q>DI#7A{#I<~zWSgch21AW5*TC*W=I5S0(w6^0fk<> zp)_|43rj?-sGzCsi?dN@5$`X<4Fku(wBsT`%y zizOI_>z-AEg=R;jC=sXc8gU~yKWDiOqOQ*59dHFON^u_zQP}p6}Mn=Tk&zs zZDG>2+|QcFK+7Pn99{ZG*Gmw_yf2`_G$45aS@j|EVBhE$It3$Z*8z_0!BErdN+^rC zrde(BW@=L@*Hm;tg#kTqoqIN8& zEybcPqd_goxKd(3?Twa%4hM`s02E_@AyR;muk}!mv7N7h*(pZX1sg^v0nZzs8Au?> ziX_Pu23>yyRd0ouu?_sWmlW~|vS#WbQg>FK82HWCm{w5xg(pTw809>~#cvIU(luyH zbRl9`d&nme!6{Hlltr3vTK$JkVnqsgV0(rc*VqD!lHEF&}RWqnmGS&JbJ4( ztC!|+BOY#>p~C%7l9MweTMpw~v#`isciX#*l5SiFRC{q|<~N7nG6pT1lB>DmH|7Eo zJz+u_K#Vb5K@A*hOKznmLSifNoBZ`*j#>+`|2j6o_fL3#nfKtjg~VXl))eoYbG>Cv z%Uyr1YFB=W*S5ze@frcfoqTQw2#@`;Wv!tE?3)+Wqc-LtahS|&pC*X-lK7BS>2cbc zu(ft@tRwk}stAcAh6tDqbwgPI?nKX|VJt?7T9&+p_e&OlG~LWR)Ia3Y<<_!|xLTX6I(}hXX>D+GQJf)nv zH`rug3yG#*)S(2{Nh_)!FPyc?7>kZmfz0dCo{Iv>Ts*MWfV8Aym{14^W4!O`kQJLY z0eFsbi7{JQ1XMu_2|Z1sRsZ(X&JdNKOL{vvsJe?zkEJ&^p)jt+vr%eEKm1fS4=-np zt+rM(=ZC=~etl4=o_Zt{(U)+D($-{KM>3%4i`f?#8mWtP$YU$d)Io3}qu)8|a2@Ei zQ4F>=<|*LagHl$+pfj+m$<=xdXa!xNZhvt#%HcEJAZ6g9(oZWe_#~(lsxD!kxXV5S zh(@^p{#KgG5^y>ci870+$T^|h@)BITF0wi}qnM4AEB#)QW zYFi9>5o9s2LXFaH(BsVfZJ7pCkFkX^GzOW)G*D9SU(kv`KrReRGS)Agy= zCyxZpQe#3#O?8PO?NlTxmVOwJN&v09e*)|pb7N>Ejwv$)joo5U(ftz2fw<1dupYl@ z$;Z%r*6NzLcJB1%1uc%lE(BGaIr;qc&vx>q29#7ae@@F^Iz0$%2C_L4*z|{S&ym=k zfk764*N5`z?3^I?LA)=?8@j*JOgp#oNIMVy{DD3YdHQ*~6JCB3alBqFHsO)6)1R{` zlk|~wOd!X3j1|NNDWsi+07)ggSP^Uh?A?KDU5D$cHGr7)<~XUZs-u|ZIr27on(i`d z1XcG~CSJ9nr54Z9{%q2(m}*jDuxQ!GoOP_c%r3AnU_h0;468S!3NBek~ z&2T}2YrrH8?1jQTl-gF&WJffW|21tGON-5jb?hFd?0!5w;Z1i*Z{?NaJWxDs_WMO3 z&MPHUiKtiZ0{DbmhwqZ$Q2QQu!rIu9TmUnPYzIrC0G5T&9!fwRG;$zDiBwd6*2j@f z<=^<25XPsp+%-NeC41EFONAHE&G_k>+(>I=tT*kG1Vs zQ8i-*^&>q9nId(U!sniVeOd-;@+SGX}A2gX{wgF$^ zqt!`#&Iqd_pi58ErVyk_OR^#tAk-8?A;RF96q-3J3;i3sQ#-SS(ORu`AVyD!7%Xcb zRICjhf>*lus%Tq87cZL{)Fcl~sDSrOKvhhQX&DL(*MVpv1%_cH0IfHHV}unNstW_K z&=&(SMt|i(IZsIwA;56`2PjKkvtghXFMcMKOI=a8@(ue(=2%76O#FdUbrNphV?QV0 zl7s~-?+@|R0Yi9MnoJ}>gv?Qv_?tOOCKUd^tu_CK&Ugy7yCz{=Y8Nh9PV6PeXz4&H)$&I)KLeS35?NcN!OlH7jZ7rS68L*jM49r00>1QGwEYC#uMGi0F$ds*1L&CkO~{LWq>us z)LU?DY5d)yfvK9CahS8pJkeIY+bFZWUN?nJt7$=VMQuo?4tN>Fl2m4^PLNlOZI6Vq z5F0&aqdD1~?ABxj>)h4eQ?t?J0gHuDs>2Pmy|xR=&W!C(yy7wQ1zM!gMg(H%EAb?B8<|!)I()i0@mozUUX2fH? zG39O3unf6nsLy8vPNDLp;aJJXq26`|OUa zjnunCZ@Cr-@Nkq}FGs|XO$)39qirK*P5tNG3ORbD!BZkkO6gzv?c}OmGF5_SLKKGS zN@+_56jDIC-)=xsK`a;t(BDrB$u0rHGGmo~UE4hr{QGUO9+V^^_YY*uTV=J~0I3C- zMh!iqO|x6DSfhiJ*o4HJTShN2Xp36#Q%+fjnvdehrXD@xF>wUYgDMzCp+e)IVnwd( zG6EnripDV10*cN=c&Rc8>+CJOAf`+(uz0h00gX1-GJ*6Vt?Sf(mqh2sjo!&?KWa?Z z{gXt{rs=^+R@y>x1UD2S$tNV05S}-t)XX{BgwW0GtOoQ?HQvG}9_ew65ogp$`cx|* zX{#%4jeKHfk1g0#iPz?L%21pNJKJ=EQG&a3Vt#R{dyne=Z9@9fj{|T7uQ4sZPE-!X z5z=2Tx)@D{`|CB#w#!ZQj_kYJZZ-6Xc3?41)^^6X(&})gE_yo0vurYGmihViskO04 z!Ee^fx$OXPMbY-&)6=&|@$5`LYfB|R8;)gen}?5Pu*aUK#9rTfhqv|lKjQW`(dVlp z70m(2#+&*E90y1XyMoYy;6G!yrmqW2zn3F$R#9&RLUVgpllNbX@XvZv-z$8}`Ae$A z^(R6m_lst-n5E4S+l}T_MTWd>&wC+x@$b;$-Ti* z2eZ~V9+u{1zjC}>jsdIedy%lgiuJ9XN!qzfp5|9PWkkq)%vY{8G63KY7y!5d006?h z#e&@L2f8?$1_yfhx&_@1_wjybJ!|q)jz0Q(y-|dNE4TM>Q`B{`oMP4N;`mv<2I=$k zKyOHRe^w17VfR=V3+N6^NRvBI$BCXFdW7%{uK)B)mQo3_*p(YGkQ(k{M_6ZT1;{Aj ze(ezUV32CIq}DgU5q$C%6j*!rj%9!j;(_0_fe4k z@-^hWC=;tltxdg4T}HNF@?Gcr)uD4Yq!8-<0HXz|svW05a>A+lUGg~0gY`h`PoprM6v(QvKGjCf^V>Hotk`ZTr@;N3Pqqd}zv0qsY6!mR&ogynbWO zwQ=jJ?~QsAJm1zC@<^MKH18MaFIIi<&Dn^3*V)ub9Q+mv4||&o69fmlM!X*TSYjLbPy8N&-a`od~tDWGDFH-g> zkajTgM{36CfrCVN>-4W>7TKhPnz1Gd)iQpII46=&%CB5qbzAtMUqqouwQh^*b_-

w^n9r=xl$D;M__BjLU^AX;`8hUQL@5Hv;uIRcCpc_R z$SX!(`n>YAL->9hZ|jzs9Qa~Y(v~P{12=R&_us3Lxpg}u#Q39V`J+H*AEH-f)j7kG zIOB=?@8;Q^J#&DJsCai#V2QdeflGm zQ_pA1=BuYsW%uXHrkhRrdM2vS*z(9XM*q4eOeC}w`mpr=#AMX){JR&| z$vYcc)3B~^c#$>E<+q7B)!u5; zZL$NNlftQVjs#nt1TkSL2ljKrTj3DlqIkYP;8CZ1jZ+O>GpDEfPKrxbK&em48?tlH z`oJ_-$YaH$AH)Ec*2-Ukl5COr;w}vbfs&dp^=IUKjRW7b4SUwv*Co0~=zrT4A+@(L zkuGBVLt1;pvs*c!_}TB}ck+2KPX2=tDqm6w#6I~}*^ zMjpmyDD{b@o5<^1-TVxtO3Q7kv-BdpUk^kDR+`sFjcqw9tWagMj>zE?*;&ACiX11h zN7q?yLII|V3^>${DAgIB=_5e5cuSG;wPyz+{rrA_ii^p3d-)(Cwj-80__M{O`3QNo zj=JG?YroXfpo=#TA~r4c%b@v5B-Ogzf?-{c140rE9l5Y1$ta#&~k>`zqPhL@9Me<)n@b+Yp12lb}xt{dfP)fAc8ZHE?P&I(50e zS?S!x$zZdl_2U~cLSZP0$Ek@Pnpa0y9@*lnc|ktyWVKb%-lH zmxSQQP}+IO+#qz_84*{uXL}M) zPJ~olKfLOzsxBdf>#&G_Z?=W9N*d%lhY3gqSel4{u1jO9bn}Q&!>vbh*pbbg{oi9Fh@p z_P>+$4OfBM*=PJ=x8s;NH584su92TMk+-@3Xm3iQJQaG^=zP`3UxEXaqU@vF-q8 zlKMUXU+B1eQMBx>M6@{wkXGa|(IJ4TcJO?63l}(E_t!arZa~30D?9Ec zl^!PT*kWZNAQ9KBbNzXKKbN&OcV^Zx!yiMws-hR?9fie3CC69#zL(~|+x z3iE&Jxgv{t-dEsXUMaRR?57xE*k&$v>tcFkFmRV{B!3CGl!;*HJMq3@w!d8Y6!AIuDY=_b$5j@&F;)`DNT6$4JxWz~b1A(_;rLramfWLp1{GBe~B zLA^I%Wl2PU31|iNV2_fl<%@(avC75wvjb?X)9B62pn+TqOSX{xscmu<%m)<#pkULm z5`^*>CIy}D33&{mI17}yrfv?P31eGtAxFJ$AZLcjQ(?N&?6kkb)-g-M9C{DV7UJ4u zzZLZz$qvXx0~&LN`n`*~fnUGC2!52IiRFRLL_{F5iZ-|pr%6ol$nLh90)duuJYsB6 zWX{r0yTv$~mVZ$E+7ub}q>8awSOk}C6N>Uc_GjNtJUw*W8{VJ#lLeY)7_A3dvY}FMZIMN$qqw@qR4(L{- zMW6#{--#1;_pdvp^Qhap+JS6ck9Ju8EI4%J7=HGzm=p{vkDVRe*EQd(YBFI{od_5} zIqoB^RgE_@7*Pf?>d-;Jn|zJCsmYEKG0TdyVRmHv9gYBN`xcWQ#!$NV4UP{e$(*cd zM&-IvcQkiatz|f*+`k+fp-q9fEaQGjL~Ipp$(N3mzWXS%{f(f6xC~nqnY$h}vi}-0 zS0_>sPtLgYFa58hS`r1Z-Q}+a1A;^`ADkKT4hXy#HFu4$U85t0&RGk216O(}2WDpsqEq|DeK< zX<_2VJgG6X_z#iWwWC(9S2PUtJDBwr)fvIF?-<=HROCF=?+}LYjU|$z@Sd6jCC=Noy#x3Fzmk0mK2?({mz1#jY7Y8^?tq?zmH#n zTm7UY%bVeyzlmEVdf&wi(Pl{vONsIsucp<)cC83(ez&<@KeFFMn8_wqNWf+E4IGeh zjD`#@NKb&F3_s=g$~6ovO{D>sIezm_Y;JZCm9bgDCPjGk%{YG%aY}!me*CCPeeTrA zfPY8M(3n4ux~z%W;3d){8;;S0A?p$>ure{?K*q(*{Ulo|>|_PI zsP7*4rB>vx5h%vRBoHweGuI5;?|q+G`&I4~^m?Z{?$G!uogNpPL!-|9caf7m)mS_X z3>3VxmBIMyvkb~6H?zBijI8Nf1qeg?uC5Lie$YqXG>XDEAVYaIJ9sy!{YAy;{c9S( zN<`f#Bt{(|*GfrkdP`1vF6nMP^QFqQaA}YTC`&~!8!V9;U9QlCeX2|d#mc-B0y4I2 zn5WrNd5o`c`x2Xq(n|#^9LVH7FHz})#p&s)8f^tmK><6}&WFaqTUu_V^{0(f>W;!DUfokq{ z7>Pqh3BK0Yoocm1;{*p=&bpRT0lMtv{yXM;V4x>j6%M1RNKnPf00cvWm;c;*Wx(f7 zwTv@#r0TR+RMZ(owaij~AP?Kh42pP)?3un2#Ap>j@AMe)SxQ>!Z8^qtkg{jzdX5#i zflIH@cFwE9fn8!uR}ZZRm+|a4p*CR1BFTAKJs1u0bg{I6Txh!PQyof4(V0M`U<4e^ z1cB1nMED2_8gJv5aBXzRqDNuB&JAfq*3 zGBWO0-1Gn#&*lJEqn>h_L>{(lvvQUPN;%wQPyJMyhSE(X1v`51UP2szG%l}OQm{E5 z0Gp*2q%B!uU7h-L44kAMVkVGZO#Ry{y2d>lL9hS>JnpBHdm?lr8SY`KqKAf)h6T|(LW&`2^x&7Hmw7M7C{L5*?h9!A zWwhlrK<-?b9Z6pssp64w8PK zf(L~r-GEC_=9NR@(1_C;@Jjcm z>ttwEeNe5(V`p@bIQ%8vI{s4}yZM-w9vTLh5zD4!x(ce{rDehaYT#eU%$=y2!~y6| zN-zaE(<^jmJsc6tBq-ZX2$+KT5~^cx?ZtpZ99?yS4V>=1N_o*i64u3-lP$aUxSV~& z$tr0m*Bs6HBq`AQjU-(K-1sWm0t&-#?WETdjYI7>m%y(AG-I*MJa-~|WD6H!zhQ3CT3XbA+Nd_Rvmq?0^1!+}$0j>EIV@ZlOfmSuh&6r6!Xpf`}2;Jsh^+|8*wwgb| zD(}-L8|JpumhNK+;+jG=2xCqz%0j~P1T&MH+G)prmSMJ(=Lg7z5#aUYF`Koc^cupK zd!QBx5Tvjs2;$FSBRB6$q8D&6=R2yeKn)TzIH_YN&}{a(6Q*{$f+&6d*-Nt>;4t0V z1P82)y4XWari@KgvMm*4(vu_JmE+$9Hr)YHYLk8l6MM8=*6jY~Q^{wY`jQ;O;bi^2 zgpq9D0rnqK6YOG%Da^Wc$at79TPCCVh8r8^nyDW31T7>7+JLkMh&efrttNz9#@fI; z$ZeUuMKW;0gP)YT0{#i{?vSnE0USLf9OMk)*LZ_%+mkK3m;~DooRS?_nFR$i?lK|h zw9KJk(DI}vlytht@l|?Phl7@}#n|LIv&K4;RGdxf_N649EV%H4Xh>7=$h#4 zccz2O-PNqdW|peq-zxEotnqMTGrXufh&X+&2#K?dQqYp1uxz;vK#1y97>JPw>*raq z##&txrgcn}Q4`6=RVDbq>By?tlC?WZAE*0|Ivu5aXz|W2xq*Fxaf-8tmuGwu)lbrv zfLL{rj@XsFWu$?rG+ZDvbSfh;&hLbmGr|Qf881lWC2GfrGNl46tZ8n}GL5bCDYwm< zVb%j}*%8V6TC&!e(LJzxk;R^MYk#dSLJ%m);TGcup_qDlFJe>uR`H7|``&>hwaLTf zaxqocFI^G6q_;o?=sBbR#o|hU^hT5Q(uQ3i@uU+sK1KS_peT|=uN^EU)$(m~OIfMf z+hvs)Dk|jAME^F$nSo(bqetE)#XEU#bL(hI2NBtZOFsYMA%7OBQjOdAy^H&~Rzj`T zR(vMw^LO9h=)zp7dd1>x>T0V)aK-Of`mGSeoEk3ePjc&Cm|fQ=Z6f@sc5fZ~QtQD? zlT9n}wC+rHc|4%)xlzztrIFLV0JG>kmxXk%*2(SL{m?g;-&}S_`!2{#4JasC0RR9^ zD}=w0>}n+%CG?F70C3D21o0y7E-ElSnr|}2NZn#+7qt7Dp`WkMgwi((q8qe6PLyfq zpWBOX^MIW{UIARie2u1*^D!-d)2M;WX7)Aik#n@n+|3h2d|m(XSU{V} zTWFMdog!xx;F_WbWKiCyX*Nr1ITK-1ZfX_-SG_zzPC zWpD39mSvyprpBbJuxV+k8GZZ#x9AADkDYIq7y(?F%gw!_`V~!u!ZSVELg=I`8zsFR zY1FXmB34Hp?BZH+7Gdv4#)sCb5f{iCK}-9ua)z0mA;iplvA0FCmh8!3CC;L8$4arT z$QW&@b)y@b&gBJ&K=W4Lvi;-QXCm?1jA;XrX8tK}a`9%%{k(FH$}-=!_g%S%Jy*4~ z{ddx2$H!ehN2pBmPB7W9Gq6=et`hByB`5}{L#P*w0Mo+(iCuuXZ;Z`aR!(#_596FT z#$6}E$OV@EDXyvg;zIG@vI0VQQ=r zjw0$rEaE4xI~8_GVck-RW>^V@fYTJEmkqVmeX z_BO>1i#Hxoh&T0$Uh4s!HhJNF8Yk`ZN-+3FkQ}fN8#>E=?;A~#OL2pC&8%$E1D0D{ z9W+90Y_*6#A!7nMpFow0L9wIa>ggHV&jSYnrJjgWy)tq*3Z$8xKmR#wmtyFG`A(FBj7G0WVHTv1ulk)x%vK-|hLRIw3UM=qE>+?iLK*yt$nJb<)r8 zqi22uTk1U0y9JylY(WgB>Xsm|zI#H1w}vJ9zwg98`Fk})i)kSV{Ekg4Mmz~%!#ei; z4VJ8jb=2VhU*t=tsSm%!ICG8qQ$z)-H_@-Rk@ft0Q$}41L2*c`|4$h|+;{!J(cy2_ zGZ}0}beG&6#{ZS{#$*Z&Mk@#i`MwC75kD?+yB91YdZO|5*F9JPm1S+wiQtvF@Z<+! zSJJHJ9Xhu8PE_i*E08rdSP;2aL}?6giuERDt;)S;B~1V!t$0{9KK)A4TpnTZ#qY*} z$}MYwKel}2!&SAKLzGftaD(8zC)ysj+vV)!*wBhJPuT0(wUdsnDOO|sKG+KeH8La! z_*Yc{ANMn`VAEiG{=VZM#h=?ehud_?#}!e(|RNuqov%i~vHgo~yx zC$>l$zCQMAIW>&SO=#Ye)I6B{Cc4>c+xEm~j5^4cE;Fx^^=c(SO&fRMPNf~TTEweU z_;=nmsfzP&s;993PsMsSp(&SW=Mo9~uh>)nkDF#6D}@5vr+7WN$3wYK{u({2we@wo z@-lhu%dgt-az>p$v7Np@w`bwE7YikZ#FzFqX17oe?n%NWKpty|)*)dx8cJ_#FP|*w zoBRmBzMG73yN_tgyCfZaHJREe0taiJz%1IaG5zCKroS%{@9 zfoB|N&(LmbHnaXSa#ETR;#%$_+|DNG{C>T*+T1a*NX#HopsDFx=LK3!@hV z@aM0+05oYY$*2?~96hLdBPbMBgt@3!KG%4xOBmXZEUv3Y+~J&Q|MHFCno6WQYm0is zXKhrmW71DHS+S6<RGc$HIKV|g z68%tF39?e=GA>!MsE7JqFCH%&T~JjUYn~e^Q>48KR@3-A~v#R9;|0DgF96Nb~RMv)^ R3K?0Ju)F|ux6I{}{{^b}F7yBZ diff --git a/summary_results.json b/summary_results.json index 195c20fd7..e714d0e73 100644 --- a/summary_results.json +++ b/summary_results.json @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.401784, + "Performance_Result": 0.413475, "Performance_Units": "Samples/s" }, { @@ -1013,6 +1013,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "gh_ubuntu-latest", + "Platform": "gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 80.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", + "Software": "tvm-onnx v1.19.2", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 1.71562, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0028", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", @@ -1044,7 +1080,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0028", + "ID": "5.0-0029", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1080,7 +1116,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0029", + "ID": "5.0-0030", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1116,7 +1152,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0030", + "ID": "5.0-0031", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1152,7 +1188,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0031", + "ID": "5.0-0032", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1188,7 +1224,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0032", + "ID": "5.0-0033", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1224,7 +1260,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0033", + "ID": "5.0-0034", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1260,7 +1296,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0034", + "ID": "5.0-0035", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1296,7 +1332,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0035", + "ID": "5.0-0036", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1332,7 +1368,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0036", + "ID": "5.0-0037", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1368,7 +1404,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0037", + "ID": "5.0-0038", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1404,7 +1440,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0038", + "ID": "5.0-0039", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1440,7 +1476,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0039", + "ID": "5.0-0040", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1476,7 +1512,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0040", + "ID": "5.0-0041", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1512,7 +1548,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0041", + "ID": "5.0-0042", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1548,7 +1584,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0042", + "ID": "5.0-0043", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1584,7 +1620,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0043", + "ID": "5.0-0044", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1620,7 +1656,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0044", + "ID": "5.0-0045", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1656,7 +1692,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0045", + "ID": "5.0-0046", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1692,7 +1728,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0046", + "ID": "5.0-0047", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1728,7 +1764,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0047", + "ID": "5.0-0048", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1764,7 +1800,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0048", + "ID": "5.0-0049", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1800,7 +1836,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0049", + "ID": "5.0-0050", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From d3675dbf5033903bd5a6de437278c364e43005f2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 21:33:19 +0000 Subject: [PATCH 04/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13904 -> 13904 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8182fb559..33c536186 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 21:27:03,671 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3266 INFO] --- -[2025-01-26 21:27:03,671 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 21:33:14,327 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 21:33:14,327 submission_checker1.py:3266 INFO] --- +[2025-01-26 21:33:14,327 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 21:33:14,327 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 21:33:14,328 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3282 INFO] --- -[2025-01-26 21:27:03,671 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 21:27:03,671 submission_checker1.py:3303 INFO] --- -[2025-01-26 21:27:03,671 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 21:27:04,923 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 21:33:14,328 submission_checker1.py:3282 INFO] --- +[2025-01-26 21:33:14,328 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 21:33:14,328 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 21:33:14,328 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 21:33:14,328 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 21:33:14,328 submission_checker1.py:3303 INFO] --- +[2025-01-26 21:33:14,328 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 21:33:15,580 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3445 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0085 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6cd3ae2db..9bae829dd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -493 +494 diff --git a/summary.csv b/summary.csv index 186d6846c..377b077c3 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3445,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0085,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b14ff30e0..fbda3a759 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3445, - "Accuracy":"acc: 74.400", + "Result":10.0085, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 92dceba79e97da734411c236c1b84d5f2ba6c1ef..0b0eea0e4ca59c186299b9c731ab14b2a9544846 100644 GIT binary patch delta 1859 zcmV-J2fX;uY|w15z##++!8U1=#33txWnX3!eI%!kzpauOEosi8EIsMzD()eg23Z)T^ON5F-MQlS z5aWIt`f--hlim|$y>Ej*jv3=fJesnT-UH{0BTZw$15JFUW(%4M&n(LmpNoHg^SQ$CMQ@Z(4K8QYu*|G#BY;T5o2vI7(^L{8vHqh%!w=ocPHi zZgX6mwL~m;G^V-F<;A|Vc#_Tdqn}Gzz0SBC#2Mv+jAR~) zfSWc;#R|q)Z`wnj`=V?_qO>5+7!9?n%VCxzS<1BegI7#N=@s6}wUu;#8HJIr%ol}n zM^nLke5ZtT=8G!wf-IdybBSG*>X^R1vbh3uygu4=yo*UHB*7#8GX9Ew(kw0iJ~|39 z`_@k>n}1)NU7Vut^gPQ0ipF<8k%{}bx$K2!!(KM|5nG77o%BQ(v$8vSgVTa%qPuB` zrcdaV7#d!orJ)klQc?4N^+H1gLnVx6v!c`*WhJ$A9Q%n+6bDb0e!uBVq7~2mSaDa* zR0=)&C4!fG?Otqgt9XbiEpk7QdZI2I1F)Nb-2&{=p4D~-u)7-ywp7)%ay6#daWDy| zmK`cNozYyVHhnJG#gk~h;K5rG)l_wb1pzUYsn4jk#U8+jq8R;ufR|6b!3A${%Cg*c zz0oDw#y#yCY*UTwjOBGrtZHc`7VX3Mv)X{Trl^YsZa(*=BWnw|Emcjc&O}R6hxK}E zk*O+ypnk29m0I!Y42^F886wBA4P=|jFv}@|Pn;?_s4DIaUTdRQBz9k&BdVeHC7s(z zE=^f3+qN0no8jeu#jQ6P-i${_NEf_a%QUfWJC1AGeT2?0&R)F4uvEKmVbj2_j`bFW z8sFT1AL&QP)>X0HIzrC|a)b;=?b}YjHe3orUo~vqLFn%0$LM1wfNm0l=ZN64{%R2Ks-Tg8r~sX66Q>bG3{ zrZnd*y*yv=e_PD8WD~#+tk-On@Ff6W0`MgOUjpzo0KNvm*8un$0AB;(YXE!=fUg1YH2}T_z}E!$ zngCxD;A;YZd`*C_3Gg)mz9zue1o)Z&Us-%OW}P4sEO*dBDYgT4N^!G391hHC`7k{o*xtbIN&pM~r|l|O{+th2X8 z>aMka*LLlju-8%S>yJ6okL{47MrfIVG;g{ejY(Sk?|1CO1prFDsOc$E5qp$UX|VHh zGe-3$Z5+2qxm??2Iae>&L{wc9H#`f0HoJotFd%RD(w=f#qiqP5V(j@A z6!Teh=qRp@Q!K>NZLoi3{D*A-7kNI%3IY=>FuVt*b|Ru9Qt%0Be_b~M1jo>U0v$Xi z8Np|70H^%ThEch2Dj5a~H?1dea__vHjD6o{eK}fVeuBSNukYiksnq*ZaX?KaIOPsl zFs`ZaWf2e8F&|d(#A^xSX^8rbx8ov*J`hO^Ko4)XQ{=@g%}H9!+k()t xq-6HJA9>Y(2CCzbyNM^jA@}tsX#N4Sb1|F)1Pj47X_FH*x&-O5>oJqjHGjCXmBRo4 delta 1859 zcmV-J2fX;uY|w15z##;jlGq)S#33txd0%E8Y|FBI2?d#=LYSh4+a|Cz`;>BQC3we| ztR;u=_bZbC!E%R?IP*l&E&KFobsx#;<8O;JL338{IL}Tr%fK39S(rz0Ha*eq@6L5! zLtF$|6eM}ZPP8Y+wQt=&jyV@dKAQ0p?Lmmyk!kYqfu#XA@)^sdXOb6bAmqP)#nj}p zf&~$OU`(WmNicTJG>9_|g?X8Y6U`-_hRQ7duVlTY#qz1vJ?3%uSah%Qh$Y9S=pLJu zVQCzOEDMS_uiNjlSn#@Wz<8O6y0u_DV~Tai3Ym_&^=2|j;*8bJf0e9=88;QgVUW&} zCdb8DL&TCdx;R5$m@zewOdL9YEu&a`t*`9mNnZS5PYPH$PGcd|s88oXoCM=IiG}*m z6BjJ4Td!rVm6&gfJY<~b%KK-3Du`jo5>^C4T^uNjhxtT228EK<>s+WooG~HEDCV9F zxNdU{tYeI=x;+X+Aj?K3$_$g7v&g)hhhT|JwB$59;U(^;wC zu}m@_-02aU1hR@oVV+ImslslV>V&<%vc3XzygpiYyo*UGB*`QBGWd#qvOKH)K6*N# z9p|T#&A%_sE>6*RcAgg@Lxa1Yh~f^eFMHuxvsX=i#Ah;ZCz|YHUUi4oJuO8pyPHL5 z{DfYKq23jm+XgWl1Jz%DFEvCm)T4MZDJ!kf7fMSfNstCa_wZQnbn4DDUWg({bbsNE zmC*BFGI+Ju?&TIYiifPyvIs(@Csu`H19l479l);a*~;z#c7H>`R;s#IuJ#yv9wyP) zaU;E86IMvoX3qsXdy>r;K8)tXpoXRMFeFqT2b`H(>;a4@i!lg)MfKDhT=)j3&#P^> zYF%P(+|#zfHr2?^SfjRyEn8YoWczUOtTrGnm32|WEvA8TWNiVrrK*|BGtp7hVZGiO zWNMi}P`}p5Dy?{RhWfYv^pNMdHgc)n%L|6!6K9zmv@GroUT33MCU#$)BZh5s6rI~i zE~zS)P1_9ZP4Du5;&wFb-3ugF4jL?*){x4ZWOHr02Fs{t$crjbEDle3-BjizkgJQtt8rV}bwO}I7f`*IV zm=&U-m!~uFZ-aRy*#xiy>n$bHm0c;;l^uA$1sqWpa70s>F9rBgfG-93Qh+Z7_)>tcD!we5vgN5;sk;1z6;I~sOs$>kVxH=GmMune z%dk2I+4kstL)d(0nbz+|SPx5!P!mwFo!@I6_`O?PQ+L34xevD;`}_AEHuhS@v`gX; z1v5HO!Hilem^27;zJ3)?lW^%Bg|pdn+g2@G$V+U0oU%PTRsnmj4-sQCd2=i{s}#al zul~RVWjC}nS}h4u8-T#otQACp)ebr+#b&@(DQ?z>!+|*sAEpNc+nd?D&(Oo3%j)O- zAvI-hv_0~8cn$jqu6+?}1+m?nwl8C?Aht`gT_O7fS_?|{DYS!g6O#(a@|t+HS|ux> zh9>`iB+z4=MfoFdR_6mW+-LCGsQrz2?H2yFP%Tvb_E0Ue`Yq?cXCYfq+Efj z+H39KwO#ur>~$3T`eTmrV>{$nAvDZD>Nnkw#v~2?_dE9C0sv*TsO~8<8GD>DWw7&d zGe)aT+Bj}ea=Et4YOY?ciD-FE-0&;}URzRVHJ}pz1g-WT@TkZ;&2QZO$3WFINgV_2 z6ph_RG}OTAQ0jTA;$s7=1DBV%kxdNZ_-k_`*Kl3edm9@OSt~wv5&GW09$ogYrNvc# zh|>X_W6;HvsNM|=c4;Rv&<)92m~|?JkpafFd%t51 zdtxVrf~+K!0QE#AB!WQPEPLBp#6PkPY2O|(#E$W++Tgr`tS7$7^Ai*aQq&?-fLfa55#n|&L zDCV>1&{13)r&x%k+hBjo_&3`CE(n=p1%U|`7(M_~I}y={6nsM3AJ_H(!7;R;KnIWW zEXOBr0H^%bhF&>yDjD_`u3C@c;|H`E&fadU@+tO{G4ZiUX=k!6|pZ zf^l_;&x?4tmie%XCtgbskNc=!c^elQ`amQx03G5Um5 Date: Sun, 26 Jan 2025 21:52:41 +0000 Subject: [PATCH 05/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 13904 -> 13903 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 33c536186..10473921e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 21:33:14,327 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 21:33:14,327 submission_checker1.py:3266 INFO] --- -[2025-01-26 21:33:14,327 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3266 INFO] --- +[2025-01-26 21:52:36,134 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 21:33:14,327 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 21:33:14,328 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 21:33:14,328 submission_checker1.py:3282 INFO] --- -[2025-01-26 21:33:14,328 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 21:33:14,328 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 21:33:14,328 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 21:33:14,328 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 21:33:14,328 submission_checker1.py:3303 INFO] --- -[2025-01-26 21:33:14,328 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 21:33:15,580 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 21:52:36,134 submission_checker1.py:3282 INFO] --- +[2025-01-26 21:52:36,134 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 21:52:36,134 submission_checker1.py:3303 INFO] --- +[2025-01-26 21:52:36,134 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 21:52:37,165 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426567 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430339 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9bae829dd..46d94e81b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -494 +495 diff --git a/summary.csv b/summary.csv index 377b077c3..333edb7c3 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426567,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430339,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9898,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fbda3a759..469102436 100644 --- a/summary.json +++ b/summary.json @@ -1239,7 +1239,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426567, + "Result":0.430339, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0b0eea0e4ca59c186299b9c731ab14b2a9544846..4d31fcaa42a8b40a167c6e20aa8c23582ffa7aab 100644 GIT binary patch delta 2350 zcmXArc~lZu7{x)7P}I;e2b&fM%@i`3NOLAE1ja2jBmyim!yTP8wNq)3BZasEN{*T0 zQdVOMYNochq=ix{i{_Fu+E7zuHrXob%uu~QzQ5l2o%`;0&pmI5J;;`|Tbf2Kr7ZVo z>abLrb8y0>)3Ovg(SBGQz$BuobDQ}-kTwe$pAK$C70d#+WY}1{01g>`F@OG&*mUv`oPV?t+;yZf;=G!c-VH)tDSimJP7 zvH+%jW71=Ny8Q^UsYqUUSA1N{xzmuk8v$6BD*l>bC|MQX{L0Mf{xoL={CzO?<%%yd zm9rw)Q~RkEa*HXybmw5K1qEn^Pm0*Em@shsfo zqta{^Ho-~YU!ED+IQdUAJi>gR<3FzCV<8!crdcRM;~VkdUO>RG9c66o1*hLwYC9x6 z@{%i&porqS_Fy|5Pb|V@FEuG|=6^_6d3N0Es`SYvC zf{YsvT_M`kPz9NB?R)po+*Gb$+}AO|Xd{g?PC#Wz!J`B;NNNLJzUxCDp$@>+ALCiH z5{5)glVIk0==1{>fWx;XP~tzHTenMJefuu+xgAm8zKd0yb~3{;t+4+t!{3vLW?k|o z68Ldk?;b4I*^LX>S3&482VBGR6kNhG%#I02p*h(4C6C_tX2}HY-nYSE6?TY&c?a#r zO6asd(;v%cJY*1cpB8-njM!-9b6N}H`+XB!TtA)TE2s625LU%rRtr}Lg z8`OiOL0s(<%IK)94za@}=M&noV<8ujP0v?drPWX#W8IU2{bq`M{76Q1R3gDAmP_ry z?lMkRI>nglB95U338H)rcHf&S&#c6qYD*!DV!fd)iDFHf+dD$Q9D@=X^(!$I$q@&M zm#vW5=WTp1@_BXIY#&Ir#ZWdK%x6{QOMDFJNG7o4y%`Cdg*0ZG_75#a;cdKQ*Y?mY zla%-_)^)o^O_xC|;l`UX@MHFk-51Ep(1k*P+J3o~Q6BSX4mHdE5fhzCObloF~rLEwP z@7^vVG}#CIgn1qpBqdJ56aih~SIf`h?vV)DjkzqbC1kW2f0a)wWE-4kLS+Ju)L~QT znFS0Rb+m(*#ih|8F*k7jq%da3$0&b{7z!M&R}#sa$UD}NS;Zh~)g_|ZF#_R|fV3e5 zgV;z3FnFvf3qY+1B)R~u;42iFLB>r_3T_LO1le;31PZ4El3JpAor<(SPqME@Chx8h zxZ*X82kn7Z$9Fr+sT|~HJ9#krnUO~U>5M!|AGeTNanOjniof&XW5My7%rIh$$<0vd z{(|=eXYZM+{?G1Z{SV5#+=rbTU(tkn9evl{uSlhnB%@A3t$^+%VH)i#1~QHI70

  • YS0?OJRm|Op`_1{rKNc z3jLT#&zWyLbx8lRBA;;*dX@hzepcaQz%70xcmMO(fCl^>J`E6-aaw5F@g7??8acd*@$|A&>?(QTqK$lu8KRs6BpY>> z&m?(qYlAi$*Bn$4!s^=mRi)0czP+rhwsSI760eEQe4a2l%DGYOuu3 zjW`!lB^B=%&beaT3DvyTsU8oBzUX%@gq2-ZRTI)fpme=$xYSHoec2rggm`_H7O0at%TR&uzn@u?3`jvTTKH1nc*QcD%^8gy|FJ4D@FpZ^k z-rZ?+ns;y2!C%Z<$mxwq9xwT?Nk7;8u4dtUzN8ntse$7< zm~;QBk|ful@rQ~YMD7@iI`Vm5_v~mb9Jj`r%JaZR>iDGVb1180piJdG#HP7k_hiniRvp1;_aLA19Q0RE>aq zum9E2@Dvqj5*E3m{rJEWI2H4$G3Vb-&&qvyjVO|})8)3{ggpHA-#=TE210scYi+eL zfOC>Z-n|xw$Ga~KgX`?3ssNPyts&JHKb;Mq8MQ*C`X0TqA@N4B_MEzfGaCHSJGApe zsf34E2)cP{Y=dRAW59q+z zJ1fQVHw!hcN3$1Zz?KIUQI3$T&VyeH9;GBLi>t3pMKGowb4ZyL_#J@{p z>PDCUWDZog%`)cE<%hOAxM+2;cljFhfO1c{kQ6AGTi7~x`9G7QTEy{{gWMyt6G20Z zpr&JPaL~6uj##)Y6lgDgz3nFxIYK;uSS{X-a|SBKY@8=>r??j9_{~}sZZ&Y7_yrCR QOcrn2z7dT10bo4;0}q8psQ>@~ delta 2354 zcmV-23C;G;Y|w15x(x;1zSG4yv%d|?3V+p|xZ}D8e3-Xvx`b1Gx{n^+-36`&em^`*h915OVp=?Y>&37-IM7 zsxQ=OmG8%y>o$tR7t$!1WYQNR86}?%+a)0cGr$*O832ms$U+9ClT6A& zB%|aLdb=b9u(FJVm1QKXBm=M-pDV-)NLYOZ_6?{)(2}#=2C&kQgq4I~!OIH4g4fh+ z5B(5WYnf!oqSsXQp>ok{jCMT%#eX>bA%h-DCjB9jQSxcoT@pfE1N{iKRvJ#Ere=HSfowibAF5@NAtzE()rZOxX_O?6 z%eTg14;ge$GHDNyjIz=mA|XUGz#d{5K$BQGfTm9#Vg&$ApF4E>$2&B^9e-lG4bha{ zdZZhkHbYFI$^`z!Xl|-6WY&ElYdT5nc$kT)`a&wZV-qElO!`73qpb9WNC?>s@P$|g z*d&i10ycfR5Gw#|`fMQswh*-BY$Nh+C)>5)^^p+L8X^msnq{H;X#HBTua@}*fx2!B z*(ghu<7SJo4p31$$)qhrGJncSTZn`Z)BszEWdKxS;Q**US%?(?P<^hD0aplGa<tiAOI2zy&u|h~{&Hm5>NPU#oSIhi@NL?3*tkO=e&%_Q=**9xRhlqu&bcjd@Q4Mg2 zSO!of1`wd?Q;1jrK-H%Z8BmCzC1)D}Qo$f1y%0~M~S3G#6nhDL?nb*23SNa16T%FM8;ydl&muBl5kX7NcANYXszVxr zGiZQV#0ud>qm2`bgz$H=q*Nu3I(u%aQ)JehBGrmi>RRR|4oZog5=p6ug{+i{NC+tn zkcwCaC=HN`j87_J1%H52pHyT(DuR}r?Kb&cREN;5N4g=BOez8-H%=;|NIuG1sAXA0c_G2Hupg{R55yEs(*^n@@ev!as2}plR0oD zYDy#}BNnn!G9n?wH9#_A8Nii2QV4MMSw^%1;Of(i3}{BslC!Pg{TBFMGtv#&40!^P zso5!d2%C?**|MQR(HkS4KV&+QODVDuXVVfBC6Z1N3t8zDkr1%SGLlV}k!-R70h`t* z710WkO<#q53x8G-wB%$faKD9(1n#$x5YWmB0j;)XsmM#~*WPTYWpW{)+I5-8=)@?; z!-%YMfJ(HKNZLd!WTj0+LI`SrO~f()Dtnd?fa=qUXaxY(=M!1*iJ&E?Taj0BSqRp= zB!r}fJfO(dtQduo`ly>NwM;IERPn`UI_p^1DOEd2C4ZJmB;_I&iltmcLP%a&Yz1t8U@7v28xj#SW#$X4KX3-s!dUPxt#T~rzWhEw^N^XH~oMRwgPGRh>W z!}zlftH=@?C6ZbZ3&m0^A|a$QKr3Pyppre_M}7^9^?60K0#NA_i!6vm(2}#Qz%?Th zlFw=Bv44!bvdcCGS9Wd9V$nLN-%FpbWtJh=cx_cLQb{Y$TC6r@XEX$;MJ!W(F(w;&o8pz7ePzTwgMN8pj(f0LpqsE1a#Ic6Frp9$6Yw8 zWtKs7+I2e152OyG$U2N7OWYJCjUpD3Un15eA%Da(z$juFz>_`h2k`VMMYICo>2r!K zI7QHsv#o%~*>4sE>C`Gnrr6`oj zM}G+nwM;IEO5qg6)d%6MQ|WMuERj=`bc$F=DmYydLMj8CB9;Lv*&~3YRAhZp5v>4J z`m7=gRuQ!1Y%BO_l!X#``V0__SRn+pX3^*d)bH^bbW=SeTlI{}g+Uq_*N0GtXk>|; zqNHfVLh>$OmxK`00MUqL08^$30j56Dh<{c9Onst}1v0e^y#L#Ys8Q zs8zm^%|@qZH$N|?7ptGj4+kHniy*~xeDm8`(H|DG9OlElQ1wJf6Z#cFc5_AeF~$yEz2gM2-|_S{OThf%DSn8j&nV321JF)SDW8w9E7nB|b1MSZ6fUXaJ8E z%|pd4<;KCAGZtKs;FAc_o^o5GZ3vcP?D-ZH^I3H0D6WlDEX2}nf3Rizhiw2Cc|OMq z0uwASya%RsBBCQw@Cj*uT{i**$IyWS9Xuu(!Dnv(r~J)^QMqs`83qeCttWAE@4TFh zecxw&Ia*_Wg1=U;@8hbe)caF$Kusk$@++9vXdz_K>^H@Yc)CnK9i<3CIsoR>oJqi YH5LIMljk))0dtckHX{ZuGXMYp00)mZ3;+NC diff --git a/summary_results.json b/summary_results.json index 5ca6537e2..b3b44b5e2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1112,7 +1112,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.426567, + "Performance_Result": 0.430339, "Performance_Units": "Samples/s" }, { From 000867e618aeb7dfb050091d5f0ca8d4f2b5b87e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 23:10:33 +0000 Subject: [PATCH 06/24] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 13903 -> 13904 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 10473921e..d5ce41db8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 21:52:36,134 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3266 INFO] --- -[2025-01-26 21:52:36,134 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:10:28,695 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3282 INFO] --- -[2025-01-26 21:52:36,134 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 21:52:36,134 submission_checker1.py:3303 INFO] --- -[2025-01-26 21:52:36,134 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 21:52:37,165 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:10:28,695 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:10:28,695 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:10:28,695 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:10:28,695 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:10:29,783 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.5726 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71562 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71038 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0085 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.085 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 46d94e81b..39efca76c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -495 +496 diff --git a/summary.csv b/summary.csv index 333edb7c3..b427c1d71 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.5726,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71562,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71038,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0085,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.085,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 469102436..61337ca77 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71562, + "Result":1.71038, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0085, - "Accuracy":"acc: 75.800", + "Result":10.085, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4d31fcaa42a8b40a167c6e20aa8c23582ffa7aab..37a0a271c7c70c0724ddcd1f89b1a3e4e329ad9e 100644 GIT binary patch delta 5720 zcmZvAX*iVq`~KKPV#Zcv8)F~TBt;Qpn=#04?3zZAQ4+G%ts?s}gFzx|k6m`gUOZvS zGIk!NkUceaMZeMa|MLIh{v5}Bzqs!6I?wAoujBX(5C#cvC_JnvmcW|iHXeCkYE0z$ z#%`rsx=ZbaiymUEmj4FG$(r)R`7fE#Chf$T-`=`&itVi2pU2#xq_*8rdi4Xo=1b+R zzbmeLp|5&KX@Q&DA7wj@XcxvV1gg)JmKvH@77lk3E@x-2kA7|sY2qLCqS^qOYpx$& zFS{IKC}n@@|Gb(ho&3775cHzr4q*Re*|+j>o5ROf7c;4yL7K~|qBTL9ACC|(lw}WX zc2>WZHN8}&9r+A<`VNig4&E_4nDacOyse429zW@!CSIU=>lqcT>Fg|QUm$#FqyBqg zKd`NAW8>8O$d3KJeB`O{|&xyy^T9% zKC*Pv|I5^V;ANBXD-;(|Cl$MzUe|HJ@)BD`bIF*gt6)*{r~7)7Ifd`GfA0AbKW%xs z-1EQimH`p~9-eP5QC4@v_WuG_rSr2rv)0L-l2^7DrDH@K8^4}(PSw_>Bo>Fdv;AM(n6x|Y5-YbY6FyGnL%)!5L z{>BupqyidI{M@f(KyOLa#epN;{A=N;6!<4CP#21Obz}x*UQUBS0}r0*zs1c9Mn~BV zdO^S;Z5Zg+Juy<$MDq#5+KFZ(7qH*fJh$wdCtAU7Edge*M#-~kTm?TgadeeeCeMWOBOQEs4aTY%rjr=#_qv;Oj; z5JU$#P)nlMOlxff^8SFQcv|Sn;@;!={6o7xT1SW8bjriM&0D&)A8hq+&yDUcy#Lot z-P25^`j1)lsM*f&Rp&Uf_rn&Ho-xAmvZ8Z^Sz4Ne+}uL&e$CZG0YLrfw>!kwnGv=R zTC?v2RDVhv{<(swzw2;|n(n@NORFZioLs@Ze)3z|mD20US(APT`&-F(iW+kt-o^d% zY|>9nyY!EP#Ou?|XX^pGi&EVPxa`nvF;P)5aN-Mhn-#JcO_%6^oPtKyD zkn0d)AddA%+}S|f!;ayhRtyDmRhlCk|AW*&-7}+*d{#0Vs-Z*dIcqeXkViMHiYea9 zIlW*p!oxoS5xALvi!*X)QI&Ah@4e0`_5lP1G zW)JK>NiT6~A!YJ*l;$&j0CGup_c0*< zS9)nv>WA)FtW`@CK2@Pu6BbOBZ755^R*Nsj%6 zS~X?>&tL|HWHvPybR`H4q7#IiK7Wk&#w>?+SvJNr1rB*``Wf=%+e{ia>W5ksA$s(J z7>bFa+BqK`DuyRRKZ@^1Daqnsev=uO4&MyH{RTy3l-t|QgAI@`Ju`|CfBR{vHlrbGmg#u zf6qAlLdH^;?_5Q{0)AxF#O&|yAr)0mcJqkV`%m8Pe|lcvzX#EG(6Ce~l7)k&9xU7` z7^??oaYAgfH~MAh)iV1mlOn$#1r8Jv&`Y+s-omvp=^1?GrLx+qKJQC#a-5Cu;Lvw! zS+^^DyAjTrp%yzt`cM4bV#O`91?Sv=cTBgenPp>Si762%z!1Hp2@7`%-b2D!+(=AP z=_{(NO0KbfBT!uQ{FNUwlovJ z`9sCjzM7}a$i!oO3|wVMNN}%ww0&eapB?W%QSKl+gk;y~g5VHWSSd!?lJ-?2oE+}W zcHdp9Lr@R!&HepckaVw=fqQ%@a+XpEp+!T4Nb38$!(Xqha$l|#;Ymd~pkVnhJuwKg zi@hPe4?(h1mz=d}R)aa{(pO3bTYt@oRSuHgl=lEEeylUW1)?u8_MVs1R$@Npz3gvwT%VlvIroGD-m1%tP~lm(zTK=Rw~=J2rX4>Qzo04yGBim zDgqXDGDW1@p{T?tdBYsE*Bj^POURJL8S_+3@<(yzZvBX2`~t?yN#d=esl-PKf@GbA zi0rYP4v_lk4(QWkl`^)fqWrprES~gk};0+^E znnOX6@*Y_JbQ&*NMu=#*4I>LN;7Kb(AxIT1IF!t!(wiR=hDp=!;sT|qM53PQu_~XK z_Lkh$Z4h5(_mxoM@qJZ+J&japZEvUn95-CEEI!4Ii81;OX2Cs^6^Q01_uv4}KZmq7B zOG6Qg83p5X^-6Bf;X!xbR|>?4lg4sY47VHh%54RpINjXOoKT{ouxNHS;G&9Mlwi^u z3YQ;G5t3n|8TL1{r0~>mw@xZo=hJa(0`DOoRw<)kAa8trJ9j&8e5qP4CmjOxs=ozNRBH==Y#r=&jTFD{-C4~Au4 zAC6>T9EctP7)96gMQR@;rm5uF<9N^bnc+NniE-65aj2C3@H=tb8J%RTM%ZhNIF1cu zns&u+_kD<3rx>o8XDH>^>e=d#OmhEYJ?i$jTdInWX-m3T9)P8v<4@j=unrxNHI}W~ zV)Sj#Ae68AFO_eO#U}iva>8#CK(hbk(r|vptlq10Ra&ZG)JO;+NJa7v9n|4mWi~gN z>pZw8B@?WBKMLK%=oqZS$n|`^@=D%P5Tv;lL5j>|9v#7wL%}n&(eZCWvOE1xoAShn zA9F>K*Sff*iUoh^B}#6iU5?8^>*V?D3o)(Q{N$v*HozXEsSn2&>ggk+N+R{T5Mhev z!x$z+GNz|QC88$`c|1LwNnGsUVjpF?A@^~P^tn#PjK zP@!rX6e`t4dk4jx>0q1)l~{&H;0%v2{m&!wY35=>d@psix}dh4sv_P%78VxXKJ>zJ z^f7Jnfnyfb$|uHJ+T3Ci(cp|!sIp>K5Uff{wPYkkp|~X@C>SU4KH8uRWWGX*If*hj zo*3Eo*cDF7HtLc-32hKIKZZ6C=#{hQPW|waySL?f?c=^^Wod+>6e&>Q$w)M|$2g0T zwkX~#M%toeffbMF0;*h|U>G{h9Iqq)^msMgi#WKN5*6-4q@I<=&fBh?c8>q&-Q*}K zg#P)P_7YjqP)@e;bonb;iOg!C1NRl;6#W8%>WzvhONm2q-rr7&!?OFhCkik-Ig`QV zphk+CG|LO{@hS^!XKNde^V3s*l&ynT42Thb2YlndS5GK5_rrB873T;O%RAPEmPVyplZ+*rRt0GBFl@ z-;&u=xXN69oe_>MT5F7ObjfI7X`yT}He#UvzJbth;5ce+XY;^*tHiq~x#Z}u{uuYX z_q6?5Zllx>v{*C8f(yTN9B=}Yko+FVB?EMRRK$E*6k`|s+&UM7W&eRs{Ka(D*_feo zAmt^L+w8B-lB$Dzp=Y-PEx}6RUAAjyv`}l~6Wk0{xJO#>1vdF&>@t^J3DGz4(LhFC z!ObotO^C6xpRRm4&4}8*Y$nzm3QXF8Nyp#bPzWx4jIhHs?0B~nZ&}0;x{Zmdsn;Q= z6*FE6r@mF-+T?w!U}^tVz#jUK{IWn*lC=+GSz8BRHh{BOR~v!Q#XXb)M#DvZ_69;_ zfBXG0EYDrHusp1%2H+z2Mq&)|9nIU{-yZbdjw5`!&?m%h;nt@ePL47p_b3?07{Xy2 zbVkSLxB23+?C0EH@VTu(;yu_pMr1_n`QmeAZa8+@sBkmA7XSSCUoNTFj!=i5Gk+v%L?jED*kT-% z*p`#Kq#RjB5)x{lP(v2n`Rnz~1i>q$*WXf%ia#UvOg;M@gM->;|J0{@^~#2cVJk7j zv4CQ%*OecS#V+Uh%|_9lNE>q`$REEbWYvkyVnA)I|KGz46v`7;$&}nFBLJNEagnJ_ zwqu-i?*+(}f7XjR)HJN}@5kyrnvYqjRfDzvPaf*4+5;2sm#d)}^7bz(S_&&x9;FgO zw%Vi!+4kQ)?sF>X=JH#y2oM%`S2!MT2;E#cby{`PXS}K%*tl`?p0!B>?3Bg1Pfxi1 zT`h8hTO#}S`3U!JZD{N9hNw_q?M9BIv<~g` z%ze4ICf)empp3uk?t`h;IRB>;%Ds0Z*7!Zd!l*b9gik%=%mW68%Y0{_UNsS#+?Tt9fhNvH!^cG9af<0vRwiISut6pKAAN z-Z7n6wOLzD)SxAP@G_iP^Q6!v`Yxh2HOhj^nQi3gptqPz!@`beSekwk0#dIog z2D;$uownfA8W&w80`OnD%KhXLIfX~Q`-QtUc2by~bHewm3lG@^OY;eNu5ZjdZ1Uh}3)UD< zm60}miEYitI>b%u`%@llGMZ2HN>?h}o}csn^6?JK7`KlFNWmywVW=}i?_9tdh(v*1 zw&r^{WxS&2xjSLLW4ar*spgD3kq!8+C}^|XyZ{~wnCJ>IOyDz}^X0 zawyFZ^!{jhioC`qFiU>G!h0Rz|}OVjFLu0CSS|_o7ghY}b{GC3Lm+#9?@Untz*^q`fiy z{d&;6=^r-vNU1-v31Qm1>vBa$pkvmOnbbvUlFt8P{@<-|q8jl3e9M2)j_E_P*F{Yx i?c~=NW&cO#;6))OiR8tL+Ds3UW!0tGRW2|H2mK#8p{Dcz delta 5718 zcmZu#XHZk?-%N-gN=Fev0STc85tOO~3?UFuYG|Qp3{8rF^kO1{1dx)1CL+CGdXpkW zr3H}Q5kv%}BOuc8AMbrXz8}srbIzRE*|WdheV#e}&V9~h^(<%XPCU<2npni}bM2Qx zzxI`;=ih-C%j8SL2jkX1%1Ri_Wf}MG+T1~-aH$dOQSApkGu%#pGH9x?*)|?HE z{{L9asYm^a8b4j%*?+%Pp7dFBh^lK;ooSbq#GLzSVBRzDU}Jr!xN0Iv`jGPQPoT26 zcd|`{-_E<~KhF5%-qX}HuC`1A{H3HGpJe;Uwl<1)08CM`{lw47-{4+XbILvY@28o^ zUfz?bHa!uiZX=F?GouE>jxk4BsGY)(6mzYpWh&J{|IQoM>m6-4cVnDM${zQ^&mZ<# zk$z8o0u&|w)THP=GD%sxl~yP^-RwN+yy;q6bMg!wm~8sY6`HsGQ28X_0N>sHP4^}S zITah!#M?elBm>nwO#Bf!Hl!yVF|a(5`Nd_cqWghA%;w~s41fHQp~98#r$652I?BVM zUPe~herum_P=qGjuAjJU*~>k&Wn+pr9HTF(FYNWQ;VPJ#G1r;M&-%WwweMkVx#wu( z;wAUd{zL=s6Ci&woK#$MqZz*>9rMaR)2q&cp?g#{@pE^1b?FuBueWY>`9b$)I;<+E zPivupZ+(Rt-p8|S;yxW|t2bQjTTj0w5cg6BvHHk2>YOApJY2V)-%kfT;XluZSS*Z7DN!l6{kfcACGa}5fHRy{3W7AG`!F0<%AYk9a7l^stP9{!{ z()2_K`1dg^ocP^G4UFAfygY#2pcJ4miQO6wp!QfEPpT#E%Kk`8n zu@s

    *|5N_sYY(XvNV|t!czsN>&3YZLMj4b6~S{C$aHGnkiGJUmaMPPpbF2Y8Atf zdqAM|sgGrecURi=hzaVJgNI>Hmz;(Bd$GmH?*|KETMkW@r#<)>)%N22#w4z`#~CBm zC3%z66?+S~s(r(fH;W6hb~8=Zea`X!M45=cx+IO9v!q8EC2q% z(?eF-<#h7urO4Bzz?oy)h9G$F1b$Q|&tm}-r*3i5CsQ(J`rXh}Gc!I3;*rvPA=)~4 zDO|lNj6?XVN<$KUU~c61E5wGrK~psE*p3(47M@PqQIPb zoFrfDnV{CB9c!fzEwQ$`|6YMdG#1VfUNwgR69*DEZa(Yd=a z#hLjQA{DF`Z$@KZYGU3?Z$3m&^%$*s01*(d0fZwQj0Cjxya0Qjy{Iz1hw=%>F5V?E zTW2PrE>iI#Z9sqFqpP-m_%~LPv;jVwmFIm7x?7b|d^%_@_idAPI+b$3v=EQ5s=p74 zarO}pN}yZoSc_r5;qwgu3V5yG^g$X|EOQB0EPI7`XJ0H-X!Ny;_whW6%%^HrW`b;R zujNmyojf~>>9&OTQpP|HFavLl0Y)jTc&AIc8?S2Y%L7~1vX;r3_bR)7POuaLdQWEl z8NoAeeBdjrAV4tHdmdy0@HK{6Q!f%s$v|tW+WhM{oHBB*@KmFAR32dUkk%sF1FTi(J>==}v+-*_ zb^FWZAVyI$6Gq)K|DEulpE*+)5iZZ%5iZZ~!VRPDI)+;frFvX5!*QnH_q34I01Ycn zxCe@%Q<&!(_f4&mEm9h z0j(H;a@={q+Wp1fWX{_dHBlvuThKe%8kU?Lpw2htMNULs@!;#4S0*L#W$J+j`Ldnr zb($2Jix==mOAubh{$=s9zVm>$Kie4^$M7+aZ^Yzvma-O4Z=4tWK;Ni>OdjV=AyM0cAJ<53@v z<~$qBzRd8FT~dne3+d1u#uDoGMd02KiytS(p?!Eq`&Dnrju(-U@i(h}h1w-j+8iwJ z%-yu-+hN)YMH^r4!pQ4Zt={~REOT+%J^V3BB3_VefKgV5gjh0|sY5}Qvbzkmzh0|X z(YY;C!d(COcH}rymrZ1;IcuXr)3{R8qN+P?nuT_vjJ1|~`!6-z4Vpykcd(AzISZOy zVHA8_Mq$5&u~#TL;cU+^P$WX6ATCr3iCU_Mggju_heJUR$Uwm)m_5-1a3g3kmTb}o zteSV0+|HU6Wck1)!2C@qPD$O3wb-1uK5V4xU}tO6Md~TDa*T-=n6J9RtYL#}$7(rX ziptSP$ZBIpg#}yXTEwwDE%D5Z?z?^(_$s>D6{EpsD29Hx60ZgNk6DoE&!qywi^0-YVN7!oyy%Y!vN4|6u6HRmX$Uw`~7% zmA8Gzzsw-2Yr?CWQg0Wv6joN3;XiPtv4q>))_JbBAhQ~zdrQx-fSYFCy=yc<-58cM zpddGzpee9snxKk7!T5o7fz8=CDW!??iU?OLkiW6+9urps0G=nWN*Jqse zUvU{LyQOP;w}VxT`r>Ie3?XYA{*$*05Ja>E{9Re{vmkLWJP}REd%b^t%}IRus+x{2 zKJ1*Nm0O6G!h3it|C*4%<4mW~m)BL}(qyDd3yv3K6bfv;?~c7J1d|>MOO#w~@)Env zDmI`=&DNyZsSrnN88OfuZ~{<_3JRWAFRs!P!qO8^X&Tudz`+SA>+)s9bc+~IbrE$a z*s4;as(pU1pZNIo1NH>tD6LhK1Ur1RW{+bSGTVR#iA2bDX+;r3*t-EyG}1cT`+_Zl z_8jGOgd9Ea;5ClBI=$Xku;(~AmS3)#ig0{5thNhZI(k%>!8&lo%8QtpAV+e4R_1Aj z;2ojd!yq78ud0YzjCIUqL^Fn3NX!QCdJrdvB*60UDwu_pgO;Lq&%fZ$PWxwH=?p5m z)frT8_mHvfc+uNN%80rA$a92yrK)T1dym(Zwr@<1arb1GnP^7R3tK0%2lLD_l`|7i z{PUm(hn|8afR^xlI1iZ1I?oPxUlq)q9(c{2K0ZRW{z4P zwc%y6wxJI`I6bZjd&XE)eNPrQ;8b|n?=j$1wR%ph{Tmz4FMZi%w!bX9H^}UByt8cO z%(L433!vJ=Uo@6(yA$dRYySZTF#elKJc$-cu$@hHUoxgtXEM%huur)VbN`m! z&nh15d1H~E(#WSqL4uVVOy)xKC&8 z-g+KT2n+2bz|Tl2O)4HLKV>p@S+=uN|^l+eqy&eUwcwGx-`gR=%cK22s|A zNP#a^41&Rev~L8{T9>xU!Xk-W0BF!95-^exgZHSJnorc{8)Q$p3u*R#dddW^Sh*mT=7F_Bg$<-T+SV zUT`yysj zc^<5j5~nq(0llcxKn#f;12hovwgl6~bwDYsk+yeP2R$RvH`7i{HC(k0XZ{xBAQcBr zue5u;l;H{6_-H@_%p58iGJkf>>$1k^!;9u~xnhnN%VUo@%V`JWR3MT@Gj+{fm5`?= zP8+jcni1}U_`f9Q>3M@=vVeLL*wEPVWR`LvOs&*zDiz!>e{YKIx&!c#J{jauP`AC}4NhCH(u%bE45{6I(!c$E_2K%YKn z&xJStYa4ru4|-Y9xXvMafBgo2V}?=#-m4M$edJy^LA1tlVBT~jl0<3=meXXFRFOh+o+?+HV2Q%q(7A_+f_T+M~Tr1Z49EN(tVNb zbKM%{Lr|DaT!*dFgx#b4gZTxwJRZ3#+?4~K@|eK7$wQR+`u6+g+o`O0YnBzfhwR~x ziv5GU4|;D!Raumqk0<+^Z+Y`B3_r)$Yt27+=$uydspr0x3->`z_et+||H{!G`NeX}Sb34_uzV^VjP6u}19$)IcwDAX^ zuLC-B5dZ)%(k@j%(YKT3$H2<#*GNEmrVsLtOO#eY3lJnPrVlSU%*#$}q z1}pC@k8^MRkZQbR^Am5;Q;oaO!df(zqvh0qE~@79ZDze{pzMZNZsEuQX~xlR>f?GR zS$_%xZ0P>qMGnx!&@u+t=LOh+W}N_mEQC_v&;K5t*-r-8T$pvD3&h_il%D%32W%2) z8TWR8WsPq?DAjD7e60ym;}Q` za~Z_OdpAeej%TDw3T{7m|0S#;`s5^l?MgrvwqDqs7`(-bzG1CNPuaiQK3Y15x{9`z zHiM>4)i_UBT4J;CAsZ9roZ|sOjN(?rH;;N93fM~mch6oIYse@bSka2D2b?LfdwY!& z>eGK$yawd0?OqAH%G&ez;D*Yz@tK~PpqKo=EfZ8X*$Tu1vero*pPa@u-KHbWM2XXj zRRKJ!XUZ+e)8{8d`<3-8^fLHTuB64Q)ep7*GbB;)9Fu%Z!K=j>@Y=-`S-oZJcT4-Q zn)zE{&dXwd3ht%9JEiW`v+^zlloq;W2K}TPc!$MvN40T)flD0nDy8! zq{Hjyx1Q7sM4>Rpn?7rw=mFY|`OD&l9075Lj%7O@H<7y{E?3`a)mcv2Dw#@;A<$MA z6z(uRS4fCsu@`GoRo^`Y@$>lQ9Yi($xtyOOLVc9}HdCga*p4p06X%~=SZmg{-Lhco zieryA+nD^)Y2q^FJ=2av3`az|gfzvJGS0##n^iG#Dm(zF9*71QSD`t*@P{rSyOXncE zbk?>?^*+5{c+gUN6_KYc${EH{6%Y*$vD`X+mnfM0Kl>aSy+&=0d0fK|fUtzm>TS#q zCqKR!VJPCB%>)=m-!RI0yZ7?J+Bhr33+0x7&9ypyIBc+HT>rPaMT=i@f?04LKE*$R zt!4Zk!V~TxFZj819SN|%tQ?#E{*yWRTjbb_c-NYT=#mt}{SPfgyHMSY#tW44&7*8- zq*G_wx6<9!quR&N;csYWeIs{d68%cGi$tJR$S1Gi124rb69Aq!7`Ee$nIAuU=RB}e zX2bjdnrb62IXv&!L>z+lcHd9sA!+DFY#-g&&iz9dmnZ0Rp^xxrV@j|87*zRGh6nWD yKjGM0`2P#+MLAPoH^o^_8K^?ZlN0$*#sRrYKz_=S91Q3|5tSEWkh)2WJK%q_V6*uE diff --git a/summary_results.json b/summary_results.json index b3b44b5e2..b6e6390dc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0085, + "Performance_Result": 10.085, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71562, + "Performance_Result": 1.71038, "Performance_Units": "Samples/s" }, { From f458aa4baa6d91da116ce9f6833dd84665ae6465 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 23:24:47 +0000 Subject: [PATCH 07/24] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 13904 -> 13905 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d5ce41db8..9ce5f0445 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:10:28,695 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:10:28,695 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:24:42,802 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:10:28,695 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:10:28,695 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:10:28,695 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:10:29,783 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:24:42,802 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:24:42,802 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:24:42,802 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:24:42,802 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:24:43,840 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.413475 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.406531 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.3606 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.3711 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.5726 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71038 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.77844 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.085 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1954 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 39efca76c..1303c35cc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -496 +497 diff --git a/summary.csv b/summary.csv index b427c1d71..e013c4522 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.413475,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.406531,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.3606,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.3711,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.5726,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71038,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.77844,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.085,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1954,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 61337ca77..c9312abd9 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413475, + "Result":0.406531, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71038, + "Result":1.77844, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.085, - "Accuracy":"acc: 75.200", + "Result":10.1954, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 37a0a271c7c70c0724ddcd1f89b1a3e4e329ad9e..3cfde5d3a15d672413fee11da1355367252f8e29 100644 GIT binary patch delta 6637 zcmZvBcQl)S*nVP_qBgZkBKE4S)Q+vTlpxfoDy3%85>+d zU23cCjlRF%e{art?sJkSxz2T8*L~fe=bQ|=54%@4Q7eFCJ~Ny7!WP-_ z-fm&{Kts{YeJj>okk~hDR2sN2l$7tIERi7EX?AWp6TURZHPwnA4BoCuVmUd`XUDwc z+R|%pt?>nUUL6Jcnw+H$u5{r8iYnPm+AxV{DW9IdYA{1y{aJ>YRt>PU?dH6YU|m%_ z6hCRbocV)nsN$>bSX~W7wih#ze%a4DJ#ml-9(VtCwt7-5=V?%Q^6_|J-AQ&sG=|H? zW#GgY^Ovm5FfX<+As6QJoju|A1XK2a`cnf#qo|AXo_yrJzOS*r!L{ePjRKJ>DqM!E zhqGIss*h*AmFulSn$3`v*?yNbx@}D@ZPQ0lk3wX`FIpNh`@Z6rklV{NRW}q5pRhht zKU#izTf26eTV=}@A@b9(60zJ>yRV6xS#-u;E;+QHdOnjDIVz}}&}ti0^fRlnrM?ln z@4PU5^89WuR8Jzu<(Z7LmrStH*Xz3k`7%XYQh#9Y=K4m$69yANbUHa1e%JKX7S`|CR4E~g?& zIR{B_#=fXoXmdX{e)@#@R}*{c%A1k9kxzoDJS86W2uKX;jq_~lKx`9Rzv@_1EfZ_~ zcVRjH*zMSzn$g&y;q4RDl=9nvzxIn4Z%{Pt-O~o5?@J;s;r}rr85VOmRsP*l;xn)| z7<=gStW+XvWB>MPEOz`3UuuA>pANKD8j%z|S=5mdt$NCfBG10$P7YAoi-C3vugRdI zl{$>Ti^rUVZ?v##)lIfULBK~IEIe9q%haf797i%W%JG07b^9{NZFj<&a;~`B>F93m zlF6SW1Cd)PwzRjAxB#{GBj34>i`gwUKW=tO$xcjw5?7z0?dEs-ucy={Gt7#sYavTT zEmfB<56)&b{r=9cj!_J^%JV3&i+vMUkv&Z@@$oe|e_@`v#d!5@rFJxB;aK`T)2jOk z9{gV1a0Sby-F32e;1aSY=m8z*5gB~Q43ove?D!7Y2HE;! znb*oY77go*L5a%7iC?A@+PTBBTfMlD#(P1}phg^fN^t@wf2KagF;_LPK*00oSx`v8 zZB$Rtj`mC!lG9(#A@N!P{#nhXoFBHE__$3xX7WZ9WR%&{*VcQeTjXc8vjmA~zB?~Nhqdu`bFb!?B*xXRpVKD%aC#bv>WEMQQmkUn(v%%8$&#sj)F z^0^_><@2t=j!rE0?2cCIeV=uEvKXd+fplw2$XW3(}Zfwveun~zWC+P0*0_Pq#1!5& z9=MEMelOfgHRqv=kjPu|tG-16s)7)=BGFVkF|11#Cjq>2T%iw*UI)Vf0T#*#JY(oX zG!TJTUDDH&6y@=~r5KU^pp;DU1~6sfhpgsJ1~)MnwqgFu;(M5ol&hu=XU2-Ydo1`YI$3l#XPqvgu84lyJd(v?)@#{;t7swp^fWkHF73jZ*3 zq>hGl3jw}gRQabwUKysfCFl{KTxsKCnn;3?Lh(HobNLkD z5Xa$Z%E`E=JQYWOCg(1_i1WaC~aChnPL2;=>H(8t9$ARCQz zDH69QJtITJ<(TooBDco0|0JGpivETwJk7`X&+cff#g9nIO&NHR$6;~lIVxB-18P^(h#`X-bPt9@^8N);sdXv zbuyCJmz*F4dwD^*ba_FuL+zT^bLQUz41eChB)=xrn`7g7D%cB|?imKxdsm}m^}z8w zXcIV_HYD7g?6$TDu{#tP$F zdE}#UnCsY9$ud>U#6I!4g!6fBAX=x5(0A*NVGt->N>$u}H8h2aeBt0kvyP~9ogwZ| zNFYs-JH8jz33A{`R)35Qr+sKj?o}}f3Ia@2)uV*c5ws;i@#YLjZP(3XoznO>=8O;M zOBj;l&~j%Aknnu6HK+)2z9h>Hox>Ao9Z}K-Lkt1asMH<523vYy$Q+iK4+$?ut4BZW z`DQF%x@#tO2$O8%HXmRsZJssM8??359#b&p9Xw9DLx&gOQ3BJen-kv$V}R!PP+E)k zI#6Lisd9>rJ`%qQfJ@de4bkO1#2%=Nq>GSP!bB*Q$oh-QbJ#a%INe$z3%rCV75r6o z?_}~Eg3yBc;c~`gYZ@ZN*VSEBl=s8IiAFxr{ekX>_ICQZ*LVsB@}s2M(Y{^6(9KWE zdms3k9iFZ2IqtIV??v``UIwJ-J!bKJRxCnag4Dv9!oS)jZeWP@4opnf=?Dr4SrLY` z50FS0QoX%U?h|9-tea3mSIq6^TLU6O7#4By#s3P%(3aitzn#)BGjboU9Y@~rGf}JenVGC6pS{}fyUq<+KI3v>x9tk0q+Jk(+UIZPu)S( z4YYMm?v1ZSp&HL!F_Y@gpZ2?hVeqlz$~fQ0eiP zq+Vn49}qC@omnY_`*kdJ$?Rp6A4W4JRiED6WH*lZr=vZx07M~7@rgoQn1sKz(Bb7; zR3{t{(J%~pV8sQ`K49AlB8u@k)*w?#H9ysW!NhDtXyf5X>(caX>e3Iy5$_Iqx>{DqR_%Y&gQdarFZkU2>v|jyGF$h(3%xXzeY;D z%h;ey_p|GL2UGOF|6g}%FevS3EefqG6bwR}z)sb#Gk&aoo$;?Uzzk(u0!P{4h9sc% zHAlL@#*{7i&Z!Q$D>X7X!l0*SNTV*KV)7yWJWSQLF~U8Sug|Nt$MQyTaaR5aGnm!6 zKoqJ@{xnpyTM{6Idk!#vFVcMzP^|p0H&mE}uri+I9O7t0IH-4`EJ2WZ2jm)EzB2xI znNx^bekCOo=2|r2;&7v6GowCVqgx%OK$xClGI_%E6w?zU$&C*M3B#DPVHQm#a6OCh zfD(J!WJEOY*-!CK?-Z)s?fF9s$qsH2N0jr)zzfcP46%N09&`lk01_4@#9D+{gb?fN z`m2_Zl1@TOSpPRA4?-x$K-8aZvT-xx)B}c}{OSgS&a=b>Ew=b8vNJXq(`AZukTT%3 zkA^-oxAah1^tvU{5#^~dI^=@XNimjW0)!f8B;-IEYWm*XNmYn&>TSuM*qTQTRp>W^ z&;~+85IIAkTpeXRyx7S@O$QU4yKH7y9^2ck2H7M_x_&^i!S`W$ z-S&6A@1p*Pg{TDhCZpEs!k1?VUibj5t)<{^y^U)P?>-!Z6M`H^xZ`&UiQTtY$C zx#!7=^vU;+m!UoRw`@@F=ymbeJmYl8y{MDoEXkfbCo&S*1Y_tb03T1Kd5GW95%MA& z&_UWh5$CHRtC6khB;+4+r)!15v%$UC*w?*#KgvMZTToMW(H#kQ$c2sw612<$(-X;B zY1QB*(GO&~?MF}Z!Em*j^|ETuPuHEhlm2O}PVT&5rYC}1fpW%RB20m<7t z<}G3jth2IxyLZ85969u&yl7cEN-I}}WZEz`dj=d;XkRujNzSP+7!Fh=>~rQ<;3wC6 z+`@(#8RmnaS2xwTW9%dhi&NJP9vQ&s@v%Tlj~At~8p_>t$8MRT3g~s4q9YsxS(0hd zY}_n3sw6$K@ch$F0-Vm_y5@_HR(_@fa1GPPS)#V)d=^ty2bQ!i9Si2l&KEz-%X%xV zHMP?H7w=w{cL~*WYoYRC-RijS1e;5!hL8lGi=7hxCfjv61S1FvnxA%qDINYP_n)lf(`An}EXFd&sM-->MIOPw%; z2RlE7SepD>dA$m8VErGAyzyXWv>kXH1nQRR6dM=;3r|7cA?k1ri98Q#=S?+L?F$G{AT$1^7!eMPrCa)<%Y+j=c&@Jh`9vfisft4n6Il<&f*v_Z`W&QFV zD>oaCutIurheeC$eo3tr25B{HvNG`_$#k7;xYRjwlX<;k(7AM!_?B0k^y&u0P#z4> zAf1oo?J1N%@9rT=iy8Wl-i`=10x%ltsp1$xJ4#I#-@Z_?6&a)Hxi$SFt3=BbF;ltj z{%vso)oyO%+LlM+-jvl(ytvp({bhbYfAgQf3Apgl|o=57R!C4GQ!^jyj6`=TB%E5il^`zz&O zrO6WQ!)agH%>N=|#V4#)L`!WE2mabAp9ag+Y!jlcY*tjRzF(CbZWe|7#@e}^Vy6u& z-kv_IdprA7xF1>lXv@P$jzKq;6IY4hVw2Y7fk>Ct#w&zDVHuK%>z1-1 zb1Z)CsA#`cuGgjiy+ND5Lb`|t(((_w48OTOc5C*cmX*5@NZL2NgI!DM27!V#9x(pD}&X<$x+C#IL66HO?Qp z`KeI+-EDNH&m-**OfBH&H2ccgJ;F_y=j%Vx)S)=h%-J3BJ~AY;U^9QeJ7CrND$(&9 zDLS$6Jb$0N`@HvVz)mOq``=T@%z_cAfxT{)y}4J9cLKkiC#l@!Lr-$wR@7S5cD+Gr zl^m0jpz-6D(8yXx7`0IBR6*9IMB?e!Tx42=h_Wp7YEpnY&5NV@`(R$VM~4y5r{7=b zerbwqjA!nq3XV(kJ1cCAXRJ#nn8M_)EUNE2{4l2d8X}pP!NKA}O{f2J4Y^1a{OHs` zTheQ-j+|$=zTG>i$LjKplAwJL1Ql+ar)+y{vT2knPrspIWK*5*(S~V)IkXSdhI(ZS zZoj`O;U)vdTK%E)!&QmgaWUInTh)VANqr#m`VtPwdwWB2zIk9A7bzk8`)G)zU4%Vd zfG1k&oBFQ}?<8JbjJM;_Jd%mQDk!3HQvu!_{keDI@xFSCV~LY;^QgM)p)*sU8pbd$ z{d1+B-+u@@D?6Y9f~!LDXHPyOdWjCn8my^eg~1@XY~zQrFU;^Ob1W__MQe(RgL|PL zhG&hSUNjTfww>~+em<2&pY+C*giQy8zjhH49&O++M>)Ji(MGR}?VCHJP}+fb z8tqgZG3*OZh3AQ;VGgCY3&F+>a2`iH=s8ok&b zG^yE^ObMiug7%HTmKqn~7Rt#M_L~jnWA)zke3O0>6#NYgPw!>a*p$k?6q`Irac`gh zWKEm3QTXLesPVH!q|ak_+>N`rB3df6%VwU>Ke1;v2sF-?`=M}NMl)nH<~)jyfzKGq z;P6=Iv*)|zvfN^6ekNT5F74h<&wz~urL1G=PlPwB5uVhN7QD3It7K~Qgb`UE7PeG~ zmzonII7+E&GC{}PCFnP$i$CY<<$|0Y{*K+twGJ{GZpvChBTK!f>IQ~>Qi+LswC&D6 zmcFlExHMZY*i-nc*>>ZAR-r@wUTrv5@sC33=X+h2a~OVPtdY}cM^IppJ9^|xH;SMq^?kqch8 zVtQv?qF*DK1{mbNOxd3Lx`KLTg&EaH4$hXC!W9E($=vhZjA|R{gFSb#%V&41AQ=FE z`?eBjbB%CMa}Dvf+-sgbk^}(;`CNnfXi35w^O2?+;#U-%Sit{TL_@1ccj^J{%$*ZN z$8dlh<-8m4)xQss1pvF7JO5X)=)p!v!LphQO8Y%6cb&?=eV=X;V@6>L)<>-0HcyS> z8??9CKd&hEmxri)h8w>PXA2G6I!EkxZqF{hYbO5EAN+A^a<1cVHEu$v(3U<*;P$pE z>92b8yC^QFR~JIr-D=B+VJFD*rf2<_fn{D%EoCw41{gRhONl z&T@4!a8U|V_R!9;awcfl*ewNtp2wXqL67mMuGdr4T!lACW-_K)28X(``!`lEKr+sr zbxL~y0n}&YmBDmwq`2A4264vtrAQBaM_tIzK9RlV!muJ7B#;(G8U~p6{(A^kY0bbxhcER9vzIN&lz1Avc@&1wwuKSMg|p2;f{_{72A~^<}E0bM?Nrlvde; zX?YJ5@1}kgb{#8_-ovZk-Yi-{Gjgg^2LXR@;38d&qB+S=L9bPESUKQ`~Yw*~)K zVcO)4yjUtWQkB2mk=c36~0>cADReoMhiuGCoD(t}s$% z#5jCQ)wJzE+rocdUSbT&+b_}po{~LL&YFmRJ zN<8-YM~p%T1^>^R^NGPjo~;jJ_^u^emT>lP)YETdOnvx);qj8^DegmEv{BNuhdh6gtow63Ldo8q)4GWB)I#C! z%29B~yY1~OpQpb~?3PIrMjTFCLMM*!+WSAww%lh0J`u;|{`h+sFeH+Q5Ikmdx4UthDCn@NRyv%At6vZr7r5_CsJs_l8T$k*~Mh;6ZWKr0D5TSfE+8 zeNs%tZs7Nsqu|^6lXq|qJa#fxb^VT$faxu&s@C!e14s7K*7-*o(|N@o_kSPyq38EJ z9ee|Cd?0}YfXCNc%cT`;k-*@|x^Q8xXU-{m5O;4+Ebi)AsMqgFTl#OM zJ+4=-6)c-{pRH#8Fwm-QgfyQj*s?IACxxdKlX8X`+Pb;UZvvPMxdnP?9j#f+^tCw z{BuLKBGatV&!8^f`^-wNs%dh`bW*2aL}4>;`S0A1)dpteM}3T>j9c&1+^}hzaY%`c z*nP}`UT}u6{d}%Eb7kv2(8?UP*H9Ri5x|N582-3MjI~zG)5!c1Pe;qT-i7d=sXnfOLDU(#wv2ZxGUI7- zw`Udadbcvy`-?S_=Q4l3@j+PHVTD?8k$#wmO8Jz8#lSt(ii~}pGq5F7sW|Iqcv}@wa4)dA zY@y%#yp{1$a%-M50GmtJ#g$g--P=lG>iR5)QNf}Q(U*Tc&Ts}{ z$kp;H)H2vpE_2YqX{Xti@58ao>tFcqJl%oxCksH4{)KZ&c1AbsHiaKny2_~h?9X~O zmo09aBsh@sF}`gFzca+{@WQtVQk`kCqlAo!O0mL_IUuNk;NHtS<#uhDEXJ;iLec|3 z4(SP2a#RcHt`SypOd+BoudwLma)7zY;!E$(z0e!)TX!qGqbuJ>Sv~!{)&jm2KZCLQ zA|7Ib(}49!^@FM*6m2flA>%R*{q3>~<0g4{I#6{o`K2uKU%Cgac7OM#$f8)GQj}uI z_R}7fV*=nUlDeZby2gw9{@UKwPtWvsYxNNO{8*aR5+hu@KTL~{1?GpK?i1k>O_yjH zEo;{LW%s>3Bt-loWL6a^qzi&NmK2iX1^|B-$gEdHB{=a9YR!Ze^c6WMEUTrps3)0Y z7?I3j_w8$vw;nOP$Fw=VC3wVhH$a=A&|+G*Su@ z%oXSs2}YMAAeDlc{-A7E^x@qua6c%4-+_t8SSsZj?u(#}I!?Wc&YmiWsU8v_g@#;APh2)9umn8u!Cl0gU$5%o zkB$yulCqb38F(54r|%C&0@vC8^N)LgfTRmyOl*`iAW?RqNDU~3o!mZkbHFQ&dh&-v zOw86<@K7-dv1;kkU%W9PJm*?{>s|fbho8$`#AutLq2V9pbM9C7_sZF4g_|6pcYeG2 zlu7OxuGr@XeI#=?Cl`$uMW;o>wGpdIkSM3nLl~682}357DRVy-umoMRpGm=34hhvm zP*OXE3(O4N>;jj+E2b%GJiJngSEXM%m4d=9pl?h zf<3t!vw$mRA;bYJ92kY8KckDP>-4v)xf#;oHgHHGM1zlo+`(FVXFv{Pr69Oq(JBwI zQQuiBA8tQc;;SCUyszv7C;}+wLW_8^^{hQ_XRO74E%@C1D#thXp+G%G@i<#AaQ-#{ z=WL_iPq3nOV8svJ zv*vOaf;#FU?l0JeY$4#1_G6$e#;g(IDz(5=gN6&$>A+&8B%+!vH<*+o)+c8))AffE z%DX9?uWx)#=54V z`I#>XmGkp6<3Z)FYO&>dkOFp)MFye>C|>rbmA*7hw@N!)p%W^E)#o6GN@1h=6Kk45z-{-m7vcscW>*gX+TmlXnphchGUS_7wc4njV_G169cCTejQh#W~GPt z$z;Q#q>m&<3TxuNvx5(u1dfbZ?Q0YQj;^01hJp7-=z#4Fs8_Say)8JqMaPf!2nlakdK_d7;v) zy_(p7UJjFI{oDD>7INyc&|PlyMAGee{Cns~to)jI%~&j4o2!f&&E!JNXRf*Igy2`D zMB<=eOVIlOgeErt0ec{(d^DiyVDQ|-(*Xz=*=Y%7egZ(7#)OncLSn={kgS;mMyv=2 zT6-UYG>fE0t@ zn=0g0m`rEOoK-*DlMR)eKhZWx8>hErT9Zr{={?f`eHS7 z-TdB2prbGAiq%}Wyol#j>Zo$I%5al)^|JnxlSC08*mT zqnx_&bls7Y<|xKfCZyD>qM?Gx<^BBqg2~ldvAj$cpf4|mMGqjAQMgUeKc@p1PTvt1 z=o&9(tRau6!1K#FF{1Q^kYwFG{!kml$|eitsaRHR}=i5|ZT704jtPQahfVe;L4F0BkB zX%|+{Rd}MG{~zU9uZK^CtZRHn%FW6kRPO!35zX{(ddzjkh~rNkX#5ORZqE0bYBGQ zqIzgk=%|^A(a75EQ*vlizu@tef1y6er6GTO3r|2c>b{34=?Y4NIcf=Duuu=-BiMzj zi?qtgB#=A;C3%GGe?79CVazwel&!AZ1Gc1<<@U}&LZUiHUYm|TB~1I<=77x}#+w7e z#wOEpP4+N}8Z&YU_L`J*Q&OiW<~JpE3et*xgwT2iGG4>PUxsU4l#FO+!Wu1ZAAU;{ zgV2gto|u7n1?-jXu7c78~^tjpr3cE?aI~wNHBcae5pRw)5?W$|_b;TTHb2 zYULYIfvj4f%Vm+pF77o8ygxSjU0NcX_R)SyB9i){+e zcT#msCI)CIJjqpal?;mK|M-K|w-Hrl9N^Nk8>25JZWfQ~ng;57G%ppI^}sBZv>=qI z+CH%Jyrt*ayup;aI0I%<*c)h$VailW+TdXMfp|pSRSP7N8hOOpP8{@5a~LyR z*;mmF1t`57SYW14_>EnqEEB^KPY7SJx}ra|$`2QB@HL^PB`h*hbfM{L16oiv-4+s= z=8P>QGR@@#A>(eGBXnxzda}5aCFQa_PIP70CdcZyaC?fbK&$wzGwHWG5kKVU-=I(_4}%8nQE9bSqgj&26hRUP_IoGLDK zO1PokOSk28`^M|dwO&#j3GsPPWuJ6ltdH%M8baFP4Lr_vjq}pz1n47Ea!<}0WARN= zb97MNAT>vaS4|XcxINND3=TZf;`n>fX)PBDm;=_)K5((rxQM}cH=y7%;i#ToC;bZ{ z!O*to#)+DZ3)?hHVIRvaEkt2#^m0ZlX%_t6yB3e6{^k1egzT=pE=lKLTsD~A@So0t zvcpW_y!*kX%u-Q3mK$8k@QukSdXg&iV{NWQ7KMD&BDWk-ara!~UKJGG>wytCNFzIP z?fX?yrya-z$SCu05MdiKwLoia4yf=2#4g8(i`i1PXA+O<)kVvu8?juKe3i|a{y~Cn zm+^yysr4e8HTbPKk*y}h{2^&rn}^=kf>M~*>VObseYhf0!leP$S{(2GCH!ktoW5a& zII1D<63sLguT^Mk-1+(bu=joQicGbJIZ zuerTua#~|a@}TM(6XCXIO3D*)w(Yi%p(iWh|Mv6~oluD_nET1c;!SM4maq5s))-VK=?P@YOmw~q^_I`7^Z~55F&Eo7U$4xa*YOCJkWt0kNKp|}KM4r|v!+Y8 znVx*BHgcd;`($-ZY^36A(iiv~Yq7i^zEvkOP~!Pui(go^nl?t? zjz1YoPVTJ4T$T15EPyJLUf{QcUQ=6NE#6I?Kb>n?JYZUS%61GF@Dg`=;5FS46wmr50oZN%#G$yaaiAt)|A48awU|67(Zj~qz3#TI=jdN6_1#!G16cN9KiJ3LF1tY!8eEtF+}MjQ_h*mm z$sYwL9!~xkaOSvw#CQyO#y$#Jv`an?AnMepCN0L|8I`Xark3p6EZf8%DGZShAVu4O z-*2O?Qp%UK46XEb)CXg}9u~gQvu+=3=I!BHiA;3mkB`Vw(Cel9g!G-!cym&K(k!SK z9K6cVP=R;GLaYbskDglr`dUQx`GJP?0Iq0s^%u@)KOM8~@bI@$0Y4HXcOs!VTK;kN z>1&xC9kngw0c|QX0c{BgNl3TwncP|YQvy`_Tt$iQ`pJZmL>AzfZJ%ARy>}N+5&!A2 zDmw0@xn(2z{znIeBWBy!wwygp&brz=q0dJn?-#Us#+DeZreEa-9P*3vS;kFGcK2Z# zlo}$5?sWxY+-@Qb0lX)1Mr<@j=}$X}`UUJmr^rWEgXVO~CnAvS;uVQF}L)@56!G|AN{_L}orIZ*rr|k~R`o0uA0h z@D$P*=CQIukpE)>^1WwYEX92p{?h=d8@DUXjfJ8K1Ve7xBmbEqE6q0Ma`TNY>3(~ z>axz}tYN2u9i4{bP1KzF>ei3XtD%c?$)Dzz_tp;l%vF+#DH&}hxI|5^!~C($otg|5 z1yHz?-Ro511E9t|-9g+=r?1BRx=~{*iv13z3$cV}!n!oc;uj|pISqUZdY}%<|L@`j3TKF@Ci`_0)Ir@f z2|DBkIZ`bAy9iWY`gN16L$qs>^6)jtk#)g~Jlr7Se=pV^5*`{>m^GOP{${|xsXx~D z&R!3HC2sw?s;#(c?MXT+Y_CHIm23Uu>k+M#dOoWe1sm$;pEa7N+Z^}Su3VMfeK=Xu z32Zyx^EKCRf?P4VHvf$7(|V~Bi)!E0v3PEe`pSMJ(7V{;jmLiNW#SPD7)nm>nqSv| z6Jh$viiO3n=ABi>%i^v1l_P8L6dZ_ByUvj5686q&9@{M0o#JL|fh#ob4dI#mOekOX zZ50j1Fd6)tKQUuz?IQ<$OW$v92sdwOy<_e1@tz%Judc2vhzlcbDgP5Ad6KVMg-bmZ1dJGyeFakI^yhuK6*rjhwr z1$Q8$Gx@p5H$XQJzHHXr8Yo}QJON^p!!X*1Y9Wl{+vyF(>)ptU`i%x~=?YZ(A z0Xl*;#rMQ>W~meD(t2t*_TxBn@VIVsiikXc`k?+Ap?Po$w46I c|GS&?9q~*cm3UKy%s*9JL6}G$n1Hj2_!vFvP diff --git a/summary_results.json b/summary_results.json index b6e6390dc..9ca0ea38e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.085, + "Performance_Result": 10.1954, "Performance_Units": "Samples/s" }, { @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.413475, + "Performance_Result": 0.406531, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71038, + "Performance_Result": 1.77844, "Performance_Units": "Samples/s" }, { From 137d9a26da2e679083de936c2ff06a969e034435 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 23:26:55 +0000 Subject: [PATCH 08/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13905 -> 13906 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 9ce5f0445..f4a74cbe6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:24:42,802 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:24:42,802 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:26:49,927 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:26:49,927 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:26:49,927 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:26:49,927 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:26:49,927 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:24:42,802 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:24:42,802 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:24:42,802 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:24:43,840 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:26:49,927 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:26:49,927 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:26:49,927 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:26:49,928 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:26:49,928 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:26:49,928 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:26:49,928 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:26:51,011 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1954 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1916 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1303c35cc..f9aaa4d56 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -497 +498 diff --git a/summary.csv b/summary.csv index e013c4522..b0ce88e56 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1954,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1916,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c9312abd9..4c2c9b6c9 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1954, - "Accuracy":"acc: 75.400", + "Result":10.1916, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 3cfde5d3a15d672413fee11da1355367252f8e29..49231e4bc5564604c5138331490f06929ce216c2 100644 GIT binary patch delta 1920 zcmV-`2Y>j{Y|?D7!65~TS}O(Vlg1$-f7?>yI1qi`uTW)QW*%(I@--A>iV9(h8g84w z*6fqZv6bK*U$T}Q!r!lq4Ft;_LgLI5MYrtJr`3Jbj*h>rk{B&%&Y~FFx&A({qR z7^U-*-u>OV;`R{Zej565meP~n6J@<`gFlWL<48Q3vXkBe=ZhmvW5EMWe5Pg#f0_!< zEXxz0i+}UE#uhpCL-s%^PhyQ=Y-@=hr9Bj6MaoZlHgS5WNTdG>I;1bMSX0td-Y18~y zLGy?*O+uXb$s%rZT%5H;EO?`fe>3!js&f^I#G%t7jQH2)%3&5~`49Reffdsv;#`jU zbm>R2KaJvu%MU#pwz!F+tDgmmVMD)NFXokeqrU6tyXzP_@#0(87S+H|~&Nh&14BmOe} zihj~8E&o0`3NicEPbr&!Uz}Z>qVM!P%L9tWcR!Jd`?$I6g=fQFHu(`-h`gQjL>IHN zJ9>lDf@h+;X^5sz=#>~6e_o-bp%T?nQSX38$7FDmk6eT&OmEF4)DBXujaVTN2e&b%g~1F_o#$f2g*_9>9pA82x~k zPrbnfZ*a=8+;+XuCECV4?HX)TjqHr&bxo{lX(blz!}zn>fVifpiw15!_oX9i3%D&+ zO{>mCOHzmRdTWuXDuJMWt&x>l@#+kXZvPn~$FU7$o60cDDS}U&Dmkbs?hRgRqgNz$ zU!5bWq4p)6+et1>e_1Zuwi()+;pN4xHyPfHM@L8(yj{yQv2Ht#YubH;&M(ehyu`3n zyKiCBz^;z<7KIw$+3PWEtY~4ZV?&inndg9%V z#@^)g>h^L}K8>XW4_ek4CQ&g$bC$TjL=9D~jf3aqjUMO5g$gxy0Pqw>k z6+4oq=9EXN-!k}3Y0g`EdA{KPwwPX&ny$R1uIhc2YJAt$^KjR( zzklyxV{cSUwHLb&qi4-6=~p{>#CNRYY!1V*z?5DAt$=%5ta0XwC*SsxAuX0?2n9uRDA zX74^j4|^`lpZAB?64DMXVFVZgbkbjCF$8Ey-?$>=S4mDA}jb4$h6w zN+8{7f8*6@mDE5jP5w!s$0!Z6N7k;+2WYs@;B`^^8}Yg={B5B+sQT@pI%f4-&VkQD zcA&~1LUz{K+ah(>+P`bN_D$I9RqX4JIns~qkfTOuxdUlVx*uJWwD{le*oO-MlzLIq zQ=}sHD5cWH&dX$s>P^~s-6G|3ZI|U-y^M*de;N}vJPUz0mXr-B#Xm#!{sSHvd8hju zd;e>o@-|7n2HGhaql;)v70Z`W&$lWrRs5G7dvbdF{7BzgD-n$`KKLI{O9KQH000080000# zv)U(A2n3w@p-+=CFf4zFDZ+N3Em<{CXi7_WLYg?D)&ohcBo7D!a7vQ1* zQjRcFqn-DLVC>iqBy~w}s32~&mDGpQw%AbiLtDUknwD6Cr<{L!iVwipOhj}-ayB8& zukA(vXBaw=puEFblHrTjgOzT!VN|Z1Qux8Zto1BT?!6O}vG4n&FGg#~pW(0d>-)HB zEY#st3{X>XMwkWawQVYVU4+AR)cJKd@m2tT8iGFac3R}v2RsS^$T99&`7EAaHY?QR zX}*ZlEY8bKUXU2PB#ZR>F!HAV0+gpAcN5QmWA58e(EJ0lbTOO*1&UfL1?iI&HCF-7 zlWjFR0YHj|Y|(76!6608(eRh(lg1$-e@%0vI1s(>uTXg}GY3Ej0d^|!RLXItQj@Q> zldahkg+X@M;lqoBru?MyAjJI0FnI955})hYf1IV# zGtKhE7xLeHX7G8={E$B|CX(177&}JdM`;HIS&@p9jzioID$?k`f(!Ry8m<3%j$)|~N_Db_K|WjgBCo9Q%;QdT$r zRj@o_+)xn5elm}n92aK|5li0af8q>%Vfsu*GI8j%2qW>ezH*SoS^k4PDPYAUiG)z2 zK3(`x>`$UN66!-wT(G2Wy_UIFV!q9@fN`EF@1OsvAO-=8S?&vUv9By1XH)U$=So(u zGoc1?#)KrJmqw(cMo(aSzv*z3{Br%O*eKbD6i3j_hJy zc1NdwT8K<`Hw)3^3B3|Se}gNuuymr^I;y{3YKUa0h0%0clv<-Lm6ncUKkZZnnfKYAXe{*JRu?H}sEJi;N zWk~|#(wt0|fAck~@QaPgb4Tr~9ZY%cz7Fs~$=0Cr%#sYJT6E5*991MfG1BgzDhC=)oMOyGzz zfg{RP(z&t&dCd zYXN*MfUgDcf29Cl3h<=>UkdQ00AC95r2t3h-6MmxVJ{ow}u}%WqimWUS8A z+PN+kiI%15(p#9i+0)6kNADZL<~z%*em}xmP?&@ofP(G(Uh}~3-Qt?M1IEjJxb4{A zzxS}Q*D9u65{D=l??46PwNx;PA7p&}DxSna<@JL3fBd;^%a+aMB{oXfo*k=zz1N43 zvAMiC=A4xZ(N(Yhzy)PDv^82S2~rz?z}2i3M1s{0Iw-|vz*Z@4)`!D^*$p422L#)j z*}Kos!=B6P=lvlyWpA`S@_2X+`v|Um5o-mp-JG^BW33>zOR`-d`vh7GO7i)EwlP9=fGzn zTTtZ>AzSP0ZIRk*?ccRs`zGvl6#M#Pj`CwW z&i-Sda+;)$fp&_T-1NoVhco zTpeT^{epEus|saFiXu=dT-Y81B#rdiqlprs8A0Mt?Ijy8N#%MSE~(JFTh0s zq#R+ULVNEG!Pv1qNa~W{P(j>kE2$5q-C;x7H{Aiw({za?c*=jNr}zkr%}hiSlCv3U zer!7cIK$9_1mzvhk_?}{9;|dX8wTaVDTVJ1+_av=*?n+gHg;W?bj4r|`7`{vd3hUF zjfFa%iUF!i&Iq$Wy|#6UFN<)vmO8%)C*BI+kA2W@yiJRYeZZp-fQ)fZ%180^+^kWZ zr}=%HW^ul3@`4!T3$iG_jw7%7&p Date: Sun, 26 Jan 2025 23:32:59 +0000 Subject: [PATCH 09/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13906 -> 13905 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f4a74cbe6..32dd692ac 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:26:49,927 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:26:49,927 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:26:49,927 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:32:54,317 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:26:49,927 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:26:49,927 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:26:49,927 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:26:49,927 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:26:49,927 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:26:49,928 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:26:49,928 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:26:49,928 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:26:49,928 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:26:51,011 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:32:54,317 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:32:54,317 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:32:54,317 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:32:54,317 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:32:55,349 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1916 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0365 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f9aaa4d56..761fcd3ac 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -498 +499 diff --git a/summary.csv b/summary.csv index b0ce88e56..f7d79907c 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1916,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0365,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4c2c9b6c9..236c90940 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1916, - "Accuracy":"acc: 75.200", + "Result":10.0365, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 49231e4bc5564604c5138331490f06929ce216c2..536ab3e1732d8b7b42274fa07725741a517f41b0 100644 GIT binary patch delta 1009 zcmV-On@Ff6W0`MgOUjpzY0AB*|H2}T_z}Eoy8USAd;A;SU4S=r!@HGIw2Ef+@_?iG; z6X0tCe1A=VuLQ0AE>rX*j3VsavYL_=e?AT79NA&UL;_lq^kG-cncf zzDl+|dfyN>-&y9(`w>=xLMNsMDA>;L)erpMEv~6MV7%OiyN><+dk-6Xqhh)xafpKP z4pcB+M+KAkLB`gv;z=A-UN2ZIp4+x;*<4&=qkn|%*|8GXdwmEgU5J}w&SXd zd!y}<$HQybM{wPXSSN_x=Cpem>jbe|lHCf~C(t@jvQMEMoEx8&K)Tb$tJ5l}fm)jU zlYc;uQ5t5CtX-WC&~Tr@>!S8I;&og2+d_3v_1i;r%<8wC1D}QLK$Sm)?5wl5Me44# zf7f>Ho3PhW?CXy?(vR(sqef_%fi!QrAB{;`{O@<{!vz3Jy{PFaQW1NUQfaXBax+Ht zCT$$INV#0wWjR+b*F;oZ6E{2yfj5?v4QD9DKSTBY10ET9r}>S&{}`y8Cdp%Fi^1$U~Ze$RZSnk@~$X0FJcHYKDMAV9lZG^s$uD#3AwXoRIkGz|F zucI9~h;zLPuW3(CZ=WCOTWckvF~$e~1GCO2VF(1#-}!-)GcYWFnoxx8KwGkEpwN_- z?u0aPM6CysTE~B1*-4xPyy|jv@0__as9YUn8~uWHLaPd8Ns1y+Omm@{3cYXF@dJvS zr;5{7E2vNp4z23Dk{QCZg;%Q$STDdu0i+yZrb2t~4Z+y4JxJ=3;7~!_X)CD@rERdG z?3*@#^E6#z37&F)>M0(8v6+bIgyd{OnjhPa0M0P9AVGPDvn0b8uLmpLY{RHrIi>Ku zfm!QWoZJT|CS%uiNmq>4kUztpo0qq7)mW(ER18pEaz>a1>b0#)d|iaYwbc1dIPp#Z zf9!)k^LAR~*atic0mw1#S@|fQp0{gM=V`u((=5)H+q@tcMNSs?U&F|o{tHl^hWwj& z1{`zWeuDZBvvo0?0|g7gHfiXS7ByD^&6935IsreEsWm1W(ck%jB>?~cPyzq|5dZ)H f0000000000fPo;B>NP$AbdxDIBL*)s00000`DNg5 delta 1010 zcmVtfUgPgH37b|_|kArt5dgBb@2_$pS1c+ZJg_TnJ8JBuDqqL z>V1`Ld-T2`Y@S)>&G`r`L7@{<0~BoM_v#0J?-pa~4j3=!@H7KgihnD4xVY<@JKa;<;_hmd(Ww8-FEq&yJPA-s?k1=|W77IisaQxbo-^ z3@E#yt<8hdoIhL_lMM! zz0vl_zBe?EGtP{j;bK1R(b%NL}$!>-06KEYM*{9GB&W+DXAl+%>)oGQ~KrK!F zNq?ZnC=Ihm)~?P6Xt>Ydby52p@wzSiZJ|1-`t6}QX7yXnfzLvApvoUYcGlV3B6Zi= zziYepP1x&I?CXy?(vR(sqef`C18GjWA6=8Q_}}l?hYJ9ddQsC;q$2hxrP9UD%Vdn| zP1<Bmw<<1Hv3%k3a&Kf1m00fDy^*cjw(Y!K8xc_}F18W+KDzcUN7uq)OF!~% z^1WW|$U&UzRd`K%a(esxNZ(p35sfiE_#d;)Ct(N#ocW%L&XeX+QOUF2CNt0q5x8kFjJ$Q_l98X*bXFhNpPqjZnc%vhtjs# zQ1(MxzMYsjDBul4KuxN0oa;ZzJzQ*lO^1?shJDtujp!*$g8bvW@> z0Dl^SKJ#{3 Date: Sun, 26 Jan 2025 23:35:02 +0000 Subject: [PATCH 10/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 13905 -> 13904 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 32dd692ac..7ac5cd78f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:32:54,317 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:32:54,317 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:34:57,511 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:32:54,317 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:32:54,317 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:32:54,317 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:32:55,349 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:34:57,511 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:34:57,511 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:34:57,511 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:34:57,511 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:34:58,557 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.5726 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.77844 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71387 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 761fcd3ac..1b79f38e2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -499 +500 diff --git a/summary.csv b/summary.csv index f7d79907c..0eeee5bf1 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.5726,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.77844,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71387,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 236c90940..ee3a7931c 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.77844, + "Result":1.71387, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 536ab3e1732d8b7b42274fa07725741a517f41b0..264e241a1899a46638d02f206ce775d721cf5def 100644 GIT binary patch delta 4064 zcmXw+c{o)4`^V21TefCoXrj!F89QZ7D$Cdg*%@Pr(g;~XD)rDzQW(O_Sdul_#!^Ou zY()<-DO=V`c`VsmWQkPY!}a@}>pGu5&UJlW=f2Hnnj?2r)~NXBxH9p0$McfSrB$+?!5hDh`)Y=6(?2@6+bdK4dfw`k z&kFgI=7XNESE6eBE~t+{UhYwQg;>XI1es#2=fy)o7{;Y-veh=X2 zAX5zUqZ~!kbytUNmWm9K`C`2b&xZvz3y(J68=;JaphbhH&N_97j=N%AgW3$BYuXWN_KZg+k_R8Y{NoUYf(8=sq_Anv#V^ zhn(8S%#9<3(+aU-=K(Wux*{Xq6fQ^gSuln-L1}qkeib#DYOECy_zydr!qLdgNuY%u zojgFj@q<0Afc)3tk1ZujI}c&Y=yYYbCL~=%OP=w*RK*Nf)nSEma^T1Z36$# zrtTwvB?Q&ydL|!{D)I=C3ae4->P=OZ8_qg?WqaC^anO)donhV-_B2o8={t)FcdjSC zhj|p*bj+ddfP8O=WC+{H5o#qnO2{E;~11VXE;V&%J4hyn08 zkLOn<@U1+mnW01h9%G@+jGJ9rN?cA-yUXvy&?+Bwbm>U;&GB6CV3m*#F|W;o6|=Mt ziWjWCG#*Zy&5P|p@9q)#Mng`!{EEbFJfphW-Havzk!uoNqY_ltnol#6rQKqvoHK4{ zy|#rn3hVuuAu3tUeObiJzPp)A?wx9u9bcarxJEiroV@OS0=4*xxJ(e_;*h9JGS%}n zs^O)Dx};I~NxpSA2&&!_=;4s_U!;3V##lEj3Gy^n!U_U)rN=ux-&B#+joa_9B@&3p z&Rp9LwU}9Pd9=t;BojJr+494cRCjk1E z{Bu}j!9Fd5-_k>-Kgy`IwADZbh|-;2wHd;xcPIG9#ERAFBseS`V^lxu}Wo@hR!)O-`gu&*#m7X=_?a zT8hFY6PE{9%a?>|r4Jq1CvZ>kKGsnZgTy^pOohcDCxqD|R7?KYGg~=mcS!4MfQgm+ zNr65FyQ(WCCQh5zHR=d4cGmv0NZT=vPrw|6Xg%9o8d#tHT4U~n`*1?UAmQ9{qxF6L zP&02F9S`R!Cb-4L!bX77cwfL(6V*O6i|7}kR@kXI4on`SW+Vmkh54de5hmdenC}rz zlpJff0KLow+MB}8Hn*IMd7-i##<4p&)|bch1{%-0)6dOUq~EgPIEV+x(31uR^e||$ zR;!*5@(fc%aBNk{g2Piie<_)>9XJd1=Z+FnZN{ps8`~0?>38;(J|f9&p@Txoydo+` z)-ckwK2!Q(7X)(r^RSMG!Ti4(M}V#UpGaRKiGKnM#p^}ifL8&bTf7gOyg&Y{!N~BE7z2t`49S$5fU|g6&Akf!xv;l}L+KMd%kxXR|%|^sECiC`$oYEe2c@eD8ox;|VNWQS0X5Es0 z>ad{8lRM=h^~`}wQo|0lV$NJGgCYx4%~yn!_*mEwkOC^|qKPscQV<6fE$_t+IIip{ z8pIdDJ2!-DfmzE7LoyKE_>{I4CBHt=wYcQk)Bg_WrSly;UVz=(L*IgN({L}O6``x> z49L!^TR_QR*l%s49U*z=CsnE+7Z+Uu)Q|_|DQV$R*20bPV~MR4K0Up$U*gxN^R^}9Gdj4246j)gLrRZ$UAH$kF1%~@s0Lx znA3T;#w#)PEl%XRB5O5{V9qV|-8nA8M8Cz6_(O3iD-h4UEd|%JjpxW zSR|c80PCK9pSJwKqGnS&x{P6YCx>9;CaY{)2t{z{`|O9P03L5 z!M$1ahcpC z*=!(=cQ+xUo!F|Yuqtnrbr;R&DEpCMv~+Po_=g~kVTket1s3Nv!Q3rCm=23MaAs$e zSgBHEfFT88HV;tVi_n}9+V#jkS~XVI0=kVR3R+f;%3kMX*{y6TXL-X|i#&9fck^pm zqsGd=&a2JC*s1AaaIiZdVqg!Iw*x%R(L!()plnM50~LaUp2!&~?>8-y^5H()5U&1! zy)}0{GCKYBVZm;akK2tOr#ovXGp&p{MLH**)Mb^pTRVwh?Q{c)T|ZSZM4)=r&6Hmi z15S$&rZ@gF!6cMl6qZOrH~VwKeCV)0wwY<=>&%|K>;;J|ol=4hCjDtfL$E+`yKO>| zb32}BWrmGS<0qJ@LU-1e3J5lj5z9_l6NT8xG<#c}Qf@Lgj2nkp$|E2QX$6D>HlM8K ztXYr3b{q7sC@Bn^ktb1%NG1AU0vKU#d~9kLkQ?tyCCD%MClvK66e7TOu_qfT?pVQh z@%Eh)Y4Ne4^llkB(ll{t|7p?ATil_Xe^x$yHkiAkY~EQ?xnC_`KL5aoZ1ugtn<@?z zCyWvjl#v0-_z1!PWqhRecC zIw0?ZRtd(4Xfl)fo`y2%V;YJV!1y)j9t(h+`SF%zeq+ zD2nkit>dZpC~(bO#VeaE6%PD7n)9kO$MEQoa?K)O3<1?nuTOW+Qk?a3FxJx&xLk}RL&!ERHucrc*8tUOT@a;& zaNDUHP{oR_7?)k#jRU?9>P|%S$D3EeV%*vO5yA@_KSdf@d33dZ%sXD75|S28%1pFt zOl=(;wSCanG9R1Z;{&Mv?V41jx3PdzzCXw-G3f7#@!#b=qJ+B%9ufVyu0<5o7>D)G zNvj7RbymF|IPz@qln!I1VflW~=j{`{Qirc6e*Chon4de7yFKaSwP;7HuUyQ#T2ZZ=usg8B|skYF~wXT-LOy*cwJxQd~uXfY0LcQ0cSD!NP?^a3KT3^9D zR(p2pNAC4M*W4*GnKR_Y+P2xZBe!y|J3qGZ`+S|f+GiDdBY9qIs4-C#k z{_OSr^TY0UzI*SDh58$twIvkR@*s`%$vknas;*{RKzjJgI_GMJ?D~1Xf4?_hq328P zyL+HRdMqgN%8g%NJIIY&uHXNgm?E?9C52G*d;hr=RkzI@3=}VAawpg0X4D(*zy9O3 z+&BIg@8%w>Fqkgp{T zV}I(>gLTB#XyqgyEvJp4)eKT3bG8I8@Kv;Jlno^; zFedDcKLn_-_7;zyGZ{NN4()jxk&KIc#VUO&@bWyiP>b;d9&jT_3()(wkIFl9u!kYy z-Eychf+>0{d@7Cd;wrypWO=*pB?$H{WxTsCA`)C{uB zkn2X*<-Tq`HoA6b<>Dk?=&R_zdln4st)aXU000Prw;}K(Mnf3-e?Q?2Z4Zbv^NaQg zo*Fc>7cI%e>nKBBGlO;XAqLDc9eL>;A#o4@u;&2)IJo`)FX~NM=Ddyp<{9 delta 4067 zcmXX}c|6mP|KFUsjZCiCOpZ-5N7^Tg88%mr8Rl&9m8|4Q2wQTDh;3NN(OOt2BNQry zPnM6|siagYge0oz@SD%)_x|Jc=kxJ=Kabbz{T`u=QkpvzB=6rn@&U!VuMjTQ-O_5H z7%WEBJgujNMD~;>+3G&_2ws{{d9)%PT&?>(?2=<>j4QPq#m00wKh3c(<0^-_3|)9Ez)%=t9^5Pu&KXp^lA9PKdEbY zE5eJB!2NvIpYT-F!GqN0Ke_PUw}jV* zGb}2|5A?7-JM!Tdw^iH)_Rrl;#RubAA3+9GCSI`!%h0aDhF@g5J9|l9_3tNOJl!4} zEoSF7iuV^^z4Vp={ycbgm)DvqM%sb~&Bg7+V>xt?o%u!$eFo@w_5{&_Qbbkm zd4y!5OBp{H@~daqM*vJQyPBIlXtjLbr^ivN>n#+H` zN%5XNu6r)pAYLZGFh(k|)w5mG%bV2uH2FRIeKovW{`EdQEwf0O{71;2xkV8=ZNNhAb7?4~HzorQPG0|h}D zbgNJ5j7H?ezbf+rC5F{rRj_m7wjmc@h#*VDY5q(g+5CJTNtcf$0m!zJ-vWSoPdDiX zK7gMI@ z{RNnZ#0VZ~XD9$vUb06W)vd&e15y8i)>wQS#n%E7{V1a`3JO}A>85BRmO}~8D0iwOJ$r3(wjPZRFKlY+?`Z1o03O?ZA8hR}8 zK#07vP!%0dG{3JLiYv*wa_h*u_QbRgK^EDZvwOWv@mwSp>Eaqm&ny}QMAIW^RM*;9 z05&?FP6X^tTN{}%`+TR^RAa`)$>b+p9sJDnN?buO%HYy#%NI%7|8_ZWGepy2&G2=+>m+_;~(WmWdXH^ zwY}Mzvdm2?3t_s8%KT7wsZZvJ>zzZ@@yU? znZue^^Px6gYqV^<)=6%ik39wcG^Dz6o``%@)J9+z=r?KAsvYo1Yl~~nwRV9P=rg@Y zn$ECHiqtM=ggAu>gjAs4ixja=d%`HCE~f%aHKt`-jJryw6HTHYtUHoJKn4d9dC*CEU4zMq$exY zQ3M0T9tc^Vf%zXhQ`ygZM{LtGND5%;W?QaFS%@+(rGTL79JTZVcfH^|VW+zaPnD6+ z-2W2=%j8PUV-ez9q`EY2Lx{a4Rb z7oWWwMC4sP!jR_m&KVDv;b4E~NtCC-@Q}#>}9J}-Yy>?|!C;7zu z+R-l>U)Qo9$NWiRx%^K(;Y2+`h3CZgCOrx*Tr3rLS@-f@0ErYZ2@r+qF`!TsDwj2y zp3oD%JWlrqhK)snqMGS721mOT5YYLj1~Wwhal&jjjd=O8Zs> zJiCL0Y@hDgopCaFOabgo)C7C`@NL&|*44F2B928_6U7@>$dG~;*B-gvyt>ewBabrV z-O9l^1w<(^lsOtdveTgh<;1@d5B}3voCYa*)tbx1o1{UQUc^nzc9DKywu|&g30Q#+7ICyrYcB!@ zZgT`Bcd(X8tDCqTQ3{MKxBiw9hg<1Q&-?_xGVFpovM9xS9>=!!`%Bl?S8~U|P>qOd zrdZ65<22JgB*27EJp~|CBl=_k^=Q8V8eBqD%o8;meC2d)h1DE?2&|1vodGs?ka#&(i-Bcu5nzvNZ#&zh!5K&lBKcs9#Evi=$-{HAoEHb=i(T6T?v~73+fET9B z2%#JNd}Y=X$CBrKp55?8-q%AM&5aNaozWSjmWsP_u42caWC+|;)E$I4Le!n@`L0!z z#U4=>HU7(@d%DadMB#y~rXHA&Ng6%&t`7?NRk_#LciFJ1s$y|xnN=Om3R-g8BYs-Z zCAU%q&!cBG7U$u17%LRz`AZpzX8vIlvLwyBnhYz;K%wRl!#u7S)@D zzA`}b+*VHZm8?Rt5lH3@es6sKEUbh4X#Tj(ZnqmBUb42VzuWI8n4_#C`i}`Mg%h{` z(E7QGLAlg&6}(`ZXg+R7tU^(~ztpLS!kuD4X{Deh!0k;-eerV8t#ArF>AvK>sr&>7 zYlkYkaFfKm-Mt@FW7;Y8o(Yt`uQ_h;IAcfaw-geYgB>##Nm&V15wi|(Y7ror?$&x? zfRb;^^(8uzagL_e!nC%4BQBGbSg`HyYEk9 zuSrXvQ!ba-hv83%eCElIh>0eMHybCAg(79?sPIC;-M=V*9t8Hanc4>JayFA}*jyX#+9fFf65 z`rEDqIDrxR#{|!aJ)yMiUBi?2%A@qVco~%(R@r4z7NlJA&^T{U>0to1ZtTs@0BD`j z!cKk>;gskk5~OS#5#odvX8@izu$>!V{d#~SxKU*CGveyFkzCvRH4QyYzG#Q^b4E?; zU%#`MuT61kU(!%Bl){Gx>0)#^41>5lCI=)Z=t|X5!MKTI^-FL#A&mP@W8bP5r{@m|s3w zxb{5YmB4K8hqgc5q`}UWtf8z)?U0Fw0zC`SY1c$j}7o>57A0Rr`bN^Id8m3u)h3H z_64)4z(c0jgG1I6ommft8|;HG=4=IhIJ7mf#oJh_OaH_UjoxIx@MyZUdE(Zknd9(5 zYV(ohlZVen3XjAjT5AtSs|%ydN9(=18AI8TA3L@}+-e)BYPyPO5AX%S{I|NV-TMb3 z2dmxgswASy5GSerE6fJL(&tIVncuptT6!wV5~Ht$=UC?ZOsSrk%XWb+$7@E4OII>2 zW3f%rOwuE2=yUvKiSEU?gwL1nSkwpUglqlIIE`tLjlkzYJ>BB3l*`_YnffB0ceyTa zoCvS?R8nkO>|$y>yp`qGpx|%T;|SzxAF0Kh{jBid8Q&JS$ei`Ge;EbmDP23!d}XP^ zn)cN)m@HEjw@5YKP2-o*=QHOhm~!3ei-0HlZt2-4=d{9l)o)6}XDGe(K?km_-aQxa zDd6%;X~R87r%vPj@}CwR>aDI|cf?Nm*S{gidj`ijU6NdEYjRBu&g^ZQ5W3rZ@c>Yl z&ZEC3lIEUCACv_Sn6Cr~jjF0Tq6r_P?$O5)PuSDldXDNI;P0Wm0~LSdJgc&QRU}Di z$G=X`*i>qbH`fsV{~vRPWgt+SwQPA%tQEp~0s*t^txSMJEQ%Ehc!*VJwdX%EN3Fm> VJ=VMx8koQ`wAPn0vjB*m{{!b-O?ChP diff --git a/summary_results.json b/summary_results.json index 96521777c..0612dd89c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.77844, + "Performance_Result": 1.71387, "Performance_Units": "Samples/s" }, { From c972e6917ad813b578d2e367c6c89828b3c91194 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 23:36:47 +0000 Subject: [PATCH 11/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13904 -> 13904 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7ac5cd78f..73feb8006 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:34:57,511 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:34:57,511 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:36:42,066 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:34:57,511 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:34:57,511 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:34:57,511 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:34:58,557 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:36:42,066 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:36:42,066 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:36:42,066 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:36:42,066 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:36:43,094 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0365 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.064 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1b79f38e2..c15fb720f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -500 +501 diff --git a/summary.csv b/summary.csv index 0eeee5bf1..bdc4e4575 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0365,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.064,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ee3a7931c..cf6e2093e 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0365, - "Accuracy":"acc: 75.800", + "Result":10.064, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 264e241a1899a46638d02f206ce775d721cf5def..60b5f451cd107c1ae71e98aa36930d2459c2bcfb 100644 GIT binary patch delta 1870 zcmV-U2eJ6jY|w15z##<6(eRg(#33SoO>?6-5WVlOP^J4WJ1X$J*ak&2U!L);E3(&)c}4JwP-rJepUkNU@= zf0czSJ~l-E*eDH^aU8JJ&!envzfU8<>&6k|MJ(#pobi+?)-lUvI_lP&=`@Z~RyY4u zusmYiP!Pv{GLM@a7iSF-OWx>z;tYLZ`bO)Ulu%vFimbq49zRk0Mah@sfpZ}>K1_6s%?hAFXuPh#CQ}O8MN>;Bk zp$2irge0Sw2QuKg&DODoF*fV=&=l@_?by-A_bu57(Ey@T}R(CO_hHnYWXU>|$Pa zN2h;Uh)i}j3(@2Wy%IxzgDbSKbfVijs=r=ph-9dR(R5mrTB9wMmX2dT@rmZ*iPr1Y zok_G5xgTrYr8`kV&wk0^)n2=oTihrfvPz5G50su*6^;eiDPXq&yRv61y9?O68w$2m z)wOc9CfIc`2`9D_YB`&-T&gyEF4*~#Y`);ZTM(V6Mn4ec zQ*Us=8=ST%x81CDiLr4{+XmZIBRgYxZ4;|nT8m}-aPh1*ATE`4QNzt=zH(%30k@^9 z8P%C+E9$UbZw)e4B@ooFHL^-8UY()g?LPzLx{ie$stvN7A^60pl7p(^-r%)2dSznw z)j6VDdQZ{0o#c{#s&d)1&CuQqE-!Aq@!)1OJVK`A-L)yjrsKL@>husgzc_pG63f<| zo{g!6yCyap6l!#H|9xm4A;;9^cAE%28^{r|T)pSGy_Mlo8hX0rm@YzhH$R5gWAAo2 z^2VoEx0l26X{;o;->}Z2M8^otc+&kPYp4=sNdn{2oJ^K~^EIpTLg_j}uB}rHxEvk3 zil*jFM5*6!@td$*H1zUpF8*yWuOyoQc3{1!M7pvo#k#Ts?>B)X$^?!m6F8zw;D|DT zBg$0Lxv~R%3BZ>CdCd@X>l1@N^1z81jO0{B`0Ukl)C z0emfhuLbabr2t3h<=>UkdQ00AC95r2t&2x}~biZ&>kUtj^Th zxh@uomZj;^TbR1p)5*3+?;FDAJIkzoKf+p2n1mXDg6;fX^T6-j;+nbx#>;)U?bzSH z_pq_oDyCf$hbS2DKn3HqR4|DjWPJT9p2R`r^@90-{JCw*md)iQHcHr@9jk!7*N2d? zxx6{%oRtdERj>ZQ1!Xt1HCinRQX7E4)vOgng4GT>D8**LRw-`Qhr@x{4Iicl1lyb0 zyU)G9_9?W3a}(1N z$aI^3c(qz3E1-rZ|0K|3l!nr z{$rqWnxu|_c8bPoBO0n>btv^bRq0|Ks{@ypxsgS5Vt3c(Mvm?{j{7z?BC=Mx*g@#~ z@Y=f^UQ3Ik{K&h>_d429gE%*<@ap#D^!E9Yp1oEg2Ir#xKTt~p1QY-O00;m806ze; z+9y*81cFDOdy_FREPq;4gzZ3EvTC4EmzL~=G)Y9Q2a;OHe_zE;oCLD!a&+&UxihF* z>}41If^|Zx8f8g}B2Y|op|&;pXqWLLikzp4(@ra>(EtuDn!Ab_!nB1qs|{E$z(oP1 z9ATzLJMRs_*s&c*>XP74LELC7sSl;yVnf+C-2%?jw8RoT<$u&ud;rE~CZZ#fvl(fA zY&QZp!_a{Q1h-U#4NL(p%$9T%DUfJY$!nc|+5&*J%|U7{vW z^M^Ri;=FA0f*2GfxzE>!kyrg^pga!wH}M3Ra$kRf<`1)TF`NSg%F*zblM^+%1cFDO Idy~;Mf0m`Qq5uE@ delta 1870 zcmV-U2eJ6jY|w15z##++!8U1=#33So+fw5=5PjdTP-S0c9&F3-}f1j`*l;>;68x9roW)qNzVkH4*w7%geeqAWe>=_>9ang&@I zrSp^C{oT3Z_7LNK8v1dT(v#j3Wxa2MKaLsWNIaUdlima8iz7{A!2?Zvre+I&nhMV> z%M+iAfAhJ<7CH4p_CP65VvS&IYl$DFJrra`%1?SWaeAmoqyGvztSq`~_6El+8XWV% zRTk3tSmT3Ztu$1|NkCISkFuuyK8-kQ8po6sF>hLP%2Fy>Co~u7Xj*S(vp7m=)BIOK z^N2D{LY(-?B5reBoV7$Oc%zGdGxUY3a}|liq0=Ia_}AvjVHRik5Bem571Jc*T#ov5 z=|{0YjpB&Q4?QtXlcx1rs^YubQ1Vbf^X0xKy8f7K5bR7GMPZS4Fm43hJ zOrjOf{aA5V&QuCL`z3;xd+lCqajSTUDlKw9kb0sn90RbMfZYP@(w^0J2e7*v3bs_$ zwQ@D4*l{okrOh+QvQY8f;UI?2P4gO{{8ZB^K?&__Nx8xTdI!25vt0r6X$#xGhyp ztIkAAQit_=YmuocfuMe^k(FBU>I{u;{~033u?=LK$}r0*f=`?(IjAb`4PI-bS0r{{ zog=ED_9dO$NiI!)SuWeQ8QPoS<;ATx8QzRXM@SdEUCT7FZaa=^+I@u1FV0@P#IRJm zZ(-BGu8#E-On@Ff6W0`MgOUjpzY0AB*|H2}T_z}Eoy8USAd;A;SU4S=r! z@HGIw2Ef;U1o)Z&UlZVK0(?z?uLQ0AE>rX*j3VsavYL_=e?AT79NA z&UL;_lq^kG-cncfzDl+|dfyN>-&y9(`w>=xLMNsMDA>;L)erpMEv~6MV7%OiyN><+ zdk-6Xqhh)xafpKP4pcB+M+KAkLB`gv;z=A-UN2aGES}r8Y}s5~Vxxra*|8GXdwmEg zU5J}w&SXdd!y}<$HQybM{wPXSSN_x=Cpem>jbe|lHCf~C(t@jvQMEMoEx8& zK)Ta^#;emRsexLW{F6YBQ5t5CtX-WC&~Tr@>!S8I;&og2+d_3v_1i;r%<8wC1D}QL zK$Sm)?5wl5Me44#f7f>Ho3PhW?CXy?(vR(sqef_%fi!QrAB{;`{O@<{!vz3Jy{PFa zQW1NUQfaXBax+HtCT$$INV#0wWjR+b*F;o*T@yDv3xPM5lnp4wKSTBY10ET9r}>S& z{}`y8Cdp%Fi^1$U~Ze$RZSnk@~$X0FJcHYKDMAV9lZG^s$ zuD#3AwXoRIkGz|FucI9~h;zLPuW3(CZ=WCOTWckvF~$e~15ir?1QY-O00;m806ze; z+9y*81RSPZa+5JIEPt91gzZ3EvTC5vl$Pv-G)Y9Q2a;OHe_z>2oCLh;a&+&UxihF- z9%U2#f^|Zx3S~)(B2Y|oq4pK}*sS6Oikzp4(?%<(PzMez>xYsV!nB1qs|{E$z(oP1 z9ATzH2k#BR*s%jh>XP74LELF8sSl;yV?)_D-2=|kbdDu>%73Y+xCh2&B%(8tvk_^2 zY%>5j!_a^POZq{F`NSg3&A#NlM^+%1RSPZ Ia+A?De;m=5P5=M^ diff --git a/summary_results.json b/summary_results.json index 0612dd89c..f5de70757 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0365, + "Performance_Result": 10.064, "Performance_Units": "Samples/s" }, { From d8ecff27c184c71c1d3dccca7c49d79c6e37e463 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 23:42:01 +0000 Subject: [PATCH 12/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 13904 -> 13901 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 73feb8006..fd35124ce 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:36:42,066 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:36:42,066 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:41:56,418 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:36:42,066 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:36:42,066 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:36:42,066 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:36:43,094 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:41:56,418 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:41:56,418 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:41:56,418 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:41:56,418 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:41:57,474 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -67,7 +67,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.16895 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25372 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c15fb720f..cc5027eed 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -501 +502 diff --git a/summary.csv b/summary.csv index bdc4e4575..bb7225e26 100644 --- a/summary.csv +++ b/summary.csv @@ -45,7 +45,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.16895,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25372,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cf6e2093e..5775485dd 100644 --- a/summary.json +++ b/summary.json @@ -1389,7 +1389,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.16895, + "Result":6.25372, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 60b5f451cd107c1ae71e98aa36930d2459c2bcfb..8271daa54aebcc218a9082c88fe126459e6db6d1 100644 GIT binary patch delta 4914 zcmXX~c|27A`<+3NCC0v#W$b&#nzD_VF=LH1*=aJ_m!yU0O4JOBnXx2_H1-$<$ySs; z49QN~R7@q6FlZEguYSKj?)~S!?s=W}Ip=wv_v;yn9Exmc7uhw%)Wn+69*bxIXI}j$ zfAw)mV9$ZAkrxxC$uLsZcUR|I8eSUF#q=}Q2?7E|)d||~Ew~#F@6DIrY}8M`$h6oR zd$o8`>ARy}D5akCcGf!EP^W~+C79g)cI=pb4W5YuEM;9@tk*2fcD=#XHo!B>|1M+8Q*4G6zMTCpEqT`Lnv*3 zOfvEH+&IsF?@vrE*Ynd%EMwEpD&jAZ7M={8@A@u!?_0S0uQukbfxZRK(j;KW;J-K- zDphB{&Q6r=TOCkVST``tkEuM)V2;jk5wlUtHBLZcqu1Y7gd+R5$8p`ccb~gzs`2%h zD{O8}EIY|WHn?`be;vnAW^mybW6`F%h-cdW*jxo8W5Rz4?=n+$cW!d|R~+Rnr#dj^ ztkn~lE@BLD-ny&i`LwUsb0D!+_1>MZ+e`V|HEYAiR>LFLimtY{isU6J)^=@#&3>D* zsS})4a}6#y&-~C8`Rhn%)#tf3h1^e9#tvK!JtG`rbmEzo(J0nTbqPyvkxR&?RLBnS zXWjhW82Nna=Zw5oX>X*_s900-f#n}Py}$V-l#<%UT`*hLy zX2s1KM*n}FCoLyG8*yX|>v)~DmdCNql(F5i;zi;`)DHMlN?R5>~>0ZVVTzpw8*tEJ$9=)QWvN{Qv_|7}97Z&RfDwdA4Czb?%` z-8%l)YI`ecv;E?))kM4I{^O{Sdu%Sd&j+xq&o*57dt-Idb$eW|NYu@NcEW6UI&x+K z;j8B+Q=rs$?^!sv$zn@#x@y>-f}iN?sU z7y8Z#DL6FfHg|g=c!kTMfkwM5z3FxRJ3 z_9(fCo`3n4WVQKf9O$xd&+I?Z*zuC4Jwp>|6up71(0=!&eZG|J-FbpI?|X6eT31E( z*t*MKU)SLmZWp&E`ET=7ha9YKG8|m!1Rky8rdKQvZ8?=kLGBQp)gi+hLp*mLcxT>9 zmw3JV&YnKw9##E-A0U-D_Sk>I?{RQ(H*v}Z0^d*P!xsqWyZOe)yXltk~mZu7A}s?f@J86X1v^ggcHA% z^F#buK0g|9flP2J!~~S(Jioj!V|@s58!7+3RHA#?i zcWtHZKn0|ex@90^<9`nFfp1W-XAeHcrZM|byzc-#^+j`gN^~b-+ATFlsG)Q>_;}(i@|+Z)bZ678n&MTSie{&`&kT z)urwKn2F4@Qyk!>y1&9&k|m(=RN)aE!dCNC&Wo20nxnQUu5Oq2(%dB_fYT?G|Nhm= zH!3J~D{p-Gwa6rI?p$UZ)=uyk4dr8}Mq9FA!Iu+K&{U{2Z>O?ZDMsxCr_ioBHU3JH zl%?P|NdfT>>Y8-_C=wi+yGT5+>O^0P(yr`Kf%;!T0>gKo%cY(PWsQyS z-j{w+0=P{_I~ATBNocN;Sg4xHXjwD}8@#N$sY8~`z+!OEU#X=%+--r@<9$els?=LE zNO*`}ayW%5J*?79kJKgU3PI~-sX}b?`#DLAR(SOej1AwkeqN4iZbMG9_M?*M?WRYh z)tb>GIC?tB6EUHo@;;BK#?VtuAoB7Av;c1QWm0O4M}oIIY~C#k>TAEw_+xJCFjD8# z-qV(wclAuQ!HizwzG5!-)1D1GkZFILr_6QLx`+zc!Td))2VbHT)G7bLSX%nmRl#E9 z%~H*Wnz2%HTuS0lx{xC4&)7lWdy)W%E6CLDIEyp4IL#c3{Wzf>fwD;((w_t}T{#s2 z!S^tcewwb|QzV9HsIO?Xd-GN#vpp)s&_viEq&V1CpmNx-ObX&!5){ADchQ3AfZcn)w*d-Xc|Z)pD%?l4nX~;V;4X=G=G?u>5ISo`D4z{s0w%=sHEzbW}aUzCdYp_-28* z=cU!YqFh8rX@otRf}OlIpjMBLdJ0wF)Y9LfO4ac$N)sD^WP|vQD=Es{Bf8(mNq`qL zXeDMz0h4$s4`Vd(A- zVYut0;9QD4Y>!GlR33CGe}36MrF|z|mMP`*r3N703nf2E%Wo!OF>+xriddXn5Uf!+7N_`@*Kbn|j)y$t z97M_QgNNc~qIugy+rbFs{PqgV`5n)CA{NIT`u@nHLYA$!JVGZfvk%118NEP{1r>0)%)N=i% z^q??branikb`~Y0L!{abzAiJMySQ}BThHaB3p;A+YYTx&x_ybFiFFSk#PRu*2C8s( zcv8KXuJVaBsV}lec0fBsg349)=wP?m@y=JNG zHJjiOdms!4fI=quoCD37h5(>B$5*tJx+g0|8+lS{hZPY)qIOOL=M-Mr|EozdSd$Xf zAY!6gk=yt+$HL%#@`U8t-9IRKQ~$w$xrS@PmYVuuqLeJ=m?9WW<8DPTnl!J2DHU#t zpf@fhhjvQHq-Q^&Lu54fEznL^xMODTeD?|8_OsCFTEk$oWR(c{p_j`CB#v^@Js0n| zZrUcuNT0tR zX()&_ge7~x>Xc_V z$G&(qfG1b?z5H80QZ4pzdRht|FA&ciL8>4(5o5c(#k z3>;WA!9>`M;G5{n0sdn`)xwRLg23EtoK8vH_8WX{f3y!8Q+Vp*f$jTGb(p;vG93@1 zO=MsNs)a`~ymnUwXKzpfpAwP-KtRvUUbqkewlh;>he&DjXMEcAR2kox*L(8(eVJa( zd%kXreRe7&_L~OyPGovENw@kf`d{-4Cey@m9i((Zm<&a1MLtLdXcYcA(7FHpWTgyf zc$l8;JH3%q zzYdqC2EwN_RCPT=3pMy|ZlXYX2iC|K zY1BV@v3Y~Yl(6pBl13zHA)kbqjScH4Cye@bEc=Jl8?8;a2n8!kAiy!w>(?;un2g>^PHy zwR+p$=M&ytnA&!7s3mNDcs-bTpZNzdxLz~!+53_oo>qt7U2<(_BT(g2IOH(S4dt>E z7oT>%Vxwb#!~6eqDoz0GV9qdcebsHNM1J_vSMy8UmNKCF+p(4ESFocw;5M?DG?>MQqLl5#fi8K_z~wb zch}Vu8RpBp8O4RN=r* z#ryG1B3*Ip+$SZ%&wYkJ&Rbg=$-REy@(LXz!ZrTWmYlG7{CfIV*JqGpyF${uO9Nil zbWfu#{OoVfUX@tl``LGNB;?(H=&RQ6-34r~J`7=>T;}-29LcNXB3zp+0w=k0sc%Ap z+cv*q-f;p`;`Xh?e3Uyd|7eSEQmN^TksR;;@3|-@zED}(cay_B3>Df76=|BCsScDy mi!`-_9->v6YW$y^qo&f(eY7v8*3ej*rkUz)U1Lyp$o~Mhs}+&} delta 4924 zcmXY#c|27A_s3@pQkI4c4T%|JU&<~t7~9ylvWA*QhLEIE(Uoj7gqg9l*pfZUXpo&E zCRvjuSrXYR+gQTqJNo_pxbJ`Nz0b$%oYy($eeb)=zt_LKmY3s5;7;TjY7?(C;5hn0 zB&I3b>-);Ok*=fva=gDBBqZ{)I{15Ua^4NJN}^J;~#;NW4{n1}Q)obZDO|!Fi8i67h$A`7doo$>-_@ICH-`VXVt!q{V+i$nJ=JA>f zN+fX~pRVnYv`x;JHZS4}5_6qy42UK!4#BdzFn(4xwy}G=ZF#_%kL~e`$Tz#W)v_`A z`rv{te6=eSJuG zg8&b}FlYRZo4xrep}*|msW@))%5&!Zn>7>K=#+Q*{xq&}=URZeA~XNhkZD7Y?oG$n z9=wO**Sx=f-3~h4k*-_ox#-m`Ozra;YI4xxcvV+2Xzc|nt{e9?8Y#;EF|)E2OrBWx z^}Kdda>Wt;C(|Y}oW2 z^4HQ=;+{YydZ*v6_}<{ZW#>l_`cWgA^7CQ;=@>#d&oz}xZL%tT7K4(%u_%wE`gW{4 z&kV%$)1J%Vm8)C+yn=SPs>cN42>Nm8=EbSKhs2|etzWH_vWuhl-98B(1EzAs^#9#D zD{bX&^~D(u%v0i3@dKHi_e4=E zo_RWV+WuFvl_Fm+)e9;HOf>C{EuUF1(8S{-lveJdQ!;CZi=`4bL>;P?ZguzVb=cna z+S{DltR7kX)zb5lqp#kDKa*E@HeFnFk5P1{R(4-k-buHusbG6)9j}e<@@adjr0+Vt z*2dUf{Xx+7QU-p?WKWq6T==vaR@v*PJmlDA)Au)6??RZ|p!FIKfU8{_FZHP+ed)jX zylj6x`09!;T?3w^pB?7Fo2t1!XuXuD56|Z7S!fzOw4JL}|9A)(3CyD0U}m5HRP0jo zv#r?dI_yl zVSmg<-;3SS!X-5_fshHAF_Y<>ZyrmGge6AW@S#}LXQ68$9>Kk>isR*~t7Q#)?>#@q zy3Q8-r&Q=OE7P#Q`p||r@FsF#KD`Y;s6VljCu)*ewx8dx3F?tA5Qv!ld+V_6XrS?O zKQVx}C+f$ONwUzX&9e!zHNo>2Isc&ei84g2o(fy2^hrIMF305je4{OOkN-Fz^(-;> zh$)R!9F_Ap#eRVInCl^uoS90Sq|$i$Cd)OMD84Ho69KX;CC->%G6J_?$X^E`W&Qe_ zS7)rUk%Q$tWp5=#!c(U}4dn2|e&Fcs^Hx$3bw_=Pbw!tOV>Tx&&%v&*IxVxErGdsI z;Zpi{Z)3S8FT>5nJF&Z1PGGC9^EAjDEYa(HF&mc5^9q&>t&r>JNmi5`eByCyciNmH zrteaoW?FasZI16 zY~0*$yPui2EmZI!%yV}sx5s~%1>X5|%l{Of;x4W@oucXgZSYl3##QRFpbRi`HoZdX zg}b>_dWCy>2(@9nynnS>teBK3F&RAeLv(tZ9%|kFbGv`ql9ZH^By+>){PWu-OGhe2 zPo6$@=&|fml!FiwF7G9_ppg$~uxFW7XN6 z!%j!d+Wq8dIYY z&`%Q(qG--+e%{bujMsiG!kcRBhDE(V8How7dAgdeepID6Yf4O>{|l(2+zeM2UX}x! zow}aSr7pXZ0Z0hfSZ$tYhNY{ocKQOTE4RJHHRD=VkT{)kCMLu3=7@HG?Ikzz<@uMX z53T9;0=_563H|-rNCaN3SsMqxNaf)gS(h`Tvz5=^NTlyxorMq>!%oT8BW0GgEktT+ z%(22(E|NP4|G*+oAf$9?11VaG`=N8)?@$II3*~T%mCxUXAy}^OJiYNQoRcUBMmzjI ze;Fufhy6vJ*Z6xmchw45P~0#!FL^`?(V8lVxu9?>?qu`E)qq@!0%7A4PsuN>_9D7y zir)_jm|uo#HzGCZ;5?n%aLuc+kF*?;Jx7oo8mO`VpHH7X|$SN@5vw zURA>Qi;MuvppqU`zr*T*LH-;e>~mkF8laZa+~71=Cnl+7RnDiEcOy2j(&K%q>s^e6oFJ=J_nC7A1Jr z;FE>uAz6F|By!rQjBYE512`hgvrY0^kdbK#%bMkJIjklyk~UIAdeBVhAEq~Z0+!1xj-@71f`>iaWIf_I3Jlamdj1GYSiG@!$6OI-f6Ux$TZr6b~@_9{Tp?!DRMEx z;k8TLl_G6_uPLzx$=E2!H*FYGPWP^YDh4Q2MW4iNJe$L$TB#E5xDQ6teF^E`d`?s(8_uIr zE1(S7?Wl3S!{Aay$xGhw!wL*g>4(`PG&TVzuLt3fl7CQesXFSC6vE|C1*Uvb12$@; z&pv1&4X>Ef@D3-`%xQexU>E%HCzj`~Y%#DNdmz)Bwe4N!{S)iL86=go3ce7&!;<-S zX9)pe{v$p(%l@w=v8=oI4={}H-qdVoVd)(hQ;r)7u&Ax$UpId7B5npMU+u^e@O%RpoNg}I3}3gYC2#=9Wu3H0kY zGIZZ2J*9+6?aq3(AoxV15UYVqeVbMja45gkhM0G*72{-Kf{IGv#F{8V4%QDcFdCad zv8}wS1ni)apY1A_7|jhzjX}+&VPJ}=49p&tjaNFSS}lR9_}P z`$NW;jC-pSKhbkB@}}(trN@=BrL#qb#LFKKJW#NAM9RTw$*C0n!-!X43TukJS|Vd2 zUiPTw!a&a1SMUaE2Vj8XcXjjTQ*vO(m5yU^R&6$Ymxsfqy3%-Iq{K&|`^mm0%K06p z8P=9oDOep7MSQ-aF{W9%6RYWJTEXCC_jk)aXRLtL42Ln5Tugc?K+r8~I|?^vp?9d@AHzZoYB9G=M`W1Rw=4b`XEops^cy z69qb#ZDXteDne&>MB1_|!N*ATq)trvPbmaSV;XEyg=>>d8IT$CFUAKO=8;aJKl^sp z+7y2^l&76T$_dl#6lYjC2go!F=fH?aK9}vmq7|bX=6j_yx0%>5+C)JSpopq!eICNv%L}(Xzu=`Es@`o%^0P`bjm;rq0G^<-x!uLDyJ&Jn>m>o_#B3)v{Fw6&1>PoK=pY743VVEOdUH z6G$)IhRjiTFbiSH!UQgKBBx$)91Md~U?!YIF|RH~TSx=K?0&AQ{Uv>glI>l>i$MvB zDvl{32`<>LY^?Eb2RR{m;EGxV%N6`TS2{Z)z5j*1#of!A1&CCH1-Z;c?nwTsIh&Cb zl{?o#+vNJTq4wfAoeZSqw4hWbQiyWICN?VhTdk_FQ#mE=1k`96wY(emQ-E3n033X5|^aZWUxGPs50z<#_u0( zOz9sjLE9T04e{yJk)moBo_3$gZT*+ZJ?=d^z&ydV6xd;|B3~;tTCHW?-M`~@B7FwG zSlKfBdFWy0-E*(4ecm(t-);prY<$lBI>Y*dSBLM`iwpoWp<6xP`)js)*={}e7pm`X zR~7)p%LAn13DfwIvZ{*RL!yJfnDpCe;>;^P?^o(?k+X%4JrZdX9q|vnb^q`0Hhk@l z^UC+hA9&j1gg`*2=ibAxs$Isw)j}#`iWxhjT=VqPzUOl9*neOr2@8$qG?X#N<^Xhds_t@|^*RX`7z8E^~(eYhf> zD@l0+_r340#isqf_W}EiST}_yyWwPMh?c17%saVgah`AVtk#i++c-^Tc=mc5fp=SN zxAJbdxbvfEoeUD#PrjS$jX^_|b@ZwDptp%SO-E$Lxh8Fmdau=s2|wA1FhhWzOl76? zOl{|T@sOy0%s4^Oc;7?Rv4IMC=nx`TCf0^vo5CV%+XEe~@8&s60B&V_(E90R50smbz AEdT%j diff --git a/summary_results.json b/summary_results.json index f5de70757..37918b2ea 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.16895, + "Performance_Result": 6.25372, "Performance_Units": "Samples/s" }, { From 492389a09a66ec637cd705c4ad1386119c22c286 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 26 Jan 2025 23:59:32 +0000 Subject: [PATCH 13/24] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 13901 -> 13901 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index fd35124ce..eac088bc3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:41:56,418 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:41:56,418 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-26 23:59:27,607 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-26 23:59:27,607 submission_checker1.py:3266 INFO] --- +[2025-01-26 23:59:27,607 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-26 23:59:27,607 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-26 23:59:27,607 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:41:56,418 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:41:56,418 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:41:56,418 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:41:57,474 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-26 23:59:27,607 submission_checker1.py:3282 INFO] --- +[2025-01-26 23:59:27,607 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-26 23:59:27,608 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-26 23:59:27,608 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-26 23:59:27,608 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-26 23:59:27,608 submission_checker1.py:3303 INFO] --- +[2025-01-26 23:59:27,608 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-26 23:59:28,626 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.406531 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.3606 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0872 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.3711 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430339 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426701 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cc5027eed..a712e7640 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -502 +503 diff --git a/summary.csv b/summary.csv index bb7225e26..714cebe4d 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2 MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.406531,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.3606,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0872,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.3711,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430339,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426701,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9898,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5775485dd..20ad875e7 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.3606, + "Result":23.0872, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,7 +1239,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430339, + "Result":0.426701, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8271daa54aebcc218a9082c88fe126459e6db6d1..3cf4f0eff44b4825ce5814464918ea42f3c7c243 100644 GIT binary patch delta 5023 zcmX9?c|6nqAD=UYFLQ>3G1oT7ToKZY<_tOJjF}u!VhHJ@qUN@ZQI6!u9Y)HMqg0N{ zIieKGF?WuH(r@+q=l$pN^L)Rb&)4($dVJmmzYDIX^K+WvyLc&W{EC2k_{6B>?X{gc z?_94YEu8>qPKfU_Vamh9)Fde-<}yrx%|}b;4tqKUXop@Sx9u=CXyJm-^g>#HJvw{Q z!Z1M2NbFgAT%p&9F|?0?H~v^%ZEjZwZJfTQeYaqR@ufYsMVN81dfQu9(&MdvkJoQ2 zTGgNCKldlAcQ@7;(PW_JV&IMOpt|dAUh0K9dDYX=Na-5MO*5nwpjD_|ek)?{>qbqB z8=TS^FdP^&IC6R*$o3G$^*8-RW7*lX-K&NYit2MO`vQ^f=fzzW#cxFz{9^3l8>-gU z(iNF&wA~SE&(*pE52K|6zmk}6AgXaifbuOUdp`=1*ZXGo*BY>VXRPFTOr}|0W?YNJ z>+dvO%h$L1Qno+tGOF=mk!`k7?zs^X@7102{)|4P5Y8{ggc$s6bBR+;>}`<@N=G!e^%gjg|R*d6IpvrW}TuCnY&MrnbtHY&AEI%ZF%Y2^`hj@l|;{ zx;`$cT+|h;IVjW^r}2HYvupp5u#8923v0yv%ESY=w<5wbML>aJ?$sUd86!n&cWZ`I zsRkp{b~~Eq_baO@|pTCW^)G5Yb(qSA4sO4XtuuO{y%`0{G?m%ZP)Ri_=osxo_FbvkZGZ z$%kLD#dIP8T-~p92?RvlRia`3;OVzdjs&U;d71w)Z8F zK4J0aF^2kexF{5Flc)B$MW^|j~I`|J&g9mL+Pyi_K(M&>lQ2hx668i zYdgz0^Ay14Bd-uR^uQJ(WSE*Tb8Y?1$4==;JzPdxBw>xT3JJ?yQ~QCp{(-kzz9ml% zOwpqo>j&V(E@LHjM?W9<)Gh#Gd9BfMBilMyW&Y=7KrQsV~V zbi+&g06bbpUV8?Oew9id>!d3rw37MOEs-6t3z;sn zGufZ6fxM(y9{{aad9;#MKy?Znt>hcsS}HwFsa;OLQ$lNTYR2Vbc{e5t&WFH)6QQje zC!PC6TkJ=utvK_Zlp#+cl$QnfRj9<2s8)A%oBw51wvXPHMNC!41;&w$Ryv{m0nbIwR{ zb)3z22-)f3H{0?VvCiHbh%JmUT0TDlqC84zl$?asJ)Mw`>+nnuFB-5g)1eiJTmfz3I5zQ6q9lkB&HONRHx zk_ZC}3G`*Pz;aHPEhmVI2q~4E%W}z;?a*W_cPNwt<>BcCt_3}%k3gV^Leqexac4mn zF?J6?Q$f73@i(oVG|@AKNGuxZn$Pq{JF@#xHI^&>8?uHDDgT^o_m#e!9uYY{+4q?eaE&5;T>q1%_U7Wu{fSDzb!}Zu}aXhi&-#*bNoGnO%Turgk958 zcEfjFPDSNqrQtjXO=-m%LMIq9xqbv3Tc2^QBIywpkfQ&}z5cqj^}$Bosn4b4@n}z} z-V}7zptGW?`$)585n859-UvdrcYv~7iegTK8~9kC6jYE2D)W>C!$8(_ID}se+|(8P zJVe5;)Etch zh-G)oyO8nF=wE`~Od1i()-ePH2WAqBpckI;H*h8q2`{%{6r%TUw+8gq?}kqfcNWm| z(#1tS5!KRJwLVsGNZO*s88@^-Noo@j!_%@=PsDORRmfOnQ}YyJ@$;Y`gYf_Ne}111 z{pHskSEJn>N5}c_wEUU_BF{t5Z8hu@K8pwG5CSaMu26rXB%ze_twfgSb85!u0l&I1KBCzA4M&vm8(9TXN2z(1>wU?qxaFs9)lU<)Lp|=$#Abr}hn&N8muj&NQh^wL;zT=v_3 z-jQMi{+4a2<)!AM?(C{TEP}j$35r4FI2kX+a1x+w>2jzX?$3>C5H_w$y-bn7HwyX$QCKU)_Dv60TS;-h4mF-Mclq<8&R4Y{+SG+@x zBi=;y-&J=#9nTEZo~6r?tyS3-p4h4uV~l&;D)5)QFmrLm%PLCEsofO`n3*z82PGE6 z2o^)gza)d)1u9CQWVQROd{p#Ak=P#a{LR`{(Zx+Wh|mi&weVDLl6!+MLFaDToC|EV z2gutma<4jCklTvOGO}s&?LCNpMX(Ho23g8N<;7?YNQhh)!xV{==*Tv5;0cXzHE&{{ zS{zE{3o!rNWK+MzetKyQ@sDpsvy&=6sKoFAd1qV4X=SLvByhhBvbM-tg~sMTEF=ug zv<36JGDS#&^*uEL9MQUmQvb z-~usbg8&Thvd!DGmQ@scikOJnRA)%6P>QOW5Qx!`|Gq^%Ky2c7cEWcyVN^YgtPXbd+ zhvWdRj^SAm*U)6vG`q7!-?%;Xwc-sW87rsIjO$-Gjcb4HRe=xlDw>1J*GjI`FZb{^ zcWKgqE^GTCO`KdHFTsN6^mb?}mfOUyAH;6wNoH*TMRd44FZFMD!=}_tSUc8j$311U zW?Wf!POs8ozQ3ASPKzY&9-0=ZgkMR9lg?!klCc=o6RfUQHOgmcqB<@gU+ReWbYP)5 z(rX~R&i~MU24*ErY-};*0LcavPtdF4K}5xuuUZSuG3L5!5@o!%lr{WhP*!Mpt8XGu zGkGy`)GlkK$c-VI!#Q%1Har>;Ni1i3FAAl>iT`)DYEB~PT*l>7zKMdv2yj~4^iZVL ztKuik>z@8q;Mx9=F@mnkyanyLCRX4_RC&(^oT(@-UOfSX7Pk3$?zCR4EsKTkvp@Ei zas0u4@$;MEm3RIn7C*MTZ>8jH&Jyaq&R+0;%s7>&*6-d=2bi)j-DB;~TQNaP>M8iT z!fRfoZ=iX>&vo|EBgXT2dc#mnmncsdMiS?=H;0D^l~P0BV%`faJkQ z%#Y#v?Y=s!&Tp%bI;!n&-FxpZHD)i%1Y`G~rF6Q6X=4j*E>usLA#If;wTJX*v)`6~ z{rIx9i`%T*dfEAAH^%7x=L13^pLq{*zh~s)*WGJz`=-rSkLLQ?^-dE00F6gwg#9VvN+_$VhQ+RPEsQ+We*&Mi!M=r#E9^~@22=>6XGuL@J= zGOF+Xj*PT#Fz8j><625^hW3(0%!+d7Pz>naheuzb$@%5U!{#vM!NbSgoCLVE5(0TI zQ))CMfiY(X8B2Fx({a68w)sGMhmhgy$hl8!zFIW&S&l1(Mo6-3fuC#ra(?S9Y tyJ2H#txN6fvl`#Nbn#B*?;i*|0*#@X5*+`2!ZoI88?pWWo!;vJ{{xy4JbKe>vMa(vvGeT}!iZpWM8j>@WV&uNN zC{{`38k(Zthu=T%fA-k(^L)Ra&*%H~db|ch2Se-I_&KI%Di}%^zcN1T^4r&vsn3di zp6l+83{8|IKnWQ??QHHUyDEnlQ7@aua&Z+_#;VO5v9>Me4cFdp*G>UQA=*^zZ#}cGjwLp<(gH;H8qS&cXa@{`-$ea8CAEE|c}cTy5ieo)b>SoY-<2CK zK5xFbMb!V&@2Xf<4W8)j9Hqv2eVRQ*eI0Eiz1b)>zQ-^s)T~Q?)0qAdF1zTrqhe z*6AOCk1_46dNb|eihmPVt?=-E(7lzsy{gS&=Z)ad&BD}{7XI9Lnd;8%pt&DYr)zlT z6z%--V`!f`Lw}zMtoXXnDwXr)zcJm^z{`9Q$Iib}JvNFlP*}lGn&|UlWyyp+6uxvarv%uZ-Ebngf9W`ZqkNL^igP*S3+$q0P zRoD00>4Nd(*JDgEof>u<)wQu;3gKE%ha9xQ9e zcUizb|2|44%(Sban`YWA6=7T9VhZRC20xL|AWh*XUSO&a#rNVuR;NAK=PG*q1awOm z9l|L#ngSzu-aP1aM$DT)Xe_;-V6xpCmfE4#^d# zw>Kv3_Qo{}1??><=M9FZLuZ%Z9-8)I`LewaUj?%o@kYBM)1^C6_x+bsToRh^SG8<> z%%_a5TX%)KFAm&6?Orr#q}1ir^8RE7ryzjC5Gft0;cG+z< z?#t^OeO0BB!{+ z`iI6$yo8o_m<|Tn%{ocMtm+D-Z#PJKky0mfm;1p?!v;HcR1e5WI~k zSNr&_ZLx&t)^Xl{(c}8Yt~eePbp=XeDz?_U6yD8|@pzIiN6$X>=DF!Bl$^fNl38u{)J;RJ%$^IquDvuKdzedDam*O!7 zY62rtvtOTXu6(M^sN{&;e&9T>T~**}DBLULj4JB5(r#w2X(ot4)3ZK?39ndy!7;QSJ zRgrxWcbI1 z80f&<{H0@zMO(P;<25=Abp#pvnB&a=rY0TKpvc~XJ4|dg#Sxo8J#jqIu3)*T9fHzX zkZ*)uOn%PHD4Q{TG-4;o&nSOS%!&{6o zQ5YOe<{QDn%~URC56xJrjG86d+20h<07btxHKvQ;6NkjVDHf&-SJI|0otOLfPc`pY zeu;fq!_)7D`nd~N)1xrvJkAuPySXA|#fT1Di%mq4!AICTK1#;E?_T?IW@J1r(av*(dg#Gc6pVEJ0pI)V$xGa;BnJ~AxdL=Dv-Xz=2} zwc=!6hT-Gv_+=9prFQgbj%lsjY`dKL>?XCY;_$u3F2Y9D=ou_E1>gyvP?n$1^;fLZ zREYIwXY+2hwr*M9or8jqIRyR^TCW|kv0)@{#Qb8=HJS8C5{#z_~kSYNoe z%>kx;txjT@3N<0+&=YxG?k8rDcqt{)Ybae+D^mf8FMEbU9@v1Mm~C4eh138QntsI$ z;D5$*0l+*owe~AmL!(QyvB=L8N+HP8$%9&x>2}O=Z@-7=P)`-RpNYbQ6y$f5;=@G~ zf&sUD5jYMy04nk`<9al#Qz{CwEB1}v?!9j0Z*BF2O#muI6oI!Z;*+0}5rh!Bz$p9E z8#T60{mdlHq!jh8QShtVS=6^i7}TgD$rvCQ+gJz0BRJ8n2X3Z*`BI`&u|2@B()7W% zUs>DK{=A%U)W=0X`QI2T1VLRbxgzfeU&=$t(+tN5wQ*j=WN1lM;UIPOeTuD94ZHJt zhTsyElmQ=8z{BK`$Ga795@xx4Tn?K=WD{G-cRgZLPCOvIk_Y%GIPMBor-w{OB>K~7 zBNyu9`z4~ScOM9c8P@CD`J49&sM7ztOf^NqQN-25{h$PbKWP8tuXv!CV~Qs@B|a+n z%@f^}6Aj>%y-d#4HUi?|Esqo2A_*QJRlCvpFTqMXs#^P8$?9%}$&h|f7VrV9^+Y+=8I7Og1Xr-{SO7Gro=~FX-_EZI`!=QY zA)5|K$K;1X!*p3D`lXJ_;>rFF3S(_kR_?bg-vr%4QynaWu$ctE9FioI8!w*+mIP$V zTU@h9EE|U54uPzTkyLY7QeMtagB~G8*cktGoeeB$GlVWQy4@RPeY8~AE1FPx_c-a& zjrwR;q~s(euZe&`O9Vj=QVIo9ik11m?zy7~a{)bNo@jxNt-LC@LOoQ-%Hs{t= zWxFRI2{y1LRu9X-eFe>}`>#sPSR7=M2xL;Y5&(=-EVLiLWmVvZ?;}o#Y(DsllsxV= z;Jr|Pi_cg^D@c%(K^v0+tf|*616Y&ddLpshUIx&{w)o_JhV-8w#kKQ~=6qYkR(Z8-UhxzD@|$8qzZ2X8>nhn6h6;$aOc1T&16|i_P_EjTv)AH**K0$ z`}G~G0v`x4pp%^6E^QXfn5*OeT+~{EG-T%rK)L0szyg33t9n3eP6<3v4N%372#lQv z3OtnxX6oWdsu+&mwar`qUirUoC$Aka8*s(B#czD*p?&Hme^Vtj5nTJM(h13s0Z=-i z1#tHL)&yKw&vZi`);w?_#RqW(X%TstACm4W-w}v!x`HD?_EI&hoHwum!(ms_-lU$T z29vBj2i}+e=ZJfRbX3R5Ym~u+5LWC|SRQ!)T%e@6f=eb;L?yD*_DyJeLu?Wny-W@# zC{ofu^cqT^=@jH0Q6!F4Wnvi2( zg@J}fwO579i2$X0lGK8*kg-A{p7#0QhH6GxU(KjRQ=T(}pPR5EF3Uz+i#`-S>y+VQ zPl=G2KJ9xX4WgmP6DL*UW1m?T<*#%`%<8xz+|M=Mk@@>xo9qKgYxV6&1n%G>?f)?O1SZZN*W^d3O*OoA!tP%%KUO*B?CKgD;@ z8hGpb*waGkX%4ED_HDC{{fkK$QJ?s75B_E2vS7~29!Db8vj@?TlaArS@@J>J9>yrK zBODX1LTf}M;Wxd##&|3F8q#?d=AzV#qxRmrRQH9uqtFExKjU@x z9=}wl^`MDV7XWP{4Z~H*Hnpcyaqi)QB7;BrjfeZ_`w$ z`$5Zmp6lg7b8X`OzCHHUy1>8hxVOhdx>Mtk7SCm`Tb_PIiV(J)kb(;mBSF?BeZ?C1 zR{J}S&QCrP1N5#Q)}hVg#r6ef6U!{|$d9%#F0kQ<&eIe&$s2m)Wu)$(XCD35)_6mZ zD)6w+I0I(=UPLJqc7*H$n^KnV(d$;lNpw?8esQic)lf}r^Glc0G>uRrWLiyPhndCd zCH;9U>M=yfgBHM)Yw)?ClLK#+JCjZ(xinvtJ`vvKdh3E85noVBz0iXgI}%6%!aOey zgt`Adn4@GzQNbN-w&{jGqNxBp-Y(rfiQ!&j{FQtTfF@QBs=4+;;#LKoklYt)Y3YJ5 zW~pPJMBHGHRQFxu6bZ~dP6u<36U`EYt?b*O!rZ|XOgGe#$NM34Sshp9J!}4rBd1Wf4b9NlH;^F6o3uWnD` zM?JQ-VtQx)YK8=eiKScN@pqs3du?}w)GS^P-8>XLY+5nN`}l_JaEswRK6G`R3S=?jmS1+fWuPMPr;h&ZkeOikF^B!PHtctcC5LZ(H@v}Wb}hwM{%~HeoI`KS z*0j)eZt{oVW2R0H+>YC59QAW(?Mr1LAG+_M4P@`#o7)4pOr2)P3YgmE1G|o`mW{`T zeP62QY?|RkcKI#O9$%&JuXW;X{rV@^C@v=i;kb5Dbd#d3E9)~5{8AT!+TP`P*s1j{ z2myV<9!9UaiSTl76CP89Hp^gw@BK|+SQc?~e2(Z*TyXo<&q zHRr7TCC-dNBvOIg$Q^Vebe!Gji;Cc9>HTd@h+RIHneyH470CHeK(bqj_q(d@X_S$t#l5*y;T4Y6-m@bCA77(V zP3IlB%u=6f1TZeFF+C&BTdzN0@feG;Rj*CRel=q`5ca!krL XMjn%3KUl&wX6l=O|6iw} Date: Mon, 27 Jan 2025 00:01:20 +0000 Subject: [PATCH 14/24] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 13901 -> 13900 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index eac088bc3..c55778af0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 23:59:27,607 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-26 23:59:27,607 submission_checker1.py:3266 INFO] --- -[2025-01-26 23:59:27,607 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 00:01:14,731 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 00:01:14,731 submission_checker1.py:3266 INFO] --- +[2025-01-27 00:01:14,731 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 23:59:27,607 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 00:01:14,732 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-26 23:59:27,607 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 00:01:14,732 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 23:59:27,607 submission_checker1.py:3282 INFO] --- -[2025-01-26 23:59:27,607 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-26 23:59:27,608 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 23:59:27,608 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-26 23:59:27,608 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 23:59:27,608 submission_checker1.py:3303 INFO] --- -[2025-01-26 23:59:27,608 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 23:59:28,626 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 00:01:14,732 submission_checker1.py:3282 INFO] --- +[2025-01-27 00:01:14,732 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 00:01:14,732 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 00:01:14,732 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 00:01:14,732 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 00:01:14,732 submission_checker1.py:3303 INFO] --- +[2025-01-27 00:01:14,732 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 00:01:15,944 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,9 +24,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.406531 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.380582 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0872 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.3711 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.431004 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.082 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.064 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2805 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a712e7640..3091e8eea 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -503 +504 diff --git a/summary.csv b/summary.csv index 714cebe4d..39ed35f6b 100644 --- a/summary.csv +++ b/summary.csv @@ -2,9 +2,9 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.406531,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.380582,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0872,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.3711,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.431004,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412174,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.082,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45726,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.064,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2805,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 20ad875e7..e958fc620 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406531, + "Result":0.380582, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.3711, + "Result":0.431004, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.948, + "Result":21.082, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.064, - "Accuracy":"acc: 75.400", + "Result":10.2805, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 3cf4f0eff44b4825ce5814464918ea42f3c7c243..d0d284c4d6c75d43921eb402a74f5c979f91b424 100644 GIT binary patch delta 6613 zcmYLucQjmYxb=({AsC|*#F)`bh?0nI)X{qvf)Jw*LCC0y(SlJDHEM|7drP!2iB6QC z5+y+h(MxbAx!=0qKkqu{J?reX*M6S;>{GrVKO?JKsi=dPDf=bB0Kils0Kg2u+DR~A z7pR!gk??+hULGAfAv$T@@mVttqL0o`sY!41UcW$`LOV~U_G^7XZPM=Te=7CdG-X}5 ze9_N3c4$@)pIV$g6V4qPnBCs}^XluTj-W*K%*2ouj{e{I zQfO`UOML9q{OnX2D%ihs$lvPCGnUUHX+N*0c~+5jTS4yQ*X?G~M{`pF%`zjDUxx=i z*=Ou-1i5>)SbyO`+gxic&^Q13U@0A9^6pnq6Wb7(F#Gp`RlI7@#QVUGsUvc`L$k}v zg}&_%jpm;A4|ad3tI18s%nL`fAC0s>;Iu*?6o1=sR}s!s?>v#;$z1sUzI{(;1@0fn z%eIJrNfY(bSt_SZ$yJ_@>M^<0z#b!=X&jiYBH-SB0(>qr>4X{6t1thk_rKRpj~ic( zOe+Fm&as%ZH#-rSXEZ#E3U5xAwUD1D_upD>RP||#aZ%6V{`bn`k8p<%l@K-~G}WeY z?yFI|_Im2OdA*1-7061_?{Ijp{AD(rWL#7ezIS*76PcN028GxvRcXSv&qOfMtIsqb z4!bTvqAqR-A#IJf|eeb619tlpv`o_AkS z)Ld;lbxEv1bqyxSef%G16U#oj_>|(})e7m^+*rr+i?(^+$vmXSAi!$nC>Z+isM-i| z`=@lF2h@M~_vN7aMb*{wKNs_Be!F`=j>28uRzwf?mmT0=$M+z|v#cGjvmMK_x9tv4 zVlwP{>YEU&x}x`Gs1_L;+ZS1J>1X>-0?+x8BwwZ=I$tR(oI7)#`qwU^(QlsmV+r#~#y!)DK-&d(09PfPq&cK_?uy-~O65^Q?iAg^t9`Ma<7 z3h&_yAD%^`^T~|3;dsYn;RlbIEiyP?sTqWEOqRr=}J_D+y1fOg_Qj0a4 zrv%e_UFkx=vT>0s!F%fHwPorywivpV2-L@@IN3&h@+&cHBdYgRjI_s#9mY z>0>ih>ugY0*p>`QL3x}n`2_jDnB!dIL{SvxI28_el{?4woDK6X3~p`T5=m|32rw06 zppjitCzW-dB+vhdaVH*mv&yoX#3*_~9LLwsby;(E^lru z!WLjTHN<-20h&MD*M6mFQYXTbZoe~kOjvbSHwY{+?=ue8kBjBodU+FhGsZo^f$epD zfSo*{zPTWoH$Zw#c)kV%tQrT1ala8W!>*4)?I!;M_cao$iU(jEtnxLa_^C(PW`8ta z?tZI`DGl&C$+JN(i@3beSNFz&O-Qra{n`}ugcNBs)NrpC*}+7o=4#^u;$Vej;MFhj zTZ)DNaumZxEm3k$JP}w~iu8;n*{N5iZXh97RdW)L6w8? z`5OB%EHF{6XBy;zfA4V&HT!(0YOYVP<%G0dh5}^)T%IzxV{Zho#l1+6&EJ{!3 zwMSlvvI7NsyD3*;kI%$0i+s!KAe zDXz$i17yEZl2KvGh7$v2*hyQ<0i!If8&D&FjLJJKl2QKsknKQJ*q>`cLs_HSU(0eOoY&}77n5DO*Ws^0iUS_5n#x>Pxid z>#~npv)ZlsFe7=+)$D;TjYQ=~MiuJ(zC<02oD&U~sj0xy7{NF`oc^oGP_wk8_S(3$ zR4Vd-DG@4+X<~b;0BZL|;DBH~6e+mx8Jh6U>NP6DmtT$V45Q)o7YB3ZX0Mil0pozkT*A!AnyrybdgPQd{mT ze!%nQGlUwa>!VKa&kdajJq7XfsRac?VO;Q3dSzol|CfIZsB(Quo02yJ^PWoSPZ`iN z<+N2}J_swN%C4#e!J!3lhtV4321k3*=4ARXVr63+2yF$Rq!q0V5r*ZexM&WVDyoOt z%8gUjIM#w5S=0y}G7IJUkS!RZH{kPBLKOx|vlH|v^Ke)oDuF&)qqu~2@zEw1sOHwc z71)r{JMIKZqS%wAGYhLs%NXXWhoat8Pm>~|QJ>u4+hg+kH9V~j-W&UmkNFSwU*bKl zLa|!TKhS=DB_-fewL)#{#&Y*mMU0x3RR)1>p*EpWBF8^L9aut>Wg!~!(^_qCh$-L= zDiF>p3vtn$@gROp-g=}V?KnQbJCaEO(W{|xFJ*M8He$^kR@*RcIC^XP; zMI;eaO@h$g{4gyvdk&}g_0Yz}1o)bK%)=NR~ zV5~-jDv9vTdG~iGN`Jc}h*OX%D7Vs_MOmb)WyuUjAM!`uM&hL<6??{b8#)WtpQ!ew zP&l3|*(ct!_O zXn7@(4RI}~9t}k{0m~(z_AoBi zm6pzM20=6R=%NMQ7LH&h=}Gggf5i@^brknx&fN;zL^K=tz(b4VnO{v$77z z5DAH907%#zk1mMrj*&c%-jRYE`TtDv2$(zcj-sGDWh!W?=ue!LcvQMCLErAmM*dZkK$BCL+qjI@FP^Hw1UF(d6?R=BD> znS8GM5kb+aD zF!u~;N(pn1unsmC5`4$!r>RO-_|JFPB{e@1LbR|!aD@;>pPz-0m^Ey9l-Bp}QI60= zp^Kk-bm<_?_7ugfzx;IJ${K~QbbG}Wp>%%r@bPsz!VE1$@QL4{C$vfpcV2F^pC6TpZK=@6eoA)>C1^iFci=k8OQDaz%}X28cFN4k(lKdlNgFSa0$Y= z!OGThQ0_FK^{h%1z%1Q~5P_8+|B6Vl=o#iI#l4WIlluhpHYPiNp!%#gQ`ky? zKAZVB31os|CdOJpfL548hgZi}ylWGv9px>toWu8kx~i z<@Fd1a;2kcj5(R2lO7$BzKslF=)P+WhEU2A%7YTz7eI?p_GF|-CK~6e)6d2*aN8N? zqUws0afG(X(J7o4w({4Sf)#=hutXnP$|yq-$lqKL`I`&3$Tj_D3F(=*H6HuxMWCL* zhN`2WR~u>2k%nmD_<)E&92#F5Jap!3RRL*pWngj2uXR&v>=mf(Rl8Ye+N+#58KXg7 zias2TF()f=q&ZEp>vgcPCGazD?S)V}{B0jWuP@P&Ygo%5A?~W}+W4MS8><@g`kPxP z<39x77L7>ST200pPm<3&{agUI41#%KiT(tV;{=jxQ=bSVms0~Dc9$VU3E^dyZs*t| z`qz4-BCLPM;!BkHTk?%^v^h_ZP_|y1moZMLW1~J3<;?gVkmOw}l-m-h=x3qED8x-j z7SnFKP!G9rs?tM)7-XMTl4zg(6hX#A zuYW4O1|%CM5u8#_q`z4kBV?{Rnp|D6{N8{uW^HpqMZzHpG#R5>V5Q$R@MP5eC=NMx z4?V<4=KW|$J~Ay@Ep3orD@IKSy1VbEK`wR#iZ>^_>y!a{v;~fLA?==ov08xQ-w~dX zdQ3zpfyEq4=za>WVk-!1$fu&6v=~*-o&KimQZ;MYCYy;_>zwzSTdL@(rKRk`*t&YG zt+kL_gvD{1-@-vNTYNn^}rbu^hDHlm=SyRR5B-LD($+w zP1cj}g`y~;edf{O@jj%YqVw}aN6>j-4tsiV?a|!c!N!qS700C8L0>>4b5*CcXGn@k zTl>bYp7W>NuMw?T3D+&!=quMfqo1>S5!0ew(}$uB9hOo@^Da_tN2dGeYWtmC)!~aL zn+9lasT>hErcJ&~m(jt+LB`d(gL!n7^TAP9EzJ$iCuLJepNomWInLtPKfFd&@y>Nq zJ2Yn^Eg8aBw!5R3*T6dFJ4HZrjHMvyCMf`L4Fv$`0RTV%Qqknf9~81wrSD|t%z?!3OVT^WP}s+?dSrG_Z)5Q_smttkwQ09ME>*=*k>J^t zS3c9JHf>M5p9+~wNa(6aW)^lG&KHu~+!Xl5l|g2~Ty%h4yj*BmIYxt4U$CBgr;TG5 z>6|(|BNP%?RyAZZYO#yNKVrFc9`_LI1Q$C~W>Aa>;puW?(FZO4j1P`fjurX!9O54# z-7uU*a`cX>LwZ3{-a#cA1=kcZbgsLvjRR%oEGFFEINjpZ}+mShoj>hIbuRHr4W4-{pOx? zD!CW=NO*qzondCGhBCZ;8q{K_{w%^wTXmCRai}2`#S)cN^_eNkN8P9=IQV@e>U*rf zez;J!sxQVSb?Z%MXKf1+s$G5#)gG%U0PW$v62FRhF)k#0EiXeMb}_BTpNY21vdPNd zJ(R^0$Mio_Kqp}?np?J`o_w;EI-|9UZqGi1Gu72T42qZtf0Ezk8eOcnk;=?!n^%+v zo-#BzI8s(2QGvxd*X2*LI>8!Hw=Xby^f&ZUUvzD1=QE64ntPiK8&k+#z{1m?@7N-} z%L1YWgnbkegYrdg6IW-3E)Z>x=S=zEsBUAUXf&Q`K>J7*&v@Ux`96(Z;6e77b%|7n z$V?yBrE|H*3Or@(7exvbOD!EKY3$?Zj?QPk3{BVT=pps=8mj%zzhewG*DtrjiZqJf zj$8NBDSzq;W8C?4S~nNkTwD@M<{^Jl#O+4eENr4vHVj7?zB&qWODYuzd-tLuDb@1# z)~$GtYcvaFpoYpZbLcLfKU^?3rZWmz7K2!t)LeY`WM#@w~l>`<>QM;Xe_5$cnrjp zebbt|;8Z60LK=^(Gq9UaYn)SU`K;9kYmCM88rzTz50i_(r>K`MiJ$7Y4qF8Ub}nBx zzH!6xgtD7gA(E#3OU>OL?!92YpO0FO9uMQzx{<*5_k=&YfqfYP=p^~S1^xdINtWFJ z!BnADz&%MoCt3Gbz$q)hj`;s=-rSP}cJg$8B{|Ii*wHP!5e4gnR{h(oJstPbEi-B| z_WMnR&#gb#cF+15{Fcumqq4oIaw{Yi`TVex3z=i_{mU7-pi&;45ivdT$H~@>7rTs) zx2Vy~q6c1IYr4?8_K!V{wVR;S26q?3DEfbvJOV2W&Ybh*^eU|%h4-zrxZ~5l^nuu5 zsF8%!?!})f8k@@62@eX2l$-ZAXm0Vf^2^XuNM2?-;kGGZ(M`S@&^IU$^dv!<4YnNy}1(NvEYy@%1UF z99qiHK!oCOLMaOn$P$}!DNr29CpP-9B8cUXfwDDr{r<`>- zUVCzqTn_^vR^+ZA8pq{M+eqw=D-BaNvh<-F=b>$QkvJ(6cp;#>0$nzGUH10U-`_xT zepBj#lru$AGM`k5h1tGHJd-s<=*JoNkn)<^{xoaJ7za^? zHr~((%Oj=YA=9Re5nUv=eKq#dw#DZu!wI!o#S?F@-Eh`8o39KDg-ykO=+VNqOBW`- zPSo$gp&42fJhFQ(Ax7VRLf9+UxkSb>Un$lmW8s+^M4vWnaxn9l;&oeG)Q~ zQzo)T+k?%)EFa08P^~g??VDYIBiF$$%#&`hlO0}{D?pu>7z^>u|N8@jO_neNGGezR zAf*4I6-z700Ib0BN-6;dv96LbKpAYIB+viqIxNWuWW%mY$^$*IP$_OQfqR4i2K*1u CLPlNy delta 6622 zcmZvBcTf{=*lj`+1*C(B2n0g!y@MejP3fT{hR{LjQp80CrI!Q(N)hQ*AW|fBMWl%I zDoF3WOTY2=-8*yt_;zNWncY9mp7%NDea`OePxl`83Ov^qGi1|h(0gyoK z>*+E1*RheTysI06)ZoXG(r=04FhDc(IlAe1cn{}8Q!ne$aQ;c(O44*fj)T=uH`S9Y(+Dr(@JIo0SUI!+xr?4x`1S+Xu)g0)A_y9I!hr;tMi~sQ2)@q zKG%oWFcug1ub=buV^0EoiTuE}T5gwwiyoSbiOydBMH3`)>QO9-iBY2U@k06OijyQ2?( zpA46vy#1RVKC?>oWBDbDNV^)z$M`;43-VC-+h8VpI5CW>Qt7=h`inQm5WTq)+6sDhyNgVi&GE;4w}$s%L15!>5BVd?k=GYfi|55PlkShJ zZ=vpMDi11534bd+>@AgNwF36@XE#06g){Eg9II@mO)V9a?mU=-`v>x}{^@>0UH8UO zA+!FTiwYl=J$aOUK=gG%&B#cENlXJ*;zRMhyIdaPNz}K*kL~$?j_i!z=F4hx7w@P0 z93r;6)!KG;jh@{)@~fWkhobOH%byJNb6FWmsR74!bILq=R(ivxxnjdW!o$Nr@{8}f z>mYiahAKQa#P+tR7JOrHS{u%BfrgXl0|{E@F%1yf-5WsgbvoK$+4cyg z;Nu8*+n-JQwus6GPFiuSJ&w~W=8~KC+oPw#mqBi`tjVcA!J0`*T{h_J-@!qx63FuN zI2K`us0B;q^5vJF1XZjOC)Obz@Iia@MJt+?Y{VM2L)i|AY!B1#kRcm4A2Rq%< zg$AXqyzvha>x09KKUPu6!T_S+!&l8dYf%j9bDyi@dXb~-W(XjkR@$7qo^Ohev5^)T`BG}?U9aN@ebu5jYf=Zl9&Gr^eTe0CI()RAgHlelE?sNhkomEY ze&NLEqht0-TcN7wN~itsD9C#4R7+|ipDii9`S+bhnQY#hCxjk6Ki)uUMw|VnpriG? zdCO2f4n>G^Q)dcl+yyW%2ro=WCoLk=} z={=)OCc4@XJDDNT2}bjwi5Vh)dR1-m2|Q&eNsXe8GQ6uXCH7MIUgbB#jhRHI?e1&0 z4EN|^%#xvyNG%3!cyTIIYNZ9Q>R+?MiyP4<{qmKK?);f5ANClA2{TxWNlT`EtS ziZfDgsrUlklcZGiYyKuUxG9+T>7Ym-Jw+ce+Hf!DSxOj41)F$K8PgVz3T>{ZBu-!e z5!sLim4>1!4Afv9`5db1xEOP%qw!q~ueSfWUF&<+&5_xl_j9i7FWsO%yYl*fC^wei zAZUQT5^@(^?nhc@gfB6=C?42Q4{NX^dZ z0UJ7)tu|Qzlx2odvZKU*S7$2(TpmZHi~t4rF+_|rgqq=|7v~CkZ*rvDS#Zj=u5ALm zNBt>J1m7i%zC$ecQAzrHbP8*K-)>Sd3otgejaB7ffo?PUX_I$CSvXQqq1Hb+>e@9# znAE)gCftjP_*V*`;&le5afN*zhk%TKP@37m_?Bfb|Glf{Dp@Fb4;y1 z;m1phAC(A@Px1cRIP`TV`&Rbr)a6h zD{tu;A}d=XoVQP_J1ej>r%})|$RZtsa1flO+tv8k#X-FY?&R~}&S;ASDQJ_eaI<95 zOKLf-WMiB;)CzLc;u%2->dYCa8){DEzUZE>%O>ONaiC($EJKfM7>v4vAwk>>Og98F0ASf1@Q& z_b&cyQ8@(%&9_?B5^<*|Ce4MxVQW3$Zc(28wf1VZE2?=>1Vy5l5r?Gm!>j$PT zueeV!`C0e`)wNW^>#2P2;YLgl7U;)yG|j8kS&B-^P;Ig>zz(0zX$fxeby3%`RDRn> z_qnVi{?&ac60_%GVaozLhZihKNf6iP?cE zgr6ZX2AxW9iPxcLlsTCvlo)1hV@q2G`hp*VK!r2aog>E)G-gah*5cgET~Ez9Pey`E z1^i=QkbeTK1UNJJ9&@^209D~ji$^Vi!_I~90T_WO%Nm+yr%M8DhD~h1fx+byd7S+> z&}L@gf2KsY1H!SX9vYWaeYC#om?xeh4tcZI#ey=?cdt1GH3fbR>C;(S@evsgoYzXz zq*=Tq%j?J&cuen+q;>l~cTSa5oEG``fmob5ndTTbF-*p$i$K;LpUx$GHE}{?iL7Bn z8TLTKQSu8sb@~BWelN%`w&xRE$PR7*L79UuEsEdD4xWjNG;9Iv=G9c zHB$D4V8KT3qKR|y&mopzEx+p;&zc)}O>5eG1^0M(DO&Fv(ZNi#IpNhvrw;bv9sZOc z!FQR=#rdJdPik`}eVC&PC+Gl^PDz#A`*4Xat@wyA8dK`{2nUd<1P6Q=O4olQw78pl zz-UiC44djs>PxaR%-cw#%R>RiGIqn_Tle^#`h*YPKCBY9bJ0mAWy7jz-%aN_NJ;CV%45@HA*;}2s5qDRg8LNYrZ56DPzqj zG~SDlx5=h;0fr0}jJO~y4!N3eSVZ1i%{GZpXy7@G-6}4^9NaPhrFBV&$bvf7Q&o~g zM)-a^6vRZ>U07Da4=ySvjoz-nm)Wwj9{ZKBC6zta6@tXBY2UYmbLYfVMQD-N9aKb^ zlYQZi-vW!+&=L4q^CkaBP~(HjGv7bVuF}5F zI@?~@zp&d-Ox!^@LG*%egz?vUuoG@h*1}D_2Ntuz(+Gm>@a~*cH8lS%Ss1SF>4F`D zRa80D&UxtpU}SlixTJrWyaxaQ)XstBx?s1K zxi*=dw1`;*V?qV~#sOs^jIgh#$SMXB!ji-6TDxzwz8W=gr@o*u7kro7n?1U4_|doD zEk-1mD2jjgEoBfN+bT7|*ZSRCXqvgTS?Ws48J)X}#9#0SoCuJo-?%#21}9-lKOqh# z#~*6T$|d3kJR#Bm<8SqN)V|B90Tgo}pZLB(C8;Q66X#IQU-_0&ftSsNfQcKp3*m$h ze!GigQNx(ahvw_vMUsLZq$v6xcLz(gvLWlK`jh^3GgBr+@8cxN<#*>NC8DUPrUdCS zfAc*tddzXS$#Uj{Bt3-zW`fjtqHsbGzZGqhXyE|}T34(Ngqwry1|29^D#8g`vnD=M z9REaG9v_x~V7KTn5u#9rua8EBB>(KC7|Iq#y?&Uh^Q7jM6_J1|fgr^{f=m!jlHqSm zNET4Cxg4limx(~oBt;O_!uSfCx+&9lYEG^i7xY)Xlm=~Csd_+e^kL!9R6t1ytqCL&{DOs451crbgfgq` z7=I9DntsDLE)UipN!!aVQI@60j{`dm($PZ`9@0`+tTH6hRCG$9A7B&R3CWT+0%as5 z%LB9UOwtkpQyk-$>93kZ$5vY8&JN+B2&+nyh1^58r!^)L^6Um4hCC8@5AW%GD-zS> zP@SqIwF$ToD(?XtfRPu!eb6QW5(vi9gsu%lI+KEi7wIi_BC`moooMD|K|a#J9hQ5l zl+}SKy>)^<%}o?J;vDVi4t{8>2^__rZtX>X}ueC}+}^!9@lO&nUufwgLyfd^0m& z_yZGV&dFVa>|MEsY;LzLM0wk0EkOaUvfG;#8=Vw2ZQ{5#J;MQUB&Q37uQvI;L(dp< zGBt-@5RrimnvnEacp)wd?7!hHI3ub?c$a!(a>}PsgUstl>EgXk`A-v_S%&bFYqJbR zLS9iqiN;C3QRZ3#a)hiFP);XkA~4PsnrnixF(ROu;7cJC2-+%u0Qc7=DSTq@V3qTB zC5+tldv?EY;q<)ZN==Za%r;9N#cMt>2X>e)oJ)6s0jj~pgno8H_zFl3iqI4yPL+5- zF6I|WaJJnVU93>V|2SJ9B^+cN|5VjJoMup%HnwSYz+d-!c5Tg$jZ-o0Vz0-juZ-EU zHtvPEIW=(fte|sGXqAVVxn#mKv&q5cxWTO~FIeb|sM?9$@JtS(Df+CKD>~5 z|Le);6=NZ*(q_>Ipj8f@;u~By+-%bQg^egqVE;(nTaD0Oxo4Ild^(o;p7A!_~gjZG;Wt0 z1d2*tQzHla3+$S7dF2~n%BwMDYBGycKWHo!rVXSDG23DG1Q3*tk=Ve@q$DjVl29m>tRAk^+V|-M}UUw>ndUHyjH0yJ?GIN) z9Aw~Vi!BPj%5tH*PR6?{%Q}d1$&X}S?P1(Gsn;ZC@I51yZ^UKRkNTLKgjnD4aYab} zg3YgaCqT8a-u6c`03*FcP}867zUMNa7ia;N5DireEEZt5iS5h0Fz#HMW_DsOSW#5$-3$HLKcx@% zq8?3Y+$kOJ=24yVc~^^x+d$g--!A|xBlP@bDf$<#H`1vCuLY|re6xQm6gS(tMxYAA z`+Y5wEOUgDv2pgQyOB$Ckt6HF8QN70b?k0ad5VUx-WQ!S!?yJ)R+D^paXQhH)Jd7< zjB%u6{C2hAdYqGBJ;hi(>x~+d;cD+{o-sf1>%2AeuJUO(>tdN;k+Bnud(%v-6-~xk ze*5cC1CKet$I-p=npn1wrYg;%@l(%M*7O>_+Nn~%$VxB$-()U%200*)hvl=ag7o|AG8WK4vG;gYPv0L(5z!}&yEBebO0fKe zscQc3`SW<2wF4T3X1U;9y6%yT%-EvvdL9{823KF7s}SeAoIi#Z{nN;mSvzQP){VI?X?$})uw#gb1CKiGF|a28~%g5Ct%Q+eSpiMMC zkgkUX_N7bqV5Hqlf$zIx3xfa2$lGr($En!!uaW#t8?W!}`WutT2(24wa`F3AYF%V8W2!z2^GwG!*j*mXP@+D zoa-rKnOJJlX=kGFA~s-?>9WQ^*yQ5KK=|{i@N$b2r;#iLmwkVAFh7QYcoaFT>W#lE+sX32vIxhIC+ zbv50tpKL!x76N#lCjm@E<-+`|29ZD%;*2`LWFcco_SdC-miPPIxR?G>?XFz6i|zgR z4wXz-k!&TzDJgY{8otP9qf44`TXJz{GBOYx0zYlgyCyB z*^-^LqlJT`lf=wM1cYok`%2Y7mzabB1Oky0PF2w75nf|*l5N%qN{q0WAn==$ld*=_ zqqFEG%-GVKexRMU`bEN!Lf~0f+&uS;aeT?U3x9vZN`)?-Q_}S?1hfmypplij1RI8) zj(wVkMx_@-4QdMV^^poU9Lryr7k!S-aW**u`ulg*y)_;A4f Date: Mon, 27 Jan 2025 00:06:45 +0000 Subject: [PATCH 15/24] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 13900 -> 13901 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index c55778af0..d0a98ca24 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 00:01:14,731 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 00:01:14,731 submission_checker1.py:3266 INFO] --- -[2025-01-27 00:01:14,731 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3266 INFO] --- +[2025-01-27 00:06:40,784 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3282 INFO] --- -[2025-01-27 00:01:14,732 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 00:01:14,732 submission_checker1.py:3303 INFO] --- -[2025-01-27 00:01:14,732 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 00:01:15,944 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 00:06:40,784 submission_checker1.py:3282 INFO] --- +[2025-01-27 00:06:40,784 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 00:06:40,784 submission_checker1.py:3303 INFO] --- +[2025-01-27 00:06:40,784 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 00:06:41,821 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.380582 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0872 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.431004 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.082 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1629 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -55,10 +55,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2805 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.97957 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0772 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3091e8eea..f573e999a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -504 +505 diff --git a/summary.csv b/summary.csv index 39ed35f6b..6f69f230a 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.380582,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0872,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.431004,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412174,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.082,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1629,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45726,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -33,10 +33,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2805,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.97957,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0772,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e958fc620..877bfae3e 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.380582, + "Result":0.379074, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.082, + "Result":21.1629, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2805, - "Accuracy":"acc: 75.200", + "Result":9.97957, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1119,7 +1119,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0772, + "Result":23.3041, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d0d284c4d6c75d43921eb402a74f5c979f91b424..1afb08d0dce5c8a0db0b15de69ef20d9692532d7 100644 GIT binary patch delta 6672 zcmZvBcTiJb)NLYCq-c;PorHwm1w^D1N&sm>453O5RX{+R29aI_LX#%aiy#tuho7N@ zDqTbn5Tr{7DdHD>@6CLFygPIE%sq2w%|2(XwfCNT2i=C;J~Y!^^KHG$E_)LIm`eUg z$BSL+VGjQ`QuVe}0ct8)Dy}w`vOV#dXy22@>f}*&uxzVRUv|qAFtYNE`^971s>$v8 zRfxV7&MW8$*+7V7Td(9F+f;Y^EK*P9$Os6|%02^>*8Pnc5?> zPgquVSDgxC1D-q$k`ux!3 z8=3ww$FVl99(*1w6IWnnhsr(_A7QB~$+coE7_ZT2nq6{8?>Y=_KYMJaAaqjXA=~qN z##;%i1dQ6f)$Y9t%8#f!gpynkNwp2ou0G-tD2$he`cCJk?-)jRNg zZ>6>I=Xm><4VSQyv)Id@$8Kk3lIHG@o?}O?xmb_jo8AqyX_Z*9taLk9>pQLO&X|+~ z)X%nr7RQ#~US_}K_cZ+igxTkb5htk;es|x@syZtP(Ah;z_3pAUAmeT_wz3t1}Ki-QxCe&@iij*TGTyN8s~ zgv}%tD8x!t90A`teRh++cP9i+m>-hN@19qL(pNtXx+Q5VQl$-#TZwYh<|t4`5O$}{ zBRl5he`&(_Y9i@j3Et`Uwq7A`u+?#&g*)stWBZ;UHMg3SJ*1ZJ?~xxTbkpn?m7HG= z?RX3uQR2b;WLr_w2YFi)AN4uCDy);5qd(u(ddjOj^K9>1!8hww@gPd&DrWeBw}y{A zI@|I&hOYSg@vqj4okol8lt0Y^j{Lm^M1-X9v|IOH{%i49tLfZ51*hJpx3S8Xw;J+~ zZIO)c!jVBuZ(N=Z%%>-hF6lOTWTp?i4mb}~tTBC`_MY0EPE4$UnY-6~oVa|jr!{G8 zru*2k_Z8(hk+@cS>iPsFa#nB+VO&y^U7Z@w%3r(MVfd zpVy_nT;R2Q{_Ce)nkm62Teeuu*oU*XZ_;~S>O*cS#1WQ)ceR!lueY#QYgq?2?D(4W55?J(Jk6I_X$EJltwjMm_&jmKcb} z1@1XW=Fkn-D{UanoLoK2YUw&_UX@*4I29{fa85?1nCQE?`WJJIFeYGY{hM|k!HKE5 zEVo1WqqP+f@TF+_&f7-|(Q`lpgOg`tmP;9dyU>^EKJ#!7gN$VWf{_T+(fA=S=fh|! z5#y@|yV_kFlkbtyCvP|?7sJZsyw~1JB3K#hBS|KJi-IS?p)t3BjJI#ZeC9Mf^!dg= z2aCy|)R6Lzw&gQ|VzU@2*HX8hr2GzQd}k*F{Mt4m^i@NYVH4^rVq{7nn%B`B60Z0p z*r!~q&B4GS2@cR_BD{ONtP4A`9|2JE5Ebhe&yXrojYt)>zG5C_b$lGk>G<Ws~cNq@EK>rawhhK%f0o`%9>8@sI zSSyZpS4XN-*fH_p%J;b?ELE#T0OL8*Ct>21%cgC`a?zK?U zubL)nQSA7l|23Mkb^jjHfT_tSMcYcUU|C~b>>9WVO7<5thIQHW+#hz*9`ewDlrq5o zh_P>m5gYOl|7~Tdr1v^cwv8EBu4`k6|F|pN!!^U?QwH+kr}*?Obgw9)d#7E=)c?T) ztN3rctgsy>Uj!8q#>$g!A7U}c)6}CS#;oG~FHfEZG+?;aKKH*=3+7PWrY))~%@v@x zuCnZNkrqRh4q~c=%%fOsBx`Suez$072{m?SXh6M5OG;g{z=t~-v${H47+}sart4^~|Fdk$Fr`TN=4)M0PQ8xP7b$cT z^1thQ8hQOZH&(ca0`AZpLNq`gL9qs9eamoL+i9yH8N;JeQqrj;4Djhuk|#2TV25w5 zL9@a~%J9=h4m^MU2uw+)zcnbTZrsS?>-TEWxLe2*UXBo|0%Czy zAa+B3oW$864_dWZwVnE$dW#1`Tt^xsIH=SL*>9;;)GBytZSGydm=mB@LVbMlZ$ zsU)Xzwvyw9m>6>S(-=#pr9N{3)@v_nB-?jToWNpi`bfZaMsD=|NVaQ_p-(TY99Cu$ zp{?n>PMZYsX0@TKru8IFurG``T_2_HEHdC~{=LGPj)#?)6QSvN^8}ydHmmBf{Z7uZ z%p9Ht@44^n%-tHNnb||JMa1+pZb5GS9}(qhk4-0;bkJN94=7&?WEL7@nIWvOyiFV^ zly!lonl=PM5oU9i4Mqby51rL@(2A6wa5a^E389hFS04sTgsmF>pzXR_>*&13g%n;L z>2+dO!V7sH|u8J_0+J2rK-^r1)|N;!$8x3k|+njLn$7@OnHvrW&Qe;5LIPx;4JV~xw}SzqcW0^bf-Qe>y-2uA6Ed!F<**^T$=G>J9t+GY zMefkDlG8Ik0H@OyWuMYYISK=Z-BuB|WerKDrCK~Z)u_GFxy}&hfeoN6aU&9uonTwu zBsIr`Ff8byF_l;447izcx~eYPCBq6-7ML?3*KgaaW$eyTn2*8f}t-ZQA4+Us4)w1X#Ol zFZ`BQrHX4k93^E&^`XLyIglzX;qFV1QsqO+0JOKf9%?fb25(-JfgEGFPf-k5%NjYK)4Yq6 zfPkTuJJ?QyXbnxXyoqVFiLeo$uc+(2>jIoPJ^_)A)CQDhnxwsnOmln{Ip`dz zU1&r?)P_s-YbQlv!$1RoB$7mR_W_X3u~qu&D z2i9xCSR$aJ$BCx;%1tDxk&)s5fdVU}p^so-)Gl7_!-9;=xw`VzDKsA|eoe;o1twr| z3NwA%FzzFd6eiz(jE7V|a??qzks#B_W6Ro^sxV0Dd2|h40a(~Lh@}0cPAVXC6qg{M z%_I*`h(pmL+{NWMlB@5>_pA9NOsqGA1X`+))STPm%yF5hx(Q9bBxNc58+=nun(P9c z29%CYC&Qc_f{}nvhcoI89j)pNTj2Ux(EXW8t>#7N+XSX^y#?#IAH{6yj&ry0#mR(M zy&12|B@6NQWQ}eU3nfWR>)1twB@O}Xq8vi*lZdt%=clc{(pfyhf|XjJAu%lr+3~|$ z(l~No16P8Z6U)q4niHm~Oi9#OgGx8s6XWqFcRlKFZ339Oll9Kjpam6&;o4NHbi3gw z3KbaF^^lG6a1;qlRA$1ZkoC@t-PtU_d3mK!Sw*PiP zj`F~%9&!%0qkpi3Ifzu7yzj7;EjP4SDAptjTlEnPnBGd223fZ zc?NkRTR-6~*5S`+ z(4`VH zUzzO2_8}LZ(;RLjS!p{&S(&i@iFk1O`e3!#MZ?chSgIdd_Fdev0;?{J|0|w{1hwiz z647O(K@mk!YH#o>=^MfJnJtMNle0kxjpILZJcpvfmJ85U=Q4t*L{=0~w6V==d<1E(Lcm_dr*#^Sbls0|I$R7L+C@cyleiTaKVz#N%b_>H0iFP2v>84jQvzntNHva;{YalQf?X?=n^LK-5_ z4y_7ZwJ4_$V}eu&f6)Amh4m*f81l|>ekQZT7?SnOfk7`K7QSOT3t~`UXuad;nw4#b zrgf=Z`4pRw}5YgVhv)`o17d$)!LX{Q#hJ3#EDgGtc4=U-K+F}ehJ{5SGmqsKv5-iMbJcDNY3UL}I!Te(W$_7DMqN}Fu z5Mn~&1QtY*SQ}F@B1G!cI=M)DRpEbV%JFzZh(Ru}m{c0xVyE2L=0eUCm0o(3W9;AM z8#PI2^L{@AZGhrMk{+Zm#-p_7S4iWTEgKJ8HLXp*G`MqupN%v~l^ICmdDuv$t3LFd z1@jd>vUWf`lbYTgxb#_s?kpWstkrB?)x8ImwSgWj`8A5l!ZF=7m-!tnieio>8Vz#F zxpyeW_eD3c<*V+l1z+t3b_|2|twgPDwp;w0+Scvm(9hMq|6H8<$o>>(E+9JFU5=Cw zj1MROw3bIRvuC%qD~PkjinDa<*Zc}L;!0n!&x&7MP5%2WW5Ug1P;-0+v$x{1$?s4% zaB%t4U;ZLMrgVU~mOHStnoFVV`{qJ=?PA|m{m&TkvvbOWj_9-+b9aZ+8?qI1+p(9< zUY~PwUhy6D1b7+iOS_~#wyzmopBd_48IPUp=)Xb9T^!r(6MQgkHpq=F zIQnyE*wNZ_?Kt?pf5bRAvzsO~keOpoYe%X}b53duHPt|@D zthacLD=YPpGe?97O^jUC>`o18x7n-HKO?HS&PKs5#>QUcK}IBs&&qEt0o z5=<2M@rJh1;|XnrU0h|eA~l?6K$EYZHaWYdueEt=#W~aIYqnBcf3--PckQmFViva7 zh+}Qstn6F0o)q`jb^5HU_!!McZ|*OZ{pXRf5!Tj@Z@=Qxs)T;p8jBEw5_wOhe#Jb; zE4;f@3Y5>|TrI&3b%+m(>5o43d&2@&f7y~7wRHWxc+jF*&BOc82~JXv`7^g{t$ZV= z6h$?}TCy}S7hzV>dGkz}vlNmXWRuodaT&POM1jAQpm60Vi;gZoB9Q`qvWwdB*%+3# zih^kSU>D<4KOI<0`PWVVTw%Tw6In5ar&258M~2&x`B8u7?0mm9)Bh9bhgI)FmNz5c zAKwb%;v4%l%N>X7pjwaYG zeo>T+ymN1Gv9$I0I$qx;JJ~mkDyWYZHAfrRpL_0A$nuD#`Wk%{Eq>u`?+$DzD$f|W zJ!&-ZsBNCj-X%lDuw$xEsM6mz;;iA%Vd83&tasV`08PXKgHr}at;kP0E2n~oe16tB zyb?9JrkS$`PHHy zho1YE`DYiy(ypQ3C+)sr76Xa1?%+c=9<9FPxL2FL$Ov8MwOjSB0aSBy7i-oxv>JYQ zVf+JqgJ!DU^yq}P&~3Wu(8hgADY+CkQ`a?0gG#&~T=$nWuMrYYdC$9Sg#FkZG9dGz z1mn^L&nBm1>5vqLtBV^rEq$Xn(7|7`p`0?;DHLuot`NSr`%Rf}FaED!8f+%* zFz}$k*!Pr9_|4#C2KwG~x5DIsheea9t zg?k>l^lUp1?RaE9{9;6{{%t?ETK&|_ETnGwONVRA@%_)60>?+a94ozkebs81*Dod> zH*?1$*e-USZha&N^t%|%J#dsdmb2@(SzU3ALuS=_ea&C?c2A$QXtQ7#-Op@XG^1** zFT}d2z%H+$x0M%+lAdOMYdlYQQSOYBr}!es))J2?eKDCgabVKjOM#>m6Mw>+b&Ie! z{X_HpV)OeGSy>1k8N3BKY{M!xZ&~ZczPspaA)d&8F?uQ2khVCvy}rJEz{}}W3D+Ce zIKdj~l3is4002~^TNwb(vEtOa0su(VeB;-lx>~6@B$O;F!GqQRC;xT9N8cJJ8pb1R zVwSqCDt(GKt&b4axYGfXu ze@cw7YVs6fz0S;6$?xT})5a0+KI^z$(a>{o_tK)R_2^x(@zau}kPTzce$Edz7M4o` z5U%%+lq!~@@7yGO2X8Pu?$wA;0wHKsA{+Tv?2Zpxt)ThMJl&Q$`?ZbM3~I#e`5)GE zeeW)B8*)ZoWcgpAZK_gM{Y_c(Sdx|OfBmaTl70xhmAozu`M+HiB*O%(O6HSM1rn1n vGV;KC$ptce?0=6K1_S^MNvBRv;^6;XAul+Rmt~ZIp2@tjycEK>|E~WJzmtLr delta 6662 zcmZvBcQl*-AAMrgC?YmRMPiFmqg7kX*n6*{)Fu?AFL%+0wfkoOI36$ag*I{4{~#NrF#nP#o;BPZD77WbS{o?! z<;DF>A?_=qleN{d=4|1KYp+EYpYbV$v8UP>Yc5wUA8Vs@;^&;j*s;P+5yR?IR#uE= zSjH3v<(@s9^t&A%BZE&zL{M(|_K~aId-9YzMhd z+_afU9?wq)G)aw8d>!fkWS6fo*UW@WFY2q)G2 zAo_nlJ^qNW|4;#8HK5!w%zHIv(^f}yFTV#_ssvfZ{SHU;$X?|jh$e*95&K8CFp*hF zCQyjALZv!l=NyiSUVEknvESon8n@qLPl}#xm5E~#Ntc8|`g}Acpb6u7Q2Vo=<-InZ zLcxXLl!sl>$*RptqWKTxg-um<&}qveh05zN0j`t(I2xG`iq6O{U#$|K&yTl1zieIb zoyvDp8vt0W9tT4o9akA3@BEa+dO-b0eqRl!URGW||8u#p?zgx9<2c;;ZF%%aU+LlS z>-cWBiEK-Un;eJIoE@7(l!z3Yj_MY~q89hQ6xA$cW&I*MF5~>*N#F&a8=C0LG(_tw zMY&T)?ojLtUI7r93V6SK9lp zNBdUomUFQ2O}(t9$<^=Pn(JW?U&P29+F0bh+eW67lEqD>gQaI>Kd$06f=g6S*XKIn zK@d#=57jwPB{leT$5tiQWC2Y9rt!MghJdBxB3Fa=Ro7RjT3M@AZ638eQ-4PDP|%<8 zxhUuJ8rVrU%^+emzCx8M^Gz?Su}Vk1s@#rvNGi(XV%aCi|HVAVItPjzo9l=j=`3@K z?LHsjSsd8jxeZTiU=J`Bp{JH!Rwb5ppCT>zh;b(zeY3{Amc$@@N{Ei*ZT8u+CE174 zV0bWU(jo)_eN@ubk5;xemtc#q+-gD{(E#-y?(4r&)u|E@Nq63vIwY*QtLg<7n)Vt7 z>&C_MZoedRBa3lQuxEW;7hofc_csqD^#(|%jwn!rfR*C_G48iwX4!O6sJ-N0;NAv8 zWzhhPy+whV1RvEHnsx3+)79R$ikOlBuhV?16}a;oT~+Tvun}=~n_sKEj-WiXn#$no zB{nd@nW@S|zbIHP8F>9m^tQY{fD}c)SwoQA9ZvvOlpsE5PIl~(s_jq6lUA=;yG1ec z(YcZ8>*Q^dOHxaLx3Qyv2TrzP3#wifx-nS9;&y58Ib+ z@BI4n)WS|c@LSq)bx_4Xe1Y0Q3^PnvjLoKiK|eaY5b_??fNYmR*(iZ#L81(wIT|9A8K?YL zH_4XQ&6yU1mTl4ES&T_#qOfOD^o^vs0&SAQ)aVW1AaCGgy^^Au;2@wHO4wshf~kkw zW9k4s|ES)bX^Dmz0Bk+)Kg&p+9Vd_)91s858ddGim-E_`+(d*?Sjs(Vq{*(1--0Y7 zTA193B#;;HmIcs|CCLjy9}wjF#hE=5P`TsGRTWXuEsb55nqO*-YkM=X_E87(>R@q^PWqm!El)7ayiqm`anIX(L z+nJD97>P$zzH>{E(T$FzUL{?um_p^{u)KW=-0G;?9+di0WfioCs7VVIVj;v5pBrSR+ z+#?F$W(I|C`Mh){22A28`CN|gC?Y>LK;+2T$3d3AyFs~1Z|@WR7>mK>-!c`-mdmSH zG!|8@JapFMnNTQX+pcNF?t(4HO)xlJgs6r+ew=%zKk?Z!cctAX_4W%?t|M64){jlC z3!ut}*;y!9nvr_-CR(C-JME&@E%vHE%!*%dHhGjpq6#9Tuv)*bP=})z1S6#?O0aYW z@F4G??yJa9lk}vvnz;2eO45L7I3>n7v8{z4wf7>hU!V@^CUD>xn()rzH7dfFPk{1= zzE!o^(SrxIhhJ!aMc|p{`If+0ROiLQoAxDnv)n#_CB{`SIJ7sOEOIS6GEs|qE!r_u zOZ2Y8B<{{Qc~{Z!!-<2+9O{2;B^(>sw@7%jjajs_TkPXEUsW~Zg2pinL3(#0y_vIH zh_uCY>`NGv;*wAAN=2q^K>9im5sDk;vL^;0GgPn&T;b`FyAnNMF~|g^NtGs7IK5jh z%lK)(4^pUr)~YVLM!d{JSz#`I1pHd9FnA)Joo0}iO9lc~gv&@n(t+^7PtWmDRp}we zM>SS0LYYX2Y{C8AslwkO3gQr;49cr8WmbfDHZP-12I)fn$XbcLG^b?CoM_F4^(Cr& z!3xE5ChHxXSFwQ=@5#h*rUUN{76t<}Gw$Ib-@t9uB82}zu2A5knGNdz3A(=(CPFU6 zgyg?6z2g3;3rjgNUt);jbZ2SN+`A8JNmSUy3iq|$01R$aWW8X8b8cn zPYf-qAh05=0pVlR2rLtW+QB$Y_iGfZR>bKqmE?$KG`P2FH^r61csspZQFHfoH8Bbf zV+a>y1YXEsYyVe~AQS3{Xh(p_!99+yX&9)QAhdLtLv)2&0r}CN^-O83T(zYelbZID zwxnd1BM6$MLmSQi7CVYTPtg$;2Z&jnjNwx>PV_u_R{~++|1-%WVE)KEik$Y0v9P(kFL6%nWtoez-Qb$0<~Wo7 z?sEyf`!2}Gc#3I#ATrJ@_CC-sAPaAGR}qrQo;Xx1L)Zb`#fbOX9kI0Q(8};x7OZF< zg5or&h97~-ZAwp!MCF8aB||t_>mD2V)b`GTb@ilOkT!o2JMf6b>>YT-;+{`HU8@$5 z0N`DV8z*7IKM})b`xVc;r#{`ak4l`1p)62MynxnLCZ7i;@zX(Y$S<4`%f7D%)d zLeK%HgiwtB@JgviG?ZT?6kwF0h`0FzJZd0R@vqIKiR2_STT!Mpp4S9|je~bDsGMwI zhld|UQ%xC`gC(oo(rndHsq*H3k(BXBxL3+}B>d@M$v`6jFl`Zp5HisGwRW&FU#fuf zK}1lrav8{4Y+kAwtwxG2%+{xu+1hC?*G`)AiZSHU^_~-z z>Jt2R#GeCASbXky{nV97u>W$GvaI4qgl`~L$RHLX@ADH2iCIr6i_-Z1J<0)^D0ul( zhc*MG-j*uA{g92_7OP*n#5?qUkT6s@EZ)0)_`m4@+vV_nU{B$`? zzlk7|{7@L>eU$)8ER}y>ReBcRMs}{JaIz8QVxOB9zzHf)uR27maKOlz`B@puv%7hR9PKH zja1>d3S&wl@2Epdplc;XgD%HUO)!K)7Vi&A;D8@a(VdwQnP`}&N;eln&t;>ZhpNRT z58}rrR|~r!WZ|zd4a0(wutXmkiYR?JT^vsS0HLfZRJ!t-#9u+dIagSpDRE!#F$q~UxdWw0FAXnv1O?-Enl|{8_9ohEj#1DbD zxKVLyi>X+{DbfYUpNrt;0Wc3N(H~E80#9;%`V*eyN?PEfu2Q5hzPwB_Y#h3U@ySEG z6T?~l?!}iV(YK_VWv1Lgf;l>^UWS8$?VEL3C?^gzWE{l8p3ZEwh}Q-0%O93%Dl*Po zb`|{E&fW2r3w@8|E1(DHWEePf^Zku7@j5X}`0GM;9e#ZDW@BUuZ(*)jp6BB=3^PxS zt9y#Dn@eGI!#sAtye+^!M)xRAFl;7*!neOMRo@OB$^{a=Yk_i^1Lgh9R2T%g@Xcb} zWfSTlGeKE$q!;7XtC1v}rUPc;f1Ayi_K}Y~miprywx)-#2BEDt!Dr7|vV!`nqSrqa zUjvf$lkiEYBhcNdi4ina9!st&UwN;`5VO8DsU&6}1)4&~C>L7jcJ@CRb3cxA8^1rq zK;msXBpaC?t&%>#rxBwf2;DpIQzI2Q2F06_+;hwX*=~d5or$}qU@T^!_;>h6scs`U z1+a*H8Qn+DS!4lW3HgNEO^;Fb-0f@3DN(VMZnT=5v&?ym!D%q!84|@X|m?}FgJwsBBTH7}F zbeulreT`_zPPl2-L|?n?8~mKpiI@@YoH-J%Z#OiTI9_m;XgxMQKv&uA?u}eN+0sLM zOXR{`8Mk<|oW};11{l_A4;RptPKU>xHPp8_o|I0z`CLv0&T|yS{^2pGjCZP?-laZ= zH)jf6Tknls-2iKu?&5&NG3ElqTPYto`O%IxPV8>O2jX-NG30hJEFM{1Gdn56)oCki zHZ|#YJ}y_rQ4$TaDXxB|Rc_pwd_NsBm5|U`p3EfVGEyKYv$e(li!+nNj0ty`vUIiB zym|s!d%<$yoj#GWMC;h@86lUzyrw3dS(CCv_#>7}>v1=sR&bFMMJD;E;4p1oEE;{- z+{f_nSm{KbPscvq*3A{efs>(gP#)3=lJE{HRx7+Amnsu%aXlgF7i76}*lolxab~&} z@wi;fuuomyK<;%-(Uolc16P**RC*ClbKfC-els31#w}-aPp8@k8ZMK6OcV~IMYR;O z&jukLzLweb2*H5D*!xFOAAIRCJ^AQKNZi@i??ZMBPtK_?pkefr&{fNX3)GfswOssa z^f0xIh;~ZRvH7}r%(=uU!H_9;JNozgNG2k&qUO6>BDyW-2*Fre^C&1{GW#W~d$OtCn^>QQ$tF*`dfN``@Z+WhHMgoUZo~FPinf zNA^CQjsIcJxMi_K2z<6T#kpgp+X6go=odu{6iF)?EpF&#?}{#9xH=ff{d874AKFw@982OMdy3<7rDzf|(kdN6AoX7z z2e~Gd@Q1y7QJ$1${(Jj&yvGeQq`#WdA#3Q~FkiS%Hvhw7{$G^Vu7xI0f14bx5BVps zoCS5bmKl7Zfq6db+XlL~T#|VyPE}QnvCV2&tlHyIg6n)pyIZw9R{CiE?>A8nub z7|O=uh^WnHn7Q>t6n)d1IuVpc1%hg}mYFGAPph5Mta&X}2ddGAB03GNZuF0mi@v9- zmMn{&X}OG81O;}i+%&v#9pnyWGp$4>&G?s?x<1-}!SL1T0)i<+D}npsfDV$buYfZafDPfw|1Og4ivv5jyS}1{&N2Zuv@5Oz!CIk} z|E|`YO?YXS8Z;XE{iYnwtGm#2&;A+wme(w^q77HE9g^nu{HTM|E!XV(mvd49g?w5A zLOQn}r`xw)>@hsvreYF4^!i%eiQcn&>}jak2&K}yw-iR+_p{g*EH^NF!JFHouyGvT zyV~qNoc^U3gl2`IMibJymVPR!Z7FIdJS@a1HXUqI-{x)2FCX7lOKbrUsnlwN$kHEe zzN1?JKEgGbaZufIstxUDscM}U$ zoSO+9xw%d?=Jf3r+c8ty%6%k zR-r*ocbW!5_ra1dMFVpm+6iu&`WJ~)Qh}HJ+N;nNgV&{Rul)V>#22zd1;zbdoD@s>nRL?2Z*>W;~PYzUV4Kp z#-!;6?UI`stPkI4batZ?Vm&*6mSCUP;s|))`WfK!qCA2JOGh|yb=Gi8Ei9jylAA=6 zB1U+L)cV!(D&XFJc>u{?MU?td}p1d>yIYgF`bl$hoEW zokI-1{e-YpY;eLSFki{nCuO&|Jl`uPgxsyA3uRl&O&c9N9QJ`GWG1IhW{ Date: Mon, 27 Jan 2025 00:08:08 +0000 Subject: [PATCH 16/24] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 13901 -> 13901 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d0a98ca24..5c3f0f060 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 00:06:40,784 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3266 INFO] --- -[2025-01-27 00:06:40,784 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3266 INFO] --- +[2025-01-27 00:08:03,317 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3282 INFO] --- -[2025-01-27 00:06:40,784 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 00:06:40,784 submission_checker1.py:3303 INFO] --- -[2025-01-27 00:06:40,784 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 00:06:41,821 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 00:08:03,317 submission_checker1.py:3282 INFO] --- +[2025-01-27 00:08:03,317 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 00:08:03,317 submission_checker1.py:3303 INFO] --- +[2025-01-27 00:08:03,317 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 00:08:04,339 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index f573e999a..80e3e6eab 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -505 +506 diff --git a/summary.xlsx b/summary.xlsx index 1afb08d0dce5c8a0db0b15de69ef20d9692532d7..b66338337fdd255406488f3a019f97d516eee4d7 100644 GIT binary patch delta 323 zcmV-J0lfarY|U)2Dlh~ZIJ?`EE-)Z}YQr!PgztR{qq{BHrfm?m18vEvfkIPSaud?V z5w(7h)H-?lDo)}g;8TyQnf-QlQ8nMGF8KxLrO`FYGlCM(+z6@L8hxx5=?o=4FfCYT zG}LGh9?hG(id)K!gEwa^xFEqN5u`okwnp0!EXCOKEhy%*=+IHz7^hf>rE9T&W&E49 zfD1w%u!6t@3k)BCsU3;vgcN*4+8@^q0KqYIpg;$Y^DM^~ZvdzKWW%6bIh7213zOEf zIJ$RUj>f+4v%VaxF@KJ~map%_s;Sh6Q*l5|B{<~{STL@s@O2Rn*D)WK@x&Vm;%OiC ziMP`t$375A3_y=@&&p@{{IXgkpr#qfbB_xVK+$?+B zTEsuH4QbyVCu!4G%8BE8X1|?Xxm@g3ll+47(&!5189@nXZiLj^3catF=>tl9U|O)o zXsFNuJX+K@CAX9t2d~apa6y7kB1n76ZH0CrSc;mho@4 z1zZp^#|i=yEHK;wQ#%sT5h?hHv_G!t0fJ*_K!FY(=UI-=-T+Sd$%bCJa4H!N7ACDH zadhv!9F1+;X6>xE#{4<{T)n*YtEN(Sr{aL>QgF&0uwYzW;>#i)u3 Date: Mon, 27 Jan 2025 00:09:51 +0000 Subject: [PATCH 17/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13901 -> 13887 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5c3f0f060..8577b7c1c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 00:08:03,317 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3266 INFO] --- -[2025-01-27 00:08:03,317 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3266 INFO] --- +[2025-01-27 00:09:46,532 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3282 INFO] --- -[2025-01-27 00:08:03,317 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 00:08:03,317 submission_checker1.py:3303 INFO] --- -[2025-01-27 00:08:03,317 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 00:08:04,339 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 00:09:46,532 submission_checker1.py:3282 INFO] --- +[2025-01-27 00:09:46,532 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 00:09:46,532 submission_checker1.py:3303 INFO] --- +[2025-01-27 00:09:46,532 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 00:09:47,625 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.97957 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1431 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 80e3e6eab..055b6671a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -506 +507 diff --git a/summary.csv b/summary.csv index 6f69f230a..84523a441 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.97957,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1431,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 877bfae3e..038ea9a08 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.97957, - "Accuracy":"acc: 74.600", + "Result":10.1431, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index b66338337fdd255406488f3a019f97d516eee4d7..44d3879fda1efff3b9372fe9a2320e140b5f85e5 100644 GIT binary patch delta 6373 zcmZu$c{o(>`<}_Zm3`la!DJ`0Wnzq-?E8|&z9uP@aVAR$F*BBA-x9KmLLrnv+4m(` z`iw!5HGK3NeXr~L{pWZ7c(3cc*E!F*pZB@%`+3hfp94Aq>ROpD3bR#Rl4k{h`bt3{ zZV>*a5(j>PNsGGiL6y^TBVP2P17LUKMdKa7;!IT|a7fNe7kEQbg5TbNk(xgJJ5m~9 z^Nk=~`f+d7LkX5|N|UvMYIq~5DD~weDdcFkaiq8XZJQ>4My<@$bNH?*_15&ecoY z3$Xan96sYIuUSxgHpiXXkFLl`x$|fD#+VXcfnIm-OhfQWfD%9FfWSfkqwGIP>qoU! zS>yK}Rn=79U9i(7HHS@FUL_I3Bd)#A<(fjz+BvzTY&!rWUl;OCuPb|1$qdRJ4~UlJ z`?&nb>}uE_^4tvx*xGnJrNRWHga{72JG?U$&~jat;ikl5pX_bVZjG7k_oKtIpYCeF zKGT%lVjMZT9u1;cVuEIenaJ{yKw83do0?wJbXz=B z`_PF7ut+3q-a-;e=Ebt@`3t?JMGyLaJ{uAF zm7Brx@JDJ6@qU61o3n zWTDUO{>{@v;^EZqfyJ(&DyoroXO_IDoZeYNjxpoa$Ny3;}{$bJ)ae|Z3<&X*gA9`;}{>%ZI-N_0msc zhkF?d4MbRVS+Ot_rTO4B7%EI_WDH#png#O`WG5_%J;_F511$Lq?HYzG4t{>Cd0Igk zd7;w*lZgk{Bo;F;LH`sAKe4@}1#BSjiJ=mMJR zzyVBHas<;sa-vTe35uw^8TU_S`Oul27%G3yCcgNMJBd|jP|fQ>vDIr4nspm&p5+{6 z8F#1EYfl?at>Uu2;h(N@TrqN`*}yUXGE(f6B7lI}XWFhAsdgc|;P8F+X>Tfw-P%fq z2{>GgXDNdbnQv(L!W~*ex*6*)aFZ#_jW{UTJn)D;e`A7hE;ZAuml>342?CTnSNGNdnRYlFW)3zVbd-!3 z?*~>L!=dTzLL6vi*BV4=i46!$%}hY3P>6C%=37b@|4BI!2T%Smb&eL=e93kdxl3z; z1X#q_Yh#3gGyzl(rRYIX1BtKUsKh2v&jb3+#tT}n_F0TyfI1gFg5)mOmr#Y`N~os7 zq}=Ds)rIQVYushELuV!PdBojQ!JOIiZYzIrDmzk?U_Q&%@b6h{` z(q+^ZjKu5H7ao-6Z03Y;{mhuOA>c=Op977 zE9u2#E5MYK5 zS+Qx8pppbnjIE+@>4Z1O2)$O7nY88TH#JOVZXlJfC!~-=CnWpy2?x%&Sk$J`cTXpH z!JKM6h4mtUyE;)ebS$*5n`@U6Wh5y8#aZBrWa+@Q(xMTbJOn8|=Iu%}D{HlI#J+;5v1=WhS=V(xuUMPyXt_*k{q8=t;#-X;u|AB)!?D;-+6Tuniwc zn&&TzOD$DIc*@Fxkcb2?Oi+&^Mx2MaczqX0@uAWhli>ILUH*k;7}r=W{Jg;_H8f^d zT#D%*QH?ezRViqjnCs7faWS~djV|HOhs8Y;}WP()*80H%V z&PXXxF1(F+(5s>9X?&yk!UYz-#3qG@(AYOPM1ue43nQ9uqN3CN@exT5tjy zaID{oFoz7@x)lj6SqABjZ-J#C6NctK)zb?0%rRL|&kK7#M1_l%Zb!qRWcv)z1zmJ!G!GihD^ zRyD&D=Jm%}3DbAct5-}zKTi&?#s9MUzNVb@^>khQ^`I<~WIiEk2~;Dq$6vqNQY}OZ zh~at3bn?ZKSz5$6oM2B~2pyuq4K1g7W18oLD_RF+3+ZqNt1rD?*Y&np`GHLU%mIr~uhR z56XbY8^q`Ulrm~_fd*o#$WlI8w`?6shUw=5#XnQm9!nx`huT0Sv+7#$$eOv_i-`s$ z5jev{vQDwAEOd}8Fe|x6S*u-|on=E16cxs}08y$9iO$U3x*B{^NahelOKJvni`kSk zBP8pq^N%XeCy=F^m2@$zo2QCPMIFv=yuBUZpPCkCOs(#eJ50pYKGr0LvS zN4x@I)*IuLrV(UTE>Jf!Gj+XoKrwXnXL`7KIyFj%3RQmX;jHw;X;;lv%9NTKx6Q*> zg5wy~C9*@~$lUz2d*6`Oo~$6r0Rvk@YI|k4O#(;St7H02KKZypG4D1e`!WbY5}@mP zki`+bUsEw7cnqnVR3xM+*H(~p1_qk-+5`a<|5=*$B{BRNGXeJd*CMzj@g9`^zFW^a1eYT{5-WW-Y?suFJjz!tLWD?Qq=N` zJAFFT8F-U^9cam`^mxF4mcW=6Z%^F^wY-?NemCA8EId|zap8TqzW7U>fEGyhxg5J2 zy%J#7*_|ESpG_y?23uu4OeGQpE-5P5xEOeb1r)pE3|+{D$!H}TXb!lQ3@~SPM5pt` z5GT1XrUlX-TAXBnfyhOYM;m7ooE71y;|#HClnNCBs&8)WPbhW>n@@Q{_&h5@=KT~V z!IF*=+3Dm}bhylbWsc?*W~gVx7(w|#6aUcHUjS8_aVQqjYa4&L&u5R8jxErm*n`;8 z8df_9Jpfgtjvo7E-(;@%AY?gPv7bGyEqP)qbD6n5E#m^2HI?8CWY$z7|A4L0XUtNd zm%#BUgyM|3X*EDpOOmK_F2eV_40ZERG*QC`{d!Q2cEBE0;S790sb7>%{LX3^mk{^L zI>CUpp3$7FkUH|1Y(}OZh)h2Zt^d&PHG{Y<2VKO0h6AD!r=NFC z#RjJ6kHGrrO5UpuVWTMpkfw>H7Ci`CUR`Kx8a21BG{yP&(VLx|XvGjr(T)8)(VSD> zN*&^)=t7zS4lJT_16GWL_uIep4AMr5Zy+n5-mmtuqQN4K5=)KAvB*vqi==*ol%_x` z8tWTF|C*S|aexTLRTgrlQsJUXtB=Un_&bXO++DmaUR}g7AI03L=Xdqb=H{su%sYrF z700>*Gz9OD`)^F~YQbXB#!x0v&W2HW5lD}6r#yFT*Cs@GxGFe`xb|ee%a2<70Yjt>Kpb`mu)l*#BfH?1rg3C zWll(hfgi@HM^Q_XhZ#r0C;3nbPk3{kOGI*7mkhCm>cPH&T&W^bwlW#$u817`eb=f? zq)nEC#}hZcV<$Ta;cyTsL5@nIitHW9WwxfaCzsi}A^xQAHsDqduTHQ9Nuph`RJG^s z%JIeviTHI{Ml%y@ci%zJb>_9QzwSnElw8>ROozwLE3a+d8i-Y^-jxXx@WYDf{SRb< z|2asERO5ZJC!BY}8hBe@w9C1{(mP-I>)79*JeWo=Jh3Pk{-eP{rjaok7^~@ah~mFD zkEy?y*T<@$YnYc1x7rdvoib!%lfz31mrRE-J1Ua1fak2=5-rs0PhUsyv5x*GIlF1LOc2I2n zw)ZgdY~K1)pRGVih(ns*$9?elH*SaJG61&DU7yz8H6;cqRq9kg@f4=>Aharm6pA@8 zmIdw}T?*&MMV0kNnlha_a-C-YRK{1#jqbj;^;lB3c5Buyx5Cy@v(cq%Ccl8Aclqx- zzaUrRSg})X!4O}J?{B`1%6|g=OXl4Newse~_=2lzy+jr|8s*X_-A%F00LUC6* zK6#alHRb?vFSx3Ji~>fu2^aYj!}s#W;_K300|h3HcNFtvgQDzz;LOHmzE%meo?LI_ z7A^zR@yY?cOH?4xD*hux5{Sgt771Hf(^`!O&rvpe6*xbwru=!-oXP*;Yi%+U<$wTU zd5}%FWpk{20u3!s?Wo3a%NS~xNNZw4b@9xGG1AGSy@{bfm~0zs@t~Q`^)HqoBAP6c zXuh*MCYNbA-IsCYf`KQJ3vyL(Qfn+06`1Kiq$_+{%p&3 z-AIhx!?XVeluv_Ia*Wu4OZH3p!8^MBOt0JhYI{F|Rj4W;{FUwFSyd(j5|_nf^Vf2g zlE9VNVMVV%e{CNl%8hjPYXX5ApCbkJFZ<^Fku+pssiaS^x=vV;{nYEA-;=A-s;YA{ zp~0Nlj9*=*I{_V73-Ej1u{{+wyI`+oEb_7ErCC-K`$;40>9eC@HczOpQ*A^|+_T$B z(!%LpA5M0!Q}qZ7D_o8gmpzE<9Sis(_%EW5bnj!2@E6(@56ayHn)zmjC*uCkU$kr$ z4Mn81%9Gf=@WMoI!xXd7eI}xI#Qn|e}1%Y)%X1?mp4~A zQA2I$38s*+VCLaRQT=W+&U?UB`8Uhwh{YSw#oi{;Mc;78hU736x{*z~cdR-kQcX(i zuk7jlI`aK#yNGOn;8hN5fm+V^DP4vZ`=bxo*HrVFl`kf}i^X1cKX`Yq&pJ=bykoMD zzB>Hgof7!@?_4_AaP(twDj&`jpy^{K^jl}XDMR8$bE5bpRQwn=;r&P=xytZFxdF&_p=lG z>xkx>?X5|c2UnLwJyiQU?^CCI!|_o+xwUZ-5vFqe6tRs;E)m8qBQ*4j2Sl{abfmXK(@*{9{ORgfZM3@XN7Y6x9R}So zT>!7F!RNy^e|~*9J?+FXCYPwgo*Y&fz4JiwSVU&zW>(y8j=Q#D8FNkeK8-A_t*Cm0WSDjgqO9?HCyX-PER3<-q~6R}_Cl-j zKbzRO^+2ocUG}>ZM$OH)(ou^-mR1LPbE2v7Q+26J$6WQU%qO3H|0qr}pgN!@91UiL z`-L6_2n|)0tF@TW@+7-jn^Dj`rQZ^d1WS|#1p(g{b)mv<@R%w6t=l~tE!!<|q#{yF zJSltPwi*qbVJ1crpq+8@zMV9F)BjJ+)~w2{+LGxhp}Vm{a}fQQVFe1`uc6m@@u5dw ze@@3UfYQAz=B!>^JAjFOaQkP?EBU$UtZEQT$lK-tRE=v*;>-8qLE)SiZp&2gnkN8H zpa1Jjb4}#eja|Nne-zA>O<6x_Y3RD|v7}V^ESr-eDwq9SD6+ENVjpn?y!F2;>Qo;% z7s1DO%T=nzOLPw%x#jY-_H8d`)at^j#?%J)?7`~ZLB#5nx=E5&Svu1nnnr@eEn3Rh ze3v!6th_LAqg$Ri_UAR-)rTkvDtcR-5#~WLE32;Vy!68y$DZpAOumbPDuH9Rq{Nj6 zULAQZZ*v1}a!#XnM~U=nJlb?VL9yeeaq!IDJ+K&DUdG;E$@(Qts@J|LL1yh^SI!kq z=d>j}?Pz|8Q;{icCTjCHHT$kZC=phZVkO-(syq!So_*%+q=FpUYr92LJr3&}e~Jv% zqKAuYrTWugmbe|w3u8V-WUk;tyCQ-$o^a{q#3|MuE1`X<+wr;wXki>T=EsU)#>DDx zPc^9iJ?$rL>8rj9M`n5;Yt>EMr+c?k|tiVG-_uoGdc$|_Q zm=#Y_65;!A+k}EZ?4bW0m`4?O31w|?8{Su09V~+{RTlrxl7CU=1Yg9jDZ{}iJoKt4 KwZN70oBj_q8p2}$ delta 6394 zcmZu$c{G&a+n&jqrN*8uX3QYFv4rf#GWK;EW6i#YkhPi0zJ#$QlwCwJwnCN~%Mh|B z5h8@_WJ%xD?|i@SpYQ$SIq&aYp5D$UdtQn(grU z?6c)p+w}XMW#p5a=d)^6c`;qM@x1E(mj>`Pxdwvh!FdFlTy8{b>^MjWnk z9w62On;Eieuo8JW&fX5!+uA*`9+`XSpKnuJ-CBPKSpQPm(Fq9?<(?zO9%aRbC_bIh z@KhCMbdI0w-R5FJB{Q=W1gY$1BM1k5HbGP0>ovPX)-WDlqmd|>yA065n4yB5S z1}_XCi)A~>2%_rm0tDsgFcPu7Pm@4gPv?L_>@_5jh>hbYW~SaP9D+DEC|%q=rvhcF zyBl#u+DWWd50Sha@1w_4qJ|`HPuawE%qjiSfeF^fF~L#;b6lJPa8K)!KZ$lY>m>Ev zLFsI~QS*Z@8SPLVreeG4cgreH&j+{shRkRPU?GZ)_^G|3jq&%!yaCk?=`9JL6m{}r z^)mYUh&!ra>6wq5TdnlTb-wc*(N4qEJ_WG`$lXY=r)qWd4~ z-u(V7Z?j9j2Wq{q?Hy7K?06GUPWZlnb7O_@i^W2hj<4!uMjmZA=_K8pkzr=?KR1Rj zD<=~dBe!)I7cRA8x$ATt!dtd7enJ9GHzf{Tj1OIOHy&!?kwk90R32IBq?^^rk=`cv zK6#%=ZP_1nEbQ-ZyjDZs`&gM4j=_iTxJnl=4mGL%Kv}zc`&TwFcGf?tq?|vNsGRrA zKxJAQ`*??z^9-}5HiW*}x`iNR8F0wpgc9_Wk%+|vrcN2`!F<9j(8S~@YSeBqH+&oV zFehjZ;b)S&1VFNqfco0sg=d3Ut)voz4Plhs4T;5mxqV8e9~2TfuX^sZw~+-_0|!Y{ z2;<`s=?G}z6(Fn3<-||Cru#u(g=S%ixis4F&;%zzGiV+w&1%-hoy^}6uU|Ne0KdE) z7Wtwr&VpTs28)?l5lJ?U^!vmML8&g8XREU?FfP><8YL#MBfvSoCI21(rNXH@+>b)3 z!(XG+HT%l>)wBr7X!pZUI~hXAkTb_F4CiLgLb&as7az`B{LV#16Z^{>Cx~fJZO>(% zrXkS|u|JjzuD?nJ!n%|3Q(Z04=r%mVwm!TQE9#bZbNO4=mq>uU3e}-CaNg=T6lMXi zM^cBvDAL`{@oQHSUYu|&V0vvejccbUT2(qe8-GdQZQH$pGGTjTmZ@hiU9zOTCUFs5 z3#Isrn#j3ibs7pgYRBC)AyWo~98d#^AR5U~!Q1|r0X*B@k9Yw@n1H8JPJ7b`~;Q=gI^5{C4 z?f)d7IYc8Cv;NosRM4dF{uz#WgL-#qN4uz>|MK!xlCUqdfS3SKLQ=0nISW~2^xQW} zGCcUzD=IowMFBxQs!AkQ+|ZR(XkN^4C1J|UmH*El;YsP7LX)yO?CT$Sg8c!l+KQ$8 zF;z&B8h-q>yZJ8Zzmz*6BtR0-9>i@bgqJ$`z>m>r(P*bTrQ6{5Ag!Uykvz1TrQBCE zs~eR4b$|38%d)-;z)BU|E;zc$rdm-@GgHHJSwaFe^l_BqiJdWfNx)A0aDMOyNg{{2 z)h%J~X@!xO!};DlSks_hSQV_&GFDH=bB!S#6v*kwSjXT`8t0xLb-y%1*I8zAwdMCR zZw>)gVMBuE5NuL|GTz$PjqY~xRz4}McL4B*XD%P=+90B%nTq2bXX?i-y*cK>d8zFfq%N1Jdqu%-A&{MDB!P3Od zTYnC7n%guDhMFLMs8>NP2YS~PatNdT5)*QH+Ls^JD}Pm9RP_+Q!=RY+qbi`(NK{`G z`;ahH9{4IjUSA|d0B?rj(uGF*P+!p%qwv84!FgEK<#-f8K58gE8@m|^5LcX`BV23! ziyJ`3T^yJ5MyN#(GEOATc@%NLtTPq%?P_@ai^K5xZ{uAu+h|5$;Ak0jNheNR1_SNF z{;_t$h0ZmWWIt>eO@$AMgz5x438ZVfrAC8pn$rf(CZRRtI7KivVVK=5Wd;RtB-bLvop z7{zn)rmIxb$$Pep5=_s=V~2qaOD>{?0_wF_*J99cYuZ=U*6iW5*{O;T{hq7s>&(cB z)tIHPd(tU|I3lkW2v9-rm0rF8i!v;x2cDjA~J zAHWbT3v!S{558kGi(S1!&~edOW?0zZ@^x${QoNqNMaj|%n_wwwCKxR4edChwh#sjF ziGb_(yVx;n3R4(DvVp=Rfd@;@H4P?ECn_BBnQm#9z=Q1kP5JHkLUInY1hteQcJ;aq zJxfO*@|ik z^(|jNI`ztRivL>iqQ|tN%=uf}z?xL7tQS@=SC$-o(#rb~ByAUiR4kgh6PmgUl!|%> z5(PM(M1mRHK@Cej&G#h{TK)mqzcgPpxVD4De7z52emWoI(P}js!ac%l*kT(zJ#*&V zd0NsmHlZ0UAd>KL@|?s)V6~XQEmzKqq9j0dk2`(c1!fevk5Lf*VFIgVvCvyEFnXJy z`D$KH?o>;SauOrRNm!M$y6+LTFzGQRW19S)AMR1?GRjXbAkT8ht&u3#$?wG3nWg-J z#{b|Vq8c#&V=s>3mlnBz%u=6&g4Wah{2{Jo3kWZ-!?>$`-@j27lQ9D!WX$Kv=`GJ% zJ$cU4G}fUR=VX#t(cfTZO<9UFa5hjjA&0`_WFL$IeB7VbYVK&$V%Y#U&47%iYjj%{ zJY|S%Rfh8p$=}PlwA^N|5Xv)%k9u<-S4o!=ZpfR-kV>V=O6xnvN2d(}oa0?_Mr5TO z$AlQ_E_9X;bHL~6$wJFRwn7MY^lp3?!8PEPv`T9%M@#Bttrb}t2hj8N_Ouj&rJ`Tc zm307HcZT7KCbXn_KSqyMgK;|sO|1^QdI|SqECvk(Ot{l?zKZ!LHBpuQU1|xCnn^Ue zbkYZ;f*aS(jcWWL8E;txy0$5MYPi9l71f*_6p!xHk0aY~0uV>Gp_LN=Sfv^hk7XkV zkuDt=LyHM)V&Z3GfQ-`DxqLmNLi^!0_IW?FW4?1lyNcCWzU*+4=B3F-=q8kE6E#s7 za-z}H=zkeRBYNEb`FR)ucl>APyfdyUEv^SL9ym0SZb7>wg(WF^`!1o>5|*kI;a6qh zGMNvVNVNQc`HmI5q@Pg0jI4~0sm@H)qrJwMmWZYnCa=jRBIL}OKeKdE7x|aDV_(a- zk7Omlctep3(Cfw`#d@m!nCGs38hV23dR31?YTPYo@Cd_%QWG-46EZ

    R)7ndT>x0d~SD9P-zL@V_2Zv7+@q@YHFzK=xoVL#WQ*CN|H= z@yknQlyRCNj5fW}S|pq{FSTukP^2@$EpS!CMJavY8iC1ujF}CFH)q1qTVG>K2fZS} z@kZI{J_H~zZ_=IIzwky5F;VmwGF2B*|MxK|lzy+%Kp*?*fHh5SFZvADr{;VsT?4^| zpM&%IR*9DFrgwQQmjPL!*V4xm$U+y8g&qeKkcGB9Q~(1eR#-;p097W!1b$YHza?Dq z8v#@uhNMMe8N+naYo?{?U`xx8(d2YjON|Z>8KzJH$>D z-Vd$(CTUl#B}(|~ots3>x_z?FmE<`QP5s;jYw|1oE72*+iL8$0$p;9%+dp;u2jioc zN-*JK0+O4Al&&&6n|kYpOFxcwSG;pw`?T$C4w5WK6J@5tr*3grL&^~5l1@eFAKK)x zmU>lNjG_%5KoffKw;68#U`ZCE{ijQ(w!q(_m}cz80xXc5L0m4|3*f*$^2&7u6@(NF z?NU^JV^0ni*3Ry`rZ1u`2JO(THqeZB$C(r1IH8ES%kyh@5VKy9R0$qQnJiFvyMY{RhE%I@!;!>Feu=L4)s;hver# zJktQ+`36`WP&EuM{+qvNZY~-!WQ+S!QBwb7*bl1co7`ZHF+Uc5Sd>koGP_n<{Q3l% zA1uOao(>C1B>RRQNmF5IUUe`snc1XGB~GhjKn zG!#zyaLwXQR8$}ORe89)C^mZ|r6=(11cdzoN)b!9$Yf1H>&-2b=QAfRey%zO$9`FG z=eQ6Td623!ktOhRk!x4egBP~!7Yr%d0V!;nhSw3YC$R=IjBH8vGmW)3ENU9U{aTCb zRn$ZiyX()3J6csF+$yx271W+xrykoCU$0v6^)C71Jg{XNv1>2x;JDcu`sVGL3#9_) zo>m5$FHYvROZ~{&^k4_aSkgpC|79A!@}%yd$h|S^5kB_%Ws~x0guO?mNfm1w zTN9UNfAiHd^OpfH+OqojtWp60e=<73B3PdRLgYoN++xkg#j!JjujEcM1ASnjy*bse zxa}j+3_w?4T$;ijVnqCO-!qtxwD#?GlB~uZ)V9K~vFt!6my`8l-2i!&k)K;b!46us zl-8`!upZ+Ep`OOuvuckNIrLPY?67QV#80Pt?rz^N176?d94K|ib9nMEb##v})ngw} zF<~7IOB0Y%A%n8UhrUY{Xk>LwLd7 zZfh+f{(6gomcwIdximu_CGiq;B(|n@V;O0G=4e8PRy1}7k3TnA(a5L#rjQVG@{xr@ ztj4C+^<~!M+eykU`78aW?pTrdTYbzZN=-W+;iR}@gfevueq!DBxkkP;m?ZrDDZ^{O zI}Fv%$u%u1bO`P@j>%`s$Hf-Ce5<`fWR*{X$TiD&}rm67m|9BybG>m3WV!{Nh|ST&ak6 zr2;?LAvq*rJaRYWDF;~VVe7N_#Y-qk0Dw6c!*vQb|EF} z2gzgh>=gW=h{-3}QtX3DQss;7(@Jhs0!I4h^ z0&g8zP(q^53HOUppTtranI#x5c|n!?_P3KFvOuiAl5gCwO|G* zQ7MvRG{dlIw6(h!BD;Z|WjN98!?#7nnfttXx?<7%@Z)j!bW+ZzM++f5hQ1p%pS(<; zcMbkNY7dUK9Y~w;1@HUt>-HTcKUxn)#Tt07UTN|Uqn({STXnds+x)xBBQ!iXV!HNe zkN$&J#&sWkhUAZ#73Y%HS_Wp>P^p(6)O~|oUkWUlz3;{iVC7y_czAUo@~Id( zI;2GrT9PL2i(jet^a&RoX(ny-OGR7Ft3_{Y-=kM37MqVJ-~{?FJQDu<04or^(U}8? zy!ihvQ`Emc101;Ly(uDnakY%wjlg#K&i}lm+b#n*{O^MOzgvKVRiw47>wrbGdGIl# z=+h4qxtJSM-O3YtVm8U&vu_165H?3nV_zK)!E3&zzPg&tUL;uPc%x%VX*;6%b@@-+ zP9UG(jl-iY6`zg9M9&-|n9ePz@BwF|rIaV=d*pACr-BQkH|X5!SDI=tO=&rw6kH=h zu+0^g`J8RFGA-n zQeWi$4hb=n`&ub+Otn%rBG)hTfcfJW+R7QOsA~2#2AHapINDuy+~e}q!MxT)oBouC z*WJZl^LGg`m8vW^>CAw{xOm(ZJp&)E zk~d*Bnl%#fnQw{_&?OyIs7{l*C)Uex8&GlP^1jekF^eIBK4Y?8q2;dqqn`Gy`gWdb zY3!qp?AopORh8p$76Ix;FooqyXsMT#+rdnir=V&0AL0D--Wwho53lk0VHME$SDDDWMs<;{^#gB Y<=BBd8B=nqK>rMOc>!usnZF~O4ZvX%Q diff --git a/summary_results.json b/summary_results.json index fcc1ea511..bddfde487 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.97957, + "Performance_Result": 10.1431, "Performance_Units": "Samples/s" }, { From 64be10e43f2f8cf1ebe69a4fa2cd1ddd518044a3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 27 Jan 2025 00:17:46 +0000 Subject: [PATCH 18/24] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 13887 -> 13902 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 8577b7c1c..483be9ff7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 00:09:46,532 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3266 INFO] --- -[2025-01-27 00:09:46,532 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3266 INFO] --- +[2025-01-27 00:17:40,591 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3282 INFO] --- -[2025-01-27 00:09:46,532 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 00:09:46,532 submission_checker1.py:3303 INFO] --- -[2025-01-27 00:09:46,532 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 00:09:47,625 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 00:17:40,591 submission_checker1.py:3282 INFO] --- +[2025-01-27 00:17:40,591 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 00:17:40,591 submission_checker1.py:3303 INFO] --- +[2025-01-27 00:17:40,591 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 00:17:41,934 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1629 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.915 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1431 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.214 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 055b6671a..1eccde110 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -507 +508 diff --git a/summary.csv b/summary.csv index 84523a441..9ced5f765 100644 --- a/summary.csv +++ b/summary.csv @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412174,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1629,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.915,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45726,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1431,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.214,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 038ea9a08..68b303785 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1629, + "Result":20.915, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1431, - "Accuracy":"acc: 76.000", + "Result":10.214, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 44d3879fda1efff3b9372fe9a2320e140b5f85e5..2332d0938fb436bcec5b484a5870aba40e9dad6d 100644 GIT binary patch delta 6466 zcmZu$XEa=G+nv!nqehJxGl&|ZN7T_7O!OL)(M1r^f)SBK7c&geyC^YggeZw#f*@Li zcyyviC-P37cYW`#@2qw1A7`zz&wcH^uYKKXop!Hbuj*!chL>5tlogl(fQcdi;2HpD zD#M1GqgSJOa%9oi^)l+PX$B|6V2=5bW$3zeRdDGW@GZyjbL^GRgnQ`xr0G}pq>Dnm zU*#P*Yu$v6{(d(TN-fpz7IocxYgkxoI&9yKt_)*{!1rbbRmhpXytSy<`a-q3) zuWw$v-h_9*!_7i7EU)IQ5G$Sj=O+g*(>k+V>f_&mFG6MG3$5>4WgSY6a1b!1xely_ z<8@k1vy0AYpAJLY&L7@a7C9;Qmh1UFa@%O!E0l|#!*#GDp>EX2N-WU7PU?M@WXV!{ zRd{g~t=(hu?4NBS$i~(>e66WY@8Ht@O3VA5@wRW99#JFbaaTLXUgu>}whx@bM;-Y% zkKjN38tL$FF%p?+_dOluTiV<)ZYjU4c3P7@xP18SYkjJ;uN@F9%01uz;w1G&fMU+9 zy1R-n{r%|4-aRfBi+E<1Z2vn4czEqm-|cuPN_bZawbN58XWQxOnHRxa*A3VCd_3PZ z`-Ek8$Ef85HtFJVW+;9*g(^vBqTC*8jcGg)oOkMgqRQJKo_7;A%yN4$UmRQNk8kF8 z8B`cR7K``e;aw`fv*Bc8BYNmrerafwZ$wT_{IqEMeeJlu zM;6*!O{(6~%SQX;$4QuO+JoZKi>sks?_pEQTCfo5R`k@b{H?E@2wvYx$K>XiX+@nt z0o9qX{U3!t9M>eDP^uCnqE}bgMmOAi1J3eg(-UWxejT(g9e9*pPH9QqR^&56#99Xr zS@1~4o-GE2pE)UKNJ?T>d#^89e0BI7i+Y?t;oxB0@WFX^u-^@{U)-gyC zvqf-&zqy;Ik8e}Mslt2)LNGT;wH!ZS6Y^6jKsgfnsVG2S!&^p=`qWwKg1QI#8jqJ> zww5sd+Ijqk8(}<>c=2q7GMlEX6jsLJ$UZ)^;Ptc6JN5B9iZuOMfb!3DeL&7GmL^Be zT}yAJ>)i|2fqlb$-MF+1KJAnizYP?{9uyQK6r-ywMS=+`;Zv@;y|A&-X54I`e3vl? zy@wsn?AyM9J$*4Pdm9;uq+fM z;R4WNP8P;S!$ATOs8w~)2h!M_or|ImaH(H;x)hS8pP{j6+#;YA8cSFR)ohT0i`72g~%aR~oB=Pd-hi+Y8Ke0$@qh@(3SQ zMEN=GMESw!yRUM=*4lQi$J6^bA-#ahg~$f!ZwZ}8@4F^c08=)r%zim7eA7oUwd1j4 z1BzbRmf^NnWgWA7dO8QOoUq+zkM*d6VVn!87<7c?XMVy!uW0Zope2N6tB&Mj#lgOQ z37`c_x2I1587YE}OEFs>Np>_OHbrGQ5GVtlgg0crXSp4;VqBRca*!vS+=9~X&m6O7 z)^O0H2qL1g-Ri42-q+6w0}^sbE-C?gn>Sw7-2f2K?SpZ-e77TfBAuTvkX6X%PE&$^ zG{1F5s=M0QtIIpodh#Yg*)ZqlX8&j!1t{Mol5PKemwkBvADJ ztclWfpBr|)c9HA#=1SxE(SgUXEhb0Sa=HVPo#gm2?J>P9syrMz0)Yi8Nj>xLQBjJa zAo2f~5;fPYLYyIRJ^m@8*7`I{6z?PIb2F~E4{h!RJV~}Gnl#8a8AW3> zkk@{_dh9Wn5mGTMUG=pE@*3b{2zB8IT>)DW_i{xw;UhyNKFb@H+Vzwl-ux8Yr9Vg- zSQMURtO@2hp&UP(491xM^XS`5^W4Dx!Xj#gPCP7sW(6(9#gTOS*}C^qBNS}T_M#S_ zz_NY*xygl2E6}dM2CMHJjZVQ2Gf{)nfRjvvfHaI@D^MbAmFX?O?tD3a)^XnVhOJ+7 zs-fSF4@bK4Ge6lx#TmaRm(Ng>NKD;nb27C{A@t9*SY?E|pCupsgZ6$Sa%f?0pI@{B zy!ALDra(``8H_b$ZkC`1rvuxFA4o^5QI(8v7O(*fbLEPPCLRIA6=!2Fg@bf8eR?T1 zgD?!v!%SJ4p7<9#P7Qunp)CQngFMfM3(d22b>50iR+SdLCbc{?N8Vsfj>o1G9qCK& z6Kh4`10&V23v&Ry1-w813W15|kh=BHSTI?#lh{c?B~u)0G!EXL=uT{L{bsV0H`53y zjCx*VboAT*_sBV9xI$YKmdeiDFWhgEhKY)>#3#4Z#xJMRQw2?^(8pN3YB?9#`f+uD zxBQE_`~$$N(a5(!v4*tZ8ZK^jsatpQ!LMcNC{w<5P6V$Z?PU||-L>3T9-2(fm@&wW zYU>sI!lm^MmZGCx4FQ&-og<8-`O)L0Jfk!pOGXsO4hY$dc`#R7bvb7uEQUe|VZnvS zYIaFje_7p>A2P}xZ(_r0^AFV?AuCaTGXcjSIORnpDdg=$eD_d(?ZM%I4&LQtyA6dglA@~ zM#wT2A|m4j5wVB@vkn&{Et2M-)IB3WQq001nZQ0^>0`#Ya-YqlZv2GZ?GEC z9z(+>3g!Fvy^kiPTlvj~xXP9E#s->cf4;GyEJr6WZh(^GrN;yyV7Tl@0TicNDjk*S zQ4A3>Z}V16eM%@p9!$s<$O{ZP(v|GO_G?5q57mCrtj2_X=JeP6oiG2b%E>(;)~@2l z`t(znwUQ1E9C~+xicmwx6m3OOOx1x2)twy>m?>QP6tpRNOlnmcbk{d+( zcRqo8xnFOpjH$BbX}&ZGA82V=J;;>wK@M(3?l72}Lnic7^z73bcqvodiqK>5d0XH{(Mau-izhLX;DEL7BKa;)t(9t?Yp2yLX>6d&E zQ*jo$j#?3B_ql`pezY^dur`G;VgtkS!U_l& zU!!&3droxa2l`MjS7c^g7e}Ol?JDi3;s--*U&1QGS)AVB^(gG;EuMoP*nBHK;e089 zHo4tjBtdj~)#xk?9Fgh_trQb*f^kMnoO*#R)H*grs~@Vd6EkdEE+X{>v=U0j(Nf7| zx0jLeJO4U~r#OzTX?pqQF%s zP zGYqS-^akzU4RS!hWU?PQP@I6&AO%3VF$ZyjX1U_%kN@S6>B5RiHId>PypjoPQ>T6i zKwft~o{(T5pjix%?9DkV!~$S)bOTPo z3L!?Cc7)l@=)X-E_KV{lrb&-tH1Oty$7@F>d`%a!=_OC);>ww;MXVC964w`@^|#6g2cB;j+^L51fGCh?lzf65LX5fNM?V&ss=T??RE9{l zeiUmaiY)snE$?fhzm=k^uO)^qHiK*Ex<{CGNvjc83_27^^mNL35rh=^uc!MT7Ai&k z6*n1;1o1cdB0*XQrjr`tk7zY&H(A7|6n(|Z+>w;YT?R2lh%OC_O*k|PPVqO!;L@f# z2GB4S;#4p^kAjeD{KXSJ6_!C;Lzp|_>{hRR(()ck94#G1EBuyO%eia}X23M($BuH} zHIK*t?04;H*evPidCx2h!(xfW-{2&sDHlIy4c)>3I~$2z82;aRBO^eX@n7T?^0t|H ze@!{fNq%YB+gglHzmgek<;;Xb?sw6^OIVsxAl$xk;!>uN4c3}hsR$4?FjGum2F=+o z$jwyk9)<+H<;Tn%nol?0V^cRkydxR0oz`6lrlkhHtz`P*;farJsSn@o#M=mRkd)h} z#l#~FS1$ag)Xf=8V(`p*nbFTH>xqG*PG3z795v?(YTz4{Y~oxn2~$ZX&n5OrGg~l6 z+Vh2sw5V5KU%_I%hGI>H%aTH-k%P;Tx@?{-8$pnb(|GNkn=pl-sX6m8~MV-8i(~>+~2|aG*kP_Tb*WbwxJ-2k|#cfPk(L6n`c6L zkY8zADSL$y`O*~4tDOwT$&Ftq&y&`&JWBIi9;qR##J3!rkCK2hhO^DGE0;$NNl zCBHC*^3YqAMYCz3JdfjdV8@8|4&I9i&3e(P=C<(c z$P~IPq2w}#wc{i$ZB=hn*yo%C_KyEEQlV~-(Eek`uKGvjwCvY%srp))Blme)527&Z z+^9Zl&O>w%Q|+|v?RaP$*Lk(&FY{V{I1cAmU|JC>kePrO@xE$Jcc{D)G7nccy=gC2 zvXq%S9D7xrE{>5N>go4yet+iFWT0Z?iXjB7@B9JOG4mG#srA2S`>&A$0Do{F!Mqrw zM}-PZt-A!1V98sQYRuPsME1=Fd`plxLIdvY4375MR(0n~4(i^s6HyX99{BCVgtZj? z#)Tu#$EwsP`;F#2`LMyDbU`8Bf+Yr2OfH3!wCcw9AK9MXl)nGKo^ljZ-`-G`nm?QT zvy}em>2e{p>V`5^je<1Yj%xMvQb;rAUd-X=F|p&QOZiOh&}1pe1;+Ro1!>`)cc3Ru zW9s1V2_(o%?ydOkx;5$U1#;`$SJT zXifp8oCKfCkjlB)t>p6BP-iaWR;HFy$L4CgAO3kfH_hU79ET}xX|WQF>gTq{WaUd5 z=Kj^4JzkL9H(?I`3v&38uo8K7Wv9Qoc;>U`b&+%uJQ5XN~d_eIliC{ zoU;In^Qq~8^*uX#hA{;Mi}&a2f|xTiyn~8$(>uIaC;wsn8Dwh53i3K>r#@mx*;yLtph}>}%Ql zgz4`iq$X5ToXWMXN~$-&n%y8J9zsR z2K9*y-5RkBDG=ddskV)1rJYW_Q?bQxFHoMjo z9LBY0moNLVn|OrG&c)ZU5)7Yf5Bn2k z0G6m6`a`PMiSu>Y4$kjWUgUUx>r!9eS?%2mQEuW*6kHmwm>qQ5m$6{}{?N+R#;liW zD0u4%D{B^83w`SAlvwIk(Vbx)obS&I*6|RcuzQ!tt9AF5U`Lnd`%@!pL9USCHl{yv z>7xB55xiE$DY~K`kd>*tL8`z+VG>h z!o!8`WP3k3Yy*3>)qVbXB2amzn2!evPY>_HD8qNzr@r#ublB;9(0g^&`ZA%_LTVeU z0RLpjV0IISrx859Od&XSFZD^bvS2VZLEf_k&bNYN=H1vb z{pQ5=_jy|{-f36B)ICU<-e)vK>ehTyr!1bPn-=LFI`>hi_U?SS0sM6Aa=dfvuzIcb zJQ$;EG{gNMNtw@~Qzg>VvCBr?OTKh_&`LCV8|!7+r}A_%5*tRdz3{!O5&%J9SO^vs zMI;_>2gW3KUg*72IdFNbqShHkK}h)aBYMjFpNF&)8WRs0C!{NW_%VdC*OHrGG1WE< z&$E}cPF~s1HazTb(5WG)t_rKZ(@dnvccMVHZ-mPTy6?6BsA1#)2u~C=3mK!goH|lI z4G#`oGtaE|{a&!__aJS;uGNmk6ccRpeNT+c!4@f#$NY!R{DLvaY&m;X{`l&C9x9_K z@o7M8kCy=3OItpps{M-#h%RCbg9z}#g6xTyX758C8(QvxgeM{Wxp26Rc~kkDKZwc5gs~ATJ-Tt zBn1m;33<2BLo!0=#0v+-M;>f}3{^Hsi|g|gvRogkM4R;<8lBy^vTJEMs#qP^GKK+U zQ&|mz7DmrYI@Yw5>;^UMidYD&eZBT$gce?Q&T}=F8vyP_zSN}t@n3$48Q7FS#r$AJ zduHB_^_00W!tcJgD;AUMFD^5K$mp6RRn>n{{-q7KFUwUIA$37Q{lDKExF{J*AQ$et z421iCGZYE{{3ZDRu85~KI8Ir0-~i56Rtcz#!^;Z(m+z0V>_A@Jl&lKS7soCqKp}SP H?<@ZS^(OiL delta 6469 zcmZu$cQjn>yPeT%^j=~ZOwdHRZ{6Sh=bp9ByViNvIs5GAefG1TQ@;Cj_|&!1UgKk`qPxim0`--GKfG>NL|XLwd1AoHe&a}Q`-e7V?u=UTUt=#>gq6OP<8_ZE zb8gSI%~kn!TxTixnE}XUnw{RvyZm_ZWBp+{hj@u^pNb>xs>dWdtMESq`@NgMMmbwA z?!d?7cXP-L@}_b@?d2SMYJYe|PO|^s{kvl_Tm@R)y)z9DSAAr-SqFF)d}t(oC2pM5 zR%ML`J*%pze6(PtiEj>`G`)?-g@oSul*=|1K5J!XpS)uOj4UnW8>-5>Rf!Kuo(%|; zS2=`1(i* z_Km#kKF!D(yMxQ;7becdm>TUxv`#Z`XdDVbv#zW)xL`f$=6&Kr%WUl2{1O*GvzQa| z^ufx)$AsfgJ+lVl1r@QEwu8(}D}^6F$n732_9z*=L>Nq1x%yU0<>ANJfwRPxNnWA! z5bq#eM7wN0Dt@}W0~4=yrjS69f60dmQQnI}^eEv36XKOSjQCehc+%ph+Z42#rrYA6 z{6~It0$NGyED7=T+$h=vmjmGh_REOPont$4z$5{?bsvE(nHS8q<}P#>6FBPs^KyiL zFE@kX>F?AWn$xlqqVuo73hn9WSoq(c?{XfEKlACsf|OSN34m3p=X0W>+0j7if zvUdLMu%C&mr|WUM9-r=A4yLMpsebeEB}=bY(S$=y`Rqc&@w#i%31C{2^K)I=ZSj-C zkWWTcSryzaI&q-7_v`Ltp!8|jg~>bf$x*wh`qi$$0BJ?v%UN#L#7kV*;svrT9R7U< zn3O6HShr2ru={67s%FY+)znJ6IO8QWAh&}l#UToppz$e=K@fQQSgNGAZJQ&3LLLxi zpybtZ_E$L(*nJqc03=VBNGdNJ>jpC@ym`#_R9g6HgTPT21PfhZvrpxgkxI0Bn{+Xe ze^GaQaDVaRZ=ic)W5yq-yC^h+95z7@R|cFFxpIBO$EzE@PcEB!az(oHm{E4_+naWc z301=?a;GPE3>5Nj@Oth(Uh0lVi)Kl-*EBzv7 zxR<6-ho4cC5e0K5F&?}Fa}HJ;84E86%!0Y`GGmtoUt}XtKBnA-Rt-ZYM}L0Qq*RbR zJXWcC$o9?vc#9dYM_4cjKbVjhW}+TeMw{%F1ZTy7R5DQq^+`Rcu8|#cG{kAlAfj;6 zBO5Sw*%nL*$%(q4!Ad*l&A5Cu%7@PEMw5CYo4DdO{S&IfK{f9N1=sF`D%Wi?AyEck*}-!+67vJcZ~7oc|@u3(SdI3=)z0MU--d3nN&&)p2Dm{T_&lLE0`vCY`tEM zG*x%EwO5yOXz=D~fwFX*pPBzKGz^uENOetbx58kU_@%4@Y$znw8w4nD4e-dC8K1!G z0kSQbA4nLyCnfo9klewFEG-oIBJFaLbnCoWSm+q$CzBu`d859BG!R`vIu$JHGH0xK zy^guYMM6DrRwU1Jz+>H(QmKb6RwVjP`cr606Dp8Vm$$|crf2rkQ$jMqqx1uy%t4?XEEc{(Uk$Q8n=hJU`%{-LuCSmd zRG+@^q%>#i@I^*uhf%8eH$D1?A8K}K6IY7EjK2TuHC@vmFiNKrexzYo)KXbVB`8k! zvVz#)z`%5a3YCN&4$`IQq8X00ChLR}!*7Q76Y!nR-a-V+@`*jLGnQ+@T^gKG)GF+s z%>Wp-^G1^!631&UT0ZnuzvbPa6i&uABO!3S-+8-L`;~%+rk!AhA_EqtYm*g|HVG<; zM@Cvo^Oa7xvy4!wRT+tyj{Z=H%*^#A^+ZAnSu_H&FJ7>q^@{~88a)p*9xfP@ZX`2a zL*Je#8#=pwsF`b(9HA$|14WylizFz)K&_ZSD3Sv!%0<6Z8P3R9%@=xTG2Icqn`!kg zQ%4XFyoGmztlTscRc>nEXnAn+_Q2RT0l%n8>91*K6*YLR*{0&AJ#yHlhY0zrH^rr< za{L@+Wqt^Fyj!GSk94FE2YvCzezFIt+L$o6=V|$O@?msiHQ%cSyVStweLzT*_Lrf3 zqsD>8^t^EkE}7NO1g)9Sy)ZEg7WNXZ2u#$`!gA5uGs5MG=lL_i^4LiFy9M_UqM%%r zHr!FKk~~uXZZi=P16M+m)Kh58do(=W`y0`S^81KtJ@bKsD4mJT(k~{gN-8L?RX!GR zjCm^pTC$e+NrDnm0hV9@20#@CJz&#tX@+W0>6uS4K?6D&1P#1s@5aJ+64GPnElep) zs%_Y*v}Gd(V;5~-v&B@}XjlmA+J@pjD^^ZE6AA5%OOMyl;<&+pqA%v;W2mPo)FX>| zb0)@b3pPIVMA&kw5MstuM<#+kJ}G5$#u`968{Eur5Nnm608MMr-Y}Q%R9%ynW{vg@ z-1IZSP`{L>IGxvcdIuK{C%SqXr>hYhl-@I1PKTn}xyj`CEL@x?k07OuoA;}!U(jzn z&x)UZ6ux%LF!0;t@Os>y+0S*^tfh+$p?8B4IK1(MfN3=%d;HzoZTUh3k06GVpe9c= zL8L`Az;L`ZSs`?Y96i)Ds*)Jb0ul;mM3E&>*akGusqshoDUZ@-)1<`*eq$WrL@#yo zS0ba-%`L_GtLO)O;z6JVL?o-O6@#dm%e|JMQxb~SO&}C1 zijjd5k_BdD7yxChbt!k24MChqBlQd5rRtEV%-rqU56=q;A|h@{%^+(rni8XdWG&tM zU4?uB2`H#JgXd+YB<~BAce9PcvRy~n_&Ed%Xy*hWB)2-*oH;<4J735<4>kb!TTtXi zW30?HoFK~uvSxbPu6H)h)Ls3Vu1-k1MzO%_l}kOWl}NOD)m)``siA(`yvpjsSQyvKug}H$LUaDY0~1X$@-wC*U~l~#aV;-#>%fPd;&tWh2CiRv_P`2>an}gEgoi` z-PytZ&2S=iuvNmvra+7M0Afj@TWxdw z38*4<^vo;Uk-p;db<^33!|Z8wk#kG&8}#*Q8AJrtRDy{Js;NZ$1)GOo(u;!LoLxYO zFX@}sd<4`)a2i)d_-UWIZXODRz_{pdn8z1}DJYI*Q zp2nC^A$5c?$%r675J7&fYX6bnDXDp!42klX91J; zhNHYRMS`jWn8-^Jh6$x6J#b4-O=wIS8M~$!@zwan9bhH|S}_EZc4EE?H0Oe|QUfD-7{kF+o6=1CWfJ8;QB#hK;+ zIoAEl;d?`j!b8C*eJHH}Ys2VGen^jO=S^ph&J1NEiid|cw3>*P@sE;d!mMOwuS>UuK0I!OxB1#|Dz~DZ{NGFKaVxYG1WJb}~VWrWA znJAwIFgRzjoOy%_ZRi*q4p84EP<0i3*cO{i;@6T!Kip3Ru5NZK>(k$yTJ)|!20#jD zgEIPXJ?5Wuf_co>K^Z3!sK5m4KE15?82WKgRy-19eguVpCWM(rs6drNLJdw#>iTzE z_$5~w=--CwQs~l#-a%E*+@umbGQ%DzWmvGM_-xM@9g!$Jwr5k%`j7x-CUl-VKR zI$n`>J<@6-9Q0^BCeeeGZ^E5z;ZumV$h0PbVhib$LmkOdX@o>&63|@{HhB8Ttc<@+ zf`tQ#9^bW+n1ry{@RuM)MVyPQZ3%fcC$lEx*}Ng{yzhZiy-IbwB}gRP0YzGSWv(pm z-4O7-8-TcSCd%rujh4gAJ6Uhd&D;pd;7^%0&+o0iv-qGRSS|lZJebD|C8+g(B;&oW zNJfe_J|=j=RVS=bY3qx!zj9crd$;~|>>p5$$VN8|uIQo4?*i(<$Z5QJBsFSF0G30*YB8 zO$%IHyQHp+i!|%K7)d%=*apV{sEn(aJ-j<;`?-i_?e?ryZiS_-a-)6MOnw1N@5;Y# z?YwM_W5ss41w&lXo~K-!mA`zwOXgh${un;pa}Zy0+-?i`@NvUc-Znxw;P1t0;O*a% ztR+}am&>8bfwAwH-;Rp5tXx2D`=>jSoQaZby>}Z=!c2Ix)?IUwfBeE^b?3Q_``jDU z8k_OhUH94)M!p-^zx*Adco8c5W&n#T7+CpLK&Bd$e<6dr*z?x-J38%|Zq|{I*=XmB zJf*w+c40g5($jysATLE||4XR9>0KGGjE=8vC1Z^_faGhoDj=hPM#TWaM)*wev%Ilb zRm^Rmz@X7zI#0qc!umJbXl!Pwil_BlwULvr3{1%>3G^!#OIJ7*q zqZ-XFuB%=mri=>I#L(+TizSWrCIkY(5^apdgGL&v-%SJfl^I0Bxi0+;ZfIPLr046< zC9P{$>P3^IPt36a=m22ojwoZnE|3_jw;y!^3FnIutbZZ<&5}t~PmtLa*?$+xrNk&Q zhU-8lc_sel9Nl@T)$MY-y&ukay~-!#t>yDsd0HJjo5^JJQaMA3?`q7jw41NDx`!Ug zW;*j79^cJxVZ7QmJahhv=rS-=QpKC8Vpk=;_S$Io`R3qB(vYXyD&FFkLJvLcwz8(}FgPllP0P*1zs(3;qn4-&=r(%n9v@2isb@bO9A z2osVxitQcq`OfK<~l&Y(Q($=e{hq&>Ksr^jih7j)2=%rH4T*$b^1RsjnUG8;g`(-zOgSTC8;4@uJz6aG; z(3X%>3#hHmJ$V7V_4|2LwRuqWz2xkl@|@S)GkN++voTRsw?ieSFB0S&cb3KLokdy= zqCq)(gXM??z?seo;gA&3Sy{fg93>|oYUA6sh-VC1%|D`TuF!h?W$#SLvZaZ8|EOro z##ikgW=fN~#rotk>K*BPX65Ti_hP{}-A_IS^_l0X8Fx(fQB{WoJ^#}3Hz{p(nykSv zG@XT{OZaXcw>jG}S?#-#0j9Rv@ArF+$)dh*L$hAPvH<%zn(c46y_LzF6hEO=fzn5d zHON=5{FG9>XNhCJOMZ7~pRQwm>g4&l=tJx4VA%ND&&zXbRd{pF&h{k3liSMzuJZkz zkI9mMpt;Ci+~2$g50+E?GG+GX@txh*CPk!aenx*{h)x`|uY{C>H8Qx2UBR;qJgnr$Xf7@Ea($i z=YJ-dmc0R@FvG}N${ktoC!x+Ml9N{;N5U*onC7nFkEro-zcTyz2{igdCDfz}kwcNAS97XCRUqw~u_zhFgWvrdwoN87pq#^4u>ccJDt?tNWP!@tj6^ z>w{Rt;&oHABds}s)VQg-)a5g_dI$RRZ=QdpC#jt~pywCXc2&7 z#?Jjg;<%&t-mOuKNz6HEse2;bGhM zDYYMZStHgK)|93;*=LW|4vs?CZq-fV)ymRo|B@T)6O%B5Kp+ajt_p(WIq+x`fk2JR z!uk{>G5PlE7|ENWfG{bQC0Z}?Niid%rsllZ(;V9#)dpJ6MP51IF-v^H>Jzt)Jo^v1 zz7{zbQTwAfs&x)^N)Nx7al=@Z%>4tfpvp~gYi}9zH{_{qhlW`3^)FpHw^;9`En_G~ z^8@UP3@I|5w@%5J_iX}ku$p8uv7S-cY3a*voSmeQV{3IMAd2*PaOZdmB2bM=g?~HM zn>=!v-PX7;`fF(BDn>5S?9!{dFjpai=3{KtWQlxhi$ltnn#~43X=|5DZa~6eDbB*I zBDj1d5^wsjpQq00p8d)`KW`0EMQkNuim}J+`l0B9IL)JQJ~Vse?^WK6iM3Otf_9Ml zi?;MNPbqsw4meieV_w9zmcv0oB>v{wQjdUB!TBE3IMC=X#X(H@-#;7}jEohS5wj`7 z&-LGS3I&0#*7tw6$Y&K8n5;Uu4dW%N2o}eb$qM~vorh#u!PhW9WmUk=7{S{DWIVU7 GF8LqooBAmL diff --git a/summary_results.json b/summary_results.json index bddfde487..725ce2576 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1431, + "Performance_Result": 10.214, "Performance_Units": "Samples/s" }, { @@ -1472,7 +1472,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.1629, + "Performance_Result": 20.915, "Performance_Units": "Samples/s" }, { From 8b1235f0072f2b15a697afe7e1898122cbf43ffc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 27 Jan 2025 11:11:07 +0000 Subject: [PATCH 19/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13902 -> 13903 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 483be9ff7..bdcf9aa25 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 00:17:40,591 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3266 INFO] --- -[2025-01-27 00:17:40,591 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3266 INFO] --- +[2025-01-27 11:11:01,750 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3282 INFO] --- -[2025-01-27 00:17:40,591 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 00:17:40,591 submission_checker1.py:3303 INFO] --- -[2025-01-27 00:17:40,591 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 00:17:41,934 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 11:11:01,750 submission_checker1.py:3282 INFO] --- +[2025-01-27 11:11:01,750 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 11:11:01,750 submission_checker1.py:3303 INFO] --- +[2025-01-27 11:11:01,750 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 11:11:02,803 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.214 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.295 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1eccde110..77afe238f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -508 +509 diff --git a/summary.csv b/summary.csv index 9ced5f765..4005818b1 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.214,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.295,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 68b303785..338598e4a 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.214, - "Accuracy":"acc: 75.000", + "Result":10.295, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2332d0938fb436bcec5b484a5870aba40e9dad6d..0ea074ddbab6be3c80a5e8c7c6e1bdd0043bb5e1 100644 GIT binary patch delta 1880 zcmV-e2dDVXY|m`4zaa$+!8U2=lf)q*f7?>yI1qi`uTW)QW*%(I@--A>iV9(h8g84w z*6fqZv6bK*U$T}Q!r!lq4Ft;_LgLI5MYrtJr`3HVr;op_k{B&%&Y~FFx&A({qR z7^U-*-u>OV;`R{Zej565meP~n6J@<`gFlWL<48Q3vXkBe=ZhmvW5EMWe5Pg#f0_!< zEXxz0i+}UE#uhpCL-s%^PhyQ=Y-@=hr9Bj6MaoZlHgS5WNTdG>I;1bMSX0td-Y18~y zLGy?*O+uXb$s%rZT%5H;EO?`fe>3!js&f^I#G%t7jQH2)%3&5~`49Reffdsv;#`jU zbm>R2KaJvu%MU#pwz!F+tDgmmVMD)NFXokeqrU6tyXzP_@#0(87S+H|~&Nh&14BmOe} zihj~8E&o0`3NicEPbr&!Uz}Z>qVM!P%L9tWcR!Jd`?$I6g=fQFHu(`-h`gQjL>IHN zJ9>lDf@h+;X^5sz=#>~6e_o-bp%T?nQSX38$7FDmk6eT&OmEF4)DBXujaVTN2e&b%g~1F_o#$f2g*_9>9pA82x~k zPrbnfZ*a=8+;+XuCECV4?HX)TjqHr&bxo{lX(blz!}zn>fVifpiw15!_oX9i3%D&+ zO{>mCOHzmRdTWuXDuJMWt&x>l@#+kXZvPn~$FU7$o60cDDS}U&Dmkbs?hRgRqgNz$ zU!5bWq4p)6+et1>e_1Zuwi()+;pN4xHyPfHM@L8(yj{yQv2Ht#YubH;&M(ehyu`3n zyKiCBz^;z<7KIw$+3PWEtY~4ZV?&inndg9%V z#@^)g>h^L}K8>XW4_ek4CQ&g$bC$TjL=9D~jf3aqjUMO5g$gxxx11?*| zj-;tMPdq=PkWFU+{lh%(Y|_zz(d}rATYLRIIffc)t!DQ95u$>A(@C14on& z98tQI&e{&}B>-On@Ff6W0`MgOUjpzY0AB*|B>-On@HGIw2Ef+<_!OW}P4sEO*dBDYgT4N^!G391hHC`7k{o*xtbIN&pM~r|l|O{+th2X8>aMka*LLlju-8%S z>yJ6okL{47MrfIVG;g{ejY(Sk?|1CO1prFDsOc$E5qp$UX|VHhGe-3$Z5+2qxm??2 zIae>&L{wc9H#`f0H;YorSB`Nu1pWFK1)hwpqIvtT8{upX-;mVboOWepMV$T?$@s2P_y@m-ze=7uPT! z)^WvK3F5Jf`jxkFkYgQ)BnBvsaVO7`E5#Qn%{3IE#e{hSKsoY{sdHG zlYbK@z%lpbC#e4bvvDz;0|g7gHfiXS5;a!=%adv~IsrYCr8Oo5Y#c}*lg~940Und) SH9i4ylO;AI1}!rH0002H>X-BY delta 1900 zcmV-y2b1{EY|d=3zaa%m+?{milf)q*e@%0vI1s(>uTXg}GY3Et7;q}_RLXItQj@Q> zldaj43xn*i!-p3M$NBdw0!bW1JC2R#luEbY^=oxM3HR~0RUDxu%XpY3Cmo{W4q}O) z24ONk>D=F)Yi z^smx@MaPEd9~*_CG>(0icv+a%?e|G2c-=T+JdZ@(nlYX*#X4r0Oh?^%Gn++W!s_O~ za+ZaR8w%psix*Lo>a1uUP&p%7}+ zr%Nx4ylEJPLVf6o3l`U{*D}{i%(q$UGtN`x{fj>pMBir-%RHej_LRlrbS56XOv&nX zD%2p(n2=-?^FRh%w{>-_VT?)L9(cl&Wg`=1_)*GPU|d}e(l|~NZpL%BAJiww1CY#Sw#~+O=jU-VOOR)Vy~~PuK*pdkJcUUVp0l8@`%2S zzM`KjO^Uydjz+Cs_otH0zc0=%PSJOEo@PEnqr0ET!aZDH_QJDfFPi*_FJ#_MIt=3)q!CtLzS7cQ+Jl zp{i@;YEH4^U>Z!jcA#Z!#xkke?73hUPqO*^hsl!amQFP8`_$5=e;#MX7JC3A%3}0< zQ9Shq=fA;e%VOI}txJrJd)hYGrW)B9YtlBcvZb|1whtH2Y6Ie0vMy@4+1yi(tS#WS zR5hbK6T6B!tk+wEOqB@)^=pl+(u!ASXn6b206C6rBHPjiX~qzI;*`ljWpQutb~k!u zV)xZKs+)RG(Yc-Ef6`LrvT2*4y%}6y+)l=Wo6+zH5y{)_S{5d@dJmjZk#liMX6e+%Gi0emfhuLbb60KOK$ z*8=!j0ACB>YXN*!@nylBm8Wi@>hc>_JQ>xQT07UpGS<>0SxuHiCq12Rd-T2`Y`(M1 z>-Qt9`8lDM0Vvqc?pk~l=cOb%2qla>l5_WYEu zU&Z6dFTE3gf3bLO+oEMNd5H~Uwr9sGVDI%IU~D08ju~f#Lb&SHAGo0GhPFnlB|&Nf z5V)GPf=IF2K?kMS4A?5g&H8XSux`VL=>fs^X7=tg^swi$`gwmyP1zf5k31e;!#;v* zU&LBLY&WOv%UCOj?UHO)$UcGAf|7j-?cm(RtN3%dOY4E?_u@4sjDAl66Cr@PTVZxNb&dbdhRhzVN+@j=iZI{(t zy<8Jfe|b&Z@GJygTT(Qj68{WU`ww_j_Wom_Vw$9mfp&_# zrh&`L+{mOl?Ye7oBU`s^+j$!sQCTZ4wh{V1yq;VRucgISe&pTcdmZhlL7bB^yt+L; zy?uV9*Ig?SgLBdUA5cpJ1QY-O00;m806zdBv)U(82n4l}#SW7&Fdu)-PQx$|gzr2> zqfbB_xVK+$?+BTEsuH4QbyVH)+#W>WSleX1|?XxteWtoBe_h%9}9m63)3M1Ej#vf(8=< zrUk_(Zv?Ny)rMXl2{u59ihWwj&1Pr+^KSA>cP)h>@lL0mwvyd@<0|iRlopk4u6E#->%9Cp~ mIsrVBrZpx6wUNaRlh8F50UeX)H9i4xlO{GJ1}rlG0000Gz?|p+ diff --git a/summary_results.json b/summary_results.json index 725ce2576..e057aa1a2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.214, + "Performance_Result": 10.295, "Performance_Units": "Samples/s" }, { From 0c1d76c14e7018d408d198ce37aa178d71161282 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 27 Jan 2025 11:31:02 +0000 Subject: [PATCH 20/24] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 13903 -> 13899 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index bdcf9aa25..843d31a4c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 11:11:01,750 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3266 INFO] --- -[2025-01-27 11:11:01,750 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 11:30:56,691 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 11:30:56,691 submission_checker1.py:3266 INFO] --- +[2025-01-27 11:30:56,691 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 11:30:56,691 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 11:30:56,691 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3282 INFO] --- -[2025-01-27 11:11:01,750 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 11:11:01,750 submission_checker1.py:3303 INFO] --- -[2025-01-27 11:11:01,750 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 11:11:02,803 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 11:30:56,691 submission_checker1.py:3282 INFO] --- +[2025-01-27 11:30:56,691 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 11:30:56,692 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 11:30:56,692 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 11:30:56,692 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 11:30:56,692 submission_checker1.py:3303 INFO] --- +[2025-01-27 11:30:56,692 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 11:30:57,901 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.915 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1187 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.295 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.86967 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 77afe238f..2bc4cd64b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -509 +510 diff --git a/summary.csv b/summary.csv index 4005818b1..88ea993b2 100644 --- a/summary.csv +++ b/summary.csv @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412174,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.915,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1187,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45726,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.295,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.86967,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 338598e4a..a4ce29693 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.915, + "Result":21.1187, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.295, - "Accuracy":"acc: 75.800", + "Result":9.86967, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0ea074ddbab6be3c80a5e8c7c6e1bdd0043bb5e1..875f3a524cef1866017ffeebccf49ae4d56aae65 100644 GIT binary patch delta 5959 zcmZvAXHXM=)OAAdgx-;a1d!eZL~7_ADIo#^(mRO$0l^^BdlGt)4$>j?Dkv>f0VxV7 zRl1-e2qHewci!jIvorhqurqt+-h0lu_wG)6k9#+E(bD<-=HZrO1OOJw004FX)=`!P zyGpA~!nxPH)GSCBO@v~Fe-G3OQbOk2J$0w8*gX5$b9LNddzAjCOUPAlq!h0wC9>!} z^tB{gB4sD*#&GFH;DwCpHG6x>nUfWBO|;dM&KI8ljepF-&TJTT`sQYhXNUX>z0+)2 zpIJ}kLn|)c&U$0y-W-1G`&XTLa~lIcyO9*~IXr&1<-+SBLi7?<`aR@(*3lWIqT3&v zk^Hd?FEcuyAF3lA_V<$w?0Dj>oG_yS*TxE=lqG_67rff(VdsZV+KJZ7GK>uV*TxV= z<)p!luoIn)b-He@CY}4CUB}ojkU-NT@iS-RGiRN{=b9M3K`y&gZds{B>)NC+FO#QB zUYC-`_UEtGPfrg~9jb_@Z>!Qmk(ki0E>Z=wQ*A2yRyJ;4{#7ls{mm&=Bx@JqRcjtt zt8@!vZ?E7A?rG-Kmf+4~M>syifK4VsFkV+#555u4&@c02Eq)bfVsainYqyaTdIEi( z74#AAYm&1C&|}5}^|d|=tp+h$N+vuqgpmxjB$W8(j4Gi_ClnGmcs;)M^~5s3RKY<~ z#6p-jI2sO3Um%fA6YNh8$mKMS`~ zlkTum(?E?@@TqFzk`QiZOJA{qdXKLh29O*_!|S288=f~Jyu=<-Dw^2SA6~-WduVxV zbGJ>2yb}FmvF`G-j6Y;339~rR1&!*#(46Q?_lvluS#NJVH!4Ty`0^cWzzPHVlY(ahS-xI5 z7H@N`tluVQ9U@1>^oEZ6R4jw<-nCEu#LodcW_hGXfroMMWjROKPVjXO>xi?e1>QmZ z4V14P37BeeE_|I~%Nl9Wc}PLjRSPQelIL2DwVkYU5mz6~{gJ3^#W7M~JR}xbNp(Cq zuGy(0t^YCT2m@j?kX94L1Ddkco$68%5HUd03!-1O0FYPsAm%2Z@m9{Vf&QU8uwcu5 z;K%CNl6?9UxoGr3wgIT1P2cSUN*dW|^_uRhR`D1g{WdRN$Ol?57!Od=Bin^?6fw!@ zx*e8c-TBliEBjSM071hlN_ggoDfV4xUi5SoZqdwz@6R8h1*xnelk%pH{XBuOfNm|t zGQQ|)J>fb&%+3GuozXv)`yqHhBG4YhWh#h~yqw@es&}dPQT?JiBn$is&aGKgDhukC>$vH~#jU2^&a&m&8MBrKe2tsVe>5R6 z$YyToDCG5CVWxgM-)q=3XarUbtFnmE)%MtyXw(ZvJFqIX)}%#?2eXpJS>u z{4D2L^pU)hcgeGba?8wFAsn#c0}LpVV~x6rCPI%a%Hc8}j0E@JNQ|-FA z8@tw3%s9&h6NUOzA6Fir7T5ypMfSBSk74ByD zU)TTw?8ZAeFSu&dB<+0SNB3ejm`%FEsa+ip%D*H8qu(3noZdq|0|Q6Ns7W|+L||zs z)=w|AT8R4hn38-^A>@_bc)V3V*ohyl;hGu+vNop(sCy6YB42E1jrYv42UUd@yi*u+ z8qqQL;Vvt2$LJUQMSQ;q14HH11f4keGCCF1s;31@SRyP6O@n@o-!^wKFm8wVp#pKB zFenTR&tTTD@Om8c+AR6T!>aboJA|J-$DHjLHH3%_A(=oSJb&cYgSr+Is1s>~UZGCe0Doj=KI%yRB(8{H1Wf@=Wr$s~ zPRsDtIfzhG2T-OZTKB*z(={b_!Xs9<%!~l33zr_#MaM;f#sO%8LJhtFApNsD3{6B4 zjdK#Q-tEsM;F|scnLjleTU=0mAWK<%&_brTVz6=9GOZ2%~XO4%N@+#L=Ij zJVEaNgA!f~Slj;|OY>8c&_Pzou0cTuXkUMbOZht7)AKBrcl7fnNeKZo5JCWbsf_L_ z?DpWkPE$XCVy_87iu?vMYRD2_fir=!@ma*~m#1JWz}wUJn(eQ8G?@;;ZOb5|_jNj5 z>mD+LEY*f<_me(Xpg1*MSJ`nD*uj*Moa}0;GTd!>GZ}oD6oF~|2XRqplYj?tE)hlq zrXA)4X_|=oE2i0`*Qg0h%R`PIT(okS`;=Ju@f{F{&R+kl{QamLNk*CjhWoB|46U z;6o}jFq#4x*v7!eLIatlZgT!~MT+pnsGsso^dUd9MY)JJS=7IBlHwuHgzLnYX`u!+ ztdImpBghc{Jcxh~yZzJiG(_6rpPuusIIA@|BW2tp5Q9|fTBXTMiBTv2;mRyvsY*|L ztM585&T7I0&mGBM z_*lfgB`^WTpbUkHptp>LOLSGnkS|?))pZ39bcMHZen$Ihn7Eth(|AIvg@jb`fI>p5 zdanEK_%air5*C1dp_y~P5CP7I^f)?3IVKVuxqkJw%ir+uOBpuFLU{uxi$(K)j@{P7A5Y$RZ;=#RVE4D%r^W58aVy8 ziYz_rME8BPyTEy1da$&dffl{vcl3YvXvcU3Ns2DRAV{Fv4@xc8 zD2O{@d)RUzEKPC}l;A4xv=C12wsvuu%D%6G7?v7hshqFt#&e)l?nzz+zZ+j>H>lCn zlBi3eN86Z)Ak&2rYnY?LLmBD-WtUea%lAK`sh%>tr&{rQW`Advx{78dQF!x{zwnKR z*DDrvqxD`|+BwiiSLKP1+_f@yh0v!72%$`S21;-S(4Ju&ka0+<`YjJpYa&V*GWvx! z&I6+V$f@T*r-fMRCoZV}w!aP&Yo_T$je6BQMe8F2J|mBn4j~3* z>AL$`qRa-p1nVsVdI{FE7We{aPU`U z%WE?NV98wloXu0dM?zW)#mqL4N}(#|rnD>GCc;Yl0;o?2^TlC^YDDGqO@e|<2!ceZ z|5c<9=^-g+B*A&9!cNg?OSxL5PvU=bh8JhzNuMEGlUh7sO$Rg73t$eLk_w@l98GEO zNC|6q5t%MQI7PTv0Pv;CI3YmO@LV+ML^gW_5$~V>r0J?ELDwZ{Z@=bCPCC$hsVR$t8=INSI)Ov78VoezJR;l&r6X?6 zMFH*>y2C{P6)H2Mk?tga7nAxx8wyF}0ZtOC34 zenE@>{e5Tdea9b zQ6~0u|K*xB2S{-lxs?uO)zlTH&x=q0>fYQt*_kgA7c51ICl;bOuik}xn=AT0-az}? z;6bIC@oxd-ly7Usw(3&j<93P5${|P8W2M{)uAL!0#hI6uedtN;O1E2o>Yi&V8LUannK}CNfoXTsNJS>{hNCSEyix70s`#ZwH;DnmPH(fu#B?1wlL5K`uExmjFZ|VFETF<1+t7-wdyl1le%SaXmW^`c;#`DCj~jkFC3z=Z zzkTh8*CSQR^RMle+{Li*;A{awo{~*Er}#oLM`_i)l}`m;zT^SxuxD)Ll(+Ykr9NIP z{MpQY_-w0`Qgu(6qDeuT=8J0M(q?EE$|3%AVpjAl>grY=R~TAKVvRm2UO`%DxQ*e7 z{p|P$aoR8&p*0e-dxI;wST-d*GuS43;pkjX zD0o%DNjVk1nk)4}a!AQF$53ZAR3=Z$@%8@aSAm()NnBJLONne|*v*YvuoIrk4wYXl zVOV&eD?nH9SzV}a!J2MXX!2>JAgO5Ka52d9U8R9UQjZ6Fv_|-DJT>cL-JQrn4%LQcd3j&gp`lTgeQrSj} zN!7@8-PX~%LdmJ2p|%0Q(~!ErbL?bJrS~naFR8DjP}C^lRceZMo&9+wX7ZK9l(_NC ze~(|ds%VyW7mmKZV|)MXb+fBP?p2(G^G9PPTw=%!Pr%M{W3)&2q4ksn<(=QK=hjb- z?0o}DgGYrYWu~n|ON6ka)MTc!U<);CzK!$9 zEAHj6wO=tFex5gTLcxGvK0>kzv~6}|u4?XHZ5$36OxF9JcP&Xb8Ay7VzMPr-eGXRC zke}Ip%oO#4_LL$gWm9=7A@k6*F zEk-CNLk_Poaux|V=(7OFl*$J+L%EhA#X&q^9Zx|r+qM)Qt-CS;uLr$4ej3>baE6BT zf&R#6i{L9Fcn~J(x+2}?wHZ9Ys=yQ>B5pGhg%S#XIKx^TL|;kdc7GO8*v|?|3Bbzr67oxxVg=P42alI>IQx z2Mp;fB(a%P0%uq04flsOMfX01kH!%qaYq<$>rs_w3z3*`s-v})!CC;s7{%0JRZ~Xf z>Hf`x=uV7Yn#woVM=EM>!pRzvKYWT?^j-I(p4XW7qo0?q`4mVO#)>1gyawS~r#`Y) z^)6i7Ew}o;w%2KDQ2i{V+NPO8RqRM+{%Y^3tboVKt4~ezYyhG8k}g3L=R?QW=>t>a zlkApxEdeVfTY)}V^R~UVOlGJMqm>g;68n4RvPF!4Xe@ux2U~0ve7<#d{h-Jxw=Ct^ zWz(xXXE25qh(4*U0x6Q@3s3Zhj4M3*@%d@F5&GP``_^oQy%G^uzKn7j|9NYfNc`IsuLy-%qbzuLc* XV+C?zcjQ!nzE}ZyezKb~e_#0@<(BA5 delta 5934 zcmZvAXH*kw)OA9ant+th6A~#>1Sx{jL+>&4CK!}pdj^Ia7brbRp8dSH`{whmciE4Rdh%CQh0V<4?e7oT z2+GADFNQ`QyIsqalLJ=XiF~&C;PO7+{cgd8i;G2bo5$CIKF@C@?S7dV8S;CF2M6zM z?l%Wpd-?y7UVi(b{! zVYL;kOAF8Ph|RA%iC+#TUVfNJ^F&>U6I}foQ#dgE{!Y5r|9<4gI;Z}#?bEm-WP{U zk6)E?>}}uuC17p=3@4vHTxQH+t|&uQ@VfGhPcQg>De@!Vea?_!JPW$^<%uyU_bZV( zSJ_L?Xu0e8quc%8Ourc>WSk1?r?&^KyEE({p>g7IhS%h2LQ93O)n8p$Uw>`UYI5y5 zM=oxMIF!}9B}oz#jYFtvRrCs$WF;3e>(?h&#NVr6ZG7(bX>l7`elOpULC)$uG9E`-1id0gBsTyY_wJga z@NM7iW82C-+;fVPO|aa7yEa=i>Ve=^0|n#A{cR$oYHO5>3CN0_w0_5!EQVZ_4VOhBc}T(*a8`cw|y^K+yCz3{UcC&81rT$O-I%4 zH{()JJCSwAm;o_dj2xHev^|jfyE(ZfCflW!G2{Ud*_`u|>uTt-MRl&!ZoWiXySsj0 z)|fM=j*AgPC>53aa&Oh)H{;w0aBVKlX*FnPLo&5d5>$(OGmwxcd=(!M?Gd{`SEZ6S z#rUc9nFmJuwu7^_ic5o!PzsV8I61-ot7~>&B{Bt<9&Ymi1xIG!@J>WasuuLs{noGTh;E#xj1SgNY4?Eu)LDcOD~a#xfn zx-|;kWjsK8uQ)Q>LKiOhopJnl@;+ew@12?H*17&~3ybc{tTGV=)5|z{e%_R$_xA68 z=!CIa_Sw3teVVTIfv(>&$k;)y#T8P(bN z9}=O4x&iMPbwk-bh9KFxK9on>Zp}ew;q4)g1A-5SimY=C0fT4KlQm_8-wh`= z`F}#3RIoZ>MKQ6(W(Plme-57@hN|>+QDh#@K8Zfdj2OHvC9S<7af!^v6gr{77H5;% zej>H`>1-FE*#5>``UDcvc_F;cu!^zaA1Z0R)wt#sKv~T)xW@R*BN@JmaaKxh^3wBK z_P3mz#Kl~=q7{ryJ`-Y62SJ|+9(XeaQQUZ$;3#uP>9Fe9Zfy=nKI*okwz9`> zSaEY3e)H_sTS;q=>skuJMg?T{3y}$&k6!~-&9vr<7MKcNYURVCt;m=FhBZBD+5jRL zo~`kQU<0}m5k0yk_jMs%Ms=F`3fdQm>D^3&EE+SzYm}~{zb~9LD_hBovmmT%N&E)( zn@5I~Ne{8-%`g>QR78$bDq=Ci8d{pRSb>+(@ZcETM3cI~74HY!H>mXOTL)~Gbj^99 zQ;Lx>=^2(aSE?y?7n`PU`TH_++};i)x>NSAg%HN5mr>iV|Aac>Z;LF*vR@0L9t)_ z={^B4)Q3@<3JHQOKf-h{1W@CKmpe!%WF{ z*kf6o<<-7oS*p@&MrToQJlO+TEiLATg5f8QeBm})e{Wo@^Vi~`df1jSp8N<|jilpk zuZB47WM=*>5TWsrA+ffsDOpE(-v2!5`H^3u#|;8gv{LYbP__BRiL%g)?}`*&XJbbFeXA(=H(6%sxnETs`ye15qEl!S)O|E>A+RRPGR>pw$b$8t;|){ zbt>qmJL~oY{3_*flSyzNHKR?tp)PC^oIF<_muYk%iRy4R{JO{(Uj{y1XCu`_=`H6q4Tm>2TU8#B+W9r7I?DZ+<Ha*d2N?MkPtvT$=OmPklrT`P#;2ONDuw z{hfI1MNGvP1(iTc@ zGD3s86--ZJ5VShK0WTg&DB|o9oPQ@ct?+f~`3)wImW|@B{8U`c{qf>Hduu`bsK9mW zM9P=G+da)2rG0`gIh9aEB9sNNmxNK>lwTB9&$`gh!%gE9LHV<96vb#W|3hx4e?6TT ztgEa$DXO4!Rgc4MFO9=d87mQXvx^yB%GHvN5C~M3k+(vupRZ3Pn?uimv!unQaj?D% z0+2ed2n_U@==8qz)QcP3+9u}DX@(uA3>WURoC80rhKzUvK9uC`&iyA_88lpmot5mQR$~5qm@EK50idfXnye^mw{{E74{l7kaLb z><}g`_I0x%sWZpfvC*!HedGJss?SrvqgUO4u5*$5W%KRk=d zjX=T$+3YF^+&b<)hl$&$L!Zr9Dl2r})(T^l+)?kEm$ql7RTd4V^1ruxi-| zlO^ChDhI(sWHzU=*AFVAo+FH()1{`zS6fI?Nioi|#&t?j-Ssbe&7=lPghd{>kUE3; zlhXWi_!D)Fhkli}aaFli_uW$Udk)|q;QaQ8G6_lk`qc3Lk0^MTqL!jNtRvk^2VY^K zNIjih&-27-EB?W&r5>~|u^j!9ot(; z=N|C>oZAYNzbez(eSF+5|AgXiZ>#R_D6lN-7#TZ5`tE&9>Gj{ocFre}}8Ioz3-sIq_)zQaq~ITPo9D&2SYs$>=dJ%04k` z%AWJSH~`eSZTY$GTO_tG9UIkd-cSn`pi1=%F!if>C(?x zc|-AMbx&jfg~6V_;MSMNY;G+k8fLeF+K{#FU!X=_(GoB{OQ*8e8mSWw2WBy!uRtO` z)mnPE-V*j5*i|}@i)VI==l9F%nA{`{)RPy4TpK8Ye}Ae?WTP7p(ph}ZrT1cUylf%@ zS(ftV8Ie!MYp76KeSKYO8=lEv&sVZXn*TZ1vUmvj9LawYNEsu|b9ub-f2f%Z`)qhE zvw<{EyP6T}5OPMTkEuhW`m?8xr(!<)|(u{}mw*ELfbN`Go3UGd?6m(DHk z?{KFdBV{kH$h+?8;D4*}im})D4_oa$f`XY&_E8_g?~YS;Gg>;dC;_; z9FchRai!ml=k_tj2{MX%9J%O}d=k8FP_LP^NEqNymp3Ps9ylyH#2+gU)AZY2cm;eb zi{YbFENvNH?ta}AN*p{Y$}_QlHS|)j3$`4ch!c*-XDOR>Gxyl}&*~sT4;jXHD-d_6O5#gonmJSXYr9^(lWm^)R^T@CoLqi0M~5i?82}6*9K_gd+uqb z#ht!v*^IsY)PTqM*+=s>He3yRI39hwu+585YOzM<;|E*{N(wk9Fjjj9I=Xba zsKi^1g&+7mP|d-Dr*RhN87;_>Z`Mrq(o5%XiJHy}eL8 zTW^mVXYeme%M%#aAZ+2^pCtbdogpQ*s+6AcUu zdE1=_6zl!2Ci~1p#0<#|xwLoHVcY!O#ye_~4|rnvKm7xUtQ+)BTG!91=?pUj#jqM| z&6>Yac4F3bRq~zFUfW)Iw-&ZIm)tYIw7+o}gw;qYq2q9vgk7+b*SSM^`3B8qT_|mM z!zq)r?hVx6BD+X98T8g;d1+-GyOWD6RF|zbUO4?IRa}*Mbq^-~g=1yw(W7Mxhlo<;p}Vh5)1}R9=P|YiuZ^Fphkb%5f3ja0 zJcZgs>xh>!NK6fVo#6N}FH=)-^+x+YkK2-e{(Qh8@Sw=O6%|N`{>F~IB{QLQcN5Y3r6wR+36_O8%w$k3c^_emV zm`kt|C(NxhJ%D|ot9K;MF;9z+s982x6bNMtOAj)BRlY8Xfj&sbc-8F%UDiLi@R#5V!xpDm z4^>;ak7%qjm&!GkC>2UNn?$#!?!g(YFzkNWanh?_e75D;7y54I=M>7f5_+)p*V4ms zD_`1m?{+ME_~9UAyX~2OdTsui4VmxJM}LS<_$%ghO3kZ*(dN3Tv6S|#@Y*6FRl=QD z%(Y$?=Kgsi>yAFCLirxXU$l+y#NC+ad<6|fNYa-?fC2BfmmqrIe=--nlIbhpNZitL zZ1Bz>NvCYbC>o)kF%li#0j|^aIdlzDo?>%XZ*Q|oWiU=4-z>B%w(2N~Nq`5FQp0Qu zz7I-8XvJ}uvP|-^ac^`Kc|=?fat!#qFacwcHQX_x^KuPvEQnxw-Z8bOF0s^Vlc+c#&Q+qdym%WHP$eaU#?_5r&QX)S2!% Date: Mon, 27 Jan 2025 11:35:50 +0000 Subject: [PATCH 21/24] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 13899 -> 13899 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 843d31a4c..229ea8d7c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 11:30:56,691 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 11:30:56,691 submission_checker1.py:3266 INFO] --- -[2025-01-27 11:30:56,691 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3266 INFO] --- +[2025-01-27 11:35:45,395 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 11:30:56,691 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 11:30:56,691 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 11:30:56,691 submission_checker1.py:3282 INFO] --- -[2025-01-27 11:30:56,691 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 11:30:56,692 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 11:30:56,692 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 11:30:56,692 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 11:30:56,692 submission_checker1.py:3303 INFO] --- -[2025-01-27 11:30:56,692 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 11:30:57,901 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 11:35:45,395 submission_checker1.py:3282 INFO] --- +[2025-01-27 11:35:45,395 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 11:35:45,395 submission_checker1.py:3303 INFO] --- +[2025-01-27 11:35:45,395 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 11:35:46,493 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,14 +55,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.86967 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2487 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426701 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.429956 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2bc4cd64b..c0556fb20 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -510 +511 diff --git a/summary.csv b/summary.csv index 88ea993b2..b0b869324 100644 --- a/summary.csv +++ b/summary.csv @@ -33,14 +33,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.86967,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2487,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426701,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.429956,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9898,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a4ce29693..dbbd626b6 100644 --- a/summary.json +++ b/summary.json @@ -1029,7 +1029,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.86967, + "Result":10.2487, "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,7 +1239,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426701, + "Result":0.429956, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 875f3a524cef1866017ffeebccf49ae4d56aae65..937c7c8b44a04ca0c79c5c5e1b0fdba1d32a7595 100644 GIT binary patch delta 2344 zcmYLLc|6nqAOFnoT{hYXp=2|J;cH*gp~((5809G6m0Xj3 zf3A*i=h9*sIp)YMk)umVP1RT7o8qq>3+&hD>c45s5c zgsu-0Mw;C&jZ{SVC1*?j{Jh+QeF=Z&b?C}1Pxt1g*nHV>7_-|0%Y@}*@Y+*8@!*Mi zL9kjFhhjJh4q8n@`T?Sa6}e@5vEOC7QoWV+&T(j|ZY%Y1au>L03Oj`mRbx zu4B@e_ za>sKcIL=}$$1=8Ht0aU+Il-mp zU2TGIKwbwm!KJ|U2@p|K?zK5sg|nY;Ll@~LUb}IUIP!`9NT(I2i9bw;MjoiQj@~9b zc4w&v`^?C@FAizI5S$)j+4h1g(##4ME|XpSXP`N3c%uFoRFRCE;})Q&0$QsCRJH41 zgP`kCbK@{c+g!mglplm5>^C|O2(qx>!wZkW1$dFa2HBJp-2~@6bLV?u~4HGQmeHNRErMkhdo~}S^hR~O$_Y}n4ZMA{0rLlSkou6D@%}+-(WRXAMnl1gaiujo9$kBA?M4eo zwo+hR5kd$&Samc8|F!0#<+&FsPX{uh2|EaeczXD?G{eUC#$WESDt#83xUTR*&xzA% zEOCDWk(63W732NfNN8nNT?Rdufxgb#Cn{g`Cp9F}WrvQBCi*Fu_NxuCPaQW-FtA1| zH`L+x-4XU%$h%hWo(%{u41`q144Yzi+qT^52$%$}o-@zsPNhlQin&SfZlIXU0H{kl zkwDLsR;LqNEc&eD!3E=QiMv10`pggR zA)6Grp+BK?(#qIJhS`+r8nSNSsWxuX#O=f9F~&BZ>aILj+#gWy-=HzOET4)fWJpLy0~XV%U1@*x zo$ouY>yIiU`q~8FqRo=fNsm>hNf_{BY}9Y(7FS3ON^`SX85oF52d`czA{no3hHTFe zEl{!jAaP^ae3Wixcp(E`98>&S4i*s1>om$Fn!8ApGVbMxW$qaYChuuH+O+)W`SVOw27^Md-Q;nP3bU$Qg zV0;eb*wvr0!a@!nvx@imZlKY;^K`3yxPD{*a-#E|tyLSSn<8+jG!smR1!txWEt zO0dd>miV&Cq^L#8&JWbxr6~80n@q}3Iy?=}{G{v_K%jU5+?HTQOl^PuK?U4ZoKUh} zn?bdBea0=~UpTObu^+@)*l+g`w0Pm7>M*ra;zzoD`gpcW(;~7Psqc@f*lw<0oBdeY zXMz;l?XwHk6Xh}S9D$czi8gS6A4Yd(+pTRbl|%nvlnYcZg7htu3L?2-3+nmz3@mh> z+>1DNw)`-2aVUlqdLiC#Q+nu668r3hz*f)n$f$VSpRn@DfZl`cgS{^QG+!INwKVbM z@Ke#i*{`QB$X=al_>nyEYh`|hHFON)k}GTa!9SQ^QEWE+EV==+V`pq!>`^}jj)e^79xyuNc>+SBXR5}rTxbNFlYxA}1I{M=5b{Fm)c z>hZIVE6k3S$-d-YWwo;H74ceDq4x(`e`S8W^hQ{>#i%6Y zwygd-+ooLi4*Tjh4V;H=`pmcY74;>0PM8oW-=jVXTWlQ}Py5XClJ$??+4}IiQ@ySc z0+Ob4sJ~4YKZ&3dvp=Q&G_WL&r+DizoQ73N%BdBc+dzRMNkoFcbVEn91X0uQ{Eq746L;4tH~d!i)IQq4rjEoZ)&h^D+mC*l>h(%{Qqzg^m5FY delta 2344 zcmYM0eLT~9AI5*f=^|lTwPoCEKDx$xUWs=i4ovO*=WAS9l;EYb;qC!xP4wjTp_%PjxHyM0To%Pw z@%DDgK7*Sk+p}o;eaM0Z(00x(h!b#RISp>w1L3y)9p90@jYLd3S@Ok4*2e6CU=xmi z&U_#Jsoa3r9^l4KJ!GEM78`r+1Q!l>rx03b-5z0=xI5Nnwa|MODO%NvbXtL%`QUUl@DdO%`vimoe)G!#uOjo_)LIdbjc=0Iiu!-QS5|u2RU+T* z9=t;#aV5D8`b7))g~LF!fXNG701!u%TTNPY`f5ik-Ch}c`~T^Vm`{<18z0+j2ycRS z71q-&y9!iSLbcIELh5FF3iIXEU5QsW%>w3kL;=e_$m+($ObXY)+w?;wP2US;_Z?5I z%%p*U$1G!*EORaDeuBlVf@+XH)QK-q#Y4X5J=$BwX}`ZsK&^y0zQcVDn6oeDbd)3u zXajy{1$@Y3;H-cNghm0N%Y;mUIY8wY2|<|#j8j*k_U0ytQR9}Fml`lFvB-PT_I?lJ zjU}JU#I{hId!t-!*Oid=W=qKZK4AugTT<8*=#{ewIt9r~;2)6EBCvs4gHL=TOy*HxA$VWnVPgS1-tH{3C~A z;AmQyL(@lVXI>B{3d}nwrp9=2F&QNn-nlO&P|rTlCIK%Bz~~@%?#EP72!o6|W}-L6 zVU~iKlAj1vdld;Bg^Sr*t75HH1t8X1-FHmQR!D--Rpx-7I5?&ct)(S^=QFeRoD=Yq ziieO}Xj3XIjAIp;r40JfWsqBLU1_b}bi+0D9(&1a>3%2It~=IkU2}h6wuJf3x-XL7 zsZaC8>UPSdMxbWGupkUL%()~4jeOgp`SsU2kqN$C@N)QixEtP|y#DA%;m?QMAE8f2 z_iqsdY(2G$>^>3CsH!3z#6-cJY7k~R?sQ+d!mTLK-TsJjxe>wemd`ZEPIxku@mVL-;WyCn>@_LOukIQZ2oa|jM_BNe4OlChWtrWllv2!Tr!OoF6PkCB@s|V zcrhpfIs_EY8G0_1(;9=YRpvZDF+Jucb}gJ{n6NO7>R-V8yC(dqE4Q=l_;L4KmG?@- zYMOJa$+dyw4QGUYjFWE|^B!{uf5{&dH5t3n|AzMTo&Yzrr}u-yd*BM8SDQ-&CqoFn zxt2>OTW|cd#ClW_kEm!pRz6*sOn84QSj($YHY@38?0-U&Rh1j*2r0ULV@sjNuEoU= zehK6U9pVC)=2AU*(h)~(g@Uw=^!`@J1n?Z#DnomD1(>MJ!l&oS)xT!1SaB*#6yS75 zdtO@{s%7?}u?h~OSJk2&VJ(G^z+Fgwg+4`FoC^6_08~d9(h5AlIWGhcuL*=~syI}T zOvDf`AP%*LZ1kI|HpyZ*SdzFkg*g*yQIe`YnL;jk>geZ3#v*MmXyp}y4z6tx<~53u z6B@KQ42K~EH?83`xIv%nJskZ~-$qCVqct{JSN+cyldk$IVN;Ip)Ghn`yjfx$5ho|@ z4?c*Cf~TIl;_oAf<+1zSc#cXw5F%!ioPb)w!vSsvThDpdbED={bc5}tUvzL`2Cn0f zhtt+B2|KCS(+N*{-Uqags;|Av`SY`;5p6NDoV)t|b7$#Ai}49f+KWvALmiC`mpYpA zPZc=#65lC)ZBI*YpP!ieTq}CbUNTjze0-~X*4azTbCBjQc{aEN?GEbV&amn6Nejyh z*V$NZ&*6=`hbKOgQxBJj65O-IK@s2md_K7Kq^orxtNPBlh9>bJk6lK-c2CZHc>B1* zWotFpC0ER~(mYCCP*!{#x?;NGA1w{uzv967GqJsR%JZo#rDNN&^v+RkipTl;)^G3Z z+x4(6v`0AXD`oBcHzHtL*uNN0o3v6F3*~=?gIHE%F71AaXW6mI7?1EVjcg1hu5Io8 zVR!OB+*S*FCRsV)joUUxgGjl&z$s1Ya9#a9;wyt$ApUp#N_6V9cPt!~FD#uo+#yv?C0eid}bL zy`A%HchRB2{NH|HVLR<_ZiPy|pStSP4Z+@d`q38srgi#3@Yp6PO;GbZiaYdW=W`vG z1&y7PL4d691E};bp3w^LJ8w?zwwCf3R1 z`5QL*;ZK<3@v^%bJ#KU4i>%O)`t{NYp88 Date: Mon, 27 Jan 2025 11:52:20 +0000 Subject: [PATCH 22/24] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 13899 -> 13899 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 229ea8d7c..9fd1861c4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 11:35:45,395 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3266 INFO] --- -[2025-01-27 11:35:45,395 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 11:52:15,426 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 +[2025-01-27 11:52:15,426 submission_checker1.py:3266 INFO] --- +[2025-01-27 11:52:15,426 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 11:52:15,426 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 11:52:15,426 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3282 INFO] --- -[2025-01-27 11:35:45,395 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 11:35:45,395 submission_checker1.py:3303 INFO] --- -[2025-01-27 11:35:45,395 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 11:35:46,493 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 11:52:15,426 submission_checker1.py:3282 INFO] --- +[2025-01-27 11:52:15,426 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 +[2025-01-27 11:52:15,427 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 11:52:15,427 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 +[2025-01-27 11:52:15,427 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 11:52:15,427 submission_checker1.py:3303 INFO] --- +[2025-01-27 11:52:15,427 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 11:52:16,458 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -55,7 +55,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2487 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.256 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c0556fb20..4d0e90cbc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -511 +512 diff --git a/summary.csv b/summary.csv index b0b869324..74cec19af 100644 --- a/summary.csv +++ b/summary.csv @@ -33,7 +33,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2487,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.256,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index dbbd626b6..7b3e5b96f 100644 --- a/summary.json +++ b/summary.json @@ -1029,8 +1029,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2487, - "Accuracy":"acc: 75.000", + "Result":10.256, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 937c7c8b44a04ca0c79c5c5e1b0fdba1d32a7595..73d097b9d0bae50e54078aae00022c0c97aa0fdc 100644 GIT binary patch delta 1920 zcmV-`2Y>j>Y|Cu0yCDUflGq*SlfWS%f7?>yI1qi`uTXhkW*%(IvU~{znW93NqK4Zh zur>RXa%?4d$Cs=nhw%3+lK{bThmbh)MA0q#^l5b;$?4;7i!?!VR`58_PBhEF8e&

    >EUjCwWv-Q&Z;L!+oaf5>XMZY)VaO6z1VUXLD2s>rL_7wClGW>6 zs6m`DA;~D_o(#Bda}2CwjIFvo3Pd2wMkdM(lbo~2yt?e=X_{x;oIZ@of3YmRaD~yPt^S4z4eI;aRg+O@72@GH)lE>|$PZ zht@qUMJ~IWMQHqlUWuXJe-)bB1~D81)n6|)L^9N)crqy~tm0C7<(Qj(b#b#yl$j(@!wuvoUT2EyAaPh1*ATE`4QNt~!fpTPR0k@^9 znaeZLQPg3*-Wp_TnLtp#*2pTYcy)&QxBv8z=eag=sou*AhTs!tnH;n%?hRgNqgN(& zU!5a{ZFCf!+et2|e=3(v+YIeZ@ABeyH0<3B`bWr;ynTmKY`LE2Q@4ZA`Ni3bm)MTs zb{tG?>|5AsP^iJp{rA3ggj~yz+ifBAY#>L-_Kc3}bykK;Y3LZXYk3IW-TdfZ4@bBC z!Dx7Tb$i*bp2kXoyAA7XN(_w9l&AhLSwl-vmLxE)%;|VBe_OLEFO;q$r`1^7~cF9rBgfUhdPESj?AsavVK{Du`z=ITtX zo$F$r>UoweMsv%sItJPH=zT-jd}o=~??+e zV>5YkEI6wa!dI{Uzy)PDv^82S2~rz?z}2i3M1s{0Iw-|vz*Z@4)`!D^ISn7C2L#)j z*}Kos!=B6P=lvlyWpA`S@_2X+`v|Um5o-mp-JG^BW33>zOR`-d`vh7GO7i)EwlP9=fGzn zTTtZ>AzSP0ZIRk*?ccRs`zGvl6#M#Pj`CwWrQ2eE%lJ3j0xt6W9xDh; zu)y#EnA(wuPDsH=r2TR201zBQ3kr1bm}CTByaAl@lMRD%QgF&0uwYzW;_D(Fu4O)~;)ypB#8V&jiMP`t$375A z3_y=@&&ucQ`K4JTqPob7d76{7cxVWr^MaE3`Y`gQ{{mE}A^#?x0mt08pP>E&vu-h* z0|lIt*d6GT4>eZ-$CGC@IsrM8qBSNPHaw`OB>?~cPyzq|5dZ)H0000000000fPos5 G<27iogsT4l delta 1921 zcmV-{2Y&d=Y|Cu0yCDTi+?{milfWS%e@%0vI1s(>uTXg}GY3Et7;q}_RLXItQj@Q> zldaj43xn*i!-p3M$NBdw0!bW1JC2R#luEbY^=oxM3HR~0RUDxu%XpY3Cmo{W4q}O) z24ONk>D=F)Yi z^smx@MaPEd9~*_CG>(0icv+a%?e|G2c-=T+JdZ@(nlYX*#X4r0Oh?^%Gn++W!s_O~ za+ZaR8w%psix*Lo>a1uUP&p%7}+ zr%Nx4ylEJPLVf6o3l`U{*D}{i%(q$UGtN`x{fj>pMBir-%RHej_LRlrbS56XOv&nX zD%2p(n2=-?^FRh%w{>-_VT?)L9(cl&Wg`=1_)*GPU|d}e(l|~NZpL%BAJiww1CY#Sw#~+O=jU-VOOR)Vy~~PuK*pdkJcUUVp0l8@`%2S zzM`KjO^Uydjz+Cs_otH0zc0=%PSJOEo@PEnqr0ET!aZDH_QJDfFPi*_FJ#_MIt=3)q!CtLzS7cQ+Jl zp{i@;YEH4^U>Z!jcA#Z!#xkke?73hUPqO*^hsl!amQFP8`_$5=e;#MX7JC3A%3}0< zQ9Shq=fA;e%VOI}txJrJd)hYGrW)B9YtlBcvZb|1whtH2Y6Ie0vMy@4+1yi(tS#WS zR5hbK6T6B!tk+wEOqB@)^=pl+(u!ASXn6b206C6rBHPjiX~qzI;*`ljWpQutb~k!u zV)xZKs+)RG(Yc-Ef6`LrvT2*4y%}6y+)l=Wo6+zH5y{)_S{5d@dJmjZk#z}E!$ngCxD;A;YWO@OZn@HGLx zCcxJO_?iG;e+%Gi0emfhuLbb60KOK$*8=!j0ACB>YXN*!@nylBm8Wi@>hc>_JQ>xQ zT07UpGS<>0SxuHiCq12Rd-T2`Y`(M1>-Qt9`8lDM0Vvqc?pk~l=cOb%2qla>l5_WYEuU&Z6dFTE3gf3bLO+oEMNd5H~Uwr9sGVDI%I zU~D08ju~f#Lb&SHAGo0GhPFnlB|&Nf5V)GPf=IF2K?kMS4A?5g&H8XSux`VL=>fs^ zX7=tg^swi$`gwmyP1zf5k31e;!#;v*U&LBLY&WOv%UCOj?UHO)$UcGAf|7j-?cm(R ztN3%dOY4E?_u@4sjDAl66 zCr@PTVZxNb&dbdhRhzVN+@j=iZI{(ty<8Jfe|b&Z@GJygTT(Qj68{WU`ww_j z_Wom_Vw$9mfp&_#rh&`L+{mOl?Ye7oBU`s^+j$!sQCTZ4wh{V1 zyq;VRucgISe&pTcdmZhlL7bB^yt+L;y?uV9*Ig?SgLBdUA5cpJ1QY-O00;m806zdB zv(_h42n8(7jM65PEifN{Zo)7SgztTd$oIr?QfOr*L24^bMXFS)qTVcf0ju$kY`1~8 zuM;4E#Hq*i%zit&ay8qjHu(kTrO`DaS&kCW+z6>RHTqc2(?^u}z_ehk(NLp3cry#bu^HyZ}!!l`7~TexXGiKBbx zF6+v{8uJspTD-mwtEN)-r{aK`N^r^@uwYzM;maZ(u4O(f;)&N1#M3_NH{Onm9Qr^c zF#tWpJt?2%^UHE2j+!DbCTUL6;$cY$EhqGT`qhuT>OTY3amc@kC%_^1^(Scl0JCl} zoC5_)+?{milMgjl0mhSOH97$}lcF^y8!XL?(k1}_08jz|01*HH00000000000DyrR HljAjLwbQW+ diff --git a/summary_results.json b/summary_results.json index 59bb29d0e..6003a475b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2487, + "Performance_Result": 10.256, "Performance_Units": "Samples/s" }, { From 6b266030754b80bdd6aeb433ffce1259a874962f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 27 Jan 2025 11:55:32 +0000 Subject: [PATCH 23/24] Updated results summary [skip ci] --- README.md | 81 ++++++++++++++--------------- dbversion | 2 +- summary.csv | 1 + summary.json | 82 ++++++++++++++++++++---------- summary.xlsx | Bin 13899 -> 14009 bytes summary_results.json | 118 ++++++++++++++++++++++++++++--------------- 6 files changed, 176 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 9fd1861c4..e0eae6758 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 11:52:15,426 submission_checker1.py:3259 INFO] Results=52, NoResults=0, Power Results=0 -[2025-01-27 11:52:15,426 submission_checker1.py:3266 INFO] --- -[2025-01-27 11:52:15,426 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 11:55:27,156 submission_checker1.py:3259 INFO] Results=53, NoResults=0, Power Results=0 +[2025-01-27 11:55:27,156 submission_checker1.py:3266 INFO] --- +[2025-01-27 11:55:27,156 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 11:52:15,426 submission_checker1.py:3272 INFO] Open Results=50, Open Power Results=0 +[2025-01-27 11:55:27,156 submission_checker1.py:3272 INFO] Open Results=51, Open Power Results=0 -[2025-01-27 11:52:15,426 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 11:55:27,156 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 11:52:15,426 submission_checker1.py:3282 INFO] --- -[2025-01-27 11:52:15,426 submission_checker1.py:3284 INFO] Systems=40, Power Systems=0 -[2025-01-27 11:52:15,427 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 11:52:15,427 submission_checker1.py:3293 INFO] Open Systems=40, Open Power Systems=0 -[2025-01-27 11:52:15,427 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 11:52:15,427 submission_checker1.py:3303 INFO] --- -[2025-01-27 11:52:15,427 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 11:52:16,458 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 11:55:27,157 submission_checker1.py:3282 INFO] --- +[2025-01-27 11:55:27,157 submission_checker1.py:3284 INFO] Systems=41, Power Systems=0 +[2025-01-27 11:55:27,157 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 11:55:27,157 submission_checker1.py:3293 INFO] Open Systems=41, Open Power Systems=0 +[2025-01-27 11:55:27,157 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 11:55:27,157 submission_checker1.py:3303 INFO] --- +[2025-01-27 11:55:27,157 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 11:55:28,279 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,30 +46,31 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 26 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 33 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.256 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.429956 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 45 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25372 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 50 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 51 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432023 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 27 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 32 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 34 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 35 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.256 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 36 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.429956 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 45 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25372 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 48 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 51 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 52 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4d0e90cbc..31cf34b8d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -512 +513 diff --git a/summary.csv b/summary.csv index 74cec19af..eb4fe00f4 100644 --- a/summary.csv +++ b/summary.csv @@ -24,6 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432023,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 7b3e5b96f..5bf030acc 100644 --- a/summary.json +++ b/summary.json @@ -750,6 +750,36 @@ "weight_data_types":"fp32" }, "25":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.432023, + "Accuracy":"mAP: 49.593", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +809,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +839,7 @@ "Units":"Samples\/s", "weight_data_types":"int8" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +869,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -899,7 +929,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,7 +959,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "31":{ + "32":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -959,7 +989,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "32":{ + "33":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -989,7 +1019,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "33":{ + "34":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1019,7 +1049,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "34":{ + "35":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1049,7 +1079,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "35":{ + "36":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1079,7 +1109,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "36":{ + "37":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1109,7 +1139,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "37":{ + "38":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1139,7 +1169,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "38":{ + "39":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1169,7 +1199,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "39":{ + "40":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1199,7 +1229,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "40":{ + "41":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1229,7 +1259,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "41":{ + "42":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1259,7 +1289,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "42":{ + "43":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1289,7 +1319,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "43":{ + "44":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1319,7 +1349,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "44":{ + "45":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1349,7 +1379,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "45":{ + "46":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1379,7 +1409,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "46":{ + "47":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1409,7 +1439,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "47":{ + "48":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1439,7 +1469,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "48":{ + "49":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1469,7 +1499,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "49":{ + "50":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1499,7 +1529,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "50":{ + "51":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", @@ -1529,7 +1559,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "51":{ + "52":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", diff --git a/summary.xlsx b/summary.xlsx index 73d097b9d0bae50e54078aae00022c0c97aa0fdc..013c4baa084e4d48ed82bd7d4d14c9fda2803775 100644 GIT binary patch delta 6705 zcmZu$cQD)k|4*!{Rn%_LL}Es@_6Jp41hw~U8fsHisS#U@8nJ5cT_yHxsTrx-t6Eii zR#A!;zxaIbci+3;{qo1_olA1hd%m8Jb@vRxrPZ~zjTUWdttb=22mm}O2LRXr0D!NH z$P0H*XDglLb%tMzf$c1N7V6=NEO3 zovrPipN|I~2T6#Ww>M=EmEl+RR$1LCWRD!!U6qcfoY>UsKi_${Yh?|K=QF7}{62SD zIP1D!r-}BwZ*$%($-(kRf5y3JAw8$9V&-jRSl?-#$8LZ$+X{ixs^&;!Jr=)Fv+Z7y zMS`BOGTJV!iJ7$8YbbD;9N$0i4d@(nKNGKMZsYhKyHMEN{vgY8%DzhYV_D~FlHN$e zT^~*K+}V1a-#<-j+CBTO%GIw2Jof?@B<6=l2p^=PU(^x&=YA?m2EZi5WB0$%A`TTD z*_0dPBcEuQ-bZLPV2=j<88v2jxbKa#D4(J+3|h+@|KbrHtglR6`l#{Wo zM9j&UuOb+k_YT4k@=J)3LGD};y#I1VA{vvIF9iV~|CNu%EWc0yBlqmt#*lklSPXi$ z-71dlep)jG-0!I(35g%egCI{vEMhM^8h?f%wCr#2Aht@BQul`NHl$TMCjWmtZ7gRGk5KK?843K*Jn(w^ol>5-a!#Aj9nNk0#^xmNgVEVE_RyW znd9K7bD*5g@AD=yovUjvDR1cdY$5rsZ(Ha~F%-!A+l$?-j0u7KO0`5>z0*5O8-7 z9M=Cwed#-0`z@U2H{Z6sg4<4b+VTU}3?4X!U9D(f#zpqp;#wKo)6_K2^P)`4 zGZ+%H0)QgRZkJ@m7gDL=Vk*M(U#IC!(%v+?79+R>d;7T$#}eE00IPS<)~2i!%tL^4 znW>Rzh$^gDMS?V_(17H3B!Aun7kgF*_hPjbJ^j*KNt0cT1H9iwSgV(W{M}Z3gSrcZ z1a+{_?p99qw&h6QWo3T9dSUrp*ryfonTH+fZu=ZgX$E1pO+p2$e!LSqGywLi1;AcX z?bMTW6&~Q=rK4CFHtvLSr*4^VBx~oSo1;nm^DENhV!*UKnqpRPjnQ+9Sbn&*ehPyf zCtsAQ`Rh38rOHLp2(5Xoci`q36`p8<1zKxR`1a8pR0ein*Uh`C%DUu*03A-;H7Kes zAv~aFfzMm21N$_@Gsl8s=RNlfv8;E{)b_f`e2fJy%;ZAusP)4@%C5=*f~##PN|cjP z>8cSejAxMRIf!&G_cc zFgUPyqKpDZqVSZU&X++ldVPz29%flKWUaon?kMq)tM6)>M&>@@X)gkkn>-(cnAL|D z`m>D5C&lIxD`M;Eu7hHMrSt?qtXAI`qgozJMW_M#O5O$@a*s@y{cDV2WWGhZH>pKJ z>Z3Pu+{P8O%=Gau&+!51?ZW2DpHRm@_lR=rX+(>EC5)kj_EvM_cBa@fkLado%O5*Q zaacJb#ae0TqfIGVAf!rn14S9C0hVV0h3)POUIG@PP0X86Zl&hCo4k%e1Oy*$r4Ff0 z*<$&^>6)Lsa1ig)!vM!A-ahE+GX9IM|3;_aw(@S|wU3iP-ZnJff!B)!R7!Y>vJ$0z zuu`umfR~*~58E6?`9i+=*Mc-tJO{tHl5a=1Rh ze1ceG^X3x1r6%K4jPsF$PB9D^71e_u#{Y(^IXbc>(Ju~(GMc0%wbTKDZ$Fo@jZrb0 z*hB;htLBnA;L)?j022;u3zpvZ;EgVrigPo@wJ0yd8R|u&x87Kr*dTDp+apsm)Gfpz zc)+$t97VIf3?#gR!W0f8?Z5&-HS)C4#sKR;E^ONSuK@sI_W78}KqXAYj%PM~_@7HOavcUfe)opYx z-|~?a=p?n{oSF?0^s|Zog@LQ#(brapPGCqX2TlVT&QZGqjPl?rLV!y7WVj}N9bhh6 z$2`iAZ<78?36=pPGc|@$KcFC#Wh+Wea=$EFmVj03C2bG$wykzCrluHDsX&V2m@sfk z91T6jl%f_T@*{~N>6l%mn51K=*TmLFQ~Ro-+L)%yj!!wsI%+xX`BYZZ=HDTbL>rNyJ+lm#^Rh$_g3xYJ^5UQfs;VVD8D zfm}%x36$odP4itmp>9^=v>7z}81Fu=F^=SiN{~uasG@0_D!BTVBUZ-xq?{9+qy~bf zf*J7qr(0+|QMqYAFcGG-l^m7ek`W#xIY2W{tyqyKG%apdvX@aDJWgv;tHl$_>_UV* zVZc*axbQ`r3j?#WX-+Awhi9BF0bUe`ec_7cdI9|qpBk3v` z0gu}m|n8=l3Gse&oVIeY2 z3j>_UFZ!x)s%(`gEQnxXz!KTu=#akf^mbdK;w~h$jULZ0^c=*fWJ>A{N<#xp@$&%y z<2>PBD!?0gv%z2((XZxRDTSzkZ+f}0)Yit7EFGmt^u~HuTLHRs#_Ab!s1RvP!A4|f3zFj5(RGzKfW@uDD57!_gie%QnnNCveQhhgl?%^! zNX9oFb6%iccPbuIj%C%L$bl0`^qzsAzmJ(nIR~XK6h%*K2qu4L=S0)YDvx4uUn3oc zu0Ykv0=-SFm}Nfu$HA@gK5F_E$k}{%c3}mI$=^58-x-KrLLfx)@WCN+U92 zgP~krW48(wq*zWB1$Qj_{M4MzFglJ=BN`L&CWWXBJ=#Q~GIXI-^!Zy}7*n9a)H*%8 z2PPl-9U?lHQKO33T|*3HL#*~7ec*czgBGKf*`gumEF2R3=aH{@{ zu7H?f%8(2rGPql|kxEhkwn-QNB{WfMI>~UTnX8b`uDdVV_N}s zOcrgB7MJFpqVsCIGfQ%*#T;CwqA!L72}$rUl31FlEnUTm{Na0d9-c?#XEBKNoOUM6vt+q&i-;#y|Vqo7&OdRbrhRa-F1^ylF@fb>54SQD52??j)tJ^F84D&Zz$;glM*qp_wG za61uJ68*p+V)_@Kk?~N6GBlu2LO(mmUq06mDHiP&sbl`5`rS}|^>1s1brQ2XSidw! zI%uj-rUR@QkOGfUg+VS4G>8s01i%p)jW`AunCfqe#X6AZkC(BS>Be3m6XLH1nT%Q4 zOh%(ts;!#tw=_Bn8OKg5p6ihWLhET-@y>+MBqyn*$;O4}8>(q=0mK%uF1MFAoYL># zE`9CXZ<1L5+hxc9cIhJIJ3(m;S7`&~eB>pO5o(I&(pPMOhQ2v$qq>EB&Sq3 zGE-|H3ErBBgE^=XRZ|_@1-P;(efA}$xtN?nT|smn>Ykx!>m9)eL#Fg^9LDnR|O>uTz6*2nxiMrO0o{}8-Nj|H*1Db^*9jG&*4(Jr#k zMfXZV@D;%_*(~31_cl?YA`0vF-$o91Yitf2QVnT4lG5>j}e)p z65((ul2@A&-!Y3{ zKF#WUW28HM76VsrqDYpdo>~b}g`Db^#^FcK1@nb8l(tYK6km zvv1~lF;Yo5oUZ?5BZD?)+0Ns#v0iEHn{j++F!Ek2cDXLwrBm|6^US{1AYrR<(?vRi zmer=FImq0wy<=lHo;S?KB&_ZBjmy@L=<||G&Fwjn^J(|X>dkLXvW>{C<}svCRWC;!EVlKROj5Bhg^--N?D&uks5B{HSNeJ_sh=LS|*o*Zb@vc}0su&X0su_^*It-xrLV@K zs^f=vi;=kIpkEi#OO|&|9X=#jBq)7skL*5IXYsexS%)R@Oo+!m0A1RPg^h@_TuXkh^|KFhkqjmwi5#!mFM(e|)6N5u82%*!D&7Ew2T439{z`g2m& zG6^otg^eYqz$a%~>Y^SSkQ++-oig=#YL3Ur4}@$_A#@SCo$_YBk5{xm@H1^Gk66~W zICZAsj04^c>*qbo72F=VG^7vAQ-vLd%gh<@CX8439*9CKXDW1ar0tZ@SiO&dn;uis z)40g{l2ylDH*>XavGeyN=zk(v-m1&%f0pU-pXkxSe%%viJeWpFCu`ViHy(;T z{rQM`b~4#a{CEPz@rlW&N>_cy6~9H}vA8?7)pac|MFlHY=-#up5mKVD|7XP^ylwXy zG!t!;ZJnL(<&$PFuiVp)zNUhUt(vg*Gu}~6dU0ymzuN+>dd)gDww1F2PMsPi5%*Jb z@72Ae5{Bh$xN;h$r*nYWofM^W)*_}IG)sz$24f!-{^MKN>h}B|T{`#>TwU+Y8?(JB zve{wo{M$3uxlh^NG1u>%{!P0j)8HJEV@X?+<2Mbm!JYM>$J9nvXiXF6P|j<~w&P8t z!cPp+G^XrDuFIm+`qm%Dv#OM}RXqE~XPD)i&2F&+jN?xpJ5=4Dl+C`RD-(`!l@c;S zx_p@^?Uq~}TrgcOjjl$R$D&R>_6sHN+)ee<>mIhx9j|K&p4Xp>GtzMp+N#ZUmk@2F zx>IiBj7rt-kWmerN2AXMpSe)a8E%)cD|-x>kzcFE`u8~aFPGEom^_c)Nc$lu)0}Rp zZ*+INiT!id#cJQK@8o-Sy0*cT4p~**g0Qu#F**;v#}RiQps0!S^qICV`Hm(zQdoHd7pvTdbr#D zgQ%VXp}zH`x!)2Y3Eb%lFxYQs5W<^(4mr}!SItm}6%AprBM}}VGf$}quWuj?xZafx z`0s)K@7n<3;2QFujevJ7T{9#n<^UV&MQ7lDJ}d#)Q2qS>4x_&22B846drzv|EXz9s znwdw`=A4Djtzk%EZS1P=p4O!&7QRNFv_)cE1)mfC-JG zEBMhMUX}Z?c5R)vfFeJ~zs8x%_p317pucTLZg1A{`Epy-Q72Y(7%Q#vA?`-Xz2!(o z-s8V1HSfuGOzY;Il{jbB$*I_~2x89tW(A6zMtZye8YcS!1$J%Jk)~ulCA? zT8I5jv_&~ZU66lyq}h3ze60|QwDPY2>Pc#+u)bFbaV5T81@vzx3;JVO9I%FtqCkXa z&Zlo_{H~|Y%-m0qJ>Db(lw=!n;%{#HzrMGM*gQhT4QW@#*X%C$%ie*0+#677kl}SR z^`rJMWXWH;WJ>*WBd&bV|Lg5JoA<$jKX8-O<((b4%%724rk!;Dts|SZoK+?d_VKQ) zdM57eLsBa<1X%1w@mI6-StzC(J zx;AuZ{(hC`$651}Ft4#c-1P%|k15G?Z6+y_VfTdfv>BR6UGWtmN>=7CZ;taaGzKM3 z$^`upe6$2vG)Sn*IQQ|V zH&1{E*hrM&GMzV?e#4v z^z25x+kJLYs{j6Tlbj@O40K7}5C`A>zfnj`!+&4FA*A4BUI_&f3I1dUiGP4)$%PWU nx2{G21p)xNBme-z|NfNt3=Y90FH6V+za)btc_`AwuD<#o+s2BF delta 6580 zcmZu$cQhPf*WcA5T6EFv+SN-4f{41xBD%G@=sgHRlob(H3sx5;x)q`mBC<*J-b>U7 z(W4V2d@JvH&->3e=ggfmGtc>*TYvY?GtC}-?$s@nRGJR@91^quz+@2sa2t?JBgulA z&oO_p+{yTIaj5D|nH_BiTPlURtp8Z=l;VeeK@G zY;C6>mj!>k{lDc}_s3j%O}GRvOzMxjBwQ5256izKn`*>w^z=CE$-Sl$>FTieHnvA( zpE9lPtvMCO20nQjBrSky%GR|*WN%C4mGjPja{uPO>si(KH`FLN%j9w%sk8bO<$Hd5 z5S9Kl$FV-H5p)qO8CPg(hs-__9b&2~&9kB}9II1po?UiG|9lkOcK+B-PT;iIQ>ydt z(EW#F?!hdSOiq1Y;_F7dkU~#=>%`t?i{an4qofI-v8y0lt-^{&1j z`>U<*e~q<$+jNZ>I*+~lb>e=G6+@YOJPIAL=3qXCZTU1&q*bCsUZ>l+S>J1Ib8${N zME?4i@WJuJUvJajvioZOfr6|HJ&~uWk^VBdvq~=Ve3W+R$*w&XIz${TU5@XAgEUy} zargZ=h%Mi)nC-95S}F6dPHu0)XzMy)>is7R?Xy92yT2Z`o^&)B~dtR`z(Z(rfWA0*1j+xR?xBV076sqH0ZXNtx%N)EN&I;uEAEQ0B_%$ zGLQPSAp2Vl%2O9b15NNrw@2A}hvnA9O$&aqQ;Y3>f>7ITR`3*GdALt}lF&hZP+WR( zIk4+FsDGmt#7nr1o;u9m9{;Mt?p)Dk@{qyCgzVJ2jMv2f9PO*H6+LKP$I&_~r* z)-2mH)S0s6$I0*3KfCWOc2cfd_#An=3VYyUf>Z7t`vn=Iuh!BzI}1@~K4*6oE^jv# zoY*4h-@ZfyHRrlM?ORAs9$D6E_RLD}%Lu%9_Mp!AUD`V`4@wb{CPvOK^Rb?l!+rG$ z15>TXmR+xIjP>-a*PppPK?PSDkS-?wm#-Wyi3O?PoRf{#5y-Q2ub*TfUm5Xa%4&GB={!aCQwMO0gS<7zn zFR+i^j>w6<&WXMH_6rpZyq(oDflWdz)}$gX*iAQd+U@+#uGQ(M??*@5?-h`t!`Q@U zW|(LD4q`c!gN^c=2va9FFKj&}uI?3fOsm!ju^y1J(!qn68=&-NfM(tQ_Tp1PSU062XQP*e3M-@Grq z@tqhyZ75MkeN2I8X1AUhs%CjD?Y#qAYU;;J`5U`1cSuf2GZZX6*JAQcjUXs zI}eS?yrC){5RJ0s(TBXIzpHVCLjrTE!gN~@KOueN_8#*F@}qVF z-vs-VKkM^QP*{QkBt(dNpPPAUSL!POLL8=O{X7JrDE=OysN7w`rJzz9hjcob-cRO* z`~R`|Y{tCP`Td#2vMXnpoA8sHGP+i`w$EWRE~+l8Y>lWv!B0Y0M&BKN7jXx6#9^jB zw?HCVF%)~6;y6LaM3dDYsdK>qs|Um zLRqJ3itxjOPv3OrqS;#y?jv*=n)OpOti%deR5wI!f~p{d*Pt=XE5;WA(9^as6J5N@ zfPiBZefJD~m`Uy8>T+op6<4;6DM=X6iEOrzmUgvoGx`nFk@FuY^8P2M=b z(9kM=k((L1%isqm?SV3LrQ3&D^m8?Ls*5lx`ux+AtH}&7SZ|;ACDnp4T&sDTl%S=8 z`i+|u>wJ{OK&69-QhU~MtOkO)D@UgTB^+K#wmZ*|VcQY`(AZgI`~&DT7=-XBzDinv~u!v!k2F#J!m?UR1X;lMqB zSfCX(s~#`r&UrtVnNo{V8`%ZfHkWhH2EqW&Myg!IdRw`oUd~H>v+GQP{);#2PR`?; zQxgXH(wxfKO14`fB8b7^QKl@Er4D1E_da?k+pk}=oyoxX5ue+P^vJuRY`0E5-!5o5 z6l)Zzq2{tdkwopoY(rT?;ng$Fx-{xUHA03f*5zpVyULzk3oSM8fuz@(C-^3Rw5l0B zz_DYqa=4a!=6|py__oSzB@^k9^j4W4vY&^lBX~uEK@J0rTFRRFa5zGwD z-@;IbGcS?XP=vurB5cmHL1sYQk&B9^*@GLyn7YdDgzzW{B9q`d5o>xsDL%{8JG!iM zAOzQjy4;!?`o-4e%qF6J$9rveTC}ocyi~2Bsme4xe9@-q5!6$FQnUlWM4YRAVvS*h zEbOhWu)*lEOjugB2QkXq0fb{v@yY)a!+?4GQN7%8YCUSHC7*Vb6)FVrDBmsJLQOJx zTS>AAVCxe#)ZmdV5Te%FxRq1~NR_)H?(K<-Mt%6ogvq0;lk}!mWv#(R@D<5Ab<45F zJ?Zq?Q7@4R$=9m4Uew7g$}P(~3qPTdN#88@E_x`aDTsPeJBTXrsg9P?6o}`>=$o;q zLn7QsZmSCsx?_N#*Yc}q1VAceFeweS6ATcRnI)^`|8R}#jmKRcmvnn5s*}Vz^F`6II_9?A5MxdaGyNaT=%wfqCq~DLuRO<vW-D}&WZs5UIRN#!!%(;e~nSz2K zl8U^x%v`C>(u(CnyafzlMtOR^7k&2)9JF+rz>j@use>U<5G<8m*~ra5@{4{v=M!vG z+6HmH>!jWU9D~UHQ2nah?~P{eKIj1$qmNR127^SDkp#em2+QzBm&+&NjW!D5NlL>E z0qa-n1>bNhR&i{+M2ee|R#%uZJ|j&_ka^)*rf{S-D=Ac|pS0ydChKnl=g8qE24k?U z5mQJKQasqTbXR1D)*LaziU{O}+D&Y%wH~ek%zR|Ekz3(VSj%@w@QE{u^9)I6Stsp# zmj5RuFmPb?9tsB+t|M=eH8PGi64dAM6Lz~##W$kSQv`>JYxde((kk;2YJ<~&d_COZ zD|VIjx)599FnFGN(P!@PwCo+JJ@gxV9jNfA$bl{E)ayG}PO0%OwV_TkPSV&yq&dEd z>UW9KDALCxYQV%%y&6gA24gg@>=o(E8L0GXa_vjlMC9Oe!R@<8_G#inE z$P${*WQ~sT>lb#-`HUm1mJR4$$%?_+>F+cyQR+qd@;ycBF!5fE1aw4VA0Pqk5T4bO(^(S4E_64R$ZwO|~YF|*j z;(%@kJwCpD>p|i`*G(oE9&`ofT27!J2=-HWIwt)G^kNdulPt}`j>4Bp6wkDZ zdlM+@n%lmPIidxPGI5`74?L8N>~9gXf5;ono^V&p*iOUYse6GV25yHhlhjV?o1oXg zPJB>buK;1V65ZSQ40)I*YL8>WXNU)EVaNUhW4}DDvYCo6$gobtm$TCP?a74rCoa9I z5D=>a)@ROLY!WDdis8*Xda0pixpo6AY&w35Jur|<@KO=2n)%}r1_kcy@cP~Hw8 zFbW}~5d@T-(}M}gXzEwoxfKO&!-|+}d)m&(t5I{17wh*kLK^x-N&AqtSPN8#tt^#L z^eb9;Ec#_dJz9gbhN2pcB&o*BxFN7DF+eL%)gA}_R|jX3MtH(@8V1y$8NS4-vGOwB z{Jv`s$7*u;oy-}Pc2~%#mNQ_-8~PuzrO@7oVhm`=P5Bv;s0~=S7-}f=@DjzGXVgvx zD3-@|hZzy!`Oo8Ewu~zOy1F2lMnoE(*+#IW;FJcvzPa#3?a3sgUWdi3@#hd4L+}P2 z-c)(KsTiO<-c(Hg_y`7`rw0=Rq{tU`hv{FVku8u@28z(!{4S6VSF7%=G!;LW$f79C z`)QGKC3R2v-Z)C^nO*s-k%Q+4ncffl$Ly&rqF>X(`>suo(;)Sxtd2vHcsb}Y5Lkp{ z(NqH7pUZI*=3aYIFyJe3+qXDh6)6=CWqa4O7>+MLxpy1g%k+;r(dyKhIx9Xg?(H-6 zFxBWHqjqJ4rYeNUy*AE1*#Ytz-&0DCO;zi|?Z;3ij~2rbcvWY}=DZ$pKW7zz5Y^Wz zE55Cq5jYqz#EZfBu1(>NZn`5!g#9R|=O!JoquZ;F$yAeyFh_hp1(RieN2FJ-W!GUAg8cMuQZKp9Ya6W%jzilNQ@;Lu~O5 zG&=7GlRN%%pw|F8Vg*%b_DqOdbf-4{bFrm)ooPMQ=IOzjTc$&|s+IY1tkG!9-Kl3& zkC;rs#V`X=+kE7vDt>k)SyvSzY!&yu5paCn^7c0A;)F=mu15%*D)%)5B)A z8E{?%CO7z}rH$w?bub%cfDw&gMEj%=%e>b1)H->Cwg~BXxkHh$U^I8v0Mk}toRGd@ zN=h73W0L^iJ@g16+-X3)8I>@AQJw=nBg^Cmy2Bm+Wh5!ZH8+D&K|1)D@T16z5A;b? zgpsl3*ZeujrDld|A}0=JUiTQpR*QHOCIa|lV_Y>nuFi1@I}=NNl6EW1X@WjT8qTwIPRYn z$cgmzNU?q$;uVd$KTtOFJ|n)QZe+P{Vp6x)HGS>x{J!t+Lsvcg$E!km$B+VL=%(D6 z%kbIiRODrG%@|*4D4noV)3@FOPIu&oS*E{Z%hd|^*&qIlKU^-#b{-d4>~_TOcf3Z4 zmN30VrE>C7$!jykZv6gPT34VTX#u@#ArLVAnNFnkV>8jb?I^6nBn&8r;VrDi;im9x)Y>%DM737{W!(;|qJE!p^Dqt0M*Brh=&TNY(X*(PTRM^E;wmcw%arLS3^im{e*LAnHY_GaxIn8Cu$MvFWgg*Mz?^!;0-KEdA zK4ywttkD+doZFyzO;j7B_9*w^GWLsS=4Qmlw%RrV-&T3Er>(JYen=1ZnfULR&|0~- zm-5eK^V!!*F$14O2Ss#7p8Dr9fmB|6$U`qvy%P=kZdzyZFtpuC{4sCVwyl+4)Z_zU zRgn*`Rh|DtSViYApct}dNyOL)C#*#!VdCS30pk3El_N}=TDb;a@TirP{Ilg z*6>Ark53&wv=$F&nEJKKcrPZZVzic2xtJI6(vHxdY?>YSZhNNp7tkN2@)=Ryf_Qgw zJBWj4WZIkiqYWKGW5RKW_skp7^&*eAY1&n?C*OyFZ(@ z$F}*jt6tgXfxoA1ei0UZiL)M{BX=(K?vuDzTYiX0EtmD%jc$RY^Yee!t#7F}{r&75 z@XRl0rYg5nGyDVPmb)fJ+;B?irKqWjmVO%K&b$7N0Q}}+oq%Y{JMIH?C1lokMYFDx5 z*p@$$47xuzwJUSBY;!jQ(}dNY^i|7}%$%%;s7wbtWlrv@BX4QN>e1;{=PbIh|Ndxh zVfYjm(0r?j{o>-3L+Zq=qh4#7n)t4)-sd@Vw3Gf!p_!sgvt z?%A~u?jwLfRkvvD?gl;)l7377#iZ&IN9=H~UeZkXrU7dhZlZz#7Cu2G%h%zEL-2b} zMtyxQL&;fkgJ|~k%!i@juep7jYkxtK4v*^}90UYVUXZ>Ep>ZRwna^$%wX5W6)T*@v zT@aa(Yo74ezj#8IOxAsng$igeg9^ThO-HqYHuHYllIj2X>ROueP{J9+8Gu2aBfeC$jaP6 z@(t}3QRNBeURn43fyZ-yyA&rR;OvAu^zrj*663f{z0JV#Wjn&Rgz4$cNe?NOf4e>5 zp7OGs3SwoYF%^{er#WBv#9?_-Ror15x-!2K#J+>)A@jby&6A^}le45(bqFxeTETz1 z0X+Z!AjO{wfaDfQ>N-O{6;dK8xw6vatUJQ`9yE=F9zEFS%9dt6bfY{p4WWtSvY{6O zI?IqH!-Th)fBpUSo2TDOoe{4Uk1&8uU(pWFkz$(|&YjaXxHW@q%&I4y%k*w>2ycGC znD@rNxR-oe%*DLYYB9>#5PPe1Ic(FwtCzjn#=>%W3(WD(LB3)+`W|h&A?I;)(-^o* zo*GV}nBYCsyS9B+Y+BIL9UGXHTp!>}ntvFMIXk~|P;y0JR0_J{$pv=JHRZjzB&?eg zV Date: Mon, 27 Jan 2025 12:02:05 +0000 Subject: [PATCH 24/24] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14009 -> 14012 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e0eae6758..68466b1ab 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-27 11:55:27,156 submission_checker1.py:3259 INFO] Results=53, NoResults=0, Power Results=0 -[2025-01-27 11:55:27,156 submission_checker1.py:3266 INFO] --- -[2025-01-27 11:55:27,156 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3259 INFO] Results=53, NoResults=0, Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3266 INFO] --- +[2025-01-27 12:01:59,720 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-27 11:55:27,156 submission_checker1.py:3272 INFO] Open Results=51, Open Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3272 INFO] Open Results=51, Open Power Results=0 -[2025-01-27 11:55:27,156 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-27 11:55:27,157 submission_checker1.py:3282 INFO] --- -[2025-01-27 11:55:27,157 submission_checker1.py:3284 INFO] Systems=41, Power Systems=0 -[2025-01-27 11:55:27,157 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-27 11:55:27,157 submission_checker1.py:3293 INFO] Open Systems=41, Open Power Systems=0 -[2025-01-27 11:55:27,157 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-27 11:55:27,157 submission_checker1.py:3303 INFO] --- -[2025-01-27 11:55:27,157 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-27 11:55:28,279 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 12:01:59,720 submission_checker1.py:3282 INFO] --- +[2025-01-27 12:01:59,720 submission_checker1.py:3284 INFO] Systems=41, Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3293 INFO] Open Systems=41, Open Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3303 INFO] --- +[2025-01-27 12:01:59,721 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 12:02:00,871 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0872 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.9943 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.431004 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -56,7 +56,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 32 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 34 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.256 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 35 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.07 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 36 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 31cf34b8d..a08796291 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -513 +514 diff --git a/summary.csv b/summary.csv index eb4fe00f4..45ca81ba9 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2 MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0872,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.9943,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.431004,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -34,7 +34,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.256,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.07,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5bf030acc..e5d373fba 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0872, + "Result":19.9943, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1059,8 +1059,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.256, - "Accuracy":"acc: 74.400", + "Result":10.07, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 013c4baa084e4d48ed82bd7d4d14c9fda2803775..4a85c3b87137ddfce72cfd61894b1e4738de3c10 100644 GIT binary patch delta 4549 zcmZu#S5Q-JyG`g)H1sAVAt7{;CW2BDN@z+G5+X$)G^MFv0}>P@R3U+&pn?#3GjwUf zCjtSKUZs5qf)u5SAm|@FXXd{+XJ4#~{q9+_-nE`Gv)}kP`M+-9Va+p>f+keG<(cEa z+ibRx^G7%Bjjyw3`s6|fwIQOg@3x^RjtOqnBsAMNlo_EF-pyMExN-12w=r<0N*=3L zE9hw#x=ISTI}gpF{mIOA~lfN*Mv!ZUSMgP&olE?*%hmRSYFcw(es6bIkLF zjfydF&7{Hc5HmjsDk=Rg>IKoWo3EXx2&lu`12{%;5NTyQ)`wMk1TMg!d}%*HqLmTU zjgUcRabTmg^GJf<`;XSz#hZ2kyAEp2()R0O#SPP!%)c98Of^HZ@#Zt>G8Z8vn=RnI zm?+Q);0X~2xbov4af1;!-vB3}xDPsLyLFyGNw~p!n`Lt^CHwsA;tkV*F72>wC!OGVq zWK&FJU}>_))K&~?Ic7@+wHzJ$XhHEy^<}6#E5R(wl-efqkdm;jwL#3rrr$=&A}>Ik zX{Oa33r|3?3eVg?6kG~w)g*(OB&_EbgQD8tQD%84BQ^7u`;|xp5l29^s}W%L;Jj{M zw~1(A>tF$|Z$B{`=F!MQV@V|5ekZ6jM08`tG9v#>o3f5yexZUjdN1Zs;Jda#tbQjm zD3?VYlwp+6?9K!jyr9-9<1RW!YU*RJ-|;mn1q6MqB5I(G`bo5Dw);}I86||z-lGj> z?|qF+121ef{t8dCJlp<#2-xnEzWX-vguLM#KoDy5V&mI>nV#H>ZYAr$h zwUAfAb~MGETm4h^9$ls4$kFOa8nQ`qv`yAewn*bK$^_%fodT*9b>pUSzczC*z_Z*q zhyptR#&`A+Y++y@`nU;oqBB(aX*e{h`tdR%-`#l`?DY1^dWmO*VRrd1gqAt~;@xp~ zclN@y3U9M(#OP9i$0!an9?)Yn8wv(FnY^+cjRs1O6mW_5NB@=9t0(qwbpQ}e6hzo7 zIuIozSIEOH=3j)Pii3!$_|W3;{6&MbF1eIcBD%{wmU!HRvByoA0{|B5#W1q44?md# zj5*GN*Y}5Eh051xv|2RB>nUjTGJ9coTJIT-9!k6In~22s)e>kvDHutNmkU$Gn*wqd z&E}NtkF5@J;HAfSZNCk*WRSqgCg$n9c`*-9Y{;{wQd zfhEv!Hye>K#)O3qjIjV6?#q?dtIR?7lrYP?dQfI2f#Oq#kq``l@&YL!h$voKfW|FN zcTr@qJV<~z8U8MhtN5n{n0I_z!7vfAEq4gtKUY+$65P_TG~8{?)2Cz|Zv}MYv*549 z6-1lWade-3ll|fPnMPlkq!QRobEu|_P^GOp1l*d2gO)4F1ml>Qu9o$w=LkZ}TR49S z8OboDI?Tm#9d}==EMAf4#$;SsPUq|W>Q3dfUezco#Ti3SrxXi?%I-GVU>k_9AkA8- zj4^Pum`fQ0M{7bcieQ!HV5A7h)T`)v=*X~hg7ZS~HJM1b%5ly37}vU&by=*|=>-&d z-elCws*y=u;!W)x2VDc0yE!x*A{3W*nXwc62$$Q{@Bjr-(8Cq}I3HHPi1+aXm$X-n zV+f4o;-PDvIRP%RRAaJLq$D8B!Uu;a=CX{HAVkeqUe(FdS%X+yNJFg{D+yRMGlCfg zG&6#+|8Wy6J+dS`z-dr?S+!DH#L&W|mZ=Y$?U=rUP(U6+urybJON(|Nvoco%F`IZ< z2yQa=v!iqAJH^U&(^x9H9kF#L>o$s`h#M4-W-IhO!xslj)2kvn02IRgQdn#LCR6Wi zgeXJ3)H5@gnZ$`d!Zy~r*Z4>bOTqdb)q0XMn`)Irib7>BIj=g$xOm2<=1^T2v}a9Y zqLRn{s&xFJ1)e)riRWxB`CpkGwGYoF&P5wh#Rus(AlX&!nai}>dYc^sk9CYYwOk6S z>ZL>Ej7MJyU?2r@MFOu6$Sa+z~RiFNP1uy`#gv0(~h7CB%(OZ@_-W2Y^t~n zklPA>q7Q59FkB1+N@NThY}*wTE?_tW5wy8`f3?84rYC;pn~&Ph5Y57#&5f*{!!g$buea4Nj|Kyee#~b4 zE~~Xu%#RRP_Ntrg-jce3Zu~SfgNYc=e8JPXJ!u@_WU1jNu$nlrv3B@lHlBWaw_;x5 z_mC&;X+pqToxTT;!fFq@hp!iq>GMD07ggS5^!|2P@1Op|Y;D2JLO!c0Ruv~517z*J zbnOI?ECi9k=W2tRzgJk_JEP8Szya(?yQ{FMZC6Aeo1~Af(W4DR$Nzm7ANBe0m*82m z(?+_X50@5hS}gBg4|yxJy!=Tac7U{ap?mOiQqkv=?!L*rlw!|PM@6r=Q2|N zqiyKaJ%pO?MwG--HPz1pW*J!bM2qY5)VsJ*I(F6 zh=po#bR?zxw3#$CbfC`He?|we3;OQ3wy|B0p9s-Cl9AkfV)RFi3lru^{GDPWHrf%~ zYODXGvf)T2k)=Z1G(nhFh`rTemyhmLO8>VITj5=$ZH}2fbZpxq1|tnm|Dr?ED~F>`|Od@$1SNf8=zvo88m8Tmp=H!9WVAi#ainA!b4cd zYFa|g{I=Cg)`V?K41TlVp$LnT*PAoL^cJ;y&p)(3%2Kpx?v7e%m**!XxJ#m9R<*F5 zVjbh9pOCIU)%-oP6QnQj98u%c{9d2U{hTtb+Ci(EC9V*Vu; ztL&-n>NNoE8pLQgj+Y@Ns?uSr>aCIKrmM_>VYYl`Tg0~sEjd|A7Cp1U|l_N?VeyUG)-oJO#Hab##u%2+zs}3^D)7aHP58YcFzJ<$NfpM%FR|9vTqo(M9i90-Tmc%+P?^XwFtn^NsW?h?BK7lP>F==qJTWfe4W%ubs&GQdv zl`EMSVQcd?Lassj7`_)P`5li4*8VHEyzpzU*;R8aP4q#Fb`?QwiqC76>ZVJt#L_}8 ze`5PYk&)`isJ6P{Jm{p<6B7@w0R|WK#)}2$?WIv_n5UA8y-FYCloh`7YSHW()*)m^ z_$Oa6-`o=53H&W8Jr<(0VC|p0$lWI4EpfW~(l_PYAEP#+<9#cq2b8DnZhu4GG`oU- zb7RuWCy-5e=^S};?!Lwv*lz3FWUBj6@U%+Eg{Y4oFK3y?;#QNqJ^cPHkNvcGg?G^!cN-R;svzpd^dCEZrhf?15Y4(ebCtWNfW|i$ z@$N!ApM;FYlG$E*@2eU)3qFzbFP;rfH}L2*)=OABBNUSN)a{|8Fq0&$K%D}z3cPS@K%*bh*Y zOe`K7J!nSDI#KdV=!3&nm*I`I3yx7^5q}br+uJ79P6)-rQ~AK@&D!QqIRSF+{D>i6 zUl;vn*OcWf5j+OE`c38c=0fghOXC5b_n|Yj-vh@kmVU3nr~0P1gZaE}W=B1%PF0PS z@(kOCx%H8zo%u6bXB4cv#t|{% zKx|&Uv(!m;uX9a&QlPb=-kBzV)a&hY5om2{ewf%gfAEE-$Ia|xEyS9K;{leR(rFH$ zPq2yd7W=tI(abZAh3y93j+0R9SxDk_to+}6cN$sCPd$-7lLQrUAq+OEs&)y>yr z{%l0lB=0sNYkF7eb|S;USnI_*U~I{nbIPRR`fQDJ7SDJOY@=rhvZ$`FVGY)pY8k%V z@ZGszBjN2d?G79uH$pu?rI^p=rph~-_rX2v{cZ8#VIER-GeG%k5f^21cFzX)bn6mw zivfavybCAsZbrtx`#+>??{gJyNm?`$4wBC~h|gF{l#E@wym#KY)MK|8e*Vo@@XFWP zf-q!6OK&Cd(%QEU93@vb>2u2rj`x|O^P2Qd_+8C6A}(g}xncKaw0_*2r5@bzD!&-G zgVmeo@sH5e(iT4Y1N^%nUNaViog9*bC02@Oh@7D+h41lurwTIOJL1f3R;Ir?m9TDr zt@O+8Po;c|(l9eBV<$Ov;$0_bFIXxSd}O9|y}xlQq_dAe-_A8@G$1r&oh_8D`*$k{@e$Ib0Mvz4o?N-m8A1a%}EB`a0TEAQr@X*7@Ao8T-P{KU<9Kr6l zUt9L`^J&~KqDw~2o=>x%@%H#si)j=oJ*)pdYc;*@+OYjf#Ff+5&YUhm>`Eez=|2zo zgvV5+MuXpDPt4Qe57O#_lRYGeP&T=wVECfP<_@e0^GXN4?{DstF&pUPl6<$zads=` zH;cUo8P5-IJ-fFEFZ=_jnmQ}W{6GIw$Ojr&U;x=c6LR|Rl4Sz`Qegmq0N}q5A(9nB zmVyY8i!~L1+T;;U9bhpTq$MkUY&r}G09Z2t0KEVG9zckUC);T00cXj{S~6@I>c>a_ E0aGS8-v9sr delta 4514 zcmZu#X*iVq+n%vy&n`k{EQyhtY!9U{7=thdGh>Y*vKJ!D*kbI=ShDYXvhNgGG9>#J zm1mGW$x#qS9BU_YBwGwe0k|VTIx!k zGfOgRDHvw9Bn1TgE-6fK=$ z%~9c)Go_5MpOvWaNkUjtKt5pN6+I56Vup1Xg!Zhu-ho$+UM}PfLFE~pwM07T4K&ld z<6!HNxDv@>`k572$*ZdQclm%}+q4DwAk3)ArF_#@^-345gYfiRC|=s%CgFep*CwA^ z5$4iB(cT270a5cT9stukI7|jmsYZh9k=FoNXg%*3XTEv*fd(W4LWeeku-sx8F3VO| zn!5C)Y*_(PZJfL{BH6y$&6S#hVupzq#qki}jCHIW1T;e}PVP%GL-LUTteB>g(r0dK zqi1l|Qbqq59}HTbk@)ac#CT99(z~BBwpvD!-v1Uky!)rV0O82*{TSZq4$tUodhYWK zCK#WB$~JDhcdH4u4Awn@ zcfZKEAnCS-cqJ}O9i65HuD<9@RI&b1E(}iA#)BkM!JK5N<4rO<6DSP`rqYzQk^>7V z85Ogm2k7N#7psefrN!+i_HoIBC)mtu^~J(?U8$5OzImuATll!$m6O*6ol{!ZD>lKN z2rmjGA`+>r2TtCPibxs;BoZ7#^uDQGTAC>1tfT2J83j++a;obM_+DHO8ly@GANuhP zBK%{Sj9FyT@a~uIrqF(lSBB~F1!kIW1`bmumE%C+nYz;PAfoZNlXhV)nrj&CFfx=& zp9s7bZ+O9uUi{fhnqLfoASd#iJFN{Na%BX=6;>iEL#Jn9QYUx7QT0jdLM4s}8n!Uu zi)?guO8^2eXYB_fM#6PgpZJ>7#%6*rs*D*f zP}SLL$KZ9y1bYrx+T4$#Iww))ho);w=xw?W?Zp>!;TgB-E>FbZPYQJEkL5$liMk9q z@L?KbFR;(6n8}nAQ0m8`=oww<6Y9;)erk$GI5GUtgwQ&1$(YZGw~o7?9SYt3b#Gg+KxnoJ(ZW#0S+cm7I1)L z7?8C>59`L4o((a?AWsh0%}?do4VD??IvX~xqXBmzT*XPxJQXQCw3vR0_{>kY*)E4G zk~dC>l9lug8T#`ijVBU4h`2|HNSXrNqw;F+3%g2{W%R2pnICJC%gu(N?lo*s>W*m?xyyYUdar(a);4`gV){-J7x ztbA{%65w8m!p#(erwJ$uc1@BU5uMAWTSZ0J7z1a#!d4rDW{{%RxKU~9;E;KyxNhlx z405dF#a>{M?yV`FVM+&g&o+f62NIi&$cte~`ZLL>p^{ITQ=#J5O~6-pr~oyn0Es{i zJQX2vyGf(FcvPy8cV)NohxGPEqZsOKEqbB~Kys2`kmV|Ln9;0_*FxL8chB~r!dmI? z@BUov1w^U_)8f)RQVip_y0R20typjw40*#2B%>h4MPrH4rVK&3%oRnur+?lQ!>Us@ zt7<*e<h9fwR_G9_+_J{Itf1$<^K5{EA$D6pb^@nBin*Vmcdxx0cfo zl4qK^i}PNPiqiCGqD6sAVH20_3eZ2P9?56;LG?&Jo(Ke$s}&H;07gOD;~$iE)#-}} zZuBCpV@a0#L=NJRxR_{@9a$=6>hdkO3fx2W;s@#v-q*h{sqDR4-)9nCY~80(Hx&zK z)MXosMKi!1Cb z2)HY4_cUoPMyu#t^a)>30g+4}ifxW-e9*W&c=#3H4-&u>mP%a_s7vZJrp7*i8F*$`s(s{URC}^9g^ivd1`X|Ab zyCs^^^?i4ZkmR~hpm@T(g)y5;Sj za!kb-6j>PI`xMqoW3-zgg1ju@;?NhdV+!L^ZEsQruOMQyAv}8Hgb0pFz&ODHU{cBi zEL9t)3P(SvFqwY*Y}xQyuGVwd3V!7uLxcLfHDN`&t&Jay>elC8oOIooh7YAj&U!1Z zt@*$78@D*{ZP7X2hTwNs`1$v-cw`{X>U*b4HE)3HvCdR(5Nh~R?dPD*xsSM;7p|Vd z9_~`6af+L=e7$=Zw?Ctm!}~K0?`%}Cd|P=UHr6Xm{j<(5O-5ho$1c}ryLKu5c=*$? z)+BMWX~R`HgN@&&rX?7Q>gZhGNstV;F%NGSI)B>s7JpK5s%N`3FLyHIaaz6c>49pK z-DbPqCD%6d6&BB-K2{$pjR}$r2v_?!iJqr@^aB;(ZUfMIG<*$YN@2s(v-O z3ZI|{II*>HQO*w+p+1WmkiAJ!eaAP|zdCMIPik2D=`LwGzfQ0ijcW-$_@hj*6g_r& zooJD$@wOwf=R}7uz|wFHl8hIdl#jgyI(57eK6>Lb{qN-KoH`6WbBcCy`q)9keV9;- z>#%ddO{UXf4Bp|o>fNfxeZ3!(*R4u`u?ei>s!-F`%|MXHH_cA6@%AOJe!)MmbI-9y3*>*0$+|G^SpAUw?IUf3Zool2Mwa}+n^24>6w|j$m@z#N8Q}H`WFSHdJ~cF zXqGqY^9H;!J?)Q_-(AK-55Miz+q;m#tQv+{Bae0xB#6gf@3PEIkudT{lQ_Y5Jib*% zI@@mKO;*pPo$<}?b9pH+;>|*j-re=k65YLDD^3yZJLe=a@iy7k+4+xr(;U?_dppj- z>SC)Vt^LimwUQqnTMq2BN>n}LpB~@LSpla`kI=~bYkTw=oigKPAvx=A!lvozf?$CM z>dHBvpUybxl@u2Z#@;IY%dfD_-RBy6I(Q#k-QX=5v$Y|&(TR2W{xH_1U(?Y!*Z(Dw z+kt`($)Pz?v^76^-l!VV)d0H3Vrr#l?h+<^j$}L0Oe{QVTU%>wy^m1jH zM{Sh}XnMZE(4VU&2J|`wESIxxoBJfJr+txDX-T(4n&Pi+H4D7Y`m@@<<2Ut6fW3V% zrBl@nQRl3BH|qUWjr;s%HAQ_7wwf0Prd|tkXDKi%bhh2T^vnvE)xPM?UBFMf0UjQ= zTJX~AEs<|+cgSMbc&v2$+-%X_j;*ptj^zhy_L#Cdt|jY{o3UKNV)DygYw2$bQl<2` zF7SW$tIw{%V^Zl?jrjjUm0h?{@3SqHnR%U4(t4!F<2BUSWVoU2N6UaELJ_##9cZ%G z*eFBBz7IV#$k)oa6AKOHbD)tOqQjM?%UeWYmx;F`8feKE+T#( zWZ4G;Y&&!NvW`xc+oKM(-7JMTkh80{Lwvv});$MAwi^*Lkg?+o-= zL?{$*Jl_&JIJ5s|r&zN!1Yfh99@FH!YqgAP!VjS?nv zZv;HMwu;y|#KjF6RN@nAc9sWJMJ3+uzJWEWNV=o_Sv*mE`IJ+h)L-Z0$_E2J2+iBP z3X%R&H^oxk)mfMMHF6W(#U9W$x?w9^Wqxaq?8a|w?$I%%v@$ykiCr)Lfa!Q!@3YXH z^SuKCepC|f7V+-f?DB9=Q{&uF`kH~g$9L*i?|&ZJ$G%z>`@;CM#XkJe_^(S1%2M|j z>5Ocq7)UwSF5yQW^j_nL*bQPX>X01<8E`h!x{)hFHLS2txF;kzn}U<3RDyp=-=&Bz znIu+aocQ`0w=GsF9@DRtjPrsq30z}~j1?`sPdv;vXc2+7*xDJVa-;JivRfTQi=p^% z^ZSri(Vokk#kW?QH99yT{d4=mQlA^v$BY**l%WYF3wXQXA4&@}SHJxhkzv%Y?fwdU z_CA?#7v7^&eB!kO;1JSeVfOy|3`b|v3C#XRVhew`ih>^SNmk$Y^4{$dinp3|_~2SQ z@S6_*!scV7sY%88GD^rM(sM|-*4EOBvIQ1-;h^?>Ia-OU&;HV3bju|8jT(n8>zzce z(V;in8*LJpg7(2!|4dRNT0UazIGZoFrgt#w4~tPw? zC!`a33!p0rqX7QjYz