diff --git a/README.md b/README.md index a77e71b6a..68466b1ab 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,76 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-26 02:59:44,734 submission_checker1.py:3259 INFO] Results=51, NoResults=0, Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3266 INFO] --- -[2025-01-26 02:59:44,734 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3259 INFO] Results=53, NoResults=0, Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3266 INFO] --- +[2025-01-27 12:01:59,720 submission_checker1.py:3267 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3272 INFO] Open Results=49, Open Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3272 INFO] Open Results=51, Open Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3282 INFO] --- -[2025-01-26 02:59:44,734 submission_checker1.py:3284 INFO] Systems=39, Power Systems=0 -[2025-01-26 02:59:44,734 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-26 02:59:44,735 submission_checker1.py:3293 INFO] Open Systems=39, Open Power Systems=0 -[2025-01-26 02:59:44,735 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-26 02:59:44,735 submission_checker1.py:3303 INFO] --- -[2025-01-26 02:59:44,735 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -2025-01-26 02:59:45,832 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-27 12:01:59,720 submission_checker1.py:3282 INFO] --- +[2025-01-27 12:01:59,720 submission_checker1.py:3284 INFO] Systems=41, Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3288 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3293 INFO] Open Systems=41, Open Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-27 12:01:59,720 submission_checker1.py:3303 INFO] --- +[2025-01-27 12:01:59,721 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +2025-01-27 12:02:00,871 - root - INFO - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.6996 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.441643 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4042 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.197574 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.9943 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.431004 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.69575 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.5726 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.406436 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0518 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.13282 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 25 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | -| 26 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 32 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 33 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2716 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43976 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8984 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426066 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0912 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 50 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71387 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412174 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1187 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 20 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45726 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432023 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 27 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 32 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 34 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | mAP: 76.951 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 35 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.07 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 36 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 37 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436753 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 38 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.429956 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9898 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 45 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25372 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 48 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 51 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 52 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file diff --git a/summary.csv b/summary.csv index 7f6de3767..45ca81ba9 100644 --- a/summary.csv +++ b/summary.csv @@ -1,28 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.6996,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.441643,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4042,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.197574,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.9943,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.431004,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.69575,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.5726,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71387,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.406436,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412174,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0518,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1187,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.13282,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45726,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432023,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -32,19 +34,19 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,mAP: 76.951,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2716,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.07,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43976,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8984,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436753,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426066,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0912,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.429956,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9898,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25372,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 484e33c7f..e5d373fba 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.6996, + "Result":1.73704, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -99,7 +99,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.441643, + "Result":0.379074, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.4042, + "Result":19.9943, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.197574, + "Result":0.431004, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.69575, + "Result":7.5726, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -360,6 +360,36 @@ "weight_data_types":"fp32" }, "12":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":1.71387, + "Accuracy":"acc: 80.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -459,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406436, + "Result":0.412174, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -519,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0518, + "Result":21.1187, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -609,7 +639,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.13282, + "Result":9.45726, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.432023, + "Accuracy":"mAP: 49.593", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +809,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "25":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +839,7 @@ "Units":"Samples\/s", "weight_data_types":"int8" }, - "26":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +869,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "27":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -869,7 +929,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -899,7 +959,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "32":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,7 +989,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "31":{ + "33":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -959,7 +1019,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "32":{ + "34":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -989,7 +1049,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "33":{ + "35":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -999,8 +1059,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2716, - "Accuracy":"acc: 75.000", + "Result":10.07, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1019,7 +1079,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "34":{ + "36":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1049,7 +1109,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "35":{ + "37":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1059,7 +1119,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43976, + "Result":0.436753, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1079,7 +1139,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "36":{ + "38":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1089,7 +1149,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8984, + "Result":23.3041, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1109,7 +1169,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "37":{ + "39":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1139,7 +1199,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "38":{ + "40":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1169,7 +1229,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "39":{ + "41":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1199,7 +1259,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "40":{ + "42":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1209,7 +1269,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426066, + "Result":0.429956, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1229,7 +1289,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "41":{ + "43":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1239,7 +1299,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0912, + "Result":22.9898, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1259,7 +1319,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "42":{ + "44":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1289,7 +1349,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "43":{ + "45":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1319,7 +1379,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "44":{ + "46":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1349,7 +1409,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "45":{ + "47":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1359,7 +1419,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.23584, + "Result":6.25372, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1379,7 +1439,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "46":{ + "48":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1409,7 +1469,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "47":{ + "49":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1439,7 +1499,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "48":{ + "50":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1469,7 +1529,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "49":{ + "51":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", @@ -1499,7 +1559,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "50":{ + "52":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", diff --git a/summary.xlsx b/summary.xlsx index 14274c0b1..4a85c3b87 100644 Binary files a/summary.xlsx and b/summary.xlsx differ diff --git a/summary_results.json b/summary_results.json index d90b92267..d16d4fa2c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/Community/code", "Unique ID (e.g. for Audit)": "edge/open/Community/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableCommunity", - "Performance_Result": 1.6996, + "Performance_Result": 1.73704, "Performance_Units": "Samples/s" }, { @@ -262,7 +262,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2716, + "Performance_Result": 10.07, "Performance_Units": "Samples/s" }, { @@ -365,6 +365,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "default", + "Platform": "default-mlcommons_cpp-cpu-onnxruntime-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 49.593", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", + "Software": "onnxruntime", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.432023, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0010", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_action", "Platform": "gh_action-reference-gpu-pytorch-v2.5.0-cu124", "UsedModel": "gptj-99", @@ -396,7 +432,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0010", + "ID": "5.0-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,7 +504,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -504,7 +540,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -540,7 +576,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -576,7 +612,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0015", + "ID": "5.0-0016", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -612,7 +648,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0016", + "ID": "5.0-0017", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -648,7 +684,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -684,7 +720,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0018", + "ID": "5.0-0019", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -716,11 +752,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.13282, + "Performance_Result": 9.45726, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0019", + "ID": "5.0-0020", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -752,11 +788,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.69575, + "Performance_Result": 7.5726, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0020", + "ID": "5.0-0021", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -792,7 +828,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0021", + "ID": "5.0-0022", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -828,7 +864,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0022", + "ID": "5.0-0023", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -860,11 +896,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.4042, + "Performance_Result": 19.9943, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0023", + "ID": "5.0-0024", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -896,11 +932,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.441643, + "Performance_Result": 0.379074, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0024", + "ID": "5.0-0025", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -936,7 +972,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0025", + "ID": "5.0-0026", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -972,7 +1008,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0026", + "ID": "5.0-0027", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1004,11 +1040,47 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.197574, + "Performance_Result": 0.431004, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0027", + "ID": "5.0-0028", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest", + "Platform": "gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 80.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", + "Software": "tvm-onnx v1.19.2", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 1.71387, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0029", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1040,11 +1112,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0912, + "Performance_Result": 22.9898, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0028", + "ID": "5.0-0030", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1076,11 +1148,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.426066, + "Performance_Result": 0.429956, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0029", + "ID": "5.0-0031", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1112,11 +1184,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.23584, + "Performance_Result": 6.25372, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0030", + "ID": "5.0-0032", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1152,7 +1224,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0031", + "ID": "5.0-0033", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1188,7 +1260,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0032", + "ID": "5.0-0034", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1220,11 +1292,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8984, + "Performance_Result": 23.3041, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0033", + "ID": "5.0-0035", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1256,11 +1328,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.43976, + "Performance_Result": 0.436753, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0034", + "ID": "5.0-0036", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1296,7 +1368,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0035", + "ID": "5.0-0037", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1332,7 +1404,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0036", + "ID": "5.0-0038", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1364,11 +1436,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.406436, + "Performance_Result": 0.412174, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0037", + "ID": "5.0-0039", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1404,7 +1476,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0038", + "ID": "5.0-0040", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1436,11 +1508,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0518, + "Performance_Result": 21.1187, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0039", + "ID": "5.0-0041", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1476,7 +1548,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0040", + "ID": "5.0-0042", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1512,7 +1584,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0041", + "ID": "5.0-0043", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1548,7 +1620,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0042", + "ID": "5.0-0044", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1584,7 +1656,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0043", + "ID": "5.0-0045", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1620,7 +1692,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0044", + "ID": "5.0-0046", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1656,7 +1728,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0045", + "ID": "5.0-0047", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1692,7 +1764,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0046", + "ID": "5.0-0048", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1728,7 +1800,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0047", + "ID": "5.0-0049", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1764,7 +1836,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0048", + "ID": "5.0-0050", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1800,7 +1872,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0049", + "ID": "5.0-0051", "Submitter": "MLCommons", "Availability": "available", "Category": "open",