From 6ea860ed11834c9ef00b4b6455ee896b1dff7721 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 29 Jan 2025 23:27:53 +0000 Subject: [PATCH 01/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14420 -> 14420 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c007c7e15..5c6182194 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-29 23:24:45,256 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3270 INFO] --- -[2025-01-29 23:24:45,256 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3270 INFO] --- +[2025-01-29 23:27:48,327 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3286 INFO] --- -[2025-01-29 23:24:45,256 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-29 23:24:45,256 submission_checker1.py:3307 INFO] --- -[2025-01-29 23:24:45,256 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-29 23:24:46,342 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-29 23:27:48,327 submission_checker1.py:3286 INFO] --- +[2025-01-29 23:27:48,327 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-29 23:27:48,327 submission_checker1.py:3307 INFO] --- +[2025-01-29 23:27:48,327 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-29 23:27:49,391 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1778 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.287 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 135b797b4..849baeed6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -542 +543 diff --git a/summary.csv b/summary.csv index 352325781..9ee81ee61 100644 --- a/summary.csv +++ b/summary.csv @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1778,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.287,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 21f56abfe..4201fdbe1 100644 --- a/summary.json +++ b/summary.json @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1778, - "Accuracy":"acc: 74.800", + "Result":10.287, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c746258d32f296912133aedabd7e10c48878569d..e7c24fee2b5ac347427c16f2229f2a53c5a1dc4b 100644 GIT binary patch delta 1014 zcmVYXN*MfG-93Qh+Z7_)>r`1^7~ce=i02 zQh+Z7_)>tcTGC57z}E)&+5lf0;A<;H;GEXigI z*%oLGRI;tm_RdX=3Lxy(@oF?mN}!q~-xBCKOoH^8*H`B~B-}Q5O~n3Myk-M`Q>X@_ zesidXS$)Ypuq|W*Rry26#xi?Tq~=oltB&il4qN=7VezR;8LU8x5}{^*R_$~@nz7aB z-=Elr3jmaIQB{*C()KW6f67$wtusdDAyxG&=eTwzr%wA9Z6cbtiEDP=034MGsMtS3 z<@o~|6?vywo%8vTUokmUM}9k5V>Mw7)lGE(`8vC5n{{KYx4D}|bYizxW`vIJIF7p< zyAc@>ZPP*Mymvjg>|INVqkM{9r&|orR3pyt+`OgS;9MO42T)4`v(hPE2!CaI-ccq2 z002+|000pH003lfV^DH$aC0wXZ*pZWcx`N*lFMqtFc3xe{R*SKEkF85h3!CFvTC4E zmz3^=G;u_&2a;MRe_zFMoCLD!a&+&UxihGoZ)F$#f^|Zx3S~)(B2Y|oq1p<4G>dqK zBIl{%w9^VIv;&9c^-akPVSn1fo7D!a7vQ1*QjRcFp^f*3VC>iiBy~w}s35MjmDGpQ zw%AbiO5Vz7q%8U9?pz7MO$LhVn*0M#XDgjt|o+eo^^=S4VNN1b1W z6R!pEr(Mvmyp4+-`hZ6v06D}xDy!o8rCFdlPxHGt&EovA$qSM{klWeUe&j{}2`I-Q k|0W&*huoK+p#HOVGo1qj`xrk8lNUF-1Z8^OQIpj-e|Dzk@c;k- delta 994 zcmV<810DR-aMWs40iziTJxl zTwc8dxO$ZYDdo_5pryb-OM!uw0s}1t23iUXv=kWNOB7F+a)2)Z_!59G0r(PtF9G-x zfG+{~5|gnf7Jsh=@U;NG6yQq%z7*g~0lpOAO98$V;7b9%6yQq%zG_J?@MXc6&6|?qj>;4HMsAcXN2To%(^yNBWHy*$9d~rHX=k!TShhWH2yx`kB?Esl zc||D@GTD-ZF;fA63R;Iy*E5c*owup^IVRKqYVFSZ;(tBwyTJ?44j6B5T20&je&7At zUJ03IN$ewF273ya!50OLvnQV{_ubw0*z;4q`X0Asx!Js))q%aQo&jSM`IeAzR@Be7 z`rZh%fxDrt&}yPwHw3s{`Q8qZm}>U#m0~?$qZHTIw*7(GHD5LM2=-}a@4jK|v1_UQ zHnh*0@_%WxPxx{FHS8m}=8IS(h|T)6`7+iBVzVThEo57uHBia6LfbnxF)DztTgR)> zC@Fzzl6*^`=P(JQ?XNnn&pK@JgNDVYE@iL+DN2Nz0a~@w{eNi2R-=D^VjnI5P|8JBO`b^G!-OeQ z!MDyBm4{T-ubkuBnVdT9U$lv6-X^Zuc>{1%CZJ;f2$km#XjJ5#W_8ZzM}EcRP#yX0 zWR2B?HB>j%0p#oKs%`3~I@WrdyIDjhc6((;=;)5)xXZB{kpa;*9fZz%*MrO6wUju@ zrxNIOy2SuZHR24<&0ES1&c*S6v(G7E2!9qAo3JJU002+|000pH003lfV^DH$aC0wX zZ*pZWcx`N*lFd%TFc5_AJVoSt;v{VmSV<}Y>WNB76j5=r>}_fh|Hw9^d3)TXO)M^tnLbC2!GQS z-mErYy#N;lkaC2X679V=1Y^hcAgN1&Lj`fCt)xDbw#J6CZ(0M+)ASxo@RU Date: Thu, 30 Jan 2025 16:27:41 +0000 Subject: [PATCH 02/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14420 -> 14417 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5c6182194..62da15120 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-29 23:27:48,327 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3270 INFO] --- -[2025-01-29 23:27:48,327 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3270 INFO] --- +[2025-01-30 16:27:36,780 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3286 INFO] --- -[2025-01-29 23:27:48,327 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-29 23:27:48,327 submission_checker1.py:3307 INFO] --- -[2025-01-29 23:27:48,327 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-29 23:27:49,391 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-30 16:27:36,780 submission_checker1.py:3286 INFO] --- +[2025-01-30 16:27:36,780 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-30 16:27:36,780 submission_checker1.py:3307 INFO] --- +[2025-01-30 16:27:36,780 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-30 16:27:37,869 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0152169 | ROUGE1: 60.7242 ROUGE2: 44.8179 ROUGEL: 50.6964 TOKENS_PER_SAMPLE: 327.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.383226 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 849baeed6..4970e693e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -543 +544 diff --git a/summary.csv b/summary.csv index 9ee81ee61..5181c03c8 100644 --- a/summary.csv +++ b/summary.csv @@ -1,6 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0152169,ROUGE1: 60.7242 ROUGE2: 44.8179 ROUGEL: 50.6964 TOKENS_PER_SAMPLE: 327.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.383226,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4201fdbe1..fa7e3d066 100644 --- a/summary.json +++ b/summary.json @@ -39,8 +39,8 @@ "Model":"mixtral-8x7b", "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.0152169, - "Accuracy":"ROUGE1: 60.7242 ROUGE2: 44.8179 ROUGEL: 50.6964 TOKENS_PER_SAMPLE: 327.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.0102599, + "Accuracy":"ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e7c24fee2b5ac347427c16f2229f2a53c5a1dc4b..1b9c97a0c87396e69a0af52434a4f792fb74b1e4 100644 GIT binary patch delta 2489 zcmV;q2}btRaM5tEtOo^uVMeNiqp=4Ce^&K=JlYD`&~kVLF8HAtd*ToPCFRg|q7 zfnB?Q&t_ zfkiw{Cl9}Y<1z9%;2Akt2Ua~JODtk0Xv;zo=pCpxQB$`O!k)KdZesMrZI{ zm5cBkZ?X!Qki90XC$g6$zqQ4PiGus$Rl9J6B=zIe&t9v1ifurv!0v~!`>_mm75P6` z?BfULgVFim7@s~1>L;NmMcYQKMlfUeA0S+xKP?H#JHCPUX&*Er1eT!>&NMD z$Bc0#9!=Ou>zVV(k)|>KnZ_Pd(|-v~gl3dxvB$-~*;r$fjCuikrj*B#Ms(fQVlPZu z$WQZxpR{Znw@{vh|KzkYmtfa!ogTCB^q8OCqydeNHGX=m6@t02@6*J~!n7)XOhV48 z!X9OL#H-SbvV=;~KFx$Xs?xjBC<+r=75|abETl}65c^&{iRu(r=QS1!-hb%o9DSqe zSVh8d=qwLH{=K@glSXOwlfFt|`7jPSm#sdVdST=Z!zkqPL$8d}xGKFBzLsph&r+W< zmP-3iewPq^pGGwExV+et7WdN;fAKQOtG6kajX0-Vkdee4VQ^Jusk)-;I^&0QVQ<(AbIpkn!GRq@UyDUbw@=v(hQdZKAk{C#m0V!GCo+~$=smN{`py4Z85`ROT8#FakqFO4d zZZB$xV5kJ)Xp|SVMwv+|9YtR35yjDmitARDaX90d7b)$TGn7nEe+lE|TzeEpTw6Ru zl;)Y|OF2;njseI`KyCqY>C8&G1IXKeTyn--j-@kP3-F=}uZ6cU)E!4B!O*e;C8Hyn zi7HRu0(|l+lGT46On-@Ls#syZPfTU#F{*8GW-y`%UeD*nQ!6-s1*c4lFu|3q(l*9G z(_ouoYHzH;R}w40Y!M9qC)nOzoYyub5_N>}x3UYunlfxQ_N2Re3%D)IyEbnQEGfm- z6RDPk1rUKf!mUqp3r108nf`s#kr@7(peM+ghAv@O%r zvF$i*({>TMxH?~GiD9X>I5pGI+gMa)x%T(&9xuE22-#Q_@q-b16ObchII3$qZYj7H z0#`L`>>%`T_kXi{+aKI_dxQSj&HZ(^cv_d;U$f3IiK-(sW^wzM2+H|&CrIdeA&!T$ z$(mH!aAB*XtJreZY&Ee;Y`49#)f86|u}Q71L@14|ZIEdRr7%GN=zvN@q9fgz8RcQ( z)im})n(>-)IG*soYsBT%OMt6aN{~_xtp{2P473y&Xn!d%&{ANarNBT-fdRfm@^mQ& z_!59G0r(PtF9G-xfG+{~5`ZrO_!Q0ACZ}YXW>tfUjKAOF6*T0{B`0Ukl)CNqlKArt_wxxTEq!ypd~V z%TZ~2_+f+pmYePCSsmE>>KRZv5pM|@qkl#HY|HPBKpVIp+8V7U%5_74yOr~*+RAhS_74AC$xieqe(fAL&gA52|FTU) z^EPqA&Rc+^G65y~N2ol1KqDjXHLJ5fKTh&1CWrFKZ!c?%Caf`4T^>Nb&92&d-B@ce zcQc4etoGX6%~oyOc2;9IA_AhV+X!8DZwJ@iTOqNfPtlun%K@5f#08$4w~}j&@zehR zv(G7E2n9#Vl6oYQH8LxIXi|jjKwGkEpiq~V?u0Z+M6CysS|@*B#jhlgRhOfC=ggf! z%mHQv!PcmoKpCsft%JzoZSZ}W@FcNNmumNkUztpo0qqK z)mW(0sTiQTooGJ{|uDlkbe^=z>xd;6V!jRcQexh1%F{is)UnIHd_K% z4wE1@6_dd>F9E@m@irnGOUj#tLbPyzq|5dZ)H0000000000fPo>C>^D9EbdxJMBL*)u00000 D!;+|s delta 2487 zcmZveS2Wy<7RLWWlp$(_2%?SNdv7B;(KBWoEjk%>1~C{!uTc}dMF|FzK_uo(aFmD? zQAY`)1Ub4OO5&b#?!#H@uDc(858t<+_FCWi_NSnUptfcjsu^W$abgXgh6)S2?rzE7 zIGS!GLWfuO&}GFN2I{fmsq>6|m{Sv|bBz1l^;jp>S@D#Vcb}peljhO`A$^4`lP2`J zw79|i0!i^$@9+_m70bzM^-xrhB7|oq7h|np3sF9 zB@3Q|B&8=R(1#I%iYY44acmON7fp1d*emP7vv>R^vc^_E}-WqD2>+I3Q4Y{GNENBrO`lX=xS z4_>IruN%xE_cR=104zD3ieDoAJL0^mQFUw1*T^l{u$?BUTY&)ws>^SV z(r*qjHu5MG@k(9!MVL)@M^P`@zoh-RFWeLhp7HOC6u_7qI67uL5$%$3?w~g^w$Zus zbF1TRJ(;?lZY$>Uizj%5!-)?WfN4zvQgGghT!bUB?KXYQz5(eX(cAO6nQ6&e`Y89 z+a(Rbfsyh0mGc`qba{S)0$4NK6;)=^ZmX52i~@}gWF!mCYrr~wT3|Uf*{eG`c=AdlR^2UiB4bSQIN?Ht ziz_NuPG-q6BUM#C#aG>ME764h)VDmPrf_A6{cYY}MJalnbv_s|l&q<#%{Bz{?Wxw+ z=jOec=qgP-gfDx#AqsRLRZI4675kd3_P&D=eQcXNBTqFN$4Wkl(p|fI{W6Cj=`nA` zyi1H=ofMP4<1bl=C4L$ZI~)9t&w9EtYH}-K8H4sz$sF4f!C3bkxN*}}A)@7v`rdFQ zS&L32J9oJT+eb=8dWxHo6RaT|AX&) z`#xlP2QE-_vfB}xsmWqo2ywJ_;Yt7r`BZu zWM-0fo1C4da!FMET~%*=OY(hG(%DINN-+=N)mEmrfivV3RAKy@r^AE&P(TAQB=XYO zGmHQ*xQt@sZ+vsEdRBYw^OQ9j1J{S>OVJwD5F3^WdJ7y1 z;Iu-M(>6C{e2K;WOb03c>B$*mnuk$BB5!-}fEv}RG9TSmC?{{ksYJU^} zGL3tt5sv!fXbaY>NRd%_<$FcV2U|)#A~N#upY_<@{0+OIn2ihP8Y z3!A0XmkXg%WvRX^u;*_bxF{_i;?kd$;i+9rnt@Oc*^3HS4NS1}T)XiUi?s8IG_9FE z`Tmwz_z86oS7EsO0yo(-4JT}4@q76>ljN#|M?oc!R%J|du!KS66<##0n26s29XSS?q78l4|?HFnFQZMXPN<$>evwMmJvXt*hYAZms}m~3X&g*6cy{8O-}!CtE^3yZ?14hs zq*rJJzl-zHya_qVS_w1ltz|=@MtY^8Cs-97%yjs}IcIE8cnBIZBT=qwe=^-n5dCD4 zFsK=DnX9MvdItg?m+apUQikVa0dHnIN? zt%xx*wVr9}AG3f9*OI?P6kOKpu1?4^s9qy?xcebWWFzX-EH#j zA>Sqmd6Z(W9}#dQ6$<21fR99bvFJ1dC;&!!p%1Am{)wq|U?PhS9CaYEu3_(CnOgDD z#WUT3H!xcWzt;@ggu1Ak+t*^|u?*+5EvEugwD$?+MG9K{YhYSyr^{Be`dznteeJKLf>pWIliAScrU`61T! zc_L%SvTkqf_HYV@Yy9>{66;-I!u5B~IgkRyFH-%(769%7a06Kfx^_x=^ZJ8&c-i6U zog5cAH9Mmi4Y<3K8^fdp8aP$&_A5cV<6gWhF+kmW0rN`8+ELBw=I& zB(VN$xqX~Yyqz-n>ygFqRkzu`l1b`jj-_hACQH_~D)S`Cb&E_5L7N0M@U?UlOeBwW zlC6#{9D5!VFSvutL$Ke(7AV~nSU#n!-iH?R&M^QVWKn#250VLxKSWw-BtB0j>8XX9 zehH^;L1ZtlE~v|Ko|7!k?zJUdaQ^y@L}F?hgl=070Fy%A3GhnixmW$`2CBaqmANEc z`zs3dPNS{F%yjOvSR;NwtzS#LY5BAkUa4+**sc10`2s&o&zY<9Xl`aYX4T*lEIP!0 zDSQi*JpZ_&@)somr7%wM-v=tsT-gx>&udUN0QKeVC<}1?-OabD0YF?70N8+kCpHeA zC!wMT`k04QQ3vVeHK+)4{c~y%09a7~0L#CtKx~{q-lmE+C_GP8RghX<>2K>lKVX`& diff --git a/summary_results.json b/summary_results.json index 7b6fae35a..2a31b85a4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -298,7 +298,7 @@ "UsedModel": "mixtral-8x7b", "Model": "mixtral-8x7b", "Scenario": "Offline", - "Accuracy": "ROUGE1: 60.7242 ROUGE2: 44.8179 ROUGEL: 50.6964 TOKENS_PER_SAMPLE: 327.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Accuracy": "ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "Nodes": 1, "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, @@ -320,7 +320,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b", "ColorKey": "availableMLCommons", - "Performance_Result": 0.0152169, + "Performance_Result": 0.0102599, "Performance_Units": "Tokens/s" }, { From 24d0058c5674f90c9b4dce508699bf4048bc2f72 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 30 Jan 2025 17:38:59 +0000 Subject: [PATCH 03/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 18 +++++++++--------- summary.xlsx | Bin 14417 -> 14420 bytes summary_results.json | 18 +++++++++--------- 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 62da15120..ca35fe06c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-30 16:27:36,780 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3270 INFO] --- -[2025-01-30 16:27:36,780 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3270 INFO] --- +[2025-01-30 17:38:54,484 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3286 INFO] --- -[2025-01-30 16:27:36,780 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-30 16:27:36,780 submission_checker1.py:3307 INFO] --- -[2025-01-30 16:27:36,780 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-30 16:27:37,869 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-30 17:38:54,484 submission_checker1.py:3286 INFO] --- +[2025-01-30 17:38:54,484 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-30 17:38:54,484 submission_checker1.py:3307 INFO] --- +[2025-01-30 17:38:54,484 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-30 17:38:55,790 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -69,8 +69,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432223 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9262 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358711 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 51 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24578 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4970e693e..fc3dff159 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -544 +545 diff --git a/summary.csv b/summary.csv index 5181c03c8..0fe0ff04f 100644 --- a/summary.csv +++ b/summary.csv @@ -47,8 +47,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432223,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9262,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358711,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24578,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fa7e3d066..f469911c2 100644 --- a/summary.json +++ b/summary.json @@ -1452,14 +1452,14 @@ "Result":48.3215, "Accuracy":"GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.5.1-cu124\/gptj-99\/offline", "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -1479,17 +1479,17 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.358711, + "Result":0.352694, "Accuracy":null, "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.5.1-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 1b9c97a0c87396e69a0af52434a4f792fb74b1e4..b3734d7aef79a37450739dca55400ac1abc8a8fc 100644 GIT binary patch delta 5107 zcmYM2c{Egy|HfwwlEheRD$FqUA*L)@##mw|`@SptUWl?Z&5%NT7{*d$-zsD$jHT>^ z?AaocCdy7TqTlFz&-vXy?mze5=e+LoexCb2=Z^S~`&YWNGuhe&n#z-W*cDAN60>%4 z&p$8@MD=-jSJ*QSD9AID33q=VNNr+ipBrZVjJT?r&($RS*)O^MPHS!R?ex)9I_3JZ zS*x|-x0;q>@9Tp)sEfl!qBS)w-wy9zzMo9mW!GVtzDiQ%K;rEZOR9Q}4l;%(YCh*w zdDkD@S3EqH?Nh(3r>;1uMODiATX)=3N@{VhO{j+*29EvrB|jiJC*h5Z<(+Ri_+c@R z)zYQPt~JqRYFFUSLU)kqCB1oHechY;8O77TRj=7(Rh&^A5)%`n($I<*mrGZ?QAay8 zs~^Sf!DbVUUZr@U;3;K4iHpHO#kYp zsEsu#A(rSjLDbU@cTuZ2&zOIt{Xc57+=QS8Ap*=>tG-dAk$#acK4t;5=;Xpp}6 zmdr*{Oo$H|${C=K7vYT9)#Kt+J<<`1ELp&6lOH)I#VBp#xHu~oV6ia;nNHeq1>xF4 zOW)aLowaY`9Z50Y2Z-w!vJRdY^JSxf9p4tb`RK;(ESP(jj2Zu`*uCzs*@E z?ibnBt<#D6Q$Y*GgYVhr9~raC{&qCGFh67x9jMibzQT2 zNbXNBU0#{!p}01pj%)<*Pwh;?>w0$vV=cpRsEJ>^zi_O4bYcztgIHRt0@s)zpF8hbbSX@#AvE zOCRv^mz(OT>bh;8?GN-E+PKQ!P;?Tg%>yHALSfrA0(1|Ih6UVffA2of5Lz1BvEoWv zOd0}&IB(toTt}d*L zFtK}W_H+?-vn+Rad(FG`e9p#$(enJEq2I5Sp0h62B;+w}Qc)6HUd`n%?Uaj37oXz; zRn0oh$WH9C`EYBs4}fgA&=#CCR!j#lJpM>b;gqV{;9+57=as$1*g+Mcz@UqbnSxbk zb1{y&=JpcoV}80OOa<0G#+t_Z1L8l@V#`3+c?0+qXPM%cjsnog8=9lSu=XT3cp^eL z0uF(0%XARoOa$!B2$U4`RT&S_8lTM2zT)tWUBBKgz~=aZ?O8p3YR;*uv?@m8?=QtJ z4Ibs9lk=ScelLF??X9wIl!)n6$3TkA`h=bkuO|cfu*}w-Y}GFxAd*X1+Ve}$IlKVA z)WqwEk4YYt2;n!7Q*h9ZX!z0MrTX#Na?t);z$A+KlX5o~GWc`Qy8Ey7T6}(CP$=<&Kr($1z%^Ibv1jX`T!Mh5ZReIbOPnrr-fLYFv}fg1ys!RNb#W^vCJWR zsD$P(spClp>v~y73q5rSRg8lbP0DREhW23^rR44Cl3`vHN#D6oD)hbNLY5kr=E+kc zkVW1ry`@`QPnn2TMiCMyDF8W))wK|?eIp?}8FNc6iW?yj#a%Z2&{+_t(vYUXG+0|4 zb(`!?`j1Bjm_{P=2;%+rT9Ylhw-|MY=?U9ppE9vCA6fu^|i6xqm8)J;`8ihj4mFKdUlfE|GJi`)AAzJ(^lf#d zrk+=J08GE9_L9m$$tK&|FLgCBVts%tAQ!j~nuR(O!wbMG;{?DGgrI-eqo=dMv(ok@ z{gzsL%a<}X5}o;7PB%$PpV7!{Zp|7Go|?*Il&Ce~qA(`KBTZR_V6JlLGtQ7(oL74D zjo5^kHs>Zwn5!7tof`o!X1NsmFnk*PLgoI&@Cv@6lwj8Dz>s?acLkb@L`T9cEMt8e z-1M>dhSE`raV(Fr*sGvej1EMCI|)1r$>pwM`v`&ZNaUDkv(iaR=%mUxOHMkdC|Yb< z`a??fgt1@Mm0eGs2J;KF-2TarVa4WIy}<^Q%_lrJ#7v){pb(M2w+iU9cP6NGBHM*1 z##5N{xMOfR5pw~@nw0!x80Ope$r0aP_}CpZ$_Bq@sazL`Dx2oZNa)^K8S3&h`=U2W zovikgUOrRre zH3BK6AE!(wu0=^hX5+5bCge^Nu9EPTzcW$|yDYklrw|;#^a*wOYkfNdOL(u+o7 z3fQO5w?|wjk?#S>1xmH4Ih!2}bedh}`u|Lr854QTAz}uS}=FlVxD|;;o^#JwN@Dy`e_`8zv?ZvEkbw8t;kkeGscHt+leGdQNT+-?#6jCwNHqhn?KqX zuxb=>3D>Z%WjM)ku6)^UOZ` z7AqYxH@3!?l~+*JD_J?)*E?FGa0G@elF&x>-!O}bA`Sw~V%!J@qcGR864nvc_TpjQ z6Wz=c-ALd5lWBTjr@deL@Zlb^#Mns!S|;3pFQ17tdyNOHPK~pJFeEp>Lg0xUC;!=5dfN5h*@)lG2%b#PkyU`#tJO>I)8f~eQfw!=sji`D&EDdHfOkOYhRalHk=D_OIH}O|2u632rlm<&qjew&B6c@VRqi&b*D~Cqknpuvec7Gp zgbb~Sv+q(xYzj}mWn9&r!ms~{x(iXK82(K2M&Ajx0FBPycD|G!anBQdp(x4R8x1cg z8z*2`8reP)teM}su7QB*D2^CQfDBHEn3ga^4{@(!d-Eep&?qY;>TzVyp!2ZH`3L=T zZ;Rdx+exRLtu}GxUz7EYUN4C4<;KoNKQ61qVOU^nl{jl=D4ZYvqhAD4XLJDLz4ct? zW<+b6;UHboQ*q@&=ttjn8nfcPPw&xz&s802R_^Xqy|pDb0e3PlKeHP69(kLR(vq{6 z9P0Hoe>CsNGe1##^w0?=)gs8HDG4GArU51UUTefgjg}k{8CZ&jX(P9e+SB6s6)GRn z3$b@Lm#yky&>()&h^c=g?`Ry^4wV_0Nt`S?JNlu)vUDofT@7b-j~=DVa!;#hydTyqE0PMj60#T1Bn)OC+g>P zXt6?{qaLsRNnJ(rmq)@`wqW&=K0~rcsYn!e)A>e!H;pAm7WE;7YY zW2Tc`VLIImI}-iaS%O9cS3{t*O14U@ezTp$gI=wnqU6b-M#n812DpIp%E5z-J9U+CNj*xrV@7ghN;@0AUHKa?@u z2>Ht!aub~Xe&1xqYwmgHJCQ1#uT!rD=xToSG|PZ|>1mefn?bb~8STTEhpu9ZFx_Z`emJ<{e%TCjW2t{JFSDsUS12zZ-5 zQ2IG>t5K~Gfc%cX8_xDtty07#+ThK7ytbCCBJ+1t8Tj+swDP=eNJxp#4T}S;*-Cu| zs@R$K($d_~x{YSB&RH_fb@BSc;NQSASjjLrV@h>=VTv2P2o6fI&7o+q0*YE0aA%7D z4T9t|E2vQeT+7r>PGyGwoa|HQ=@aW?s*9Iq#9p+JNHSj;o|8=OkAQFKri?$d#=Hrc z6NYIc(T7eDbh300x1K2OGeLX7-TMlM2g#;)oYm#1Ly3c3)486BVLw5 zGYr25hLv?tsr&MqXYch9asoQ&K{$F`qTolfb4B z=uRb@ai8TUpd?O#qa0vP3lcb^eBzMfVl2zqcdvZftJUa{D7RTX-L;KP)bfVdgVUT_ z1hVegH#nDfELU~Po8U*6onVbJ_{1pi03b2i4WL&)$9{{#9ZFx7qaNGVvZ1&1boLoI zv~wp2wVI3?U#sNfRKKvG+}!6ZBYK|Jg{DwM6WU(ZvPblAMQCFqq8}@pF2Z2ivgk#+ z+eGQ|u_P86n}974yepx~$0KmE&=(E(m#9HsJ26QA4Sec4+eXWz@1D8Fk)v~%bOBq_ z8M}Mo<_)|hZ?4!xPrJlg=NC?QXl|GGhQp)C%+KhhB3FytCs@%-B?YKJHS|Qzo|Nhw zc?49tPjttK9X^rCt#NNytqMYI9S^5nK;7|exZx4-iUNL6rnOnsIX_~$=>((3E(pUq zui(*}PLOs|ISCX(W1|mwb5{bz$0K-xNpM7K`R73@J2|AHIwyS@)G+G{TW3y8eCc); zQ);Pr4|grL+#21d_Og?IWR$@rTC@w;$d?^qocTJ`3C{(^}4*rKn@uJ{r>BNJu!-|~;LCKdV{~`yw zDGyqkj(TXs2icd8XD-#WzyiTc(`=zEBBZNGNxX`vAGjXm*Y`R?)6BMsd$g0YP5|Sb z^#{k>mQ1j6Ncfd?4DjJZSp!MvzrUT^4YCBNE8uB`UF z;9x6Rt!MR{-gUq=rnlmR6yXmk-xph8=T+_junt_8BxAz;ME3G(&Sq@kuk(!QM_oDV1Ba*%`EjbB(v~9tugZk`=4V~F z0TOr&C`?q56r_&r+dXsgx7vCyHCL98#K1HP8Tuo9uI+uIt$)8Lm2HIRhPA&~p z6LbXv`rpsFWCc552kDoBD$tfBrKkb?Nb*xu25ORCDvDnGXC)8-FlGP%I065E5F&(0 T6h##9K1p0jm>DU5a^?R3PeGJ9 delta 5104 zcmYkAc{G%7*vDrYgk&ef&oT_c2=lZ3EMwP=c*{l=A*e#d7L@h|xvxHCk9q3mvn$YPcP$4%u$#;cc@^Jabiu0c9+2KLx z1OG;8R(l6!aIYL?^|x|9c^0$cdRy(NVpOsF(^5yrN(1bhDm5!>ej(gT`IAl8aOly} zUfGao_-g*^hl6MX{7R``?Mu(0exJkO(D$G3f2}Gzc(?vNzTcy&ss0Jx3pNshaZK#A zG1MGBakADvEJFsaH1!8^`bAr7uiFr#dAz=A#cDw8nHs`}b4p z&D^J?F5+U9k2e{((7T*ms0MJ*RnceNpF=oN%3{Lu&g;h&3V6HLt9Nqn6)uQX_c_L=IE(U#;pDg0TiQZQ_zqtm#d)$Qu^p4jTg;?dE zx79aX6`d{FiqLOUI8D>18<0ckZ|({Z8MDJAIks4X4Mb-@AE5Lb!eXGJiC)nhvWe>) z#MoT*=2#9~5Z+Yug`8f6*cSh(j%ak#wA#t(NQ>6>7X|0}uGuwoYLZZGJ|Qje+2+$Y zu9ckJM72}__=wKdfKZ1aXh7yL5#m?>BB1p+ zo4Wq{Y~y>`{R*2$t<+IT{jS-BAD$IzO-Ez=y#;H_2KcRMcb^)o6W=AGj<_r{%EEH*_Aje%abJrJvdhe?zAhYIueYgc%rr$i#qZ?1`0!VHPG|N0 z<@w^n-bGX+zv&f9x0C-w0k-EX zH&KD-PANX9zk2WYe@Au@@e=6mt0^FPa8*=>;wy0ct262GgxI!+$A6_ zsXaaY?toDzuT7=3&-ixW=M_Fn0QNqa8GEgZUl3m>@ix_pGRu@P)($jG)qfQO&PmXR zxgn|7Aw*ag$#z$sa7P+ zr?^0qp&=K?+!gQO@JOs!RGr)`r1^)La-@Vf&mH0tMCXWEgU~Db?(g%DH@hw zuyC8>g6}*S))jmA6R@MXp~F#2HKt_Of0dwlg;1RYtVt}7GjRArfE=?}A2MKV^xykf=Qf!K}V+OQ5l`4b)I+rs5- zn*e7``lDh1yTEbm$8&*B%gPV&PE7WYXJ7JCYC~mk6P9YiKGF^}=|2*ReDh^<(^gH! zZULlrjpjQ>fE|~Ef`-H(d1Nf12f!4IO*W&_>obespCQ;{c>rqa2sPq1$Tvc=&Hut^ zVa2e;scOvPaV~~ND>W+8wFnwjF-=`%O3-7nbj!^fU=jtpGli23EMkx{SyCNEQUAI;)4c}Be%7AV?p27|b+on?eOi@J_`@In z)K~4gXjBi%=AAw?HzAdXWL#$HBVrhSu&~Fa_FfjHrwO%cV+K4!6vc9Dt6AlcfAYE- zjeESR83P9^#@S0>#93J1eU&9;E&1Svm0ANNYJiyfoDFH%!REP(WVB`Rj=?Zspn^sT zk^M_-{mg}V{RQk&?!W%|KuK7mH^JMvDT{P#JKuR(kM4<4n$R8W4{ZEa$5$c~ic+7p zt0{2_g&5VpR?|nxD0I3^c*N-C^&6$suF)Ezqqm*5!CNf2L%a)l#Wsduyhe_7qwjcG zrt4&Ey~JLF^^_avQ;To~4VZ-8+)|R6nyUSbI^CE)H!J@^>3n&r-%*E|3tvzS~(+P5UgUetTXs#sxKpW zufw3~ocu~;h zUZfO8_tu#afnqooMl)>w?pmY->))^5jIAm}^iM?&e&=w%ZZhvaNGr4=Xckb2qYLn7 zU1k5U%tdsgCk)=1x;7}Auc2)I(nB6bj}i8^+|xm_kU`(3!f>NKb|*x-gI@i52n`VI7MGDC0`|CgnE9 zQ|;k3m#|jKf3U`#Xj-0s?C5EKvjqMqDP`9F2+Q!zZQF9^U6Us$+Kw3VYeRF36$wa2 zJc>o0fMGyl<9h%^1U(JNiDONs1ozO=`Poss&_y%FoN}JFZRaGLUw39N$n)$5xrXwc z{ucPYGx+M{n4inC1zxcedZX~{Pa01lkT$sX=`CJp=HOf9P@@2 zI?#y{J<~~o4bVA6?Bw3{$A^8WOQ8A|_!}$b)COCD22dpa7`Gfj{$xYPJVGIESv6EQE@efdjYlAo>Jtn`)Ca3JB z)k8*m_*c_9Lm@?hH;pZi1#9G!9VE<0Jr5mluu|SKujZ4*3yZ_>cRBWI)*A&5@G^nA zSH5mqnLor*cEd2AjW%K;lRE($F)jq73J(kE=ezD&c>vwz$}mNY7A*lW7mJ16lyBld zm-UK;Z&dE#D}f6KePJHpo9yYa!X>9LTIMj9|~Vj&9r(HQ-Km0Qm-b~K??3>ccwD$;N`0ZkpE=`2AMv=avZ>P$xSp0n#<*7lB&InYv6 zQrFaC=$1_jp2~bZWR*S7zh+n0g*l(4uR+WkY1J(s3Pv)1XXy*ZFw8(N0DxJDBAQB7 zoMjRoSa`k$`}0ls5U87>_`dx83VPhlsp9dSAZV637tRBW?Q5{*!Nmm)zK^IP3VK$^ z6_8%AUHE9Z3xQz&)!w}YzDb7dfjo~ige?K%yY2`EMd2}0d26)ihX;r}a1Tjj?@8v% zY+=xB&1+3ptF>|ozf$3QYOI_@3@55KCJOJoyr0ZY;+xrj!;*}o>;XA(9&Azs=+PG; zkSiO_(#-P4T3KZo186N6F?QR7H$=#JXM;bWIzf)H%e1oHaPoTHI!K%qMx5nDRmVg* z7t(Of;2aw6CJOxv2*bJ3aG0YuA&~PQ+%3@eL|OyFwfDnM*-X1D@V#PhLrFe1UB4pU za@Qmcn9cR%v0fo6i8i742vN@N$I0vuz&DTIuy~_J_JLP$6*QW|5mp#f@mJhT3)5Rf z9uJ(c>)^>f&XC4Oe@T2M;*W@alYVPzd{9#C$Y`oNIT@@78H| znRAlrOUl^g?V?sT4AQb+43d-T0(}V+lM)B#3>}%!4&w42&kuWf+)d+8tWO;#&KNT< zVy2;_Vnq@6TC=m~NCfK2D;G})@~a!yWt8>l(|Tgw0}<2-hj_IxZ`{j$J>6I=VZRkl zfZ8WYQcR8-gza>~$PWeO@x#YbO+|Tk9c0w(*KU%3h#JLd!Jv22g_s~ufwOE~fTwwS zpYwZ9p7Rh)sYeeFx0lvi^8_cCYNOUku z@>A8MToyM{(}Y3quN@#6by&P^O81ebIURAGg~?M()uG5tG?p zFj*#**sg57I1Hf148h-uNUOLA<#Ofz7>EzRTgh5mSah6b;R&=@J@eKUx|O-ZDLU57o$ zvs=_$HNrdtTx&g3?tD+kg*3E9G+Jd3gmEBKxdSn;#OQ@Mb?HEKG;ws}18RcAoZgw| zJOCZ)hub>e|EilBf@GUtHA{ek4jl0vJ)2W^yV1Q8ucnPn%3M-)0&W#~B^lZIjQUBe z%7?%>j8hduFvu{Tprpv;Q9vcllWc0{SvOa#Lfq(QhL`p+-XXN2*pE;NzBw#3z_N^; z?xXh|`MIw_>5-iJJ#N{$^w?B+8FocUFe1s15kYyYZpVy(L4VKEPGyS)&mM|l*u<{R z1F{jVv}4i0#U;^yr1^)ZT+XFFb_X{G*j6huq+IE#HXG-0z)6H1!zDToYtX>!w6;1X}2_BAmJwAeaaRXfb~90Qd@Vkv4rTa6-xSY>a=9ZhX!BT6thg zu;}Sn=1Z!DG!>uE=UKFOJ8F-sD6#d3C}!qjvdbY`BFsy=PZnYVroqGtv0#+=YVt|+ z$h0$cUrz=pjpI2|2>dg7Lc)u#@sgah)I=bFG+t89C#53o+MKYMyM1=iybS5jV)gfww&=h;~jY?{4@O@yAGYz>`G`iEuXfvRwa7XdxP=# zl~*uj1n!|YHb1RvCI{d9A5urytdL`L|2v~D z>A4|GU!LhKtcXnfxKzw#XMNe&k~`cv^Fnn|Ur1%5u*JNFbykg?PE_Jd-|I%142X$+ zVD(qR-Z=+u?M@C?>8gV%_5{I+M|oG6Sb5q?YDh;GUj>y@af5<08m5;p5e4|H;5hT< zu`l|HqW?WjS!}P-B>c-Fi?o>f4(r$)Uu`i8sC;g7CClekhyW zkz}n_>eKYvuwl%r!9u>Ack>DI7l!WIr~j&8jJaF5^VEB`1aKC}a`d>?C*?U?Aosas z$a-EVw|Q?j|8P*|bh|t6C3mtcAF>hpaeGy9`}J>@)VX9lmt=GD@e$V$zB1rnFzA18 z=F*hyfi2`MWep&Pd__eE*hBVKQ3Gm|i&P{8{$>&g09c#@05|~u_YlG%Ccz{0VwWzUH||9 diff --git a/summary_results.json b/summary_results.json index 2a31b85a4..11ebdc12c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -732,14 +732,14 @@ "Scenario": "Offline", "Accuracy": "GEN_LEN: 264", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline", "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -768,14 +768,14 @@ "Scenario": "Offline", "Accuracy": "", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.358711, + "Performance_Result": 0.352694, "Performance_Units": "Samples/s" }, { From f1d3cc13a73d4f90ec253018fe1b179a808c89ac Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 30 Jan 2025 20:37:41 +0000 Subject: [PATCH 04/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14420 -> 14418 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ca35fe06c..c1a317f01 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-30 17:38:54,484 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3270 INFO] --- -[2025-01-30 17:38:54,484 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3270 INFO] --- +[2025-01-30 20:37:36,073 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3286 INFO] --- -[2025-01-30 17:38:54,484 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-30 17:38:54,484 submission_checker1.py:3307 INFO] --- -[2025-01-30 17:38:54,484 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-30 17:38:55,790 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-30 20:37:36,073 submission_checker1.py:3286 INFO] --- +[2025-01-30 20:37:36,073 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-30 20:37:36,073 submission_checker1.py:3307 INFO] --- +[2025-01-30 20:37:36,073 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-30 20:37:37,379 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -63,12 +63,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.287 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3041 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432223 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9262 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.82 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fc3dff159..f50c65e51 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -545 +546 diff --git a/summary.csv b/summary.csv index 0fe0ff04f..0aa8af8f9 100644 --- a/summary.csv +++ b/summary.csv @@ -41,12 +41,12 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.287,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3041,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432223,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9262,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.82,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f469911c2..684682369 100644 --- a/summary.json +++ b/summary.json @@ -1269,7 +1269,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.3041, + "Result":23.163, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1419,7 +1419,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9262, + "Result":22.82, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b3734d7aef79a37450739dca55400ac1abc8a8fc..8136662a9a96980d0caa2e095a6937c89e5f0967 100644 GIT binary patch delta 5306 zcmYjVX*iT$+@3L+h!`rNEW=F5*!O)fma#9 zNTS8Q%T{B3$N&BCzRz_%&zJLD_ql(+`@YXP*ZJZ*R*wZ*87PHo;cq5(E@A?u zYWQg@<`eg0M}GO$?v2kagsfa|z?sd~wlv+uY}+ed=3Jul`eV$LX~9B>wSi@s3ZI$#cG)R&F1W(1h)9~{j+bWB6ZhI-n^e6(O^2C@ zYk8)UzlE^d)OUMyMK)h{N1s%{k3gC%jwbI z#X;3?(aVXZO3=h_%4$4Ld6iE6S_i47X8pD_m&8$1{mspp+1C06gciBUDThsKZP>JB z$5YlRw}NG~@6wcCgXX~Yw|W1@$L2>(yBSTAzQb4QUyhd=PBg5_rzhE~I<@j&>#JQ- z>@92Y3H|5wns;?ga+Y#4v6r6=oPWP9UXbUJD?lC$U7{@YF1=N#RBC=ZX3^7Ctr;rt zx7TNLs%ic1?UfU(aO?=Br8Jc>Hh?t<)6uPOYkNZmW*?>T=>4dPdG|GV#{kaySBp)Og@20Y|7<-rYGDrfMVL1+2Los24$7JPi8fG5q)I7%9@$Q;1HUNs@+R-fZPQLOzD4dkf`yxlRxZ0 zF;{pKveYw1*hXsIwx2DQ>6zW8eKLz^2(alZ3Gze&P!3*>%aUVozmzYGOBnqH$S>m7Ew}mHGfcX<)aMnN+r}A zV;(Vb9pm|DVXPB)$VJKPH^hX6YKm|$^DnRxM|X&^D`0i=O(*<)5v9VysQCSLa?9f_!$xB}tP z6Pdo1E^_6t>|6UI z&I}&Lxpgq@Wu<1CR~lwNn(q@VN%{P_cIuZ$xY{#mfxOvtr!c$T^98lJ$xW%FV?2(t zBG%wATbXo_<_{#JruQ3;dY0kwRLX3nt{}91&&9GfgC2? zecB0c&SlP2J{^8KkP2wTWjsU@JN}OHy-~TPQ0)VVJi4XZxmKxV6Rf2nL(=;sI+2QU zWwJx?IHwYbdY=RnySs!!5?P z@*U?DU1EhR3T{1jx9jCxpRKUbsaKgj%^L>h=I1o~u;tU|+^(`kXUr7JkfiXbqrOiX6N5t?9ZMfn0 zI~M!d3@!EW()DZtr?C_Dj4f?-nFQN72Bd;2v$UO(lCPqO&HoYpIczP6WK6m0|2+-E zkO+U`jHSQ5J7@NOjyGnoUiNqnC@=<0DZCl^#ULsPQMZ}qxMxgCJuEddj4jV z%RUm3?*3ILT6cE(-ftoyVXyHLd7cf8qJpXskPcU9d{TG^iGdHmmKURw&!Sy57_F>8 zYvL1x)m3p9@xruh`3>^zXSeZ2a}cu!W!%QFT)))5LYF%KTxK;^!3R$9>SL;cJV>eJ zCPZ`qiFJTS5S+?7v`|aLfuDY!1E(?=78j$9P6_7zWUiigcem6_!9uI=8SPNiR)+kv zY$zXA*SIvk_!?N-~A$J3&Nqe zhr0;=ewSTi^h^}gKcd@r{nZylXmP^jRBxq+V)KD+s(hDIb><3% z9fjzi-55GW=sX4m34k)`b>BF9Os-$dm62#h0cK8>$eTo)F96{2j=oA+I zy2FIGYB(?L`OJtgCp2-x_Nm{dpj|QaTT-zW{g_ zI5boQDr0yn&W4Py5((-1ds&jq;7pWoBY}sCT!Q(QgmZFK2sUon%zKtBUVKMRU(w^W zHy9{4e@xZFg>@PQb=Y*GucFw&nKN1nJU7Pl4)YvPwc7$Y2$~7aSyr6K1(bdJuBBLa zEE(aa)%CI0vND9kldR&sl5(y^D`;)7hRoX~3`r+=hkUf}tdrZ&%FmeO# zg&lJ$Y`lxn-H>5sZV%`_)gDkPZX37S-XNW!d6?XBndkb)^wPwesgHPb)6CC{i5zaQ zU2R0V-P3lkWAg2U?IU@X0SoT7BdHwxB1g;MJK>?{|1fV@(l26IfrhcMZl-F6iF4c$M;CmiI8yH|{#|19# zEx@V61o09egPE|g-~6%qE1$y+8F9~hu0Aa;vOb3VX4Jc~bp&gQPv;v0}CMP)ebJ_`-jKtaxu6rzYa zlo8!if)6yyx`@O7fweV9xrYz2ep0jL{iJ4H{VvKm;?DL<5p5iVCm#oZ@QLHN23VJ05IWd8>z~?WvUi9%*U{1#uetxf09S~~Bc--qV2==lfeZi8w!`J;1#G7b);?J=ZUMUfb(U=&HJ?wHSeKD z?ElH#OeJqdhSa+D%tiWYnr)U4QnWYSEjOuiro;Y@zy~(@F(qqkPwWF%Pg)8rpU0V2 z#rhOWUI5F7ljwDO>|PXrJYuJ>ATGM{E)oJmtRh(HP?Jvm{^Vc2_!fR^U?;R3`y|mI zha81*&A?bKyF9}s&=o;I%RTOOVS@k7rT*AuP@m2B}I$zNaP>9(86|=1*(OglfFDO^OM0S>V_t<$b1RT z2s5S(UvFEClmNukyqw1eVGcNGmS)vaWXad95Kw0F(8^_=Yp_h6ZOd;ZCvSzn*3`B?Xy8Aj zsCYt^x1*Ay8XGhoF4S80kN9BP)z*;f$eN7=dGDFP(4Dy6+x7q7;u1KMfdutGEM*l9K&d zF7x1-NQ#$*OA;uJZioyBHa}VPIBpKBuZ?x$2*mC8vP1+vQG{%=TPI+zeOMh=sndV| z@3AE3gPpLt{)-5>a?yhWcCXIK{$i?^`cd8V^IfG7oE?=(*X*XZ<(?g2iS0UTX4vC( z^JGp3m*vo&3jkMnL+3ljx*mapens%lk`Y!TTxw9_=&f*Or zI@Mucrc#Wc=c9{ok@#CDfvT-|^*RRU z1-TGZmUBsdv{o0v4Wp49Q4_coG9U zOMPLRu1=gYv5gKUh`(BlMe4+s_5LFb~GI&&B_67C;!?>N^af;M467+}n0F%VC`O zYUTi`>T6TyugJa@v`FMlu}unG-gW21*APi+7V&ENMT{Lf5}x2x*nM7Sa5XEj`)xW4 zdfO9S$3R@%B%*@R$CYcp2XXy__oj5i7P@Rv2hmnEB4~!*b{rd~v*+^cY^Yi=7^=J>=gz_PEtT*+63Z-f^&59Fl5M_E!frnZYmWQcftm z#4$F3x78e?Y-Q_UCR6#$=@f26s-HvjhwezRL5n*1NZbpgDSe+dftb+uX_M?ONY<{2 zpgmZ6I~y&N^&iK`S_goO{=?hRL@oxCR~WY4z~Y9ChoW(PL$88M^sV-D%EeW4dTLR4 zjSDYh_TGH<@_T%@c5~CsTfSeb=B53lPsNOZ*MWDA`DEGlh4PT|{eP7xNg_*YApX48 zEc)%u@J(5@3W=jJ`H*j+LkGqNORfcJqnUGd-rF-m<7GD}QX9aio5t5r?|s_zOmJl$ z@WcUiX|t2CJ#%wSB&5D)^Vea3=3&U^lAf_?O3(5fMPI8vZ(siA!Nx7qBdhVU8Qs?S z*@E$9w*$!1$l+?|y1W{%q?t&q+lsmx^(3#tM$dNV^w#q1-0;5ZrnN(5cvhwD=0b04 zU(GLNeEE^vVm((HE$r@Q@;01yd?BUsbnggaVGKEu_DPYq_7~g&cH#^V^KzPktN_q^ z^Sv}IdM*4^!5fLO=bju55N~Qpq>i<{|Cyl=9Q6??UsOZ(y3KwgQmm89vJV0SEfs#0 zNYWVRiUt+B);e564n+qA< z>-J7+fa4OX=kOMvZVWyvcr9aA4Kp4ETzmd0-2$JF{#13p?F0)ed)pT?OaSJTCVVA% zNuT(WTh-!eR96Ujs#b|;edzxFg@bYi>pd2w3K2j5wdN-e!++ceSFyi-coC4e_M^+j zx+K=x4{eH6o?*jDzbsjI<6s|^5g`BRxWN3|Q(*hT?JhmGa6u9`k^gsIdjAH^4Ox#i z)lS$;pZ3JsGCrGuNZeR*nYKCNdGSd1&{Q4F)>x^fy?L@WNR9{c|LzyF6|aKZ)BY%` zgK=q+O4{JQG+!lEFgmSF>C%Pc&LY7ekOcz>!~y!hhZqS@TUOEp-%S%&7KTbI9FP1D DeAU{d delta 5326 zcmZ9Qc{o)6`^IMs5@M`1DrT&ab;^=ujD04u?^{{3Z&9{sMk$n!eJQeU6_cGXn6eYH zjck!6O_ZHxgzxCD-}O6xoIlQW&g;70_j5n*=bSSdFdk6l&UV(;F341#3jlCT^=A{p zPNVpqZX8${kK2DT$7^7~DDfFPxtE{l2cr8ty(;bL2NdKP3AlTI4z6yZsb3pse2qA( zTh7-e{M|3}`=GV9`F`qXGLx*cWY%VF_`SBZ)Jtbj7jbdeNVK-L_50xinFlHEE=_!c zhPvl>Y}zLOX31oqOrP}pUZpHnu7u@QV)=BUrh+QN+H4&V-i?_{JCVw^4c{{<6@LcF zk#d*u&#=-%(*pxK2<55kS&nrf+#%CrBd#REyG(9P&_=A?NgW0S63uyQIzsdu=U41 z<*BXibqNiy!=SPMz7+O-KRxg8NnMdQ8>=QMqqklLhrQ{n8dqa zb`ig_=l07&zGZjXO)cllZaVt?F`oV`O}#fMKB!Vsz;05{?l0@4BO^ERUwQHa;n6p` zUsF+yd=wnVz4^?3k8=X}D7u^Ryj+EO{sIPs2-&UU51nUj7s5>k^t=jMd9gxW+Jd^& zJBxFV;|^ZPZd>f}ykV6u)xz*=9o99+fDEo|NpB>_hI$jA9DxQ{5ss)`eNGP5BVEBq zW%Fol!ehteSfylGafKU2Ib9w~aL#eNduii8ac`FP`m|ClNoA+Pi|?4D+wxSJ!LVsrY(#k{+*|iCAgtg(vKC?54sPEkw zjI)fuASQnE;49rzCngsOo;I(#C1(^@uc>rRjfNEDY^_WT{TvIvi^#^@I4-u8@FnMD zf7uE}sN}qoKbcbPvnuNAs-X<{9j1Mn!j8+8E`E}iX>OpX>$QKiKhSq*=fuAw>n2cI z21eEd!?$bsX&xA1hxy&>{_I9-2riE8SaBvVBo6^X9oR#^@Dq2gZ#)nQ4bsw8e$mVB zPg}SnLtymn0fmm=>=AnWZmrq-DK#zqJxc=v=RCdpd97+K0;je^dw*Uq1-SaRi1Zd_ z$SVik3OHLP`GE^wod}A*5@*bQ4GD=u^`o>+pC((IQ>3=A$ftw0b!uuN zstf7CIesNt>&6G*AYqi_Q!XiAcp0Q>)@ep?Vv{WtwpRND$Q2gc66S~#(*+EV zKNeFsqpCKDJuGhOytcOxH>e^Q6nwEMTcG+}KFTrQ++Lh*%vaBZp~$+&Sku^G;L6X8 zxC)RCPav=23`6|lQ6Lh2Q*%@ZMoe}SPLvdi5{5vxr8|;@8F1*^QHZP1Hx=AT)+uzv z6^HL^1`T$BHphIn=kzaA^3GIeRMRK@`Bv)Eco#2l1vMq>}>VMKfA=vevwgluRjOA{Le*^LPNf>4`d$pOf!aNeaD#oDl}?h(;VOTxuAf z!GreS14szQFUsAV@Q|;;>+ZkT>k5m5BbxfJVt04#zb(D$yy+-cv^>+Ir`db84^Sx# zA~-W*!zK`peOi(W6K46t%z(=2o~yh_M>J#T9wMRTTl#qN!Mc9V(R@#RLN)zhWwUbo zw4r^tMmb?Srfir8k!s-FcQx#z#C(n#r{?KXB9H~1YrW-LThADhtc++r00`mCuEl`u zn+Xvqs5^4eT$19^ToqFfodqx|jTssY*ulEeXg`Acq_RDr%MG{sOx1Y<6(m;_pRvq| z2lR?SpqZ{X`%lm&lAd%RJI?+eb2)>BXrP}UK08|i3ww|m4R$bP8mXW8@3)KJCJCMa zb@ttBOR?zQqSw9giuXyzqe!DmNd{r+ik+Nuz*rhG!xd-!30hs9Xl;sxrgw%j@1lXi zBr~WG!)Dv4@Au<2&IV28muMk>N(Qx|Bp9)1{+_n{5lAN4z*hI_KPo zxe4%UhEuT*#j9~y5Fbz~T**6>7Q(Cp486~PkH4ivbR@#UGR~*b%>dn4K1w!@<5m`X z6C8)qg@|({gGV9xT-B_fAy97dJQHnZn$WUSp;(M12TiFcQfx}$c+C4RgT0E0hB{HBLHd8{MEjc#1v=wqSZFENH6{L@xh4X*zKuS_(4r z{CZtN{uJ(dDz@rRR=Q!AMVIlUBs(y3LS6nw-_F3IT9ftnHFX8_vzObnVGaKXfX|a_ zOwC!P(2IV-hw3q_WtaVubvLvfQw%R9#iwBNNkTB!>qIG<>uJGEki9;gIy%{(wa@GR zf>yM?q*S+}?ybc>U3TH*d_+#;c|*?=aKw`Ej*A%sc|Bo+-+?K3*~6#Un6uiMTer z|E5`NbkZQeEY=NoYZT@>R>nNSOe`JdX{KSbPqB}D`cDgS7d_?m+M5^ikSW$q0?<0) z27LWOq{VYQ1gknZ&IUrIy7?9ZPkA63uM+^G@w6L$uqOR5!+9`3M^*uzZ&ojT%!prS zNV6qzQC!22n!Tm7N5+#C6jkE)d3jH7h$A>7kY_;Yrpsh{iPq7`mDFO>BN)W3gsP=U zqftBHI1#(sXR3Ca)a&T(kW;aB*L~QW^s(LhbZwI7KBSA-6rX)hzp6L6{x|v_M4fE- zE5i%vA7TL-o%3_PR2X&N1IbsCZ0>~=E~*&Ep_rOjKjW+!-@C4XfSCyPSWAF3MldNO zVTcyLUPoffV@uE|GbH-Squ@d3VV4V${j={&-VNJHWt^)qalO1I>lL$J6xYjz#?Hh% zsi?!Cm|(0`7;8qT@a3QVB9MBc0~pVpm(sVR+A<6WX-E&n6~3^~K13>`;{7ilkwGt2 z9cowZ?Nz_GB{TzfvSnUa4g7fIM^0D%ac2%z;f8~)R+Kh-B+1*hEUPSmBUr071U>hwuu19e>}O$5fbQ%AfG4gK z(EQJ0`$=@Dg5fo}g}r~UfQ{2m105u!Gad|$OKw45%-gq>yT!t`bHr6tb21tNCj-BF zU1CdwF`|V<_<5g|bZTj$+;N>9aHObvl0jaF7BloE;>qgo%mdd)A3NACUQxX+>|a;& zyWh+PYsqBjrtdO+)4Dy5WHtcRGQ4}sW^JUj#GyM!TwhJ1wJg0CmRuKZeI>(^rV|ge zr0K-FhCuTt#P(=9%ZJHw3%mbx7NvELdr98T1@r(crWSSqNCTKEWA>Q=h#zA)Q`(KbhqufCx z#TSZGj}Wr{q;-la=TKQ3qMjA*@m2CUqKiOw)R^vMQ<%y$LyyEfaTcd0fomX8Y87i0 z+VUq*0h7xvj`N{j7XW3k)3!t>Ed{3=0;ipr6mBqRI-Cd$w42~J!&*dCpkJ-&WhXq= z7R=Bd{Y8`3Op_+e_@1UrIK?6ir=2%VQx=D)?ZHjalr?aP?^kKDe7t-XmJQv1k=2zO z^W|?#w*zfQ{aE9*|K-Z9v^9kBWR!@Bz2KYuDr3nBeojqE&^UhvP_mC%9al2E4r2i8 z=Y4TOu185>Au2x5*%7e44P!5&>ZIU}&3+htrV;$N7vwfL^W(nBwCC*0&JQBhy33Pq z`Drv?TAHOnKD0DT_e~>+B}V%&h9v#Gbr zIL@`o+u?zO*_p@MoXPWckwncPwQ~NmFzjQXAFZOabK+K$S}_3r<6Z>od$lSNmzZ1c z9$>Y#WEB~IAS%FL*QS)`^g=_+yl+|@pv_hqvJj=t)Yq2gj@Io|i*=5&an6f64@3R{ zSG}}C~7BirvjSj<7`mYeF-ZKIkwZOG(?UZyztne>VpE`G+ zSRX@uyc9k9qD8X#%J8g2N`I8_mR{QULu=H#&{-jvHXM2A1VN@q^>FEnV!pEDp(l== zRt5eVe@|DIHtjUYjwx8vtY!ek74Zf8W?lWkbh&?~peyCcU@qxj_COs_vH{Z%1 zxXfBom}IqPT3VPj>Z`_jGT0OX-KkV15?mQ#TzK;+W2Tc;ij1 zQKO{?Z?k%~Ya5-Yppj4nCBnxwIb(cl3o6gb`5rev&&%?C_~{eyw}sYIQJT>v%YW58?0Ac=K-H z8!|YuLTj_Sb8f_R(+Nh2n-_w0UPEp=L5QjNR8T0Dl~&`;U2z02H&)SVtf-AVGHQ$wxKZ?T-3_}c9ps@ziXKITRo-Wu7b_PX=($S9q2K|&pE25OwEqRl|0 z+x!4SKJ_}(7Kq#nL-MLZPm%df0Pnvvjc&2KF5VdrHQz*LIalFHi995u3T3u6u>_@b zxe>iOY#lA(ZMSGGV@_+C6gT1Y3a#ynw%F&GX{|TjSvZ|3zZl->+MpgD{lG;7 z%xzJR_$Djp<{nWmztixB*fNy{PSHykvZnMZNo%^S5Ys%a*k&Om$CE|DXJMs4QCh)E zKP>?bFrr{Q;04tn|AN6kjYJ!1EDP<1+rM;JfTebVXD}^hGq$?i+ILr;6U?uQpWGtJ z=lnYTOXC=CS#&h#*`_oQLFwwMk;^?VU^6MT(V*~5sNwkI;9Y@n>N!$d;`zhuVMEci zN+7=j;gw8j(S#RV#ar^E)^nS%Q*lt1KTgUg(PyUYmDI2QI>(04-JoUN# z`aWORXaAzE9|{pRC&cFZ_J|wG1BtrNYx{hr-uD_kkJ80224UoGsI(u2$+yq`aK83fd3_UCgCDA7I&pf5g9SRTTL>&uy!`>?S z);;3PAr^#$SjlR=z&31p)dSZU-e19`34KcYvCs;;pmHC8cHq1u5gXwvvR6h%fiEea&|#DE?fK;6bialIvv;pd|D;CQ ztX(nuUBs1I?K@LEsN&t5_igysfL)`PV|?U&^A?4Q3Lon)lX3wsG0gkQS{>f_nc Date: Thu, 30 Jan 2025 20:39:46 +0000 Subject: [PATCH 05/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14418 -> 14421 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c1a317f01..ac6d97ce7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-30 20:37:36,073 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3270 INFO] --- -[2025-01-30 20:37:36,073 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3270 INFO] --- +[2025-01-30 20:39:41,203 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3286 INFO] --- -[2025-01-30 20:37:36,073 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-30 20:37:36,073 submission_checker1.py:3307 INFO] --- -[2025-01-30 20:37:36,073 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-30 20:37:37,379 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-30 20:39:41,203 submission_checker1.py:3286 INFO] --- +[2025-01-30 20:39:41,203 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-30 20:39:41,203 submission_checker1.py:3307 INFO] --- +[2025-01-30 20:39:41,203 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-30 20:39:42,224 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3379 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73908 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74722 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.287 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.95025 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f50c65e51..2d78492f5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -546 +547 diff --git a/summary.csv b/summary.csv index 0aa8af8f9..ea502c9fd 100644 --- a/summary.csv +++ b/summary.csv @@ -15,7 +15,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3379,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73908,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74722,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.287,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.95025,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 684682369..c7f0fdcc9 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73908, + "Result":1.74722, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.287, - "Accuracy":"acc: 75.000", + "Result":9.95025, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8136662a9a96980d0caa2e095a6937c89e5f0967..2528578ddc36b48c1f5d1f1ecb32d03987a32f45 100644 GIT binary patch delta 5312 zcmZ8lS6CBU+f9f@nh=WALuf&yOOPsr-jObbfJm1nAkraB5S1z=p(q?dAfOmS3xZVX z#T-yTP(Tss0gnVdDmL++A|};W5LzlY^R=M64MBNY!`q@6M+Zo zzphz-aQ@Gptc!(lN=-Q_KK}0%_1=K5UzPJIs){nS1$XE7FPRO~{pp6OTUMMk&AgQK z!=KLrJM>pKUQZl-%%d63*|ni97b&gfeujfaD8YAD=P8ud#e)cih!Jm(Mu9;~Bi?Pt zwvUJT3I!*VCw)Jb_GYUVq6(`30mJx-kgBLQ%ypE1L*CqWtOlljF>mkX??GCuq7XR+ zkRO_)QVmfW6PNN`>cY6gCP#-o$6G2l0yu0v;-WS9mlPJ~NWLhi>%5Nhyrox%j~4dk zLnc0d)q2eR#wp`jO~2Kz{C8uN>7p9H`d<;M2ger&v=z*?Rmb)B)QV1kSI2!MLaTRO zdcEjD$mqjwCAaZK>6P*hL5oGeJIR|%q}l48)$?ht6b#Iy<0fDkAKn&U?eY`3yx;*SQx_6Zfyi~0^7;l@{SPa)M zuUV~jswOwo4qR^h*7|v)xUTs6&QhgaeuSC6aEYJrv}2ibK=;Gn?E|b zV=VagUhJpXKM)u#Mr9OZ8=4X!78f?<*Rzr${aZvhZd(8n9PvBmoE(}*Mq+oLeKj>` zNpQ(ZQrp6Ea#Vd4O-?E;a5G3PjWH0LU1q!JZg9hekhDzA{BF`>1Y{XR8mx3q*xe-gq3OFVkeKwvX4=1DBrb zH;Q@)aVxB41zZwo#jo?(HpoP)wyf{bj@M7%$NT4Fl+_NS?&#IY4kZnpCvB55$eXc~ zg)ZDDPk3%#?y<_-o85^-ULNG_QFn@Kd7DR?TNvx3c{ZYs07p2G;$#!^s(*Vh*&zmt z8vAjMT;)v|`#9U;>-cZ4?3C(~(&e6sk+9+?n+sz@E2E(|Q3Y7_VA9+o25}0IDvA)A6!8@D= zR&nDV2ELm#V4^FHr>IcD#2_~M*i4L5bA%tHu!&1t18Z7s$BA_i1d$^h_HmI^BgmF? zlK3H}p8ve|ccG~4-M~rnw?CwXGZYk5iDW4!=Rt=aFY25q$-@Mk*?Pq+(@@SobUuH^ z*3vOW($ih|44(~WG29!6N+Pm~*F4{HT8Rf(3j~=SKZ`25#GBO>awi@zO81nW z6s}3*`vy|l+M+2-jh_3$U#=Vy9UiACP!%MZz1T&$J3_4@?YYOcZJRcvXk4Bn&3a1X z-f2AGY+pL4>wb0xV`HmPMw*#ehz#ej+I*nio`iSZy$FfKCv_7H5QThT&1Se}jrt7D zcN&l!lqkH;Uihg+T+Fy3t9g%O3Y0Q`dL>1s$1P&Y#=hgixMDVpyV;97jA}3*Uy`Dh zTCm1ow0W#1!Y?68%Yx1HBk{($Svq)DSX7>y37(;ySfx{RF4NwD*+kY9JhLT#=gj#j z@@rY(EcroM`r*3-G!@NYau_^XGevRW0#*@W&Vtf$z)@r80j(lS*4>C!>B*trFE#*5 zML=ZTp^o+_o&md4uETau-!X$y-}Z^|EcYv+b2&byLGbZWgx0uVHm8-fad3+Vv+lZb{0wKE3L) z*EPDtTx42aGoPdBma{$tXA=Fh3(vZd!aqOuJF8_v_0-QM2~1(`!x*^XJhloQz~XKCbQp*)Hew6UY#}&DPd) zV?7he0nigLCu~6E35q4@97qEThe_c=qnSO4pb@4**p!^}v$qcVyL12KuV=W^8ETT2 zW6>>WZhHb$u2xp*Guir-5=WTC_`QhS273MOIG%1~yO?*TVG*AJ^GqYz>a6=|Tu%sC ztwACXpCx-&!vOQm!qi3XU#V5@6o!rOnv0s@=2uF7y?DtvLG-dPZ6J-%tdqGlB(Kvr zv(k{Z=fy;cN;;D6{7;p1F4&?u$CM$pip4{rI;;-0#r(fb*?9es~HH8xud3j;5@Sr4<5mpzsWBfiqCx1X29Xn_i(Q~!O`lBp6IB!f_S$$xeI;+))UR>2yF-?8EMGtFaCkXkK zR%>g|DtBcL(R5VQwLQw#U{V?(reJExW@0p;ZM~WWLki6B)UaMO)3}$mKQD#>^X*O; zVvggt{=N{gG0Z@JOOc7GBlInEN9YTGha`A={iPh#QAW2Pzej)e3#wmMI_3+(l2=Od zXbXO=QNkle*V&OVxCeG9!T%jV3$&jo6gaJLw2pO(kCY!8G&2(lc_lOs7PS5iG4K5; zMt&>qqardGJnQKPv-nq;q??gIdE{K^$~=bbeVR-OU_K|T`K4|(vbI8~JkuzVC_^~n z6#~8}tWRV+A?tZP?NkIaz17U=D!Wcu-qzRoYztl{%d_b8$UY7&?cQ{@yUZ$bCHGYp z6gF@{=Bx;(9w&rVBp1v%QKF<%Vg)WahchUG5X%bhFghV?d9D2%t>y^}U?$OmkY9Af z#cfal@R3ONiqu8-UX`7DF4$ZtE3^V+7(FJ+j0xjy9A0Q6?xCU2wPhjb+TAdadC`Hs z3@8`1XKdcJ2&?xDP?({Pz?>cNhWR-2;JDDiAuF}KO~Q{4ffdjlhil3rPY@Nilc1Bu zVBc(`CR+&I3T05b6})gzx)mgxgc)D&MCsHq1kq0}>+{}%BGE4(Pbpn|V{UWm5 zM1P(bc-M^GVy|2ncG|BdAoHQvDmu~TvZwGBgF=%@l&ojjn!X;R8*Z%&2@<7Qi{*KK zKz&KH$9Xlyp5Di3C?{4+lzz$fpVy^;R|P+Ra$|uEcx{Ca=Dr*2oAujVf&LfN=ISC> z%F~|HE}j5zmlC$G`Y2ygWcBwx zLVEpPm_r>)UzxWmv61^=ya>F@pVD^erSZ+|(n~WLXJufb$V(7+CpX2hEoNv>1y@J~83#iCIKIbqid`)l=r#he629Xj`~5DPI- z{t%@9TiXm3xwWk|3rCvGml#a40H^7*#QW~FolMw>z|2lH>wB|tC<6v-Mz6DOKMf8s z`dg1a;EvqFFC!RZDmzp3R2)sJLPiq6l(JkZo@|+2Y?~^B3)? zA_+#vXd5j&0a)UGktE}^SZiP zW21Vta!Mx|fSAuV7>umC6{DlI{`fCai|#-C0+#N-aeZcyTokDfHwJM+hJR|WFBN@V z&3X~*dZKSfp=~qD7ZIn%kZmIJ_g6~m>>89R63k1|`q%BXT#L~!P>6fZ(B~*wh%^2^ z7H3{Wql4w4M*DCyI#@kdWFSKcyYGEgpmS2oyWnNv`qRHqP`d$F_v@Omphbe;TJ=eS}_z@G0VfAYa%T+Msg3Y@5wJ1+m~Cl73h+Bq(9L5ra^NPa-n?VN zIKjFRhlYMa2f%={+?G0X_xZ~vO(RJo^|k@3hBMrp2$Q%D=fOw7VhE0KL1lVwM2hL@2d@ZJd$WHB zeEjR9^lD;9<}xu*-g{-}m&Os7E1hb_zipyt^Dodgt23Rm-xw1r;DUg7O#B(K_kKDU z(YqpBpYjb0gHIJ9l>$la9zC6!ZxNN+&+FLY`#9qbOyd*pYuL`vU2{=qhThvuGHEo6 zkv=UjVAzh7CJ#59?wZz_D?+n-q4V7)$l&#y@2u;s;{m(&x<@W9(Q>7%O)O5~G4>-V z&6P@yi4z{l=n^9Rt3_`srauO8uLb&uF8P+)-EiEsqDgw>RG}LB?oRztDR0X##QbNb zw~6FIwERWEZ8i+$nR>_ znE{>wuqO5aoutTpZef;`Kaz6MQFto)`aLW9Ya(z?&)r`o9*+3A{`>SRR(Zw`wNT*Ok4UwfMt@T7+2>3bbY?1@W8Vk=n&Cd8I6@B>H z8MKM~w)|p8pYqF!tfh;}`2tA09`|>-NaNq?7CRhrWMLkB5P^Oy%8?bX@Z<+f zB~~*JqsPCx7yZ~(= zU0-LF_!O>$OUU%p_h|Kt5&|i?=V2^iL?#2hBwrj;Hxehc9SO24Vlt2cm7};WK)L?b z7@SL`KX3mPJ*p5Pe5C--yyV(}I|?2_Azt#4w*sEv2Q9lU!tSkW8qUYK zOM*{1;LUK+C7hgw_Mc^ArZC;wwP$^&;gYvT9afU(qP^cdx!#z+W?yi4cY9=6V1Zwk z$tUpMhXMht>=F%P7VCUs~=UF2N8_Fc8BNVL1Hb45aJfen4J z(xhS|ii7Q6o9vKD%9SP*&Y`<)wccNf_yVM_dv|NU&ls{sFFi@>9L4LnQ;RMi0M6Dm~C f3;zFR)BFAR!o?#9E2=1PBtc3|9IB-J=jQ(b3Gn5J delta 5264 zcmYLNc{Ei2|DG`s5krfmEW=E;$-b|Hv5b8Q!yq(d56Kqpl(JSB3=&h8WXrB>p-4ma zkVK1pm#xP78=vnVzkANT|J-w)`#i7bdCu#e_lW2Pv_Qx(%v?{N0!^e<`zX?3I( z-?MjX|9JdR?)pKCT2Hn0RX|b-w>HvmQ<^B;I~N#4GKuZ^cp#WP+p3%;`SOGHQGciB z?Xap(G1Dgbi(yv!7NyF(rtUkXXUs0b)#(OT+X5xILB|Q!Ee)+rXzpd%U zjXS>|`)u0yg$gS+OZs@s9X70{&E*UY#J>6-JC<3QmyzkwHTwz2;h8S9ZcyO(q`51u zVcE#DHYRBl1xDy4G7a`C{=G6al^q&#>vxN!1yAv;P~Cu2Vu~vu$S%~P_-A*jL)Dmz zTC$gr7Q-|ALgVp^$fD+0h_T2YiFIOJsJEIBdw?-HH4e2$AJ3Q@CCuJF%nuiw(vrlW zFH`yAVoV6LHk_Nd6E?d=xsMtFx)9$6VtKr7nj)j!~UNZB0QYH zUf5as9g$V9vJKBx4*pjo?OkE zkg;o^u$1aV31KOoU~jqfedhpwoOH~Z+iH4!mkpSD2EDo^n>S2ZTG`aOAxBmynx_Iq z)N#|JVcmKq<=oGv)w)rAB;JbZ*`?4hs*8%`Z)%EQGOLLhLG7&1}mFcVAespddQ_V1x8{Z^jy`|1aG~ z*5kM`WX9rKi=U)uQ+b41aC3 z?A*RmTO5(`jx;1KC9dER8t%pwubckk92@>C@ZI2(TJPP0hPcZYvzZU90^#QdkK)`q z822+%GR!IrvYssT2^1#}57$ip@`zM@A;q6Nm*x~<*UOn-lath#GB(cbI45icjm^i5Gm1B$P-($OH9?H4Q!Iw#zH3ojo?qrC74vKCHv?)M z+eD`LPs8)!a7PZh%XWI3pJX@;h+x56vMd-&Q0Df9g(^B?w zt$052d>{o-k4t}qAawj4<84y*map;ygr9ipbgoxuT8C})~BcFfP7i10JwKTyWG!t1i3}xEH-IHyB)#-G# z4mu}w`tUN)MoDC|D|jXyPq1B|e(UXo;SKA5H!u!pzP5gZ@Q50jrH<79e$Q+_m#(Q6 zS+bFZ=P+`joU@^>tq`G`Cx8@CMW&Y1<0M-oq4__;KS!(t5cJ8mf!|ZnbcwKM&KTO= zyL)c`=R`v~iRj1g!d;X@E$vhF3>i$(L zQhR>p?r#Dc z#0yfh8sQIuh%5u# z0^k&up~V^k7W{m89-KmFP*j96JR^|vlc{R*{hbmYd2`La7t|vW8)?$>(xE(PZNu`! z(pz9{PbE#@p@N}cjsVd@&rl2g0LkE?g@hno8SJ*>wF*GF6MJ6-Y(QA#MZ#^|?M&4&t8@><$^^fZG-FWi_9$u6nmg1}UNOU3CO@&u1MSH$L$Wf36+Kr}B zWLQ8WAwdj`y4^R=pOEVpb)_emk^yTpjU&?lCYV{akj4?W!lKUAc{M#|sx(i&HdoYI zJkZffr5W~cC_e24n}Hrh^Xmc;TFv#7wTh+EK;?@9haOoIEe2C=#6J&3R!bJ3b(a9&e24n-U?nt9 z`T4MkH3B~EU_VoW37mlxYQS?-5X(^i;z$nma)E|z>jlr^rOWS0X{)+C_WA>5W=|=a z*oaQUkPhoklr54CoH45@&wX=T_bAr^S+m2R4W}AYon^$iT|im4?pTO+$CBUys*P0g zhpl1OHwO73f|3~4(9)XM%}xD<=sLaqr$!ZzwusB<7XF2wb-I%$qsxjXRKvDK98^Q zI5W$ymQyetT3yWhGFdOSKq&QylNVT(T>qMOGF!Zr!g8v+l2Nw}#Ib5&O9%Dc61?zO z;tYz49A88GcHK5vl_JR5jq35B;Q3{81OrA4ixdINV1Q5T1I265$Ju~5B3YA3;e9Y4 z48;jm?53V#M%srz@adjp3+jB@ukI67j=@nj?pHqQ!G5$54tBIt`2z4imci(kXwo3B z0S1g!Eam`~p%TBm3?ga#jkN(GBR7*sF}*a%ai_M7LfG4Cc%mFQ!p$%u9(^S zP#vN^Ra}&;Xg`=^p0k!|e2!DmUBB6acM=Y?02T+Lm{t_Kw1ADgLw>Av(a_A=+x}rfhaLggs?Yt&=aa{ZCcvUCT1e8JcUZJTg)_=EEdq3TG;d7^wY zL7P0Pb4K8yMrjves=4kiGA(qdoHawqIt*YKfJ4fB#c};}Hyb5{gu~EzO?W1p( zU1T@@)%L4C-g5NW_=4dN+Eqc}28^C( z{ll>g9tdv;xLl(!_B_fyQZEUdf4aV4r4m+-JXK5F0e+scT?3o}F@^9bmZ!Lq?Ph7+ z!kdvq?^^Fki5tBB9IJ4mB^z_61p7B|Ner=1YGot0xmYtQ^We+|>EVtlgJmb=9kW!_{Y&=L5^DmSiZh3ScX=;}#LhURfQ zQ!81XV@QhtST2%C)9tZ$nIH0mjTUnk9XXe){6j3l7^!gMPQCu5U;eljKC9jEGR(6? z{cKV++BF?*x#IEyn?Q2}&Jag)1WrVR5zyo@nj`TVObXEFQmlTM|XR`&G!a7$K8zFtq`> zNW2z=VH5$}xG|zV28I|0kvfgR*z+L6Nc!-LX#NvxuW;6D-gl22Rg8y=pEdYaej5-W zCIvE!apM??3SvUV3FHPhcsc}=mn3phH;1*iM%rCyY+u5*(&f`0$X;r*ol(?CFMmQ$`5mQ4#evVL0E%$ryGRUyHl^C} zU6MbMB&e<9zr<7N&b9(Li`R!~S4Di8PBvuVd{Brj(+{zKjjI~{^#}X=smS6V+K|#( zjZ%yGE&+(U_f88Al$hDW?h9;wP0-EtKgBOA>sQ)#qw~F5cda8}?#WA+grJKj(y=ku z6}0JS$2k+)XrmRt*%l$#x4G47=>!(6LXjeBm9$OSo~_ccVfR(A(AwKRJg<8a!Zn@TFc8ws&!a z=o_`iKAaJb%6I_n2erq#fz0X+Sbr|cMtIP-4Hx~xnn%#0x=R-hKHL7#y=_keD1&m~ zs+fW#tFBLTUXyt%V4ld6Z1XsHWzU@hS4|+Qn#Zf=71DQTU-bm1K=1Q7gR58w-S5(v zQ9GWfS~|kg76BQGI#I6m0d(;ntT(wEy4YodJbYkDRqAxcmGg_2k+F4`N6jt=h~3Cb z@;FIoL^dr#uTHi>4`3i>7dtanbHulQ5>m^>(t*VG{gY-gJ3OvVK3E&nU;@9YO+F?6 z8cW{@-cfakwvnlY8c*jjrIESdDFF@-K6c*|9W<|%yN7*+Frh74V~8Ovvt8utUd;e}M@gkk^8#L=)Fi=#V{zxRQZ|F^Ev7Y5YcA2m%zo3bPr%bVHCtP5zHiw0+aY9m^k}VfLr#@P!c@4%ZBn$B38)sOy`|W$LjqgOT%?(9Cy1eoUM#tb=r^hC7u&P3{!R(+O@Kc`-`rPa{ zMDIs`hL1rtgQ6~sr_OORt)wc*2moFmq@WMhBhTbFT^)bv$zBifr4--Owz3aAH}sLc zE-LwpO4xq4>2G+lRZ?lzVQ{d8{EuP@D*b%npnSLNeU5?ejKvc?k!2Sbg$yH{?JN5{ zUJdf8Jyi_3@Fwlj>05nRf35pwf(CcHeIM7uunAT3IP=dp2VdmBl{T${8jZOEJ^vKF z!JH31SKeu|DR0^+e~$L=DW^ zP@$={b-E^0mK*ZFbHj86;D3Ikax2<``%(iGRp?O2)a%F#sX_A4ODAJRfI%R0IuM8* a^#2#(9xQcLQ3HG@6{#e|ASHkD=KlaqNY4-e diff --git a/summary_results.json b/summary_results.json index a74b502ca..7a6bda421 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.287, + "Performance_Result": 9.95025, "Performance_Units": "Samples/s" }, { @@ -1220,7 +1220,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73908, + "Performance_Result": 1.74722, "Performance_Units": "Samples/s" }, { From 47dab668144e25eafd41cb9f1012ea2bb5912627 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 30 Jan 2025 20:40:19 +0000 Subject: [PATCH 06/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14421 -> 14421 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ac6d97ce7..3f6d3547f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-30 20:39:41,203 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3270 INFO] --- -[2025-01-30 20:39:41,203 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3270 INFO] --- +[2025-01-30 20:40:14,684 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3286 INFO] --- -[2025-01-30 20:39:41,203 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-30 20:39:41,203 submission_checker1.py:3307 INFO] --- -[2025-01-30 20:39:41,203 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-30 20:39:42,224 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-30 20:40:14,684 submission_checker1.py:3286 INFO] --- +[2025-01-30 20:40:14,684 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-30 20:40:14,684 submission_checker1.py:3307 INFO] --- +[2025-01-30 20:40:14,684 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-30 20:40:15,787 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 2d78492f5..a16667d78 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -547 +548 diff --git a/summary.xlsx b/summary.xlsx index 2528578ddc36b48c1f5d1f1ecb32d03987a32f45..12b85be0317c995d3dd26638f9305dcc3008af4a 100644 GIT binary patch delta 323 zcmV-J0lfazaMf_IGBN~;XMBK@HZmZ8YQr!PgztR{qq{BHZb}ih18vEvfkIPSaud=f z5w(7h)H-?l%1+`W;8TyQnf-QlQ8nMIHu(kTrO`FYGlCM(+z6?+HTqaB(npl|z_ehk z(NLoUcrmEe>+V8OVi!q-JST+4h|#uINOh^Ir; zC*Dqr9Q!~dF#tWrJu9E(^UG=^ftrF8vy|j%Nmd1+_k`xNbwBc^{{mE}A^#?x0mt08 VpP=~zvv@OF1O$p_e1MbIH-E+!of-fD delta 323 zcmV-J0lfazaMf_IGBN~%2LC~mHZmZ8PQx$|gzr2>WSleX1|?Xxti@%oBV?F(&!rH89@nXZiLjE8htG1=>tl9U|O)& zXsA&S9?hDYid)K!gEwa^xFEqN5u`okwnp0!EXCOKEhy%*=+IGI8>d)^rQ2YC%lJ3j z04@l*#|i=yEHFF(Q#%pS0V()|v_GyL0fJ*_L4gh)=UI+V-T+Sds|}-a=2SBD7Oq;4 z;^f|WIT^dI%er#3#{4;6EneTpRa2>lQ*l5|B{<~{STL@s@Ocpr*D@a#@x*Hh;;E1N zmAAtp`#um!3_$mBkIHBH{IZ-RqoyFmT}tw_B+G)(@{!)=U&F|Y{u59ghWwj&1nhHP VeuCx?vv@OF1O$Ty|3Q=1H-AnmnsopG From 1a053ab30e1b72b6026f94930e6cf34db2aca334 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 30 Jan 2025 20:44:48 +0000 Subject: [PATCH 07/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14421 -> 14422 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3f6d3547f..e349fb912 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-30 20:40:14,684 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3270 INFO] --- -[2025-01-30 20:40:14,684 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3270 INFO] --- +[2025-01-30 20:44:43,274 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3286 INFO] --- -[2025-01-30 20:40:14,684 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-30 20:40:14,684 submission_checker1.py:3307 INFO] --- -[2025-01-30 20:40:14,684 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-30 20:40:15,787 module.py:5492 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-30 20:44:43,274 submission_checker1.py:3286 INFO] --- +[2025-01-30 20:44:43,274 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-30 20:44:43,274 submission_checker1.py:3307 INFO] --- +[2025-01-30 20:44:43,274 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-30 20:44:44,371 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -72,7 +72,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 51 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24578 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 51 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21828 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 53 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 54 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a16667d78..c13c4e8a7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -548 +549 diff --git a/summary.csv b/summary.csv index ea502c9fd..eacad165e 100644 --- a/summary.csv +++ b/summary.csv @@ -50,7 +50,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24578,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21828,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c7f0fdcc9..7b0f3040d 100644 --- a/summary.json +++ b/summary.json @@ -1539,7 +1539,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.24578, + "Result":6.21828, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 12b85be0317c995d3dd26638f9305dcc3008af4a..24304095185724d2050255337bf9794591135f50 100644 GIT binary patch delta 2289 zcmVrm1_BQa$=j@~yYfVq6@nAl(wMdq)^8&9)H2AB zlUY-idbDY2mv3jvN{{N^(@NBkNIE?%6icUvgpk$%r-x;LRq($Dp~TO3zzUR(jSaxkkIz?K0K@C(%wKDfO^W zETtY2LO27Y9+m+($?`t{r_bpj1rSc3(qlmBfh`$3Y@qZ&vjOWH%$saEADB05wtOB5 z=hHNwS_T<3Z2^!ykOx7F)0Rjmeb;GQjI$8NiY(?*mx+v>s9bvGiFz z2CN>~lCi@EwjB+@g=K*01N$GEnoXZaaQP(Nu9ooyaVbH!PI zQuQGr$>UMbudT_5&5OX2P`BPv+9u$G8iECunaI5Aodua*h2~+gFdmxfY<|D zGIrSHV)sWh1ao76>%$7+t)8uOeT0N}x~5bmPns&XRr4_w%}2GO@_(xO)$d`CIHi`z zDDr#54_HW2(IX+Clw~BPEF&o;8KBhqL?2QBDD@TSTM&I%c$t|d5Y#~V6sEVX2f96(^R1PGGKBA=c!$NYo;~ohik^$Ba%K(yWEuL61 z>C=8_1^LG!z6yN{-hU5l$=F`tc_j_Wsgs1@los|sv^9G_pC_}E`R64!)xXKZ%34Mk za#Cx{+RyrnO1h?WLRw{0JJ=+)h?3q93(47~M?%PEfcL{Pz$WPsvgs3lXa!)?Xa3y% z@}5mFf7o8&nFur+kY)&?Y&9(PcI!cW+5?eVMi>O5u7YSfk$+3+u@0rj5~D;(rH6&& zt^FPeA&3D=56b|EY>^!R(dYEg3IL){>aig8z?O{c1s;j8p1>m!5<)b=>9I9CJzg~b z?(>jZ1{XwS)#WidG0O2xUe@9ASR#ce>GH6Uyo1;yA*3?E zNsotxbM9xE2+Q6_QyL8WyFJeCL~N(wwI)R6)Y34bAi0Rj)p0D^2?osLA%XYkMp zfS^y|v7qq4mW=HMZ|V@wrz0mul7V}#WM_EqwPvfQ3hMXr(rX!I&|TQF+oO_J+;yGF zI?NtR#1bXV9u|_{L*64H*^*pjim;Ef(wvWGN7 zI@#BA0e_t}yFHJk^NII=w$$)>W0NYK#igc0@UafT#}ao(OTmYQdF!G~o4 zPqs)7;OR4bXa&I2r}$V`?McgK~m|f(2v3Ufh`%^3tWw|ko@k^0SN)9w2%Zfu2}`T1@${z zgMZdCv=B_=vI$h27^IPLT!hBTfho~Yv^0TOs3T1v5<*M^Odys4Oqs_6nEFg0S^+Ti znLsg^K(Hladx5iD7D7w|zSoNu0!+6=4*GVdwzZ5Xh|bElc&S!e{+q08I@`ELWOmkA{%6zVdFNU0PaUm`jZfcPUF40`*UNmbnSZ$~UYGS?n~jq%r>{=-n#IS- z@|v^-%&=3 z(QWp!{(*ICfA?~9mLH84XOr2oagm=E1C3h6<@#cBc7FTwYJR!-DQ-4C&R47X^!8up z`RF8Htp8=2jnnyjb^DL?A^ALC{Ixlt`~L;A!69)F1h)zWn3Fa#AAieg!!Qs<_x%c^ zy)F6CQiSb5Te51P(3FT-1NoVhcop6yhZ{DSk+=mzB( zK?!JXgw&e`eJtncBT9T=TCmP&XwU#2&Dy(~Tgr`tH)kxkAi*aQq&?-fLE8{4#n|&L zDCV>1&{13)r&x%k+karo_&3`CE(m$R3IY=>FuVt*b|Ru9Qt%0Be_S^L1jo>U0v$Zg zvmBqj0i5zT8%E{Asbm-|+_av=$-VP(GWLC+_2p=d`E&fWczqvNO{LzSiUVqE!6|pZ zf^lt)FN=7%j`^^NCtgbsPeasiyd4)g^nplX0D6dfQa;P)mnG#KwFN2eQ?~cPyzq|5dZ)H0000000000fPp2G?KeIFd6O(S LBL*}z00000@OVFi delta 2331 zcmV+$3FP+HaMf_IW)20YT8E4?vuh3%41Z}GCyr}L<1ly(@kb)5@UT!U6&?~oG6NJI zmH{#`gJk*)9#R0w^a(t7zq}_C1Ri$SK;WSvd7G7WSDwhSLU1Bm8q-$7`c0&sS_T<% zGHc3Gk2Wps^6gAn=~3N#T8SDGNvDT}V(Ijd5Yigp^so%jO4bGfT760nDS)*4jDH>j zMh|Ss*kMCjSqNw)>;J$BuH667)NJ(p7&Nz3>3M6)O3(Tv*J#(eUB)`#B-%+Nr5+ZF zrPM=02xoxQ!!iITS^fv$^f^7G0K(~0dJHH%uq9)M4U`^eHeh{&d6O;Y1M_Chmd_*M ze46G{%OHd1&9dfG%$us?y8etEXn(SArjnKq3&qm%At6LF!17@kK$9%f18DjrA5s9( z^f^8T93R+{vBQRF!dgD08BUU5^q88Bo*%>Ewt79L>h)NYIv0;N9S4cTDv6}n!$PqX zdq@aL3=n%*21q1J@&JiGuZI*s5`9{a0j&qNWNatW>S2c#Y&#M{Ho@vKHGf+@53%`o zb-P-|7ldV1>#_J)48qzyW}B>@@1Wb& zGQJ=NQ^{arT{m634hD%KBEPiwfQ95@Ry`6z1_Q(%mH`F>#2(`ldq@Ff&?oj75PM)t z#txfY?EZ*`U~UX>eOMv9)w6Z3kC5S| zMSgGi0SiegdL#stvW%pZWhA8}1C&~y=tBwsrM?1v3!)Ef$=G3&3)>$-vjOXyM3R*T zB<= zecBJLApcmzSD|mg`+tEg8QTjyucRS4b&?RA(!&0Swr20=^JI21|Gea;`Zsx4S<47R zPHJsg`&oZcN!OH4NULmW2b;teQPTTiAvwGBNC?>s@P1eZ*d!f7Hhtm`tpIHL%%8ho z-m?kj58Deo6M<#}(hNbAt%jxEZas)kdmvKF2!lY>RS-=la(^j3)}i!RVw5PU^stb; zwcjHl1TjGAVHp6CEwTe3`kWqG0YLOgJr<-M*piXGz#|dX6L=&-LWm|fJ+@}2$BX9Q zeI8QF;DV^Ex;#cFMmfI8%Q{>hOQaAbT^<&acMy9dgj5E&JS+oLvgLJvN}tR_D*%-~ zo5zC916wk-7k_vnLPK!s8n9*$?0;x$ws}IaeA45PS_T({rC6*-cRQ|IvDV@7SmK%} z>G80TT$H~@LWpI6$HOv!C0lC;u=I&Mv;tu1GkGkSJg_Ardw~Zc&}u+hA%YvA%8?LK;U5+K#;Ag(~$`J3?5nm z5cDZL78D-XlCizuO&#L-bmYWHGH~ye>%<)pV~t!0Ixo`$Aa4fTQas6ywL+o_K;>s zC;NIXpntPwx972RKJos~mKr{9Y*MAOxYTqAKGq@lSmN$zDfqCE{KW5(5aJmi_^=G% z$ri~0Jbi`_tpIrX6dwzU4{XWUUO+FPA-K>CSl@>gLSkz+fgU9G@Aw6^3@!-NvIeA8 zD~Vm-Okyh*PRCF_vel{N7i-AyRD7lh<~lWZS&a&wSI%O@8iY(=ChN@`a6I}7cb|F z>1ef@%+4Ci|7@B(?>y`MsiSqV@#&kZi@dS=dYSJv^Dmdh>#`nfvvKm}^wr5;v-mi< zTukPR$?EI%-cO5aR~MuG>HOs9O{cDAlmA@hKmRoTsq^foap#wRHuNuzy+$$g+f-;V zy3JnJKd^4??_Q41@}tq>Y%*ImF7nf2pi!&1TwhGi&ToHS%`Z1U#m&aY`D!(v-u~-6 zAD!fj^}kHBaXO!`ZvU}9B%kMtzcvSS|G!X60|b-KHX5_wAyW_pif4R)lQ%LSf6Z#c zFc5_AeF~$yE!l2L5w-(u$*F-tQ(AHp(k2nLevs5UdHc#v;w0cxkE@yec6L!U->WwH z1?Q#FHOe!B642ZTskb%ySS`{=l=#53V6D+mqXT#}Z|*8?DK`$@oU!171fN8Z_LSQi z?Lx2=W6yV>n9rg^M{#4EVj-4pe~T^S-)sxGAmjln2u!fRa1TuFNJJ;3;3LxhxOM;t zj-dqwI(VFCIlg!UIOQiB2Ib1BWH?xuw4TM$z4vl7c3qcs Date: Fri, 31 Jan 2025 07:03:20 +0000 Subject: [PATCH 08/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14422 -> 14422 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e349fb912..e8fd06a05 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-30 20:44:43,274 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3270 INFO] --- -[2025-01-30 20:44:43,274 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-31 07:03:15,828 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-31 07:03:15,828 submission_checker1.py:3270 INFO] --- +[2025-01-31 07:03:15,828 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-31 07:03:15,828 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-31 07:03:15,828 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3286 INFO] --- -[2025-01-30 20:44:43,274 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-30 20:44:43,274 submission_checker1.py:3307 INFO] --- -[2025-01-30 20:44:43,274 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-30 20:44:44,371 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-31 07:03:15,828 submission_checker1.py:3286 INFO] --- +[2025-01-31 07:03:15,828 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-31 07:03:15,828 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-31 07:03:15,828 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-31 07:03:15,829 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-31 07:03:15,829 submission_checker1.py:3307 INFO] --- +[2025-01-31 07:03:15,829 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-31 07:03:16,874 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -67,7 +67,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432223 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42546 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.82 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c13c4e8a7..75e27a6bd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -549 +550 diff --git a/summary.csv b/summary.csv index eacad165e..12618ab6b 100644 --- a/summary.csv +++ b/summary.csv @@ -45,7 +45,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432223,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42546,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.82,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7b0f3040d..09a570d1d 100644 --- a/summary.json +++ b/summary.json @@ -1389,7 +1389,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432223, + "Result":0.42546, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 24304095185724d2050255337bf9794591135f50..b51f13ec1171beb874191e1c5a53c94167054ce9 100644 GIT binary patch delta 4673 zcmYLNc|6nqAK%OsV$PbrjxqQ2%|deLYOWN?+((nMQ0{0;${E|(R*I1;x$i_-MT~^G zJ0wS%BVyE~{66*l_4F`4g$I5gbDj_ z&y9a7Y8(;PLeJN7KoMVLaRB*?IJbnlB&LpDmXE?A*KhWBK07*v$Ikz`o%I zJw^R{Rc~sITV8DjU5!7p8{>9=Rl>ycbH>OVV@2CrVlkA1`=;()lSOrrE6-WjVF$$R zs$H^N9a9)QrsB4`#W~yV()29P{+xHlW|@D4bYtkp`-3YI0k#u2qhC>73@RN)@;RZ~ z3qNv#D=yitz3j{S8)bETR)SwKTH|$j(%5;ee=6GOG`4KCT_TSVDI?O*5@bQ zOFQI1CFQh#HgPd8hHarnoK{&yn~cr^qt}7oCN_D}QacMeNvYh~k-T(U!x`VJpuv$2 z8hU;HR%YZ8GOokF7#t_rC3;QsoluV4c9vmZ{IOLf{}^(150!d|Ii9OobD?s!Vc{EL zOsfRcJFTqL`+%aOf7w1O#58}gKAaH4_-AJeR4)dES)&B6yvBVaRa3fYqe zB#+ms#Y$}!8lcP8yuC^=`2$DMHCsg6_YnDsznE7xKTxTNe&kO2DC2~OE^X2=@J0t8;hh$nCctxpd3LgZim!WsTwE$ z%A%bJ;eqeCll58k+@560&B6iCkEbT0CMJrw)tb&Yvbc$vC`SPWI9AJ4&=2M#dbIa} zgOCEx`up*6-g+*&UmNKDJF#O7wA2Y1h3fD!M6J}D+$aG{aBQ^f71{Pu<>9N&=h8x% zsIroG&aB9X76SP>HIZp(E0~%%5z2%Wiq{Jb!Q^ogYP2&J0`mFB<*AY&{ls&kYWT=9 zQ>D47FFTvx8)eU0za?R#;&{tg9;RecH>?Oyq{{#l ziIJ*WVpcR03yl(z70s-q}( zxYdM^sTRt&o2sjiR-j7NU8>ZJl`_IaD-Fx(7K1+*$c-q%XGs1CDpIixE5miT*+Jof zB!c=(AW_pf-+Yo({}gS$%%8U(eUx*(d!j1R#lu-BYG4Bz?)vK?!@9s*YK_s2^2SXf zG%xg>%yI4Khx8_t_uHJ&YSO-)^F?zTCp_q2@Lu8ebXo`E_@4_4$AONt>6$=Cxsm*^ zErw+i*O8^hjHX?D4*Y;T&fKp#QwwV|=HHAyS$(NWCYTug&{DctoFgVj$ZT-bRDgHs zm;%*wawLDmRvS<~vy_wZ^;%qZ<$Hjwd<~NOu^+#&HSbS33h(+}|7S|PMOgECYLlAD zaLo(WdaAa%@O#w5|sPjq9Li+Trw|oA3P1r3+)G`g$~2yhs#030;UfJ!A!@&FBybcQg;Q62KgIdw|J=xt9IityISVml^zNd#axCMRZwgpA5jz{sf` zeSGQQuOdm1Ep4~B6zBWjRIU35@z)?33ifm*J(DLF7FF)||FDh;ZaoW$udI|NoFa|t zSwL2Y@Y*(to)uss;0#0oK94em_`zmK)gs&;JBIRZ;K%GVQ#XD6lsYBFtKcsgq%p?YpjaWsdU1IIJ0?+iOLPs(SfQrXd@rtl)sSC@0mx+=j!GkI=V$VR%BM zD$y1dO#5oCPDgn@<4V-b&jUtt_)?$FM&T5GYyYK{x|`SMQE^W$^e--*k7$mKf_y&a zeWaE$%=3Y%cGct&d9tLb(4TCRvR#~JEM~oya?XUyq!P|IWe+FLUYr(G6fyqOmrg_gf>P^ylc|5eW zEFVpX(DCy+V@fxwvOH-Sy26z`P)P&{=O!ZPTCqidnjI|MsuMCrQ-rwq6VHjL;h{Sv zvJgj+P{yP$nBFrW20Awn7qF`}|g;6j%Ter)@>V03q8t?ni7Z{hun_NuCO+=d}VGHG*? zd8b8eJZzRdis~g?adnE0Zwd;cUJB^5%3hitv3X5Es~aIDY)ey;3FDyB6o34=25soj z($RP{xMWO)eGnS&WBW?@5VOzBRKxW*-8F*9yfmwij7Rh1J_&`#V|Zr}m3yH3#-%9|rh181+2G>8B3>Vc4DeQk9SLg~ zOYoVla@~kS+;?b92!>&YI$T>4(&L<>{-N!n>LK>q-De#)e2XwP#)#xVUbHb15=e56 z2ieo+0F(2KyL<4ne?5lUWBO9GUs{VL%Q`gc~zS~)7*I#ZNGe#8f$jQ~FMgGK-! zY#VAs!dc>pXz)B!W#=F?5%3|JYtWv)39lxcz&z;*%^BU4`47oQ9cEXxToy9%Twc^- zK1NEej^Kgoe+2a4gP(Ybq66po1TSFpgmyY)Azpt+FDK-K+%WX@ZSW%zjpWw&mQ!1z z!qcogN3vniqxa|fd(I-|#?t#TmRve-8O`Rp%|7U@-EZOpx3_^DWO+@EmB-u{M7Ce`$(h6{*u%J=@Oy3coguU-#O_!DS&YQtFF=(}pZZ$Tm! z!7yKF>JxznEQ`aXJ^Hz_wwYo|WLYUv>-JSdvOKS<5fUQbpc4(UppBSofKrVsdXh$g zDQq|pi+B;wm2H9MALhoD_l&QwV~2O!2hD?F9#FC~wdYHhYD(QDnZwyij;W->B7zKq zR3kAnb%=*71*|@`q;Fd4@EM?ILmN2%{4=2DJ3&LyBDh}EeoO=3wn>xDJHsN3F)ISC zc|-7pW7Sx3yUav2W_djp8DXExj>VvB7#|hkT`2mTFN}^OX*NSJI~KBT;5znQLn%Ai zNfPlX9U>a2BVP?Y0gVeCP;k&oa<~g?b)KpB^OW@$n^;t8PHS{-*I=hK~3 zs#m}24cylkln=7xrrDZhyK&!bi~>b9XuFQEE4=|2QVYHS7*ZSBCNncuzC9#BuN+*h zpV|JCKb|l)J>94T#$-Ip7yME+wHk-acj(w_EWI^&=-yct^01CfQg#%-G3Bv=n}9JA zu{R6M&o?1x0)}@T8!`_nFn$TJG)%;$XY+LbrCE4j224uX4Slpz2dT@?%9eH&f&&Q3`mdAA+TTU_V?jXak+4@d7XZLt~NN zBtGsxtD3fuPof-Cs6vQ%Ep&kCph|Xd5^5RSseiGP=G^1^t4p*Tg;U;1rHlg}w^{>< zv7r#p0*N6-p|2S%ht)g$ERsZ!p9%ar*dh04HKUKI3NmVCWe8BbjiB}Exjg4W_tTv( zbpkcb1)pmlzGvEma}==J*pAIyY24TRGpZRZQaVuIcKQ-XsJlgS_KficTzOybkm-}7)BYX5AMt4Pa~9_RAT61LWL_jsF! z-RBI(((=}{Lq@R6f#P4R-T6iP@|p`ddmalWd)mJ%Hdo3c7O6O0_P@8ET4}N01!_GR z@ESL%yR^FImGx$$cz8Ax<*s{pW5loUOyy$_Z@V{d!f;*#{~j~x9(dKm3D%)a|6Vqf z%b(AX-s-e#jL@gmUfQcSM$i^BI(7LX_c~A8Lb!9@=t?)vz@4Dn%yymcImb|PxPjo8 zN1(}Ti2}v#nrdZ1qAie%jEd`KPHvaQnaZNg*9fEfaX-~h%_3%;vMcZYijF>`^Nywa zlSeLBN9O_03D`m3Qq(=^MC2kzvCnMYYGYjb;jk=Mo=yxo+zulWHm$#Y=0vYpFSoMC z?G`QlTMB@z(kuOBUZ z=<7v1`;~d^1k?YW=J!e&zU9?+Iau<$uvK+HMEF{gbf)Kor00~o)3x+YsoJ$)kJ?Zv zPWw#lm3YKuW1F5;tVa=rCpMr5SY07J8$mQSv2{eD$C%(MK+lR8>=braYtd0|NkLsyr-)N{!hY*W4QkT DNvo=Q delta 4673 zcmYM2c|4T+`^IMs%95r@GRl}?B#dMb*|!*5gfxaEjJ=X1`!l3tOJ;_wS;AOivPMXU zB-^nw$WcxaL$Z#JwcpeE{_*?g`RjSE*L8pH>wSOj*TeE>`B!-e@vR)ZvQs$@fjFkz z5pv-+Tv~lyJ0hgO1w1s{+=NJ(C0=VZ!N;rLRwNpqetko!@X`b48vK{4y`XH2M8cnL zyH6I_@>UxM%PY6%OAQ+_ZsNj<%QQa~bPIh&)TTi`RPD{mHv3@Z7wTZ=Oqi(_{Usl;a@o-o|+w2>Dym6gYlYuCp#`WMgnho(#yZjjdil!8+G;_ zP(Xs?m9SS`8-4NiVG%lGKTa?!JyOQTzr68wc=iD|r8cju*)chM|9Q^Z+*tqONT8=q zcErVlBGQYgVNJE_uWR8!8W(*JYD4Ii#wm&9jr8d)+05EsrEV4-nZC|}w>E$$ku8ls z(wicsN%5wVoqIrz|0cr7Dw5@9^fsMMbqDrmUk=pM3|p_baB{ zjR2*#Sej8UDfkA?X3f}Kc1VzMca=CKdIj+uc^u0K$O?l*{ZT5C#I3bssW)vky>JvcVtq(wbw=u_g0o~ zyN18rFl|WC(EM8d#a)gvpz#Nyb$++*eTgOMij4-Nd}eYkIOv$w+8-BN@g?kir&Dr~z<%Sq zU3M7_=W5J;>3+fOse(v=xk)|Y{4 zDa`Cscgi(7)LH4_8y|_vS&Yc)qU@k259QkO*AoKhN-XS6s6^-Vs@Y;!*cPMzVC5^R zlNkN&+Vz|pw`9(>r%-r<$sz1y*w_((w)M$e+*OKG@Bd5os*4O z__%ERQ%Nn@7ikW!t`EJ@E%!B;tMK&YL<4jEw=zP6;x~DxT;qR0-qQg=IH-y9piWGf z;A97%z7OT!G=*YWwrDAy)wjd9z8$T=L$HoPvOUcJteumrE}z4f$-m3X6vMB$4DIVIzYsSBl7ce5<4f z^-`Xw5&ujh)9Q#T52Ygjs@@=dn@+<;YZ#IKu`qR3cqY3foWQmEUF(}xkhx{S&leSl z$s)3aX+!xi+d5uYL*_agL3@n1IU|WFQr44kMdmB(ods-k3gd+JG$Jk^lZOQER7l!D z*cMTjYaBCV$>q;Kx_T}QoxNSAvZ?=+`J7`>ra@cE8vf^qZ0hAA!3aIP*+H9eatyTt z@*s{(xitLzJwY_$0M4YEiAC_!=zk1g{JJNcTZ&c#1q5{$=rAq8Cr=n9RxP{dLr}MrpS&eJ6**{vGf=Y2M$+IH0bXq3llRUgBaq603`$1rrkO1 zX94CPwj`N7QeGF(eg0qEec%?)RLk{~AJ%3#<+|{X;XmZZ*ZT3FkkFd0TMDiz6ECJ3 z#}>(3VnE$218j^RZ-xO5=0_L8Wb0gwvxi)WcsV$&xH6gQh_65CZhPSr)kEDJ%<1*o zg$+Z!ziprk%yR;I9XKE}u<-qwldYy8+A{4GjThd1R{jVYVSs=MqSK&=$pU4)0xM|2 z35ro3tca0&@314j=A^ZcgVk)x9Fz}}9q<#B{I&tf2F8n+$K;Pvi)iZi{#a3=@)r7ViM ze`H`^zmD1p7Hw9qb*B0XDA2^P-HNo6uQv78@0EKhv}|%b0!#mN zRKS6~PlekcXZEWEtxfHZm`SJVEXN9i#73}^d6y2=yMM2W;=uZNEABwZahyWm`batJ z$fL_8X;9v~4y?$!+YBMOB*PsgBUF45r`^Dwq>JUYt9e{RCu5&tVRmA9n}+<5r*F9; zVEsoyOmD*Eo=;L#9&TLqQdLnE(D^LhM&cC>_~7%xoY$f7m3fP@5VI405=#f1e|zJA z-b80Opf>_bkc&TpbX1liyiK8O3)|Qygs5LhHP})9-lpm?bZM8ZjHC5;U}N z!saEhF;o_$B$iCEuekspT(k`(K#RsaH$~5fx|n|uIyKy5peOI}>(FKYxr5kZkHY!R z{6>b^fTOK_C`+c%xS6$#XF+9#E=$UCs;qGg6v6;dhZUhfCcD>lAteKB3=jNr-mse< zJ4ykm>6gE$_q*MHW&2%}$WZgG4@dFMkErC4c#$f1(p2+YKiP?ob3DK6r9>z{tqUif zO-nnIAI4DYgS8lE9cSG$6QACff0Ok`DAkgv}7`A_#q9BvjB#yGq(k zlV*QZv(9CT2Cw0>PcEBW42kRevwUg{{}zoaCnYB}l?6=mnw=&h{*9`dbai&%}QlJN+_kjpwec zodhGNDaTaY2|+fBELnsKbl7k(m|46Y=O%K7j{6RC+Od$Zfi!Mr>hM}ffMPkV_m0CO z2=eZ5)(@pAcTV`B|CE$6%y^G4HL*Y5KTi-mw$11`gu|Y!1s1{aEQ;Q$52#d!p7?q{KhSy@hi9m(V;A`vvfa& zTq2%566wXW-6{E&KSJQsZ%CYHLZpv$m!-taq>60m)~SZ+=E>BiY|l%pjs4l(-a|4L z4Y2X;{W!3bXx3C5Xh0&{YEEG5?JT4=i2Y>Ic z=USczK=I7OD&Pn|T;QjQ;{+|~hku08KuPX`m(gAtG;?;f5Tg@OXk^NWeKgD52ffBY z%|Nelel&5&(ER2gKcv8Fu^P<*Q}~?+6EDpwh+$s#P_$omGDQ*r1YlPa2WO z=9Dy)de$tRS^umPB_Ew-uUApuJe-ii6yaOvZX)yP3Y9Rs9d{H7TDm!@CF1uxJuR)r zkBLRwj#E^_Tz(lk*7hAwm`g#$j^baVaUHJOO;dNL6J7}=cOu4(Op`%_1$){E@UqeZ zxx)dIBuxs#YWDdF@K)${YnuhZn-tK3N7=)M`HykaI$5~&7^#0#Gw(X`pyWlqT{&}i zU!QURuvXe(tQo%+Zj{CaG8YZy0;`SoyPosJXwY+Yke`xo^-p#FDp>9^&z(j1K#7!$ zF+D1*(%mz7<&{Ty3IF2H^rybY{bCNen&J^*QpB}1(02|_b$<#2G57%2h%C0d7rk=)`6g@Ocn zwC?Csz!8qMe4FTMpi{Aw?604nrix~gOuz4VARMbH!(rC(ngtA=_XgtLP#Qd1%mrF4 zNa=Z-Dxk;l2CqQM;sym5tOpK>F#UFj$iJ9AY7d+Z9^-zD&9+L1iz8D)z9Nr!|1`_$ z^j$eOpdi`Hd%ubrp1nypTpGPvVyToRxJ+2U zEWl;LBC`{gv15TILC%d+ zQp?M+V10&aH3eV1VPR4}<{rc%9(az-hRi#xScx+eldL{FU7t-F8}ALT%mjc&oO3$I zB|9lav83|yR#r!6_na&KhK2o&%Ab)WyXGd*Wvh7^-8&lhRkQcz<$i6KtXbuF>LzLa zxmWuf@cyov@cgrJ;?S+FnuDnOzQE6bz%+>@5zmy#n!B_-5m*wbV{XSaASiNo!zZ$F zTlV$^o&Ia(+nZZA=C>EHH!8$_k+YIUw!`o5CUY>`S~vDyUCiJ8#ba#_<4)3m>Mw;(Mg@ajSkai z+%<~7kk@?lrni=2(&TT)qAj9M>_ip>XzF?ZV&{r&A@wIK~v<()+L46y)6DkXC-Ka`x&kWa$&b14}KaP-CU8{_dgL5M?Cn T|NjuxW?xW;{_hCK)qwv4bZVmd diff --git a/summary_results.json b/summary_results.json index 9ba84ca0f..2aa57517f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1292,7 +1292,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432223, + "Performance_Result": 0.42546, "Performance_Units": "Samples/s" }, { From 0a7b6e1ed00229379e5a1b6227d12af87dec39e1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 31 Jan 2025 07:07:29 +0000 Subject: [PATCH 09/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14422 -> 14422 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e8fd06a05..5289b2a6b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-31 07:03:15,828 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-31 07:03:15,828 submission_checker1.py:3270 INFO] --- -[2025-01-31 07:03:15,828 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-31 07:07:24,681 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-31 07:07:24,681 submission_checker1.py:3270 INFO] --- +[2025-01-31 07:07:24,681 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-31 07:03:15,828 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-31 07:07:24,681 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-31 07:03:15,828 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-31 07:07:24,681 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-31 07:03:15,828 submission_checker1.py:3286 INFO] --- -[2025-01-31 07:03:15,828 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-31 07:03:15,828 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-31 07:03:15,828 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-31 07:03:15,829 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-31 07:03:15,829 submission_checker1.py:3307 INFO] --- -[2025-01-31 07:03:15,829 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-31 07:03:16,874 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-31 07:07:24,681 submission_checker1.py:3286 INFO] --- +[2025-01-31 07:07:24,681 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-31 07:07:24,681 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-31 07:07:24,682 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-31 07:07:24,682 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-31 07:07:24,682 submission_checker1.py:3307 INFO] --- +[2025-01-31 07:07:24,682 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-31 07:07:25,742 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.95025 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1992 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 75e27a6bd..0eedeef9d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -550 +551 diff --git a/summary.csv b/summary.csv index 12618ab6b..5f063fbb8 100644 --- a/summary.csv +++ b/summary.csv @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.95025,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1992,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 09a570d1d..038ed0c6e 100644 --- a/summary.json +++ b/summary.json @@ -1179,7 +1179,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.95025, + "Result":10.1992, "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b51f13ec1171beb874191e1c5a53c94167054ce9..3ed63216e4d4f2b8c13f4c70c01bbdcc253475a1 100644 GIT binary patch delta 324 zcmV-K0lWUzaMp0JGcp9-JnV#%H!>f8%WA_g5JmU>3ZuO(KS*1Q?Lb?yYM@Y;mh6Ny zaYU^Ll3K@qU)fHa1iUI}Bi%b^?hGo{N6`kqU@TX%L`f8*03=n6%cevh^(K5ofptVO z(kclh>cOIQbyrZGQ)S@IC=JGOuz?3jn^Rq)gL68^*wOVri z-s=l*r$MGVV1W;iPjS!6XZHLhQg2X|#_1}IlQ2u__#yWvUG0XJH}w~wI5qh<@eG)9 W-+qGX53_kQS_A~$JnV#%*EfI7NRf8&1%Ci5QOi23ZuI%DW+)=wgYX+sewXMT5=Q8 zCK0uMkkmSP`zlW2B;ZqztC{_Fc2PCk={EZXACxsUq6I-27-6L{n;LyA=lLVbLgYsB z)*7hM9s-&*ca?BVSPyUBI`C0J$P&l|CR~lSF*=5E5L?hZ6v<(dxVB#Nm`cBY!H$b> zz5zTTWQsLJE_vkm0Ll222ZzFWv}VhtY;!xpHb1_YOv_ zXK`@vf*Oon*A-paTT_0DzZS3W{i?Y#hf@haO(g{r9(c6Asql4?4%hM!7wN=n1@dX1 z^pUsIBF8?ESqi|8anH(U`TVjaoTFw!Cigj^c}bV#fsvBY>FO}@rvC!ery>6)o&m?) Wx1XT-1G9NES_A~ryEXTd*EfHSn3`|^ diff --git a/summary_results.json b/summary_results.json index 2aa57517f..fb6218a6c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.95025, + "Performance_Result": 10.1992, "Performance_Units": "Samples/s" }, { From 6438c6cef3f5049ff92ab7efbe249bde9a07a479 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 31 Jan 2025 09:15:19 +0000 Subject: [PATCH 10/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14422 -> 14425 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5289b2a6b..9e699c451 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-31 07:07:24,681 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-31 07:07:24,681 submission_checker1.py:3270 INFO] --- -[2025-01-31 07:07:24,681 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3270 INFO] --- +[2025-01-31 09:15:13,812 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-31 07:07:24,681 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-31 07:07:24,681 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-31 07:07:24,681 submission_checker1.py:3286 INFO] --- -[2025-01-31 07:07:24,681 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-31 07:07:24,681 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-31 07:07:24,682 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-31 07:07:24,682 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-31 07:07:24,682 submission_checker1.py:3307 INFO] --- -[2025-01-31 07:07:24,682 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-31 07:07:25,742 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-31 09:15:13,812 submission_checker1.py:3286 INFO] --- +[2025-01-31 09:15:13,812 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-31 09:15:13,812 submission_checker1.py:3307 INFO] --- +[2025-01-31 09:15:13,812 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-31 09:15:14,930 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,7 +50,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432023 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432666 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -68,7 +68,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42546 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.82 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0eedeef9d..f5e47edc5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -551 +552 diff --git a/summary.csv b/summary.csv index 5f063fbb8..780cfed20 100644 --- a/summary.csv +++ b/summary.csv @@ -28,7 +28,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432023,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432666,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -46,7 +46,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42546,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.82,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 038ed0c6e..d5644fc05 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432023, + "Result":0.432666, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1419,7 +1419,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.82, + "Result":22.9348, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 3ed63216e4d4f2b8c13f4c70c01bbdcc253475a1..e9a19490aa600bc25f7f2637e6284f3018ae1250 100644 GIT binary patch delta 3927 zcmYjUc{r47+{J@%l-WB`*;8D=XyuO$HQxaWF*${^_R^Q z0KlvMoXiH4(J~ZKaJa0XPSK&elOSVQjIp%%8=zw}b+qJc5LW&tXPbwih=XSGIEPh2 zh=FG6><5vb*VF{g4M+SDM(vQGnc1PUj%9i+#>dajtSxWFnE1T0-`|&dTCDWODYwKqrUgcvgYk~yDu>1wA&W_V4pE^ z>HE?tcq5URD2o{X3;dy>R);)W{9_G#ZLBrVPCdID3+m0aOsFp%4G(U@u$-rtQVGyVlgN{s$7}Mrd5|_c445a zd*ipK)tZJUtDs*W0;2@jnE7*g+{M?C%RwI*%MHa1k@1gu^cZVvUT@zQ3{!oKN@gY> zTU&PP34jL}XsbUgc7)&~AYrFRoQnD-fd>;W^puG{Hn5m=!u7{*bB43NE!*DOe)FR7 zdaY>RgBJ5=6ZT$lM=M%aQt$RF5je*K1bVUUx^He-$Xo7wrHzmHEdxT>~iIZALEM zntc^PZ@}CL79$!6fJYeUj6Y1!NI0J!56r|#u6aOE&0atd?fmh#9}IR29zVvg=Gm5M z)&GP=A}^#ggW}4at@xHxiDSrg5q~dGzQj#Va8S!U9(%;BQ~n=Fnl=K(4P?6ijjFB9 zaCa=yOJ7%<=ZH`$?-H#frxQyj|K3j24!snK<5<03(O>_=b|Im&JX}3MqS)AAD7WdF zYqhaoDug$nm$lDx7p7!p_GAVt)1D16hhk9dqvnaqa6KS$6OVEP7z=BP-4qB<0grC- zd}yCjb$Ri{?MFq~O_6`Wvrx5D61X}m5*}tdh0*d5y_nX}p382f9da7gQ52gVkNrD#nVU!c=S zXVZI=9p+X#bVr_{6KUTf)Ye4MdB@3?l+#T6P8{(x^8_!!N=}Cvxmlp;3e2y09OD7P zT#;LW&4x}ruB67UmjfjyryN7(Q*=E^OSz60HD8&QcIG zJ!+6|EE&PG5r9D@{ zl`huNcqOi8qcV+@!^4_?O{6EJ%Q7fd9_7ke@OP?3Ao4Vb{BUy$L=ImU9E*O|`obj) zlnU;qEaGJ(T8K&6mbHcc_mh6X)}pwX7DJxME(~^i8yQ1ZRZ@tZh$P~AbR|wRtvhkR z&T7sI-z0-r(`&h85G&Pb2aD$tyQ2!izTo)o2&ZFHoJ*0<&|<;1cM3NEy5r$mfx|%N&%7m6&#gdXe9H;}6bqul*}F zp1lM6l3h1);uUst3l48~2I4AuMNSym?)_(f1pd{q43xs5|G|TR7N$Z$3wHw0GzOhX zZkm_QoOfadi=hVRfROSzwRY~cWwC79sKQ2IEYY&NY-;<&tI7>L|HB*8kh2IMI5$G6 z_t4_=YCYT3&61OQ7pUv0V1g>yd!B$=fSKL&+0zq9lalT!cH znTWb^1GDx{%NHQM>sJ+C(zh7A!;PNzAx}nJc2oA*oKY)IiV7yGmS$BCGj}=z$%3Cp z6hUgG!v&+eL<|x&;8lRvCK;u#ev355&U(U(-c<$TDdEJW#1`_pZoe@6FZ)r1t#@HF z4ZA0>q}zPdZ&+sBS|Mv6yGd5cBjkG~+)~B&dyO=aVxHZtjc9aWd_ybw<{F$zddju$ z18o~@Hme#t5X#?#nMIAY6VK=3z3XQ$tk*uIwYET3^AFr~>zz*yTgZNb9HE9f(gy0N zrKInJWMn$;TZg4<+z|}R85VN{3w36@q&P~HI($cq!KsDoC2gBY`iR4!zT3ol$kHAn z!hd@{uc}ZooAHM9y`0WG>cwqDR(X2A&(r^l^xf$H;fZ?HU7KOkF*h(!FV(`YBuU`L zQ;|gIs4O_ceo%uu<4ltjW7l9)6H`6H8})V?Oy&H`Gxx8;Jk%c(5PzuoWT0^rOqYgo zo#QeZ%`zk|`yrT_SXEK@W~+uHfM%?PSil-I#@MwTutjx~J()p8uWN-xuXFR%HD*@w zA4fMz8+AT0reC@{BdwAiy&$0tk;~tmTJgb(x5yIE`%m{MT)#D!lPQuL%CRe=!Hp=UYY zk$HJJw~;9yxaJn;?8X;74$Or$oPBE$pdK#&3Bz`kmFf~Mg0zkKiy&=IA({9YA%)WWY)-@-@1yo- zybbs4(r{9Wue7;UVV!?Hgl=1(oXp zmBT_iLFKT5O#-vDe+hJ-ZQ*DDap{+GMU#VYFL&5ILe0-RaY&>!M1QQfVd8#O-kU+AVaAQ zmN36B8ufOx{^-xYkW$J64Qs3OtG#!nkma)8&=UBT=S?d#B+x04ATS!hHRDUJ1rNZ@ z8^=rEv|!X|2c+6nM={7GLo--6p%VUjaol{|+CAp(S*ptlul&5H1=8tDg0r6Pfr?Yx znqSkL&ZJ+}e0FGyc?qLrAumGpuyX9du^rhEYE}O#T8!xk&aznluBO1{8bQv#8AapF zbvsP~70QlU8b)53;Y=mdo+~Hw{<=@Ozw%~^q08<`!VFqxH~i^Nsg9n~LX&a)O+N&Y zt^ZuXc5mW@Y!&K={=15S=IL$zPR!nR6J{2e+lbqT`28e#)_+@##U0LN_?&my&c|1R zrx$(Ln(Z9tPI*1? zDGQ#gQnXeh&Fpp^hrkoIViCTNh#fJ<$mx89Y;IWubsIL_9a_R6`&>(?JOF#}G7pzY=@nD8nKB#8s zfzM>t<>_Hl-$9MBp97xwZHaXD#tj5`b-D7D0aAg!^QE1;_${BqTU&^Gny zk-J|9B>5Hn|JF`_{ne^-Yy3P59zrO1TF8k+8ktqh0JjMpASry}l?<`2U!;nD8-RNS z!?2$_B+<2|`9xXy@6lVyG}v#VXFQ<7`#GJ}4>(guH5JcCfO8>?tOwj9$aW|15(TI8 zHxp^!OB^`LY|oLUbF~iN?0qn(vxrASWclmYn%A?#hoIHR0HEzv&LpI0x{g}wF+_%r#)s{`k@@0Sm!Iz_EvDUkDmBY(`*e}H%!b()QI+$j81+%cYS91tspaG!bc1wpRuAGKUK~TSqq|3R z03asxz=g=zQ2GTjpg`al{gno;c7ctYeN9wga`o00snU~llJ2{M{x=} ZFL6?h;Zpzmh?$hY(LbaGGc(<}^FN63^`!s+ delta 3862 zcmYLMdpwhEAK#1-vYA7zVjFXs>D3%_K1?=}nq-6!$)QjP@oc0_=`ov4Cpl9wM(7be zO(|vxbM7FWn?uCtA=P_(-ap>^kL#cNbA7Mh_w>21dm?@`o_|bblhX;2v$+}w6mU0A zWfelRIpq~^RD*w_Mrh23_28<>veSjW%h}I{I=ru4>IrRY z6FBO+6Ex=XseAI`>B!Vo>9s=1#jnak+jiUAI%;1YyQk){^w!^5@^Ne=eV4mWQ;jh5AKc&MY&0Q6V zeoYKactJSbjRn@+bq7UDe(gmFJi6uMzm=Rm-;W)lE_&SL_nwK&&;EB88JhA$c`-V@BMkgu#D@k50cyA@gEXRd zu-@OSMo^XC%6clkm5pgF8Fzp4oENd**^FmYw!LAZMP-Xq0&=QNCx6_Q75L1sLa&Wz z3_3-I_UIweTg*uH%LSgVAm}ll4al~SGNwbUql#8%qDYgk^)SRDb(h4sA2ft>66|Q12UKd@%Bs+B)a0=~ll_|;iDy7zfKvdbrWq-AR+p>(#UspKZmkl8 zm@#4&$fO*RSe~XG*>o6F{H(IzUb<(-ONtFD0Tb z-uP#uY!HWgzO*ms&Azduv9U5)qvit?iEKe0fug8`3E$dKeClEhm? zMOrbtJ~5ZH2Wg~M02d=m)f$xtkZ5(Ik^@9Vbg@GXLjz=!aY)sOo>=WvPTyQ`m^&I(wy1JMwLy_%^|ehV$Roq-Kf!6B`^2)tEWDPHQ!#)n@f+GXytx8 zZ#Z{n7OuzZGAue#tEEz~$JlMM)pp|m)Q~P_iWPw^!t1t$Ysu_v@zyEVz!v?jm|@~u z>^jbBd_Z!YQMk65WVV|5WUMaHFVI&xsc#V;@Bj0bkbLuq=DhF;;RtOUOEWpv>(2HU zDL{JCYeXIgb~T${W`9|ssq}l>zS6loncIQgy>VdaD8R^EU;;2IOjJNF3*DM!OsDT& zYF?)=Vq%5k!If=EMAyY#n>KYO_9}KJ-ibNH&=$0sa~yuUhPCAF3ilX^Xd$`tPS$Bf z7bM?u+t#ACia;^*XiXFo9ZZ*n6M!t>Bq}5 z6|Q1VHC$+D?1e$NSVHzu zUB#eJ-Udkz70H!`arfRo@GhnPwicABFM2_OFv1HtQ1~WHE*J_IfoQq0gGlsX4QNo& zsk|R7ru6U5HbG7(8Mf3JED9It*-Wpt9{}W1D!uPvjChpLfyTebaTA6w2l;VTmfb7% ziaFCTV8Rg;!UYc*MsdOTi8UK~RsYX24UlKadPP;}{{puV5v5R%7wUKk^sF84ANy=@ zL-Y^%3@!TJK}cF{?Kb8<*041ZGB-ds_s|Wx4KB!QqCj*o9|%s6Fys`A2a^q4vqJ}f z|B~i91=!f+@bz@hg6CBNHj|9Q;hKtKi%a(Tp$kT*-*fLf#I;Z@d-fD0)jhtY)s(NV z40Ji6H&WA-UFySm@$O2!PnTx~@87wemNY(vs+J1brzdAL~ ze+>PlvRT+(SJ%F1$JNMM8W&$_MU2M1lMWMln5X@HlGB=_qKZz^B767bPfrecJY$k@ z_IP#As+?@*D5xqYf_|vU96m5@mPP_sju=R{!qY-LpQ>yZ_qsS4`~Q-E9{2HAPW7>} z&)ubMReSmm`&g!pUTWJu7tkuA%HQa+$*mlE?9!N+MpAgI9G^yoPGM_*gKjuf<)}MZ zXY9!bSNyACCT$SX2d#@UjcXc751FjJYE{V2gGP<@1Zd>i6gZl3sx{jcUjZh5;p{frDWpP;R)JHc-N4K*I9=>0Keq0&d40rAMD+Uw!QMXN=|V`V)Slf3o;>0rr;D)xAmE4HH7hQZ6ZaLpfefHc)6@ zP6Wua={_t(au-mpOKh;bmH9vAdXmj$w{XGbuvyyiXw3}m&X4Tac3QyGEnQWn8HLF& zlJ%3~$$6@)oICOW+7RG_2s8xv;Mr6Ui(~r(}RKNDX(>CLxO?i z(Z)mBF|SdB@Mo97cVNcZ_tIMTEvu?bO0H2jc2Rd;9TIgP#G^(o_FkIydw9YAU17kx z@}7pxjv*MbTYfg3!YwFl%Bw64r&DH@(n;XbVZYM2&ZSHfq;2e}pdY%&EkiQcq$V-F z^0m!!j?OCfUMaWnJUXfZ$GWp_b?;&&?G&0URn_C6G+DJ;-$@Du%6y=|%y7?dT`V%t zU*n$si~r!HCMk}5y`yI{KXc5nHTw`*J^|yu2S#Uqrr&(F?^8)p;KASeFyk0xXb+iZ z65}{CZzQ=U-W0|0nhRl%>v*I&;<1rM+M6)f!IOa7*DGCns=e3CH0WL>wAIt!MQ1HL z*F(={M(S)78pCKB;Say$$$yK25u=fzaK?SbatU!~Rkux{wr3tfkE5;0y0>x`n~jDV z+T$VUCW~Ycv1G{A7{qY6-JLZIq_AmA3ieT&Og@pMFeqEoJvu8*8C-Agca25{!a2T0 z-Cw#4bNDBBPCtyC0lD?yzp8z#qD%q)) zVXTujQ=8;h9Gfj(+LciqBnc#wb4rgWbt4t+T4Oxw%t8u2!K+Z3LM7D;-{@kx2!i*a zNCC^F1wvgTY6pP%(yMlwd$q?|1rG!e)#Dau9-QC|PmS%<@wU$LzKXo(JJlE#q#c16 z`)tH>ooXba4^XB;%l=ePAt|UT!&9hGc(`bqEsJu;Z(9W9Yyckv`j#qqc9Kf2@dUw7BjBL;N_io)rMyA{*@w z5#Ql-)&Gb62DM|WZ5!xd2^9Fa-g1ETWXa2J+?33ffefjkt+w0F#5H>VDAT}l{#N|C zwF9+r%@K?kDhV63*%9c2>M;+FV+4twA4=oSL?0G0vcA-%=d%0I#)#I{*>eF8J70YN z7Dii|5QhKMxT}c4tIEE%o6Tn15KuHx^kY-F+xMS}6%R}tNr&_1oDMfdm+mj(&Ssk@ z5P9;ffNt86iG4R@J8;K!U)x50RmP3F4?-V?;QgA2Wc3OV);CX7O++p0rxP=ihy*ci zliqY^c~X+!{$ihqsXoKaPZRi=^5pA2LFMO8+jNhMAqru-^UUB^#=49pKiW`;*B7fO zP)$O5zMShe{s*l3c~9kLs_Mw`uYs z6OvITC6|N7+=46vGJeU@P8@mDeZX~Rj_&~{-_GOvHp6T7`cBPUIuhtv_i{~RemB2n zaz-hx#<|)iBooH;PU!EATv^wl#O>!imEiK&rk>OPV7?m!&0XMHEpI8cwpg|LrhMhp zn?LTRn(cu6|KF8;)6x(8h`nNI2li*1SviBn>?A82Fpe+KscchgU8uRm#+&~Fc7EFy diff --git a/summary_results.json b/summary_results.json index fb6218a6c..5946f1eb2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432023, + "Performance_Result": 0.432666, "Performance_Units": "Samples/s" }, { @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.82, + "Performance_Result": 22.9348, "Performance_Units": "Samples/s" }, { From 1dcad10d9ca9a9ef67fa2af68810176e48780c39 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 31 Jan 2025 09:17:10 +0000 Subject: [PATCH 11/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14425 -> 14423 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 9e699c451..90ee83e01 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-31 09:15:13,812 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3270 INFO] --- -[2025-01-31 09:15:13,812 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3270 INFO] --- +[2025-01-31 09:17:05,313 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3286 INFO] --- -[2025-01-31 09:15:13,812 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-31 09:15:13,812 submission_checker1.py:3307 INFO] --- -[2025-01-31 09:15:13,812 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-31 09:15:14,930 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-31 09:17:05,313 submission_checker1.py:3286 INFO] --- +[2025-01-31 09:17:05,313 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-31 09:17:05,313 submission_checker1.py:3307 INFO] --- +[2025-01-31 09:17:05,314 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-31 09:17:06,385 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -43,7 +43,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412747 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1187 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.73123 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f5e47edc5..68d5f38d7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -552 +553 diff --git a/summary.csv b/summary.csv index 780cfed20..8a9c7ca34 100644 --- a/summary.csv +++ b/summary.csv @@ -21,7 +21,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412747,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1187,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0219,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.73123,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d5644fc05..1731b6c59 100644 --- a/summary.json +++ b/summary.json @@ -669,7 +669,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1187, + "Result":21.0219, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e9a19490aa600bc25f7f2637e6284f3018ae1250..a02a2a0f665c5597f453ea4eee9c709163b68217 100644 GIT binary patch delta 5325 zcmYLN2{e>#*q$-=Ci^y(u{0yHZ`m@&5;K%On+d7tyn{Qco?ia!rq^??>ilXQdU z0``mPnbfTVXB!74;tR1U4k=YXJxzaq;wv@dmE2i*QBb{AKMoYl^E;6Zk-nACF~5J# zBucp+dbnvle|Mm|*SO(mZgnVT(+A<1d4000J;;RaiVz<)U82!EPUh!C^#Z$^F+3He zuMUGBZB4&i&R~Qc{hm3|DE!@}J-q8zjg?jTYh#+4Kb&!4WW_r=-Wjt!b=* z$Nl~F@r$#`8yS&WjNMsT1Bm+*Wlt*NBq!UNn&0o0;cS)Ku-ngg*)-Iuo!{QoL15IJev%w2grF@_Il;H_2lx}vO_(^r_&0-Ko4 z>^z_2tG)w0zfz-owp25|k)2ys)zvaBvAa#{$NoZJN1WhRLV8w?E5h#7W|Np4R>n?s zB2F-D%c`?CIj8nT?cf_5AKfXkYGYW~_kRI`=AFZOFzJOeRFuiN$yJy%A<G;?iOUVuM=MiNi|ux2fwdVKQ! z%T|lXuYUye^?TALzBqm-SW4f=&Z+0~b9~qFZ(JKa8Z2 z+-^sq&bm$LAcmvgA`W(zj;dYOOPvbISfFpkCf5HRz#dUOuPevl4Qj2^MEW7)wCI?q z!+%tZtR9hezq}-U2`2xfNf#HN%O+nq?9{$KoLz9}9n;x*tO8MXA{%62m$Jjp2Pv(_ zbR))tYDUSA9+%L+MsyzA6=^Bnn|yF(ml5wz)^=;Z*r?vw`pG4+XyaYQBy~W-(YWbs z?5b|mA(y@$L7my>E2Kms8X7<0wl3 zot3TR(gMlYoiKgVToJ6k9`?cbm)ccz0e-B0F)58TJA%E(;_{+b9Dq96S%}(RPR)!^ z$-{Q~l|o~L`}h;#b9*F-{ri{S>VDXS`?cg{<3&YpOgQC^D3gc3)PMb#;HX*#7+iqC z2TM{l+ox|9=~Wcf3t12LA$U|uU`FEa0DdrOx%2#Jh^e>j*Mlh#7IH~TK+SqrDNsoo z|8<4scjmbvheiW$i0W&o|MJ&KdAkb5_DO5`<-NU)XchPQtFj|_GF!Q*XMV%scJOLx zs8D27YC->y1EAOu498J#2c}yWYLt{kn77=yUbr-4db$7X3gEo?-RW20MYOblwLng^ zH|utA+;-mzj!cfh7HAYiH|s}x@8+kSn>PtvH46sG5=EaW)(9PKhM3v1m16pBjBJ7n zY^P2Ox^UdP_(h?n<&;?e(Gt!#H@VE%2zP!M1$8lHL8~E`LsF&C+yGKH%U@1NyEa>- zH}k2W<)NF(OlZH~p?tu!a^$inuQ%L>OY+A_o7Sg6 z%IAnzz)~47kOC1*NrU*>9%5kD{J1ODca-D-3Ku0ZFi+vsYc8hl>ia166{~&5E1lry zEp8ke;lQc{PKskD+;_e6`$zhxnIMxu+WwAA?eX}eeM9Y$c?OpgUTi<~J!9c9BA;d9Rdfx_93%UJM~ic9dEkrKfC^1`#FB8w`)~v2aLD3w>FkegME7^HrSLWr zdB9CH`yd=5DXNaDq32k90!e%(!X#2{dGD-B|*18>tT9}7RnG^^aMQlcu9udxV z=4HauFdZ?TLl>89z`4DFwgGa32YtYy+=E+I3ghB93=!y!W>4z}3!FVlgJaodi5RAI zIb2^Xw{IC>!MT0t)4cO*C)5S?z-7hHcAuMbM>&ePomKNCb(hp02lfVh)fJ2qLvnWs z#MDP3)yzGhktj|Y4DwVR$s!FYo4gLJV?mOsAZa$jRqH#MLP*_Lg5Dh|-yCLM)Fu`g z@1EFJc`Fz?m-|TtBz$PWPDjntBV`S_LvkH-4Pm93xpz^VF5t1d7p1)yM*0ISV?R%*m6Hh7eLk=#&ain?E(kuBg!HNI#d^f`L(YM!HXp+=@0Evhg= z*FidE6z$e-@&`1XNfA7jj$vN|^WDp(%Kjl&Y~RNXz&=J)rZG9%KP7F5_?t61Hh+Y~ zs@!wCc)H8k??!;8{R^k%_KJvTWA1W6Ib#VzCQ;(!8iB7~Y>}r9OhM`9igA@vB}#G~kU=^Hg~sV?G`Z%&a(`Zu`l76Rk^v#7bdDcEi3r%`T{ zV^kWJXzHE&Z3#B}FJEAz-Ev3=h-kaBt12$QqU2(Fb4ZP7@m^}t?g%FBYIu`YIW}!K z{oeu^8jbg#x@2pRBrx&)<}Lk>S3I2+c!33l5}J^&hgW_)DN^2<##>EzSH!D4%*#~4 zy=!T&5)wAHnAA=W{k6J%+}Nu)vG#b-0!#d}ASg&&z{nKzNje{BipCRci`|*tD@Amm zhIwTrUgu|SUU}qZrU8>65%?_0Rm{=uow5cEwy>uy8Ni%|#EdH({&LEQmz10A7;4bZ?;!3*|D*S=I?PREKB^)K&m+CL+zvRf7T*+2W)J9%~P z{)NI>FYdqy)a(RSNE+f-E@xsi=#P`@Q|pnT@zvApQv%qoO|Ej!|Ju24)du&8GrZ*Y z2ezRsc>05I2Eu@~Pf-}RuvQOnct=F*nV+L-HuVA~d|l240&+6zCjin?Fx4J_H$*Z= zQ5OVb2uZpEQF}T_*L58{1%5sbSe{QeLAZR+;j72W-TwSR_y*#Da@Zsj0`EQbT5X|E z?fz`tx3?4j{0hiL&)S_xF&Q$9jne#Zj9xVy+|84Nve#zN$a0NF!ZvCI^LSaj*=Oli zIQ=RQ?o!6p%DQUiym6d-3u3T-bu&K*8Mar!UPgjx@{A+(G$gYoP;Nm;SZUQ14#n99 z#*X1I?5)1vK)?b7FSDFV3DSi8`=e69LxUbY9Z6356WoyGl)DlA-c)0;_;;UL%WJ4- z74uKm+xm)sQwhRvl%@{~qUZ1_RgDQKPAM=Y0mCi{6E|NbX3mn22;3CQf?A}r1$l+9 z*|)LqX4d6^_dW?pXZ3=;B-<>X)O>{Qv>n^HD|&b&9gc{;tMrX9Y>9ZgkR1fOB&QhA zh9(t4RI$)+XOR%^leK%*#iwOo;`i-`07q#*>*TTBjni73ui zU&s)A7^Q?}Z`D9-;)R%!*xK2vBt8=BdfDBLn0;iwUN!b2?4nbAw_b7{SY^3rI#}t9 zZ{!BUw=q0gKXp(tM<@<6XkLd;=6emO!(;sbM)d)F$hF;A6fk*=OcxuAy=L)-=j^lF z|Ajnuz&t7T`o*Yd!f@F41xK|$zJt+R;U`YUqRO2&Vk2ZPr*gt_BBC*}`8>Q_QV)SV zyu*Np4-C~3xOwGpz6|h_InRY0E_PnM;A`7sgih+)c6q9=iqG|YWDQRO z>lAAGRhkv1MseM)*w}lvFJ9euI=yVF*QhQx7R4EOnh=X&4}fQYfYk^xUI^)$V4e*u z=VY$Ns@OrRP@^yfX*Kd~Y8Cs`X_%w$y! zF_YB?wF98{V)WTNq0Ih1Mpfg*UB@<2oHMM|0}fU6PXj9-7*Tnd%ACViwF9P`4sD?z zIFCa5q5!%MpW-dIPvB#Qbe{sghJJqlR7mh(b_By{go0dogNHtPf#&b0F22U9A?bHc z4y^iLP(iKnxna&jKkv2*)nrdx{XAO6qm-^L{nCMsOYx>L2z)moiwp|*Z?w}sP$}UB z6V09BAOU*u2kq)&8`lFwnXnpX#=(P-iP-L>|B%Ei;YsO{^ARC>Vz7H@$M3&@M2M9x z%ugx=tP+P|7Xzzc<1(omQfvgn;u)WwjXR1Ez2KWoN9WJnZBs?=d!=kUQQr?CUcc6J zv3)@_v&x%WKYVbSYNlDTQ_m>$?1P4zD~8)Hi{d3Hgt8K2Bc$9g%G|}&&Rh-ooF2n$ zgw4AASkvNRPkyHOrMCJLevNN+-DsFgAaWw~F~c|~J{Rj|GxX7j(3pd?lOGtWbmW1h zBTCBXy+${?mY7E=BJe%a(8OTXCm1!_WMTl@VIaR}N)6i<(|p0o+nvPJ!6Inu*+DD_1C)S{```uzq*1+iJ~F(I`6xp=UEo5`OrFKFu^TZAJI|jthUYT1+g7;NyPU%DEN6Wi*4B5 z0M{G~S{+aMfIz7Ux}1qteZu6L1~g}KO$*Kw2aK=ojd24?tk>%#=}he#Y|wPdlb-YZ z0!A9IXm!GR`W_lD_hV4-q*K_Eo0DHvbyeR=Rk`7ZebR+W%7%N5c6hw;$%MCn0=ys2 zyt$jlhDM4BV0$`oeHTNCVQvrmEY>CgoLBz>iEJ{>6ViXlS~fpB_MF|&;$4bRC4DVw zI8y$-?q#TjyC7U2%Ix{hZ4_r6csm-yPVvpy1mq)f@J;}Q1et6s8;My;U*&Bit1BHP zZ^CB+b*mc`yuaw<|0YB@x(f5q>>(^Neso5c6>7mM0B3`W1flORQ~U~WhnZr-X3A9y z={5D}05;?pc^hLB{fFIMt;n?1NuuESp`I@1diBl&*ShHjuJwbR33-U$*k2#(=SqpD zpQ0WQ>Au54E&2rDBTSB-CJRzh%bI`kx0t8c3hHD`L0y5-^frBwPk-{)wU`CPG~hLi zb!QH40Urx@=<}BE<{V&~r?z84NLIQT7FHkB1@MbL=^drzi_97-Co*YL0k4=et((nx z7HR770c<^!KpCt{-@LseH*V9YaR>7ZZY3#TWC^mA%m-RN8~{CFSTautD)fg))d&A^ywl(= z!kq8^m?6SqiET0AVMLLe^D<@AMzd;v*kv^yKdh41EzD@R?1?z5z;V<;icfrWduMm| zN{F`nFWV;Zm>hA2$@Re?7s)46a?Qjrjeh==yo;DOsftt4os&g~bI88FcILu6?g&_fZ+G-@3ybrSaX2g6AKu`Zs(X zF>t~!PlAJg9WZd;fAiW{fLKZ6m(DaSN!jzU(vDTsNV;kg*x{gF6}~G6C)}l7igHp3 zoO5@YeW#KC$SKZq1rpO-d0|cWe|OiDw7GnzzpgFl+`4X~^O+`d#3qreDO|#42odut zS1aW1suelZe`~aCy7T7DGaC-z$hSf}ZJ8>(bC~$gXOg~LzAS<@Mss4~z1nGt0nmHP zDrk4N+8E37&)91N{aT*^Pxx+TXqqU+EVpv3gu+TdcK7&ciSg=yvYs&o&xKUSTYe0CE@Y$}n;-h_Tiv#xS{DjEfcS0eu%rHW-4U&B?3E3w} z_AJ?V3XLVQyi@Oc&U^p3|J-vw_w#(7&+|Oz+!3!)uW}bQh8?x4r^>{8Y+}HeF5lCQ zJqwdTo6k2(u7yH5CC07gQa{soM^N0{%WUYol@u8p@DG0PNp9*LOxBJ6XUtjg{%lp; z;ckKF$GPE|?9rof9U z21tJy>+aS@sg5e7*;V^-`;86sISn_IZhCN--gy+H3R{v{oGWreS=*kqnm?O+tM6!G zf8J+wVoE)gyU9AfprXroFRgc|YBICJy>`!E`OmQ|MMDOwp**a$uabEJ)Esvf5kI(8 z$JHYK_zXOs$?+y+#+6E&-(SoGdd2Q870s5nuZhm>xBF11I($tpV8449T({p%%OCx% zcEu{Aj79m2n3&l90b2R|T+xC%>WF$^v6|oPXEyZCtw_jkq<-8~X>~c^C@ADZ*Zr~T zvc3A1nV3HJ>ZbQE08c3;AlT9_Zn!KqDBsW9vSY0TC{`}}Gu$}3x%f~k_gv?}a_E@y z`ya+@Ey~e#m6WURW;t1ZZDOZrTOb?!E@Z_3<5Cico>5aswfO`#s9Nq?i2*=}XEGLKBk&aMU!IU1R zA>1x8O66B5Cr8;7B04HJ-2oGm8;lW}U1pPY#Ms*rqn5i9)^!_nfP|=-jXE}<$#0WA zd3gn_lf+x*6Ys|QqSqWN)=(%^x`jNsM-#y zTX4{gLx0@MAKV(Yi!lRplywDr?j?`?t&8@haVZw?0^-8KHgGraP{LFD$B8!7S)YuhUWG$)O=@ezABHYm%Z!Ucd;W+9(JTyTrDyl$k_MTUHDCOFZ9H2Oti&y zQb$+ji}!mM^&0sMrAa;Z5626fz8vSq$?@DR4eYDwIk|GMvsiDxdXVfYA{u>x>fh3b zg{tq@2?*=QGy$*FKf7*?P#L15nqD;#qUhs&8Jn?c(+P4&+pXQYsXFozyY$MMEO9WV%iDJgTx8&gqk(n667i*zTk z7brKYH5_4bEnt`Vg@Jpf`I7aQ4erfvuO9{|*i9LU_e9=h$V;)eptk=GSt5G!_&w~f zXs6r+W%{v-;CuJnqK(t5bI88>79Xp+x7LMT&7OIH=Mee0>cF^(V_E3wWp= z;P{|<02pX6PQ$9b7f`DRypD`DKULF+D;0{GHu;t=ZhB~L_-&-QEX`ahnMO$0X7!oXuX*z%!f|#AipkMlN z!D;%pLv^FhqpR$#$88;}dm$6#hr`V8g$dt}5iK{KlKyQeSwL3kZa zU5nxH9YWt+Q87fj|-L)2QoHA*gdy4ra~ zVG7<(%aHFa9B-JCKM;D=cEr{`l9T*ZM=y^HBmYTPA}S)G9ds$m8L$0WFa{+T#oheC zI+L%B85FZX=jDl=Knh#Y`$bi^q!Jo%DrUnH+zivBfu94{NokmkTUH{KdfhJwo)&on z*_;x-M)mz7LNI5t0Y+2zF5ak39(@;YLJra4kih#Myc9AAK+D@!+GJ3FDqV(Covhg> zbFG^8FJE(SOqjYYhhMbTT1z$c6B<#>Ydvo&o}mjnz&j-!HtWy=ZU_1QrP!a{rh6{* z7+R8`&lk#^NO;31}r%bLQH>#roJHb#s%2?vL{)#1b;&R}!IK6nO%clxS1Q22L3hNeXkxK>qo{2ge7{HEjf%YKWWGxXY&o|9m!78h-8WDRiFN6c1 z+ZolyJwoq8Yn?n~`X^L#5c@A8B5tcXKe??sq$r7DzipkSpLvX^z;&FH&s(j;k#o(J zhuLa$uMP@5ABzr&X4loRD&$Rs&hll^fTwA|Zx1GDz+O8PZU3f*&Rd=o;F}%Pf|t%l zjy@9^e#DN2{62AhfK|__!v$c*k}_Q_W_$xMd^}u%W4>tlPZv+$#b0le2mReuBfJhU zG_dLIR16I)(#(@4JNZ`NUsYg6FAdIwMOV2fE_-|N61%w!vS}O@yh(=Hv+TyoKU80N zxJOEm`w%ODzFCx0QgS}@G$I|G5A6n}leN0?jd^l-@P?U^zI)xlN(-de9}J^Oe__86 zDu+x)qz(?D(7KzTprUR%6FIEtcX4mEkwG?yF46fvGMHvzeF)9MZ6FBEADG%@)`Vp< zyeorX-xA2JaOpw|DR!M92{+8O?-YpE1iJDkj!Z_1_tlygrxe}p!R(+UZ=Pe5_pxp8$|L5o30&aS1lg^1D)j-h$+m zC_2sg5^h^X<@(}$NWNNQ{RaGD=++rsF+KMSIgcN@qWSU?O9%0PGXqY9`T(#ja0ROu zW%v}Gbf^Hd1Y~o75xm5oq4kZB_yO;Ks)UiuBhSRbu#(pPs#ErN(mQXgZDt)#>bz5K zm+G*?AlvA5u0%0HEjx>q+n)re=DFK{%30C6s!3fUuo~k%1kdeMY5T{GB%>ahYh9HW z_yWIrG#@Z3Km68$_DNN@8s>PkoK}*_oHV8Nr_dRH&4yG1FR`|rOS|-3>yI1;9!=hB)ie*%K+@Y(#?BL* zkts-s4PWN27BeWNkuDT6%uXvy2Z(g`C|)7_vbn3Gf}$j9ioXTNc+TgG`EB^N= z*nb!7;t}$?$aVqVO4D@&B0?6u?f}5>2~vFY(1Qj6QxIa?O`Wk?;2WW$m2s6{;e3Ki z)?`J0)?{iLzrg&j^!NTXta5E1E;~Od80c?j*+>)MXO2hE=^^5CKg&X~SsQ#3GS)-9;@ z#l>)RwJ$%+*4_!$XaETLGP^BsP4?Ha``4v2Tk)*1A-*9xrN@mHk`reH69eyv)4J5&wou$2h zl@|8p=2Zt6o>+VW@j4&_w6JHkuc$M={NJ#R*cHx~H-2oO3{xKmR=M;nC3>NHsD;i6 z*QhR|REb5yA@7jg?lv}sQ42G@p2#PTh5HH699Y(VIct{Q@CsNVl*BH9cRfJDA%uTf z(Nb`bgZHzF>6ru>^yv4-&v;q43z+c)7wcsdT~Ch#!ZA6?nErK#Y|AutWCW7&g}M|3 zNl*;LV=}+dB#cItb>c@L*7CJng*)Y1EcN`fV$Rxmkyi66^4sD2M%znsEG?C*JKtWX zUmW=_tL?D}hmDZLE={?1nsQQ*Hkxu$lmiq#uXhc>0L5kw-$AeaQ!Z<~4{BoxFuq2W zpJW$W?F7=N82|d=4zq`*wCvm%TNzIs{3znBGoSKCfQDuZ=hT#j*uY6(Y04*5bDr@I zEf3ht)jQ}N8rt}2HD&NXWVI%v!Rg9PC1?cG?caZ>6e}LGsI&gIQmwmy6tqG%S4xV609ZTev&7c62ST;y;KAP z-jr6K485mnoz}}K$v@(=M)5m_LKJ-n;o;gK{MkHl0tmx(@P?6}ztMg3Yii;m;nQPe z-}R*GN0z#!{u9CojC}Kf0}M^L)WL-n4IN`I5!AReTlBSV`l$JD=X_<+pYT#ukIBf* z;Y7>2YL6Nye>D}b^SO1Du2xK&ObD@>+Tx?)rb6;PeKGParJ8U<#~AUIqw<88wn4}U z55@pMLki+SU_+#%wj#|zN}-Fr-(@yN>8X)4@Mv@u*zKv{C-YX*`*i|s1?td3=ZI^9 zs~4m4x!xrSteFH!$>}HE46@Xn9NS2QhMmjWE@#DenU8{@p+@W_Ai)E&0E;Qjgj ziA-y!)i@Cd4A)Bm>#Bh{ZR!S!z0lbUxOUUrRv?!v_d-j(-&-}3E<(RvO$xaoxU0)l z$m$hn$@to7Z4H!+N~O`e7^)P{kWN$6LuL~b@pO7n@phlUx0A2K9@GfOeR3zFhE zcIntLSwr}0P5Nu|6^b!xGfXIFmkIO7ozId3lk`kGmtK}=O;=z9ZY47>;n=4Y%iu_^meoQFR+p9Qcloax0c2AdS zixnDT$++oMG!J@(YNfGM3w;*P(Eg8Q*4eL6z{ZtPYO|FYQK6WQX6O9le;9^iJWKCx z1?hj2+*Vp9TEW92{^3;eTVWgyEj5_POB?2|!MrprU$0xg1KrGI#7Kf(qu9H`PSMkv z#90N_+fQiirjRx9kw;Izv3Rnv8Efs(K}E|!^sP>Zqc)BLdC`L?(l9#-N{faZMk`_O=`W>L|tONTfM8CWYGWTa7bx zx%+$pP%?bw|7z!Wu&@5x@9mx;?Mchf51;qa_EWlN{jAt)M^*NH4uDmsGONQY?`#Yf zV<@X@JCnvS?iOcrk5&%nmkbL^U5F<(3mPX1$A#4O0`H|1Kxtd%`3D`j?QvJR`X0Ms zjdIT)8+LJDe&zPfddOQVN&gyf%(_wZ*2do8%a{9rU5|{iMte_3=f1mQ=H&638a!h@ zW~a%Z%3C$F%=4rIg*=nF6w{=1%J-zn1Ph@j_A2pJ%^=L6^ex5{Mnn~PIM8F%DhrL+ zP}80do_(9gRE$gp?O;TZVdx2|;B;J@M&%i|DFJ1-0rOAMb3ram8Mf~%f8I<#45Rih zb0$h%Kke!nF=!Y)xppfko)i9pw^)%+zxq@>X~^*7rG<`AP`g@ci1-soEh zl?66S+0u)f&K&F`^6=rUw)3ZFz2Q62b#p>m@F*l>mevIEmmCmODq`_EZJ|m#jp%E% zdia>YAW~-^nP_ozH%pamUwMoEoR2*aw#r@=bQuQuzmLZkO4i_Z;DMQcym0AQ00-4Z(Kpdd|U--x3#0_N> QID{yzBFw0)`1j<00E+MwBLDyZ diff --git a/summary_results.json b/summary_results.json index 5946f1eb2..444b97778 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1652,7 +1652,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.1187, + "Performance_Result": 21.0219, "Performance_Units": "Samples/s" }, { From a072cb6bbe913b48002b85ad07891fd190529daa Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 31 Jan 2025 10:05:27 +0000 Subject: [PATCH 12/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14423 -> 14423 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 90ee83e01..4fa13e52f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-31 09:17:05,313 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3270 INFO] --- -[2025-01-31 09:17:05,313 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-31 10:05:22,074 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-31 10:05:22,074 submission_checker1.py:3270 INFO] --- +[2025-01-31 10:05:22,074 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-31 10:05:22,074 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-31 10:05:22,074 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3286 INFO] --- -[2025-01-31 09:17:05,313 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-31 09:17:05,313 submission_checker1.py:3307 INFO] --- -[2025-01-31 09:17:05,314 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-31 09:17:06,385 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-31 10:05:22,074 submission_checker1.py:3286 INFO] --- +[2025-01-31 10:05:22,074 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-31 10:05:22,075 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-31 10:05:22,075 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-31 10:05:22,075 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-31 10:05:22,075 submission_checker1.py:3307 INFO] --- +[2025-01-31 10:05:22,075 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-31 10:05:23,134 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,14 +60,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1992 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99473 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1692 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42546 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425185 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 68d5f38d7..b1f994ac9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -553 +554 diff --git a/summary.csv b/summary.csv index 8a9c7ca34..1b5da515c 100644 --- a/summary.csv +++ b/summary.csv @@ -38,14 +38,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1992,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99473,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1692,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42546,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425185,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1731b6c59..a972d84fa 100644 --- a/summary.json +++ b/summary.json @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1992, - "Accuracy":"acc: 75.200", + "Result":9.99473, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1269,7 +1269,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.163, + "Result":23.1692, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1389,7 +1389,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42546, + "Result":0.425185, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a02a2a0f665c5597f453ea4eee9c709163b68217..ca3f56c4b2ca316603d09d40b5e11f40f26b5b12 100644 GIT binary patch delta 6905 zcmYj$cQhRT_x`R$LWtE%qOH{zA$nO|bVBqND|!h+Sga+HC{cG=y|3PTO?U@EM0A2g zbkRkZ5Wh`+e|)}k&dixJXXcrE?{lBmbI;6-_l$R~(`~Y`ZrKBAS^&Tn`}8&!dQOw& z_5PWe;iUDaM^(xwFvh641tFGBh9kpHOxm{H#Cf_=1Ohh>_RSWt=L;?Ko(42^9gGd} zm!~CO|E)|f<nA)I#0YUc2ghL5M>?K|1i<;cKvsNNWi$0L)~3{gV&4= zu5~k)mavTLx$FMDjoHhZKIB6f`i^7xCZY!U&e|64-ojXTgb=fCbV}`Oyd2*2fHMmZ zp+Aqzy%|D^%3DZfSU399`OjgO0_Qrb4?Nh69G(ZsveAn;6;^18+SoFtGcxu#&R+Fh zocoS_{jR{A+GQ12A}XSPmN7Ebu$Nos)_fK)^ZU;|VYrBZqLiuXg>>c>y6IPc1@^5| zV@xyWZ{Kfm|8@EpW!lJ-WWQp?W^D^ob5wgegE*pnoZ}Mhwf{;(~RQT zb9uN$P7S%#=YC$^zAYuGJB1Z}zLFP5zs$CZyZw!)+FUCj{xh=f{CE8QI)fD6+0sdQ z&dmhdoE`Le%Y6NwJpz|n-$pCIgd;1=G!!P<=5<=THX}vcMvusdjh5PD_pB3Zq7B(3KRmxEVl0 zy`N_Wg~W#>sP%3KXhN2MtTT-*|6ogsl5R)EGL5CqKq0@xIOJ$*#@L~V^WRlH-=DHx zs&WSfg;5vvRea@6N2}d!VT5tr{#uhZRv;&0!R%um+A*A{7}MqOJO0O}puy3??!MPX zscIXiGb^3QZj#3&?_111laUnd+dE&+F8!|0&{Ux)Y}Xe1pH4WZWwNuH^UB)ARR6S# zq^p7O$7}tM4cUh!Lx|8I`J5bvqrN>a3?rHV!qH4QnQv_x}6wQ10 zE%6u@y)9bMc#htl?TckVE7&DQoo`M4|81D?++Aq22a=;JvX}u^h37sc&g7{y0ybZtk|e5SXIfxc07&3VQvX#b3h^GJL@X&vHg?kRjQ zGMJ>R{+kO&&9&p8VGDoD<0v+kt!&9v;=Iw{Gq&|y#Stzq$p7}TJ!|;EhFq>+MPpfkv1-s9DCB4F&Bb)& zLGOB43AzNCAf6iN#3o;de4rNZ#T9VG)eNK?yp34iG~1!X()F>cNsiD&C`fI=kqLtb+}ji!bs&MJ+#!MD@BnXQ zY8wcxXvsmUN(=*MHx z3$E7`>xK*h-lQorq$H>x(gDqn>3-A^br&4gMUluJQs5kWB^+{}JYING!=BY+8r0$6 z&6e6d*AKAjH&-eCoEiV^Q+Bp#vZ!;Qr+ygCsxvU?i>!XSCGRqEgem^aLhjnGsV5+&h5 zKpPaaiXa_qg8N1X16G5`kIjJ`EtUW$OJCikbz#cEjt*$V4~$Aw^*1j+_Gd3!aCOAg z?o#`L7LIpg!hNNM$c0EQa-=!Zw?e(5xZ6Qs9Wf=?AfPTy8R5W=uEQ(wL=#d^BGRlZ zQPL-0L>0|{xNOafqth%^;{-3^2N3)14;x* zSf3kEx`o>dlLUJqt7f^Rt7R*c#lKA#Lv*kszb^k=N3YXG+Rryo8WwJ@z8RJWo{K@)w3Y3zVcIe|moxdT<{S9%`5CM&aqoYhUx#Qrk zG_`s*z|E^3B9F&a8sM&^G;)I37BcM99=G&f7tSI!5Ute*+`A#}v(31lulRP$x6okj zs$RaCeyzqH&T*Hbf%)fuE4tx8(3LeoR~ScVYNcv-(W#bNlC*4pYJv2iI*lvFRv#JKdb2V0cBGFbdPO@)OOqnn`uEvyuOrFnCf;@&HB z(l1a}6-iV%xC-F$%R0ZqZlQs{g+RkVvBxy;us1HbinQHnY$GzPwTNic%7#AdqlMUD zK`;8JwXqxphaRpei4;7@CrHUbqe26s;J^WCW()2s6&L^u=)=@qmf0^bEerF;Ln3L#FwMhYTInFxIUEjdUn>i;93a#ZIFF15L8y zfUwzLvSCFdSNZe+o!ra9B6|WNnn2`%qwCFT*dRo&n_F73-D!h2Yv^1nz^&yGD5ye8 zf+HJR<`YFamk)GJ)(|TLCdJkpfgqf#VoD$c^CG5-irAW1O8_#xOVu{CUeOFjU@ldo>JEO1RjV16z$2*P(Oak$r*94 z`_!H}EYXiINk_~rcTZKITg>Jik%CZmhUyp0gx5!j!#s=J6H2e0+E?MLV#s!P|K83 zn>!i48%d|chnh|xMQ`6|Ufrn;KLHK@s(cD?j5;ZFZw{5ryJVX)5CTyw*Oygj16$@r@i*8$aGo+DUaF1npzS#YhuLUU$U&dja6t0Ga%H7_gB?zi zM3vkbhwklgAsIQ3#f86(NxH`>NV%6?X0fb%ud6M@YzVBnDrf8;C@W<1?x6Z&Z0Udr zk*H?0Y@ctgEf0-&LaHeb13e+E+Q%~TcmQV8IB+i%X~4rLPssMYs_LPJ;C8iq3fl42 z;-hA<;;M*~Oc#YF3b9qc=|@d$o|bctU(?Gc3+sa|qwmsaRYbl*xSk|}$5|eq)KZLU zo8BO@L>W=>hc*7nVgu<@c?{0lt6Eyzbm0~R{6|*(^}2g<=lGIIyyejr(!Jbh#7!Zf z&_*zTc{>an#hWx4WX^}Pq$168C)*J-TIl(P}gJPC>x z26+$^Ej%~}?=CVpCG?9-k3pf3_Yjz4OvDo&Wahk7JfE?n7lVX{MAGD&ksJ=FgkcuPxwU>?c! zvG`<1Xb}{YY?-^MN)9L@sQFg$zgEGz&$APHdOK4MjY%Pe?oJOX(hc$sk~GB%5o_Hc zgEskQby}_|Va~a<-4tO^GkL&~`ED<~Er;4JW4>EuPH^-7vYLezXs#Ds%K!e&qF0UV~Rd=G~_u*(Yo|m0)g1 zHU>-pm@r*EA{`*e`B6Rn^Dyn)VM2T<@a+B)5y5AZnK>-YT#%80j$#F_3;?kj~sEt5=-6(N|E1dYp!oLB5LsUYl)3dx`}ylLTleN5N#!eB zM37J+Mf)wZm4EP~@tFHF>6JY@b!xDO<2&}87bdHJXW4&CqU|h3uDiluH=jPtEO*}F zzumtrerw@JxJj_)eLNPW$~bD#Fn?q&n;UrTtV5YAB{_?4=$!OZ;PSZ%$F<5?lEY!p z>*WBgn?EVQA?Bn9PqtS9Iq;rHQ-C}wCIL;-M_3v>MDiF%!PYLMu%A8ToOKeJN>ntz z11?_>YDny60vaz*>8%#wt-Hfu7(^xD^PsUY%)FFKh+x%*AQ-`_4QJ1Mfu(Rt!o*^T zVoM~Uy7?EV&o;zUNmom%gL!F`8Paik$)T z6e~;yOo4HgJz>`3-cG!HJ^dY#d#Wm^DS)C65)J(8D#76u4;e6aigUc2fD{Hy-Uo>a zrIx(VNcU08*9zU7Z6Q&Wj@4I}Fn3*$cu1Q_R% zsR;s#;k10DwEtrCM&pJEH*}i?kEw)h;s%tz?@c!2s7c1!eyZD+rxKeQfE5 z7<<;d(E&!EMq<2teCk|coz`+Iw{Zbhch;z_P)q9#RBk7v?%(@|u7-U0=LthjeF4uL1lS;rqMh z2oY%X$?Te0Z1}^bM|0;@|5jp1&hWJ*`{>m*k)aNV7&{`#(y+uC2_$Ltu4PxpdUA=P zUlSFVg8AH9CBubI`PaT^AH*kAa!1y7g*qS8VXRj#_beW;e zo{#elhmM<(!Pq}=Zq!dX8;itt+B3bf4*ikh!(vv{@_q(0d}gs_{K3fedT^hnV1FXY zi7k~ecziijc3H%h2YDj7R!_Z=d`k?|%0*HA5oTiMXVY(jT=X%Bi1SWff52ZD{`}cD ztrcm#G|5o5RaNSsi^uD8+G3P1M~y$AvDV1E=*J7ehD5T5YIKyB;>@ocqQ8BeqPFp} zumK9-5=`m$JupOC2T|`%-WBp@@brCWe8A~6AB5?=*qFMQVs6JXx~>~f1_!DXFHBbq zF~|!ZLIxAQsGaZPx$D#DXUAj}hL8gFnWQe~F}PmC4N^VIlu}$w`TQFe2$KL>S(;@{ zz3QQpk@kZm#$Yv{A#U))4KYgneVz{b>*dW?RpN@*M_(1)D zRUVEOK%1Xa(O2NAFh^>ZXsW{|#SBqr>3jDjPYVD3F|B)zJ4%B;qMQmoPP71WJ7b=l zh!OH7>n3ePC9%Rypg2NcrL!;8?+54M7mRJlc@OYMx_)M%(^HS-Wf=P0|m|04EJ( zTA)PSRiaa8@!fdd%Y7Cf)cM+6cgA+;o@XryCu;lDR(^f*i;N#5oo z@E!4@4&Z+W^8bHI+C2lnR`W8TGXQq9i|#}rdSNwW2NJ+{Jl!+I=l?DWrXCyJK4=2G zW9XhCI=7^;V~O4D{+*8V8LnqX0Y_m5pf6@v+|R^zR!H6_v=F-g_$oKX_-=i3aS3*9A_v&7Ge8 zW}80M9uU`A1_SR)8+KE?<9oaVx_HvG|EIY>C%69Otd(_W`1GXm_73WVf-gI&B_ejL zAhi83m}``DFS1#qBHH&Gc+qPuSW2y6Xclc4a}pM3^woO)_6?`0kSS@7FD2=3f4}zW zt}Z&AALVraSTY7*>PPp?)Fv!uh#=xPB)2Elhs?R6hA|$^iMwIbO5F2gyNh4XO9Aio zS!*;3@x>yNuHgPiN~pdIVf`&xU%uvn_AH28;FV1#{iU?6ySYiA_kGG;Je z{&0SXrI5^B*Ch2u;?o8}Gc$i=Wm8%Qvl=(9Lo%SjWWG6j>PuWyn799({Gq+w#r9x9 zX!9|D5Y|gZi5eT%+WqXv)HhBv^cVX-YtWZsY52*ndXr>qJ2?c3l@Vh@54-i;14rzm z$O}7!26>8Z&WAUzPAcW|3%wsqr#^+sb(H4p`=wsJp!%3W zI`$%uBDU5jUGL?B*_%tCBd0GH(wS1+g$g)4-(e}uQyFRYl=Zxn;|bXlBHC)@c_!8# z=_nx=wwTp>TGm!INZq3E=*RtiQr`i>WNrN{kW_i!5Wu<@q=3sPhHNu5L<)fK9t|g@ z1r?L@IX@+?=^1{l+~(~@e|Fo;(qMUYfkcA&_dAhaZnedibZB9_h1Vv}9B-GpvAixIz z&;kCpg@xN-MWvJ}C;0(@Ko5~;en=0;XV`8jIxJR-g-8R2wTIDTd!@j4{_y|=01Sx$ R0P6oe4+snA#qLYF{6BqaCnk5{wc?%OJ`qA5nu0LJ)mMlpzRaln9A@di36j1VKb6 zT8J-17lI*Lawoa>|JQxjI&VE|pS_>w+2@?K-q{DU4{ALaXsS+?35u)$fCIsgK@js* zpF4j4)W*zG3SZ7OO~o(kuCD0efqyHfyPmNi&IzhF?Z<*b7=A}lXeRDQb}XIT&i?zw6@CEuIsYWLP7+G+BP>aT8XIbJO-a;kfEHKQ0x3g4cg zpYPASSxY+iyZAkSrI7o(OJ(@Ty$T~D^UvHV%iKF1r?oDty| z_a9?tUJw0c$EXT}3Wq}P7?@Y6f|E;$6|~58enk&q;Z^5fXr<1)wW)7b>7h}YD2^qk zycB5;;?>eb>(3?cn@yon^){IDmw~^YSar3%=~zC!@c-pye%{v1{Q)<>yFY``a(sHS zJt4LbyPFiGe15bbq6Kz%DeW|-d6k}u9Lwr=PO>qLZ#e2df73M7s*=_2m`ks@KV#Z; z;&s#RWfkr0==(zRZnbgZ!a+b?p6Rdl^UQXs2Q!vU)r;kbrDk8{j1)UH$1aG|=+TbS zNJUq5_)m20jotmxd->i+LwXKz zuYXxIGx-Yb9BhAd0HrI2?MNh^SdpkDvBQS|;v?Mb-Zvz+8+u-IET-Y1JSTIQqd8TOpyrXaCCP znLZ~ReSJgtiZ1=RC6J$gBehictW)LQaBB9MOK4~7r3_fwx>PF(vzqEJ=q%LJ#Tw;1TR+=IQ5B3|L8-MGko+4iGeYJoA=*#ZZ6+$&w!8LVkb_ zKj>S>mX?NMvK6vJnl(aPjrxovlTpCf*r_tx{;Iyn@oxty4~6PNnXLOirT8Sl5xK*L`H1$E--51xQA zYgQK_FvdEp&0qLVZ{8XdAQT2(nYmE;{@ljGV7Wm$=;TZ4eRfxx=T5}+OK*GsEsCYJ zOs$)dj7YZ6vUrVYGp^z?GRhe1a5dHUXcf zEXk7i^X?jw+QZQsabrtq$`%8$Gd%S$kRy=*p}z`fwYi?r*VnaQvO zR7a@Okl3mjeMYaBxu@vhX&-PXXb*`|Hy^ODUwYlN`8*P7%Pi!aR?qf~jcHnLfKYO8WDym~#qsj&ug!D+i#LhFOz zaz>7jAO!Um6#PmaP9^{@nz{$9BZHTYfdnYhZX=_U+2Cq#SzS8fb}Z-1YohXWkFLxs zT_m(^irx7=BWGJMGr>#5AQ5d^pA1ViZD?U~#$yDvE&cdo6eX&UzJ8N61_8X;LVT@V z38+O$0XCZZOy%&m%y09dp3TDqP-x?`y;3+nH7ZxeCdmTfw=3YL)_V zqlWhFdVfF@NfhbF6H%0#^h{4O#zg**%fHh{3&6b4tVkenyz`2%%i&=};@CU~4wHFe zFLtfV#{Ish;+=Bqwf2(05MA10R#9Evv}8Q*+-4e6J=Y3D9epiAErW}uaE!Nr-NLXC z$!{@vpc*R)UZi{0I9_S+Ji2exsHG~8QM@@$DCR$O-9t^3&Ck5%?k2ST&`+I=V5{H+ z3|`+QV`mk*@E?;`qs5v}2MBL|cqGfiLMCOaA21|`H~z%GVsQZ#u+t9Msu-WKnE5+E zl0xItSGK7N1YT4`zfnuS)h#C*2}WRcF0Uf^+u5xjFY}}iXK<#IE+r8%Ihn~a*pDsk zm2B*~##1VZzP~mOE*pC#CpTXV8YA(41_XKYu;}Q6J_}?4^$l@p=J^h!+AGxTKnydA z@V?7R-n;eO-arA$OGslfDXk>6c9*zK2-^70)~XiNdWcK6#PT2Oq`+y<0fITF2=pZT zQR|04Vfgw%2P+=GLU6179pwf1{RZv(+Td89E)|h`NO-|1g4vVaU&*kSYv%N_f@n8D zyy97f@5J2wT8X+A#+BE}T)P?Zv0UxeLy;UV(^(Nj3qnd$FjV z&Y%ZYv`?-_Xp5rR{}#oeh5+NEEIPfgokuHo8B?tf4liqJE&>`9thO~)YnPtDPd0KncB;*6@rF$p0+$Z;Rp>F5HGYl6juJ}m| z&vkVY+_C4qa#)zX6IJbTn^q&)-7Ny9Q31{1j4^(HLe?#D?QJH^wuq*ce8a$I_cHb& zSZn+CUY0l9@3@4rhyYb&n1HJ*@TE*5?7iX8!pdnZg1U`<9E+lCb=v^}vo#qLtN~Jy z0%=)KNBPl1gN{8NG1eNBwBXpd$ASGWW4c@sA3wJiSC3tn%(_-@?#8ntfprErB1FQ@x=|qLQ9%hQ+!U6MDRfMW?+w)?r%E zSWC(fEeVh%-YUj|rWkp_Pr+oAPb|c|J&%?ItE2B;!5bhu*WqB7tIa1>`PW3=;7;xg z0b*Ju?rYg8FT7W@=f<75CTOx4?vwZ)M9hU;aI{yI5qz$hl{o>*{5+OXgEWh=Qprqj_3 z_LtVWoYI~5!vaNa##2Mn14B?ESqzLc{5e1d#$iCtGi|v@T1HW<+d2KqMW^L-8cIfz z*n{!3&an^e;$vgH-V$pQ3 zPF+SAg4*j^S{RDb6P5%5R%w>v*x+`NMybGJs%E#{TK*~t@s*k%lnZgu#Ii3`+ zlPcWm3% z4?#UoUNvA@N&GypK6@UVnJi5`Y+5s*zh~Lz3xY97B(AU+*5Tq@L{HL~NQOSCrN3+V z=@cjt=}5{5>bwyGvcoWyG)0#AO1DNeo&F^ik)6K1sOz^cBQ`udofmAgICgGT|snF48%9Y?q%RO*?1i2r#$-+;Z;|{FHJ`HKUd#7$|UXC{~&0O3*dv2JQqiQ}zJD=N^{fr}0!jBNESjO5_2iJ_!Km~D%XH?5rYWF_+`wbfrSE9}&D8$vO* zUhqlZ7w5X^5gGPoLvuQ5jp=X;@qwWVD+XwyW*?fng`8duV=WA!NslZPV)1+$@g41gYOaosppRqNfTu4mORE66FMOD zOp63=LjIhHkS8~9-p&r|Ao6K`k4C@*>B-Ff9(&rQ8*kMy6c4Zz>KPW3z^blDToZss zB(4eQOdi0*#_>2Upa8jD$44Y--=%;g*1qf!WM7(1}SG$5p$A3j1**U=w@ zpeSqIlJ+owEX{PBH9#U#D3yXh3Zt)R-Og7;+oh|SiC}tp>!09<{02Di6*kyh+$<*KEjitXR7ok3S$=WH;7@tYj5K` zddRdYnON-WXac3{oK`t4MD>DVias4lFJL;;WA3}a zI(qDf?o2v3(F@*4hTT5GN`$~kvrV{mFTnaI|47;grmX>MiK|$PGnM|k_Eo&%mHDWq z8=sIj{KZ@gQZb%hSG8#oxq%4kGB>axX+aUQ#jTw-{DNFG^4e)zkTzT?Z!bt@T%>bI zsZYB+oPf{(%No);IhyDa*S@kIB<|Ubk8MmkeRgTF`NgNHfJV_B#fri z{IE!Ayv(T-SIbRmxap*MU4rVOgAft*{L$gj(Jdbp@n3HLG<7c7i9em&-cic$w5d== z9Z5Ov`YG|G*f^2hRKdvRA}NQzg@oDJUpDw2HWG`DvY%S_#|zg;j~y)>4&Umw8PjsY zu+tkA{XP9J_u#43Om=TVMT_UD1`#HX}7gbwQZl+z2X_DqY1$9pz)Ve@%xS zPSp~JYZR}HeQK=fuY~N9eY>(}7%S~BIyhsdEyvm2{Z4Y~Q* z4M{|#f7gj~v%{#6@^7_as=|ow^X%80v$s7OzKv*E z{05T!p%BwK!=0R?q4W~j-#PCgccy4eV?8oF>IVAX%%fZh$zd2^cuB@Ux_;o&Cl_fG zyFuhpYx9s4boWe#K{dNdGdIk6?OLzM+M#Yu@>Hu>7khUWx_itb zUoOk&0vVOf6q&EDaMap4N8mV}V#fTC8>(m#%}!A%bx2F^<8gjqen$>R(;cNyD z_BZu1)^j$DRm~}%0tZduzQBO3k49~v!9}b?zaV@gbJO;vV$Lm3` zLc!;WV#qIK5r<;<84ScE(YqYw+>JDG-;^!)@%NB%DWG;7+vvX|u7aT`HA#|rn^`%j z^d3HSH`m46ukx)8i+B&3nf|x%+_GcdtP+eRDE3E7J>9$Ka%iQKqK|QGB|ioU z(f&`YuS*=quBbQ;73(cb+f@y^W-(rm9oAY2Q9{y~Z6oDDN6vInC%qtl5Kr*N`F7O! z{%2yl0X1DsqvX_d-JXiiR4{XUtD2K%qVwx!6~(FXMUUk9Vj~M1K z)-*vkmGb-QzM`!hWHyTEUB2T7rw!qYyq_P1$NZWfxe96tvCCd^Y8aJCEEiPp@W05W z{%{tA5k_o=@jt~i3_Z78c;H-7STPsy39VRXgl_4opJB6Mt~0Ev(i2Ik@=5ceDVBL_ z(SM=SWu#XixRe9_=KOU=;Y9cOkLLj@FAv__@B21mjIFRSO0!_7qOzlKgP zM$LOB*3=cv_oA5$emTe;2osbu$oN?$4Y}@s0cEd(sDm_CTNgrip5(i_c*Cof!kr>k zB3nX$3n14YB{PkC`==U}dzoooS$+Iu7>@_9u;2*uo;j++A@46&KB*d7?mPSb>RGva zJGMM?7u#SnHD*5=aN(^h^zeQP{CX?z0(X_rhag>SN9O(#Q=-(p-`b*|KuSjb4}H{c zJhM8oTE*VY_c-}}{XMWFYr#(!S1FpWe-X>e=+vwE_@I^WkXU+gLEp(2ob#It!=6k+ylt-6+ zfs{L2I|VTA($2%JelgY8eO%^v-8TKJeD%DtH%rSC6}6406i+MAD+9l>QeR5-Kr5b~ z3ZMK(1}Jv0oLNh()i5bjV0Q?7k14xZR35c=LwR&<&Fgo)sos&2fA6S0woncTyD%ub ziGd3W?kiR_XlWga3*in2{iN?F3suTm{R4q_k5n(Tmk+yE^?x4W6RTUe|4`Y!$!F0v zWhsF=q1jh3aK4Mj;gMflUzX-Aze!0d&w6Ig5Z`5^YT5V5-=IHh0~-K9FjwYmI26^T zrr5fak%JM`j^3} zZZqFDmsKC$GgJMtC3Hc-o1w^FK%ouhaxRw3rR}QWIMeu`v!;Lep1aJ93OKTpYoQ`k ziL>$J`B}#2M&m}tTCKCl#nY>jAn6HtvL<8bU|$nzQv3yTcc5SS3*aSFK$4=KROnhO z)w(aV0A%rmc`GVH9#GUXj-_M;l~D8x`;pi5&cti0KJ;c+r?ROs@DJR7%$?%1$gvk# zTXM;Uk#bs&w|eM)euoAgBuc>w32E$>`VpGWFnhq1z14T0;;_#6JhP=&;6gLambT}_ zKlQ`F@m46b2~LbP@lgHuETei5RQM?QWA}MF=auj)P5LwCAJlUj-IwIVhEREK(Es=Q zDNo4;*h@H861e_%nWY2(#6$o9Qh@*Wz((5 Date: Fri, 31 Jan 2025 10:08:12 +0000 Subject: [PATCH 13/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14423 -> 14421 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4fa13e52f..05f58549d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-31 10:05:22,074 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-31 10:05:22,074 submission_checker1.py:3270 INFO] --- -[2025-01-31 10:05:22,074 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3270 INFO] --- +[2025-01-31 10:08:07,634 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-31 10:05:22,074 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-31 10:05:22,074 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-31 10:05:22,074 submission_checker1.py:3286 INFO] --- -[2025-01-31 10:05:22,074 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-31 10:05:22,075 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-31 10:05:22,075 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-31 10:05:22,075 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-31 10:05:22,075 submission_checker1.py:3307 INFO] --- -[2025-01-31 10:05:22,075 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-31 10:05:23,134 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-01-31 10:08:07,634 submission_checker1.py:3286 INFO] --- +[2025-01-31 10:08:07,634 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-01-31 10:08:07,634 submission_checker1.py:3307 INFO] --- +[2025-01-31 10:08:07,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-01-31 10:08:08,663 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.73123 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44853 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99473 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1811 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1692 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b1f994ac9..3749383de 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -554 +555 diff --git a/summary.csv b/summary.csv index 1b5da515c..e0d7bf243 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0219,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.73123,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.44853,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99473,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1811,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1692,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a972d84fa..035a071dd 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.73123, + "Result":9.44853, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1179,7 +1179,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.99473, + "Result":10.1811, "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ca3f56c4b2ca316603d09d40b5e11f40f26b5b12..6579da9f94cac34b8085bd9c9e5f891209f092cf 100644 GIT binary patch delta 5383 zcmYkAc{G&o`^RUDEygzC)97QLN%qJVW5yOUl(BEwvSsW5Pj)f< zye{8^%^#LVW-`R`bHhxL3a)C3Zf?YOdF`dN3CVvzFRynIKq2gZA{dbIo8fP!5B}A; zBl}X|a05FX6+8OHzwUT)nH;*|jC4r6HPqVdp|fv;z4_ZqspZ+i(4u)tXav%V zciQ3EBU#@4lj(uiKc_tq4WSV)E%D_!fxjP`w>G_dGxOuv|2NL~sHu^!HX_U}(@1=K#f{=4}oy;;U@#H^uuvK&3#Xn4L?SDOEYeUj|* zVNll6i2a!QJtZslgW%;lY{Ov~M*=>feKp+}TQ9>-@oU)c8(A`+eL8mDe%7m$=c^Te zYtpgXt3!YDpQ_usN}-a_5A71#%<(OpUHW0@3!y!Mr(ZVj|(%$kn^Gq_X>F@JmpOA*I-VC zv~M$6A+cvf9n2k49zFjMSc-La&-MBK1#nisg`x%%nc?e0>-;;k3=>I?&=iR(nZ@XA z?J>p21@{|3xb)r{h|KS+!MOYbFDJVD2PV;eofgnd{6>+4aD}g=^2d*T!$RbKos7n)WK~_1fz?kALA!4len0^UvaDd_L_A_x?p{5~m|i;br0!3p)3vTAp7w#>C-}z^t#4N|n#16H-b0W7 z-3`tv7}MQbxTUU}#+#eNkN@bty>ghS(^;~78QGKO6JsXKl><>yQQF1{1+ zp8N-j%_9NN)Gj=GNssO6M6zGXhiUPD1h~RP#6`I;B9Gmb!O4ik zE-umiN&J_rQ;%|j%DoqGTUZ>DG`YYty&}dF5Ega^ajx1&EN!V{C~7c4mO{N3(zt7^ z{7_Y`gEH?TLFkis0BKu(Q`lMJt(<>nCCvxam-h9ENX|UlZ2}j(ebte13&s^M7LP{A z(S;NXAZoA7(fq~3Kf!a#n6O?mkd%U~o{e8|-hit+bVc*}wYffO4<*iZ(XdfYHoC^j z*If87mdDk-o{@3n-b?}!jUhz#Gr-4EqM2${ zwli)%?JjS4G8SLtnZFxO!K=rmC&CGxKgRgqT=7(_#rudo_SEj$e1S3#L8;3TuYSCk zt);@I;SA60(jLKnkxY{rT3(qB1wq*u2BC^)B!<>uGdp7t32(7w0BR6MHMJQ405F_N`g;oVjA zPhhI$@2$PE_qT``sZco9YVl2-wpNcM8SnxIYH?x=RYcm?0(Mr#`&}6bme|NdZVh{~ zKNfOE>VaPrJjO3daT2?J-*?$UzBCX2WHhb+k>$Eg_la$nE1~2e!pRpCAXkt`T7Xmm zdf-r@3>Sj!N1>)sJsE!vf-N>uMiQ_R*;4;t9=A~3b-&5P)&WJ9;Xo*QqA-g=8?bExoCJ4tBG3>N6%I3X}ftn&`*SlLT+hZB|9 zrQ-stE~}_&eTr89QMIomK4^MB=%OZzcbbl>CafSaEeg$S4epP!VzlZ6zx;OgJ{pL4 zwO^)5v))Puu-w>rU7mQ=C>AaGOq^U1bJ;ar%NUST75l8)rwp@am~JjI>2uC>a6@S& zuvCX7JJqaN=M)-`W?ljJ$6GOe19Lu1>$@nzkZhQ1+Q|aQ#v?1@xHVMI=B2Lly6IEP zUo?(G!!A9%A;H#a>FVjOXj*Qu&|DlCqRmotR$N;!IgyC384wnBk!$03f;y%ImU_by zk@z?ZH0%*F294=h=4^EFl64%^suRF5LewO)ZLSDfc(Gn*?4UX=->a$Kx`GS$q&5<_ zV=iu#uX7gR**sIn+l%g~=hfaQ{Df;U;lU7$5Qld>RP%?at)2}QX!c`^Iv}gX! zqiprx_oT#)t9&+k{grx8$tbIPUHT>S<@qlpNwEaVCzK*|@$l+)&NJDa5y322*R?}O zc}eI^z!TpfYjKGOB$&Ti^I>ks?hyMcbV0C)EC$bx)+M8i){;5p`Q~_Q!I$(^qZ5x7 zL~MZP_NiBGen;*Idy&WO3!8#;w0GproM{j2WoZvAm$Zoc-rlH?j{Kd}DFJovNh_y# zq&()sRq}hJ+$p`v*Kqa_LKEwyiL{7tr=o_m6SOToklBB~>1sXQ+-hoq4?Z-uVv1PX ziWjPuqJQng{F?|qn3ZLl`;+lWdOvQ){RV9h6s{LcamJ{*Wt^@_TWihlB&ck$r7_Kc@OljD9}rmetD7IXOz zdGU2#EIfh;98opL%Oh62a6lMSB#zgS@6|Vri%$LoFpYC1Tptv687g5JU}-Dv<2mJK zopPhRdfn6q!M8B9{E*}gk6@0wApvL|a0QlT!Y6UVcMxO4>>zZit9K#r6bGW=IsqUW zPN(r3IQg3?!JR>9U>SfbTRwGfogvL;^z5Nalb%*jF@7rC!Lssjs$*&FX?;3amYEWA zmZGBXJeFRheKK${wb0~580uPdq^(869d*bP;5QtYYWA8?^>mIasrY&YFZP?X2lk)8;V*r+#q>q`0f=m7EW$-s_1|=`Uwj9qGiJbw61j-FoiP! znuiD0g8j&&Oms5g0-RL<+anzzY5r{OabijAXm|}EPFK7aIzGsc%s}vk3)^;$Cawx3T zz6GWUk0w=*A5jTAcimCx;7f6omj?wwqKehjL6v$z?f-{sgH!Nm8Ef4DjC@(MzW+a{{Q|e%5BV^P!f8R{LCwa))bTq6U=bqtz ztJfq+f72ZvfQD5G@IEXWp^4~87@>(MY0b~{wGw0~lA~j?cJsNFhdM0UkI`LS3gky* zAOH+=N3>alo*~P+3B#Zp@^{8HRwLciN0n*UbQJQEuf_Tyg7b5CJTe_D?>-}yfb43) zm)VqkJ*bT5_j)N?Ld@r2?K0V;wb#oUdpDtusg=VZuOkG_X6hae$*Azx<_BKy1CWK# z&qSg9%!fref}r`va&tO?2vdX2<8<{z?B1hPxN9j)ww{yxcXSbn~9|3 zA741pp;c=}v$z&)M6%Cnr6c3 z`eFGXnXV_VZ$obG;BOu`&75$twNIVPJFd|Rd2wqkQ1Hu_?8=Mg8+zB%C-ELfnQTLP ziupV5kFp%P1pM`-YmFByV+CuH3-h(LG{h-reQRA4eR++t{U^vmfYGXCe}m)Ef5D9> zT&ER>fm!1;27R)BX{X#vvo#`*YbDpGXqZTLlPk>GVezQ^!h;U2fUd9XXM~Bs2@VQ? z=yfH>N`7TG!kqzs_(S9^iyiCysU`Mvk#IJ0S!(oL8t=jEE1m>Y?uVg)^`R)r>g)o} z6)pN2r(5}hQ$nLvozQ}f z0~*qTjq}Rp2Mmbq_p<=<`&Mfm_i1NeXAp?5$!Qbif@mpduGaDxsM{;NI|xO?@3ZCy zm@dgRd#8P+v{Y&@)poKNn=IWF4rQwS+r2)XcCQ;ZnM(JlhXJ=_lqoCuhH*ub9D4H?tHB+&wF{>2Z`k^&(F zNI)rSfEu9)yYQR`V3138ZeNI$hEiD(oRJplgDhP1Dcu%QJJF&)GoSe1U=WZTT{14t zRxwju;|!7FLZ{)mL##^S3<3}>dJsJgMo%47m5$Hro1@V*bKg5_excDOS!afYbFc=6 zi7?O)d-CV48P6_@z6+(Z2G?G=q-h4 zP#5b;uxwml-&!@j>&`GYvV36Wch~XsR#gsj9Kj0=f>&H-`xyc))(;SApY>+`P)$0P z8@zo7t!+J;b5d031r4V7r5OZyiMc;%XGVbN1xY{zRrLy?q;HEBB{p~7S#$eul;0Q> z=>K^NH277ch4$*Y(BIk_@yNl5JQt-Q?Wtj-I8nPs!K^=ENSlh(pW>317UK25;929M zIfntPajpQfgH~HWITn=912D^Fa%l%eot~VJvE~B@?t$3@G^){QMJ%niTbyyytulgUC_!N4hK30lFBRblU*@BbJtIq7F1>N~{vfM@=j;T~x* z3u8$KXl9L~7+9%4zDuH?t#tMaUp?`8j3Zyh^}P8`q4>nFA_^Al*Ww@K7oF+UlE%@c za60ih($#_;dl3R{!?)-LSO{f(^bgzV|Hk-i@g;44Ys9{u)3F&I(Em!Up~^)^GtlQ= zN`?n)@g2E2zpjZk?pdsB52$&kMqVxn6+@vMh9A_;<3-oX7dJ+2{5vfiULP*B+O)Pm z4c>Ob*R8m>9P!PyA0|-m$y>>E4-HMs=LV&gREcH33#^w=LWG8g=8TaXWOiD^ejnzC z*je1K8q>|io=$VRy2)Ml5LrnX4%G({CT4FCg6mkfg50s}=4X%pG=0SX#{D#1BV#*8 zW#5h3?#VW^+Q{u?xyv>0+gqS&d^R@i!G^m9?Y{yMcEg`{Q=RgR3trpi(uP4k5%!;D zMp0e+6S35QD_K*&)}#azcMew_itW{&d?G3TvEnCU>kenvK+S(pMg}1I)IKG?x&>il zPI@s26I1FHc}cwZbNIQ*$i{_Xjps7bt!HqlaW*Kl2miZp*KU8vR#{$xLf!0V8HrK7 z5!k!A7KLUV1<{MZPmhwr9P)pJrB)lwgNC9Mb+zMFx7_Wfd8U@8R6>G2M6p8dK{O|p zr%2y8qdv(U`~o~Ylu`}k4*np{J3q2f>-sBe11QTJ+Qx#XJ1cc*3S_FrH4w*rPisPO z3=C+OQ^?a`O2X-Gcb#t&N8Pw|(b5pEee33E7_Zom0SbA%(R);e$@3$2d--6|`;dJVlup2*_;>3=E2R3^c{Ayr zc_duHZt&?j;mKjFYA%pB|_M#Ay zWQnL`mo57mc}MU2KEL06p1FU_z0bMle3o<0yoWj zx@&1VWc%?}sU`tTGOMk_p;B2WEabr;U3YeW;QZREUC}~&Un%{F`0AjunEcwg zCZ-np$NR^BZ<`;OWt=IOq1Wj_Ny{JI zLu0J7%2^dZbxKHdENUp8&n@ooR@mP>uw2Y<^}YF}!L1nPJEGztb>7#fIZ*A59gpJO zv5}zLyK5cymB0VY?87L|EfLj_GLglWx@v=sXk_k}VRegiOM=HwPydHaIXB`QeYY?x z=|6S~%qP>^ZAND@zpvSO?U{~#mihB=M58w zb#^t|5)O+Cj@M~l^3#J&{hAZ#pZWz!j8bYO#0vDMjKE`m^RY+5Bf_hAgC4v{{2d26 z#jjPU?tVSPmV@gX9s%Puk|vIPE9sl5nH!!9jh{dL+HiN-cdNRRZrpN#XeOQfYU|VW zWq-N!thm0kW6vy}p2>Yk#loPhOdQv;~gTb(tt} z;Wa})nYeWMMa>>@X|yAjkEmv!5Vg0okBHV=({lkdw}T+Hr`OQELD38*US0a9Z|-;6 zC@hjKH7`t#RP~DN-&yU-Z4Mf=GkavY@9z`PAb$7i=%ITfQq~5-X)UD98oGSfEDY&x zovSL;+2Hs=%5Xj0Xn7hSY5*JRU^mr;kx*QTNq69U<{?BwwS+qlMWLH=tBG?kZ>zg9 zYkl2`MpyrcQqGDlJL=kDUW?;V{h69r(E^wG&bI!=M_PCc8X@)iurZ_e+Po^&r?{m0 zO5;V7{8M`+j*`IOHztVSF&eK}^36sL^p!Z52>%$xk3iidhT>;}tz;c|796(T&ODJC zu+}~oT0krah?h(8cZQ%V1Fq@Bd5QXMiq-;oy4jMV5gnSfE0RSHJWB!w%rVxySAU=q zeY0P}K1~HSh2CgZKclCsuh$4?P^yZZNyNhw;2@oGya0&?xU_)y?8pdDKFVuu{he7q*BJg+tTvacJgRzNRRJwu8+8N zJGe>@)e7qdJbSLmmmGg}g9m7}#q+E3RO^e4$|y#a4R#8?UEY!EjcTF*DO5$-+za-o^)EH8smUC5cxg%a4-#JHJWb4Fgmwq8m_7 ze6DHtD&BGf%;vxSFMRRNdJJQNB?)NvI)2*AU(9z#?T>-W@$Q3CN2KOv@py*)U5T>h z?DXc1$bvpz4B5!B8xipC;?K<6Tv~SeO#Whm6TDZtir4=|F$sep0L_7{Gc>@4L0nLp zfy3o+ZlqK=cgbi7PMDxtMrZA-KKpzK|o!iz{UVskN~dWmoP*0%@NSi4r!a z;kh%jH&VhAj7dNh5L|?$sY2ibBLF@JXc05O!WK`LK9I(ZhSu8D8z+}CEp{r){S@v{ zdC%ozj^%wk>_;T1rY@D0YqOZS=F$dOV1jrWkB(LsE|jmGPQP=)RjPmv>qJAT2?vfb zw8I^!R43*|N<=Dx(XNGoA~Ijd5e*QblB1D+>>MvR#kJ9mUP#>dV%|R)Rf7YbonbC0B|TiD0qP zcFB=W{hBuDfNNgYw?`+4&O6e5WlQ%J2@g*x(QeJ#(buVYJTqXKQ;=8mklDt&mfy`) zNR$2qarKJ5wB%3i<{cU%NKRl%y0E;mR7}r?nzB7lF=sk$)@a`qzjjicF-(J>!&{vE6fdZml3FUHxQDI{d-@?n}j&f{HGy{`J&r*pX-v2{( zbu#H6D&~+AR3f+~Za#6CDzWAWypS^_WN5FFmzl`U)4)QXmw8O97q-wmD7q8;!5Y>5qJ7==rYgJ0b;_$mCNRBb4iPRbE)cs21;{n};)=!#6W|Np zP#r3WmX~!xUKPKoyS2#-c^1P0C8m4tf^JvOZ35DXBeYVyzoymubgTC__?pU)LW}Xb zsc`Zl=Lw9$Tv|xbkT3;^96eA8&FeeL;RP5{4|DVG{9iq+;oX?aklnEBtzt^*jn4BD z89jUYe(rU*K!L@I@*v*#BwbV?Ffq2u33NFS2eH$g@#{h-MJ$en&2auXozyB{?@+Oin{&gl$vyAn)X-Hpw|cajcjY`Hiy@8 z#nZ4ER6wIMv1fkKG2oTIFKABD{!%%mpYg{)VW4_Z@>g0__N;jFnU!nY1UPA&lg)bJ zC;@CHIB`_LZmMs=VIY=-S&8J)kR%MpGcBF2oms93+XnR>l-vX0qqcKBYC{yV4d1XiH+h?-o49DFv`SVBKQ3Qm+tmiR7pc zcP0$5f3nK?(mGVK7wpIEA?VPhTs&$k*R^Ps7k3)+9=aodd!jS2dY%Hf<8hh zK-TUeO^qCrWhi5U7Inn2btz0*S=Do5{6v^czl9JSzcUN1r!+6MGz44r5W(6rsyBW8 zRitmfSu5M`pIj5bCFqz<-N~^sK*PhGnDo#{kQ2T8Zz-c`0Mf!u|KA}2CgPH4db#hk zwKsGTOJ(R}{L_hBwW$2k@a=S0wO8z@8K2=>uON4^WBBi>MMJq&LD=XET>8b4PjPPB z3E+Wq4%-#%pA9TdpTZIeW}H%?H3u0GkTIuc_vjsMtTnT%7y|H?rR@80>(KJnp@3Z7 zt;GP3BC|esHNV{32tUE4P;!(+;!vQKBn8XK0?hzoIeP#ZPC${+L6_}>Q_KY>iV zTPYbr3JHP#2%o!No6Y$#T|NjCAHM1K!bJKBdzxbL3lnAHy&~hedNmK+>Gb8ELfWr3 ztx6uki5bC!6Eh;V0qK}NI?W1%w0L0eKboQG6&ajv5!G-#wS4Eo`OeV#Qup}LTR|oI znM>EwHtN54N>t>cUhUhZS1y1TdtSpnkgx+JjTx?c5l1`<^=VR;*HY zW~2y04|bPsXc>?vk?v@j?lDYjp2-efFox7+M~xkNv~MhECE!S8jTY0LeMt3nT%X^i zf1#cfI$f$PYbr~9q6v%cT9w8)KpV2m>GSt3pe_<0Ndi8KrGFVAH77lg(#9KY1ublK3g!vb^dCI z?;em%{~~<|B~``JlJM|6I4B8AUC?F)yVO};48!3>O9>N z-Hfkdr5W_kv%p_@XEbAHHAs7+2JY%exFwptcEJR*>HZbzrc6hRkuj^07Q_T9|GN1v zYcSEl@YBvIPNbWjaVDIhpoWfNsrBJL{iWd}Mz+4>@AWK;Vbr?>GS+50#NwwpDfI)( zf_VDaTk-y@NjyX=y-zaq1xx|dPRCP0KQiGubzd7n*Ut7leb9J5K4{xH)u%(3qMSAu zU#yu|XTv!)2#Fd(5#Nb`x5cCKxf?PdOZNkjHw=<|0lcYquqHUbvlnKf^w=a-u+Ds<5_cDZCk^EcSVy)xMr*tv z(puZJgrwFIz%T)kH99q2l;H(E1_BekT;>BF2pj7O)pl$7XV!yNym4^B`dx!Z5xaIC zwSfwnM|QYC>0I_kHXGldTLUpyMp8@KmMe9TH%>a@zWL0_O8l=S+KrM_TZ>Ira(F{6 zsoxymuAC^GZTwf5IWb7@3XM$A=KpM6{cY1qh3bFoV#q;NR2ZdIHxK!!i9TS)Q0i5& ztQaKxc*;+od7T{`Y{j(Zv@`?A!n8$N0MLY(c*YKTXYiEG=Kl<~b!9@Xx3NMi<z`m8nuA?dT)badAn_!7fG z-ziK9kOW3f=98GN0&pulzUlKbPLlFnF`O;W)w4&TXS=v#dQk(vO zn^YG15|xkPmSp1om*ag(5B++pWl+CX=mMon^XKoOTFP0*{)Y9dc16h+>SW*REdlv!{xSmK zl(qXbN1pUn&dQSKJGNE~7!Mdex_PY_mpiCv$$!;xf4l0d2cNx6&j?=rtMl|s&%(*a zoc;M3Z%kHiXHWS^kh*m7?c40T64hktNM>VN-0?&6!^6Q}1)r*^x|#NW?M6Sh$nBh) z{d_NX_SbNb^-*i%tCk>TG$JlyvwGd9zh2U~Crr0=!<&G?sJ0vhC^y+2^C+GdcIlb* z^!`)6iSsx6XnBn4eZSyY+kDiym=K)RdyEzSd~|%u)DXl#?o~KfJq5i1X6Ro^3MMO} z5X5frJmzmI1H>!p<)rLsXkyX07_996jdzkjB*i zIGa7T!YuXrlX1&uhd?~kTQtChL(Y{G*!!&st0Z0$X?ag*@1=+ni_ z(yox0nM?Y%Hst|Ec|RRGI~0EcWLO$H&%uZ(XzyUaoEPkcs96llLP)S8fY$msjENhR z&)DH|kDZ~nZ_(Aj>kQVC5Tk>g#P!yRBJ$XX`4ZKa3Rn!Cyksm0k zoYF;-9gw_aA9?VF6ExvK0KoJV0B{!YKSyYo1d6PQ;y$^F9B~EA_Wv)1h6R$5N}^2i I@+bfNAC9Brw*UYD diff --git a/summary_results.json b/summary_results.json index 5f0e1966c..ea2fbbaa1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.99473, + "Performance_Result": 10.1811, "Performance_Units": "Samples/s" }, { @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 7.73123, + "Performance_Result": 9.44853, "Performance_Units": "Samples/s" }, { From 7612ffe3d155bd96306d1be481f1236d0d1c0c27 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 1 Feb 2025 00:57:45 +0000 Subject: [PATCH 14/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14421 -> 14422 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 05f58549d..f92730d8c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-01-31 10:08:07,634 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3270 INFO] --- -[2025-01-31 10:08:07,634 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3270 INFO] --- +[2025-02-01 00:57:39,416 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3286 INFO] --- -[2025-01-31 10:08:07,634 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-01-31 10:08:07,634 submission_checker1.py:3307 INFO] --- -[2025-01-31 10:08:07,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-01-31 10:08:08,663 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-01 00:57:39,416 submission_checker1.py:3286 INFO] --- +[2025-02-01 00:57:39,416 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-01 00:57:39,416 submission_checker1.py:3307 INFO] --- +[2025-02-01 00:57:39,416 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-01 00:57:40,637 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1811 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.027 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1692 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3749383de..6227f000d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -555 +556 diff --git a/summary.csv b/summary.csv index e0d7bf243..dbf9750f0 100644 --- a/summary.csv +++ b/summary.csv @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1811,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.027,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1692,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 035a071dd..01cf91e7d 100644 --- a/summary.json +++ b/summary.json @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1811, - "Accuracy":"acc: 74.800", + "Result":10.027, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 6579da9f94cac34b8085bd9c9e5f891209f092cf..3c30aaca50b61bc90beb4a5f8050e21e4d9204e6 100644 GIT binary patch delta 1881 zcmV-f2d4PdaMp0JX(I%3ybg(zZ6h*&#S}0_g+NimwFzvoPbtS%f_HpmEjfh!^(zzS zz;cHqIP4Qex2&(fR(GrI&hd{~9HA-ASePa!Ev)J-L=!&^!eo5XdVIK0+AYMmmjqsv zCiJBBN?Gg2>2Jr3aU>p1*h%Y|^U0B>G5?vy9#hi^O@wBYX0gY`zu8!0lZ<+Q0ehyD z$B{;K-PU3+Oj^iK^Ms$YY#X;wo`nD8v@@4r*KVC2v+(qopWdVajgB>bdaMKuKe>R3g>ap)`$LjJvfy0Vi-Y4(%8N?`dg4mp>tKAU=Bl;;X;S=saTH>@)|2GT zKd;WO&d?>jNHd?J-oq0zbyu%0Tkx#NizGi{6XCa$mdIjOWJl}tEa$1nZW^HBD_Rmm zof|YYRH9lcs%|f8h+wFH1mS3u7qv#2NhuvgUhEOY(T9rbR+Vu$D1vI8ZfBbtdSPu~K3@+y+me;!PUYN}XazE4bL=rO8oab_@Lq6l8k=fzViIDZAF zOp7qVm8{Y>#z50xn__Bjtie|jE5K|K4F4zC-d>#7HYE~ugz>ks3&NT*Y&Q0!yL$_` zEz7$$Zw@Re#nuz4re4qe2Nv=*vQkaDIY+(Me`$hTLN;k+}7m+DnM0Ifk z9d_sX>VDAg-1WLg2n(;YEz{Jo?Ko}Ib`iR`I$vmsVX3w_HPg`BSX5@Y_V?}{FT3~% z*;p0vgAsZYkRxO`s%txLDYzB_S2b+xAoOtevwPbg+;@9}{@Km_b+>q0m)&2p&M=9p zBQ$1l`z}Eu!S^!^5;!A@uoi`=L9hE2Ija(~R zj!N6Zr?HYI$!sviDt1+}ZD+DVShYP1ggEl&l7T;&yrC2bnP^GEn92Y^1+7D<>lw$@ z&RbXf91~LmYVF?p;sfu0yTuF89vF)^t)^{%zwcpfuY^pqBn}ZUg98Q3;EMvrK9EmV z`|fXh?D;8Me~;U-+-zUZ>cHMt&w$d2cuU9_E$U}mes2WY!2QtHXf;u;8v@*|eD8)x zbUFJEO0gcWQHq;u+u^{hny(rM1p73zci%7$*tOJt8#-i7`83*pC;WK$8uk%f^F^!? z#Abckd>Ly5v00MM7P1}C8mMGDp>X9~D5@spHjXl$1amo!*` z6eU8<0Ik~Tel%l$tI@wdu@4sjDCMH6CQpRzVM3*;;G#1|}e6y(N=4G9Q0V z!!Qtp?>t51dtxW0fUG2y0QE#AB#J=XEPI<;#6PkPY2O|vY13BfiQ{@^znxvVT(#E$W++Tgr`tS7$7^Ai*aQq&?-f zLfa55#n|&LDCV>1&{5nNr&x%kYp{Q1{F^m^3qtO&g1`g|4EMm)PDFG-3O*t2k81~j z;22s^po7PGmgAE*fK&cz!=Riwl?;0eSFJ~Ja__vHj9u4d-E6SN{5k$yy}S*prc(E( z;(+Q>aLOI9U|e0|^CBLuWj?Ipi8m6&<38$F-VTe5eISw;fR1sG%E#>KdA%e=bwP@G zN{W=^YeMM!p5BtLe&j{}38)T3{!Kgr#@v^mp#B51cr%;>1aiC%iIW&Nx&`Ei>AfYB T*EduFd6O(SBL*}z00000@~E64 delta 1883 zcmV-h2c-DcaMf_IX(I%Tb2&IsP$=BQ&KM3)AGJg;l+UXyT_qn2b+aj}I4$+d_pS)0;G)(Xqx)kF`QD7xsOc zcv+ZM<&R0oSykAhERT3qno*WeN!q8Ga7R^oHyTA@LaX9Ga+-ydX%b@JiziW?;_AG{ zV!<0-ouhA59jiz<4xQye$iG*AS9a1U&3@8X2`nGRA?LEyXHzeXykQuHTz=@4aT-^p zx5C$w&G%XAQ^rzh|H&bVk5(1$Y?1;=@QA+kzN06aCdJqwY35VZdw4>o-qx$j7CbBRBFT@~MELEbC9;?m+0i;Z%XuoYn+9n3ik8Gs z=LSs;m8h1As@sbiA{Z)vK{y)aMXgb0Qc6dW7kfl;^r6yjSCw%%_Exrh-RY7)3*ShyozM?p9fQ-nkrV9?-NrQdW>pYoEeN_D1z7XdGXW=&R@YP z(;`f8C9AZJG0-&FrkL6rYw(rC3NTv)!~Y4kw-@KNO^HMuVf?M^g0Q9xn~go`?%o1! z%kr+xn*&QqvGqi%sn>J=frWgHtW=Y3&Qb66Uz(sca@v!8CNF>5mTr)}E+SLDi0a}7 zI_%E%)%~E~x$AY05Efo>Ez{Jo?KrM!w-LIyI$vmsVX3w_HPg^tEGn~H`+IkfmtA~> zY^;j-!3ezx$PqFewQW1?QgAH&+cu1aNq3>`e!%y*WKc2U3P!XI>RKY zj?kFJ?k^FP^Xq?3kkIu)91mxcHL0}W!d6FDvE{7UYGRexZFgm>DXt=7lUiGeP#Rm? zAkz{`VS=f;I-nAf=ty^FMtPWcHI4m{X1t~xjwk%@8gY5`65#5U5~P$v>w%U611$vx zS_%xb6c}hJFwjz9fG?3eUCIHz1mH^mz69V)0KNp^O8^JH1mH^mz6O)6CmDa50ACZ} zYXW>tfUgPgH37aRz}E!$ngCxD;47E(QV#I70KOK$*8=!j5?>mO>AWc^?x;KwZ{%9p za#Y$LK8=+$NoIp7R&iS;+jb@^gjL(KK!_uME*bcf$s0<6kcpNgjHwI&RM0wvx}I@d z?Ywox&oMDIpw{laFFx?TTfBb&?SZj)(`wrG_xm2!_DaY!OX3g#GdNJd48AB}>;w5^ zweSA6$DW_E_4l|P%gy%ntPbpb^$aMTh_{4{(V~93@_QrD2JVNpMyrW(-4Nhz<$E_o zqRZKTP>S_{jZ)lP+YSe2)qK@BAlRpwz59l7z^LKq(hhHF+Xz4-+a)1s9z$Di5iuUpvQ*GdX$MziboH zyiMG&^A_N!OhC#05h~9g(8$Pp&FbvWkNk?sp*-^2%NnBzYfM#_2as>GD_2)_d91aV zyBS0!mb*50vsK%+oz>Wlh=6c)8==eY?cln5D$yxFx&%EI}y={6nsM3 zAJ+^3!7((TKnIU$M)AcPz$rglFeq0}B|~Rn)_N8v_tDGA*tTufE(dGOPx0sW zmAYRQ2UHb;=iC7c##IHrzQn~f%!h4U@lJwx?4myNHV$&C1Chi4`6=#M`B*+Z*CQKL zEy&_NB{W^qnv$HX;`!=Z-qc@!YHadv;u&zteftTjKeKr=oC5^7aIhbf88^BGClUt- VB$L=TQ~`LCEjS|vGc^DJ001MmqZ|ML diff --git a/summary_results.json b/summary_results.json index ea2fbbaa1..e087dc556 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1811, + "Performance_Result": 10.027, "Performance_Units": "Samples/s" }, { From df5b01d68723e2a7be58c928258b6552ec245e69 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 1 Feb 2025 12:23:07 +0000 Subject: [PATCH 15/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 14 +++++++------- summary.xlsx | Bin 14422 -> 14417 bytes summary_results.json | 14 +++++++------- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index f92730d8c..1f31dacf3 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,29 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-01 00:57:39,416 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3270 INFO] --- -[2025-02-01 00:57:39,416 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3270 INFO] --- +[2025-02-01 12:23:02,615 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3286 INFO] --- -[2025-02-01 00:57:39,416 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-01 00:57:39,416 submission_checker1.py:3307 INFO] --- -[2025-02-01 00:57:39,416 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-01 00:57:40,637 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-01 12:23:02,615 submission_checker1.py:3286 INFO] --- +[2025-02-01 12:23:02,615 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-01 12:23:02,615 submission_checker1.py:3307 INFO] --- +[2025-02-01 12:23:02,615 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-01 12:23:03,659 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.383226 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.278282 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6227f000d..0ee983639 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -556 +557 diff --git a/summary.csv b/summary.csv index dbf9750f0..6c7f293f5 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.383226,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.278282,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 01cf91e7d..568a07746 100644 --- a/summary.json +++ b/summary.json @@ -42,14 +42,14 @@ "Result":0.0102599, "Accuracy":"ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -69,17 +69,17 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.383226, + "Result":0.278282, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 3c30aaca50b61bc90beb4a5f8050e21e4d9204e6..ec256c5a088a1676a68c4826cf239ffce55d4514 100644 GIT binary patch delta 7946 zcmZ8`2Q*w=)b=oXC%PyzMjyRK?*`E#7$v&sQG)0*M3jVJFhmbQM6VGhK^RdZdKb}a zwCE+@Fi~fX610lf4!$@wBDWt~;K$g&c!+fpZx_ZEvN7Fq` z{evYf>*d;4_}o{WFTMpJL=Ij0@774YRM*XL(Ri85#VAC*_lJfvVV_t;L4V$?d%4md zRsW;!=->hj_ph5ejJ~!_O-_+$7s}_~yM3m0r^L2wH;k^<_4cVlxX5SE@tg298&%pGJ5^F=Mi?3_bp{(S8=P`ABhDB`Lq5hP1JMvO8@n*>u~yiDX6 z8KXApihdL=RR^Ore^yxc&#Jk6VD_}uGH;#?GIj5Z{lGN5BJJ$SR=(Yocre(PYs+Vl z|Dtg>N;Ge|kKfyDXkeMK>gUUKu#EeH*rF6k2#3~5eyEjBsNmfihO-~5cruD#WBb~O z1d?6Z()EZ-ikMdym=Qhj5qy&mVtQUhcsV-iMf-ZZG2PR8LH=W#^HQs-c%K{yNtQW; z5+7l^L>ZFXAKu@lS9sIBdEdI%mhg?^3sMOz8M*VNK9<|utuRNo%9+n1Z}3D@hiZ30 z#IQ~s?|G{ctYcb^8&de*j+B(^JU0bEFG51f{30`kfa0x?IzB>Ag$uHTXM2avcmafd zU)U-eL-6of3=eDk8af3D{~iCjL-JA^CKZZd`4cR8aK_ntmY#xizJwP=^95_eI__NS z!BllN;6=}0i^$L|vmMgqcyi5EupZ+%P(h}o*EPJJbku3V0LI*460T1?7^M2*XYAIQ z$luRb@*K8F)d&?dIZu8~`np;(d?@<~ovumN-=rfm6I=WtdP*kF00u{z4OFGfBcW_@Oa;~9^cA( z97S0E9QQ4}BC6TM_(IjgCw1Kfem^Y{pulgvTYzoeNvgp5QnvagVuhTZ?>*B|A77h7 zM>>tSok+jFPEp+@A? zpIo+0vvZee@G68os~PRu=3M$3lQTuC8Y!>kE;IRr*uw!N&NNW zgz+C*Dw@bZDx$d@Yk%4RpftWC|FTTqGww8rw_T;y7XPNP`hFM=W&H4eI44V**1 z&n7^7l!v7`Qg9=tp{iVya?Jd`WlOA06bXi6@jpB?Ul1|9jp=#F72p4pWLX>~@66sx z<<&-G=4|%!?6&Mjjt(DLovgDeU7fa%8eJVx8qAF*A%-Xw*6pN5Hz7|)N-P7Z>*=%) zJ5JJ^mVn5MOToa6tS091nw8>au%b{KI*SIg+$z?$rmcwJy2Kmf=1B5>r&SB;H4&db zvFi!ZXQbA=t4Y&i;~*dS^AK*ScvzoK?^)m%WeRH!mQd#kL&s?>@=ijX-Ac$UKVp_ z6ST>)vLSbMcL}kqJ){+PBrR!%32z(ZUpX~i{WSYTGJNu_B87Bj2NkGwT0h9?Z6a=R zV0*ny|GM`6a|mVqlwiR!^AEv*P?O~d=<`R|&S3`ctMkqq`$m8a^u6^~GQ7s3wvn8FkvT)W)UE_H= z(}EGo>-a5y-)Z>6eHCMUAOSr903gAgDgbx4?R<$l0DxVZ4;2K_!}#Loa82QRMTnXB zdr^G%{veORLCABL*g@gId>e{h83rKL(x>Yy?gN9}rq+d1n{HhW-eyc_h4sOR`lVYC z;!5#eO~l~Ft3$6=3yY-pwgY8dGj57X3l;5QSB`w;Q;Y+h0a9`W2FIT-Gs8{(djC~v zD|md|&g2^MrILXVQTpoh<&1yG;ky|dis7R^`;n|it7FTed<_@Qwb01C0vRTz<8U!6 z+3Z}|K>DWaw9a6w_74Zn91om={3STnhaYzh=qf+5Wq7l(@!VzP20L@^X!7$Xd(x}+ zsvgsv4Zg6GvTfas8u&!cRohNPJLg6LV@ZK$#oO?#p)QTRpZ=~i-hcxW?}${OP#k((zv`FE0cWf{I5>Me1Ov5-BPEjx4zX%@6mU`y7% z*jAnL>>9xO%UtYi8*R6Cn3+zrKB)i)&dGc8vo!@^LKVN{ZcKS+TZBD2JD9P*H~lkj zSb1W7M+quI^teJtTWO?K4%#&_sbZ015#ed;6CBa-);!TE@Q?EEoQ3soX1{j2*=AO& z<_`>?1l!I2fL=bG6Btq`c}qR=nA6tj>q}e5B4MzPE@Uwg!<{<$cvVSzaN)I#w&%GU zCB3m@V-`(ba9S`87%|4Eal$ae>~TNtQY!XzYn_8i>}_`gMgzQDewe7)EgoSA#@}Ls zfbB2xGzp-Hl=xtx&k$O_)iwy2qg{iJR!6ogo>Av4UbA~-&!Id13kFO(lBqnC8k~3w zq3tU05xx!g{6(H2S4+%nVx>qc4;iED3elGE_VX%m(tZU*oJD8Qe*1ftLlI(f4V?Q{ zR3@r4)Fz8)ds9rc7MmDoy!#6#v+Yof4!&!f+FHK(=x_A~-xcEe|Oy0~5qR#fWSz}~Zt#0)YGtYFvT`hx4^Z!GU3 z64^|i-WxtXK_r+z4OE`krLPh#TUpCo#<-j0_CT|zHY63WGoxm2^L}C1z8|ePyUOH6 zD&M|m74+WB+nGM}l~H0Tm_$@1AFbJke|i76eohhe^t8Y%v0U!y(5vdIzZ;uVI~9@T z6*@&DGJCz^h>!O|y)}&#r>Mn>8iS+u?6qH647)-WDvV^1uR?q48KM~@j78RL%SH+)WXPAkdL&s^D zV^i?C!p20@7^N#ms?aO*;=j^)&d>WL1rzy&znjQs;q}8bA83P75{cbK{rd^T5?)Wb zK@nT-h((?ut<~Cjo7~sVt1#Z88`^<*+SA_WysuA%9`G_OZ1FMphlU0zGFOZ6rLT95 zMUTD`9heCZZay}VeJQWdHL%Q}gd7%(rR&&ed)Hm&En5`it~cEwxfWb{9;vB*zQ zpX#Jo(go8Q(WM4E5**xD3U++jF*b(L&(>EoMNsEYV|cbO3G?5Rr-_Uq)W(8~Tz|&A zbxa70;aFpJFt)$-32nghy>WtJv;W3IohYE7OS{1>s-4yuYLWtB_bD|}0Tn&SvX;)8R zMC%5p=biVw%0F5Y@A~0YMv&B+CGt3LV}4Jc%+Gje>TVq!@zz{Vj+wsr_>_)21e-Wo zzRIA{E7S+5fdDa%L?IK(wtWWBCB%ej;Q$$+X7-y9qpGbgQSiy)>$dfbaa~CDrCp3C zgA-}C-O2j3hqa$j&_TaXzOq@ckwhKJ!avST`LhBf27{tF(39v8dYodhvcf>lRg;hZ z$Q%%SLf;F+Tg|w821)I3B;kWUz?}^@-OA_wPk5e_WxE@YE(Bw%18s&8W;R5@A9{sP z;iy+EJOyR!bn5=PM7=yLTuhR%>XTIcc_7!+e-J;d;fsx-oXD7-xx;K1ctDz&^y&e7 zLB&@o^o_Kg$$fG{N6=RC@E_V|*0f?|Jw85Yeg5IQ3h4umJAi!=%qJ*S~ZVA^C z;nb6dn<%pQ$}&}iHrdjPlAOVX7d&1sf+aGvZI6Xdy!8*mJ3jg1$bXd-S$QLsXkKmV z5QqQH7@8_NLkuyEWNGP1n^r*b^#Stn7-{;r@|7v$lSmH#yit9Jzq}=ol8PVCm!U35f|HBGG~Z1Bm#w`ZwrP@nL1df{uhp!vYIU zGOj#Q#1Gvda>){}2@7j4N+o8j@I>rlMZIiewt$(C|L2N;EDf@zw1*!wAjOFRaWE1~ zirP3h5gfwq;YnIN%vHodr(0|UelU7?FGiSUj=ZJuN2N}7RL^(4GV||o3|H!DoxaV} z_c$ws!VRr{T79f?4hKDG6SUTJ@0x`K`IfUYB2u+QwIc8TLOQ2k^P9!RAobCyHuZ_C zs6djIapk0V^Awv)!GNapKq?3Qo>TDW6mGXk?x9s3M!UMva*~rk>?MuwrwfNPlYBe+ z;l2y7UBo;@;Bh~3nq?meus6J7;DL#FBP3yNWC%euW~8tSPFhdEv_#@UIPztchI%i; z4f&{BqJw{nMq+#Hl^db`_y&{v1?`%~G^*}1jA=DKb~TBPE_y4SA$sef1(bH-xwpGu z#PzpsK^CXp^bZ66sjpexE7|>1gL<`Cv3EiU|6`)*9v4G11^5*20+R@kfkaYUk9>S>ldPHLJ9wkm>kn3XcKKPez+PI&mse%nCi7_Eu|Ea1(zhK{Eq!0o}9@lis$9P>j zgRJnT|IDa=L|~jupY%=JrWnlNL^lr={dMY`0z%^REv!8M@NVMdB+Sntf`XkXwfV=CdJ^> z)tBn&<4xeuDF}`Cn|c6%zQ2G6 zjX0vN0>Cdu^4)W^jwe4toZmu*+ z#u8H!9e{Nl)?f}FoNp*Kgxgga> z-!`XGTzYXnz?)f+8LYeaB)uU6;6H?=7lvp5rE;LYMpkl{R2sHwiy5DCXa zj`H2Or9|56_fWCHRQ|;rC>=DKTn)dRK^}#{RWc~V;s@$aiUYZ&tNTXfDmgWFQ(+z| zQ0J0U4fPJ50hoxTw!f}KN68_=Ql%2PI%-kqsd^bD%%b4c!6UVk`q=j?S=D8!W7kHT z2D}fTVQ4wD{z1+Bo1owPz8F=_HyRcQ%8{DfQs6X&<^eO;X)T2@3{8no7GX;*1faic zgbE1g*Lrd=He36bMfIg@h9b!J@c=e*3w=X}17mRIt*KeJ^iy!oWjb4$dD z-Is13f-5U$+h<8VP1=N}l$)9{xercDpNty$htn0Ee&Fyv95P-XvW@C&<6Q_{ z4Fu(Z!}eeg1cHdG?m4hyt+fe|?8gi6ltV>CVNy zaCE;)q48(2zpAD1)wN+?8s)pH#Yc3M7@dl9U3H)ncL~B3+$OlIQquiAG62& z!_{pMX7RF~GMgIjnFx>@G0_7IqVJRHHyw7Q_`e0k=N0a>}4kEkEr-!Gs|HMT?geYONRcQ!*R5FTO zkj0t=B|``ESd$BB>W_jP>ZFfH!eQVaAdsen>XieFDypi-i6>2YKh8Xblj+cG(~1Ux zLwbsjkcGO#4D)dQ+*tefOBMY3LMs3D;{rw-^S`{$goIcedUdOW!&E9ng>((vKg>Cs zDojIK<5A1h<7HTWk$W`4l3iVh7dQx!qKv0_>YkPs6W$mBQZog0-EpPe8(FrZ^VyI zZe@Saw-jAVKX9_yxYkPV$CX) z+MF~rCK*?_t=1X(M^1Lp`b7hJ&q0Tn!jG0qlpRrRSQxS_DP(h6y+4DkzUwQBmRtJ zQ;Oz^%et?Z31$NhH{G;l61M$Z^kq?rM?aAxf>m#1U<# zb9&nJnj1Z5!R6WyZg;;i6p4l47&EUwu0R$C&18 z|J417<6$Q3NNf<5m~~S4ShpCt{3*{!xrstLie0ytQW8A8P&c%cY8Ezwb&7&r(`>7Mtg8^yF2{ z=G6d;SU(6oA`V*+EDf(1C&UZz&)ccEQfq%YwZj2|Vn&*Irb5?*IaLE%8PKo^APTr0wg7a*E-JK;*{BQ$g4e5b2-k1M6`*}X61H*-pHG~IE zfG_CoOd==weX*Acf=1mtP3ETv>I{xpBPSjM)QtU0v)vEQb23gZYPTI2N&Hrad4LFs z5Dftui4nUP@eDj9nN^(e6ySA*cVqSWJ>F8H@!8L3SmMQJxV-=+F~%ALgSp$tpIx)I zq;DSU-UZte5tC(gG7%<5a1~UZF8n;5J~nN>ebDUmY357*MAYY+e7ur7h(mOWxYhV@ z^EDkwW7O5#su7PO8(D%D>x)Io)J^|W&U8m5`(k?8sphUIOX&iBDw0$8$NS2#yS1L> zH$R^CW*@mX8l4{=NbhpDtiOVD#}8dLUPw%mI5t3Xgq;-%rH%W7)D`4pl?9Qtrr#+jXbR}ag3UF^iHm5I35;tS}-WV^$g#%|o~&5IETM@|udalckzSIgo~yOf%U zzl}aZefrj(E0Zt!NtUs@V2fL+Nl3NX&FqNJ-IczvTNl@a^}MxMx4e%49hdKVr^&`F z5=d?BMNfCH^n=3}jgw4g@qv2*G#l zK10&vBsdWsU1Bsb`w?WtZNf99B_wlE&8y?8Z$*SZle(wHTdBZ2*xMr*`26S&1inoS zMk=(ux7vM}c^!2;y2-l6qDJcWBxXE^HKK9g4=s zf%BNg_G&h~Xs1opC@PK(%h$t;0KvpL;3iXsh_F=kK7s+T&%kMz=f82N}SXDu-lV#***SpaLzAPgEJmNR2bL7x7H9I z>7L2`UI6A-fCBS`2v+#)7&71ONnki9Yl3_jL5}@l5lRrf2(?*|dlN06~5L;NK3x{~gFEi!?D=HOlV- z|BZPX5aQ+X^xudylq@U0Dm2X=%8(W)O>_4@dGG)LV|)OB_WwrXT4WSRB27)=&cBz^ Y;v{Lv|L++xDj-c*j+;nS`d`og0DYOtSpWb4 delta 7896 zcmZ9RbySp5_wR@98bU&PMnD=tx@!QD5*U!~E@^}rx&?+*Bt*K0k`@G{q;qIV=|)NK z_`dh9-(A0Ft>^r4p7VUxKKr}(K5OkW;W6t`TZxAavs5Wht;G|Cg<|SHfA=T>8Sbik ztVH05UG5OC;(vXmMPxCx%-L=N>BHkmd{~~6{=n=cICXyp#QjF8(ZZ}Mt>ZXU(8b2# zd+#to$;>>;kimKKLD%6n1sd?HF#1o7^qAkg(LtpGjcs~r8jf#9 z^jK!dx4w|M%X4RC8HJ(tT>k=Z+*NALw43F+o)685L>2x*|=2hFC0UWuM`%} zg%a9cdAzr;33CC;nhU{@Kh7+th$uNo{ja$qLS!<^IU}w#U|Q2kKT&@5wm+fJo9f}X zP#PTjzIw4cGgIAJ(x-ZT(8Z^9b&YPK=@~Aifu6EvSWQhSiXZj%36{R{a5Io1Sb?OqX zyPN-tD!iSc2ZqgU%%V=M600d}aSnl8m=2~T)4m>j=|-QE9rc)#MYIwizhoVWtnmL+ zw0@cy=a^%~F-4Q(NQA0j(W?}YFNRu%zJQha_Y#tLKdVYV*ep+N$^A7TAM_<&uTs0K zkck{9AUcXqKs#L%fldDQB?LhDC<6P(Hw^{M;&EzL449%U2(*T4MhO&XW#`f9{>^M{9-(m=Y5G zKt^wGWD5CeOy;}^7RqcUi;Un6OG~vNv*jBsh#NjZzU>QajI1_rdsoXPP08HXyWdCq zHr^tgum>oOx>QXXO-Pf;NcPLe>Xyl{r~v9TW-!mCUC1U#i8)mWei5gXjprM7kzHFasotbWI&<%bLAdxG;>U+zLF<8~>$>G zy~&gqcKNa>>_elO6PbcJwMnJn4UMxTD0r7Fw}($q_$~%V*M3GjdW6D9WLuF)7mq=u z8B2+=fsRr>w7abjNZIRae;+x`$X`GW(v-w|`(a zIodSnw`ECoM&k|#jdC%PdlAELt#LhV{Fj9g2|fv1(xlu4Cy7l`9NxDtGujtAq;jaY z`a2fQ9nCo8jR`R?iBA%J!z;f>P=_Utvk)Rjm_+2Atx z7)IFP?_EIdtcVMuaK4~H4+60fxG4|^@>nG)8jwyEJB6fxg|hKKbN;BXxzOq4G_b^u zJH0S^pR80VC;}-6Gyxi(>!(%Ghnlwf!;lI0+D#!uL)M#yWLtdh7h&76VRHnppKK;V zr>0rmq2U4S!ZD%>;r{Hxt^?Ty{S3bF6~O*uVlF%n9jkr}e}P0!yc6n_aas#Ve;O`d zAO(Fr>|6wv-0nXO7j&LodtqP-Ur?9AsZZHw4eb*XYEJ@C>BcHBAi?irL&BU|m$L zs}!&6%A2x1C=>&~7=mne!$_dy9^J9$rQbopw4G)=#&eB(c9U z`C4jukiQ_9dOe}ACR@=Vrec3$njU}h*u9Q0E4kxIB}ZK*=xbwNOl;Wgx_+BS1n6>O zdd#fiZJeuIMb}X@@)k+#OMNYC=z6e$w-Y{by+)fSL$t^7iDPr)OUkxUW(O7CE9kwO z75{u<4&Pl25ytr$MsjvBXW>j|?wMG8>KWuwWWYyT&t~!0LW{PJo*87VkNQ#Dg|vvn zhtda_wgwh96#xLhN58UwiD;z_A5R_IqfuFsIaZ1h@Bmx zTS##`ZGrm{R-I-a9Lt~dZwQV|>~V10^3`J{U;a9#>qD*Ou;|&Jp7pnjJA(m-)-vV^ z_ER0LPD+=?GK`}t8@sz^_sffqB-J{aprpm0yRTh79M62+jK6We{foL6&i>mWHhgYd z#R(HIJ(wBG7>?%{*|ze}(87)Xe!!MT^Hu!)2uHZ>>Daa78&AP8yw5(L2Ltp}bt=T^ z5=t&B2(`9ZMeH~@HY^`pp0|J8oZ8Y)2?&va(kz>QOysAzykDMZ{nKeJR=J;dv~=r@pK{LAl^sx=6r!umd+%X-+WBw zJ|#(OEThjd-cU2D{=DzzYvW+ESVpT^HnGg{jAqA`qm*%O72(|I_s=?=662Q(Ctk>p znh&iv>8*SoGp`z}7t0~bO)nXSRCpO{%#!$bu6?q%XUyiL-#&+uUit0RYc*a6lEufv zy7$s`v>NzGhdmlE$7Z(mS3k@#n60{$Qg1*RJCbhgzFd3i|B*OVc^<$m+Mdd=lKEUX z{$a^i{rsilFU1B_^jubLL1vaie|E|j(BwiwJA2BN-KMn44^#8!D$ai#NT{@}8nVdm z-@b8QKkSJ0`~76)7RbQf__mN{!C6rEMQS2B5sckj-#jJp(|7MuEo+B7x1*l`O7-?( zNf6r#{TM{)@zf58EjdJ}J!m>WnDyqn4xJNHWqD0p{2yMFwICPX_nm&roO@${)5q?q^J;n~JAoE~`N4;9bX+j_c)-$3C2*HkIM)Mbe1Gz(T__M#a720>AsV+`NOo zr>hU=bEx5!pJV%!1XB)Pi zP$qsp*%#mAXLiTd|5z`@d>9VMt8=8&x4y*NKrozZqwJT~cn7hS(Q_^08)IXI0UZ{u9~K681S;C=nyN~;%r z@14!OUGHJvIfZ9QKhP8IGz#agU|1;$bwXvY;*Rn5Os?~{XOyV zB97cc+nH0go~SdAfpS`1fQ|bCc%UFkq;^^5G1+ZyO>yf!pEr?M5 z!I`5@BN)MhAg+Z7b+OqcD{on+NGStoq;Xm+0kx%Xd8|S+q{40KJV)l)>9Ryo`*=IG zMX(h&#@c?I+AG#B1Hp{svh!B!xi|I|6yT!8;q3M^RV^)QnNhXMP3mXrogI>(h;{_= ziMmi1w_UQ~;YPYtFQ(al@F(TL?N%6@={9h~L(sCR(9<+Jejf;s# zif$(cLK!Sphugv6iUdVLZPF6uPem-Nlz1S$l`#>dk{`fI4L1gjrFjN%IbhGAf`E~^ zn5Mdb*h{Jb2u6F2cWO&pO+;1lOS{Q|*PTV3u3R}}V${`EJo4V(Z6#8pCWd^T90NiC`*54IcB#E&@=2JQkeodnTS*8~GKOh|1Am!^r$oq-lZ#2m1@{i)==CL|?)MncBt6bbYo(H(I^P z!{@u+rAmZ3sjpg<{!wEg_`Ag8u~6J!#N=^lW1MU_$uINtdIZtOXrg^$RfwrLezS8$eV0j_$?+C>X{sIfs))g#^THdE<~ERJt(W4x3$$L;*P=2 zs3yc|B(?LGdMk#ML?Z>&0Asrq417W%eNy=1){R#3y85e*+6krkoax>i2T{m6ed!<-H3 zdvR?c#7$=jc<=6G#Pu{P1LP^$dY#vEK1KQk^4X7#UyT%^GAOIPpG+=9zea;37EqUn zNVwT#Gsd93L@66Tgr2dw49=Ssw)iM1o(U2VbbAjGc6*m|8^j!SHwmZ9JS6o#rL!J@ zrIimmrM#netYmab@hw%RZ=}2iqPSgLujo2~F45X4uj{3L6CGe>wYV^$n~QaI{`6m$@j^srBsW(`agqw9@h)YRmNL?=`B6$R$7h?<8oMg3`Dj}R8>?$6k zMspdn|6#*ihU}y#iS56X~wgZ*ETJx3IoMU!RCgQ*GoOya~~Sg ziTUw|{V9jV)UD^zixlrCI8zIC?^)?=i*6JZ(6pN#AvpN8IbQ8Wi);g?#cnFBLD+@V z5^W(2vy+FUI~{CPNYISEqiT3?7x9KpcJJk3hLfE8MKNyS;$g3t64v23Tv(mGY#I(x zF)G7IA@`!o$3PN$49?0JlvsEfa}?(hDIIV>ysE%LY9V|goZYy-+-QM6 zSuLI#rg@^qJPbCU2P9@Kr%6$&lqnpDCNdb2k>hPLxdN@lZ7s1JL}EHaL&z!bNbzZx zkZC8TDR0gPG;+diNaQTvDCv^K?VvU?bWOwqH^KY(Og)LftjF@du@$rU79 zKGAeI8J3a#N}_>T^CATjD9dA0^jlE@VhjJ>BP0hl65&kIzG-0A1$??NlBKU?hRq|8 zJ-D-E!u!Zhj5OBSr0V$_6#Oj6K{lOOAZEDk>a!2KNHuCwtbtka|KoLmhar4MDqd&@XJXco2ElsB_48%cN*g%qPNm(O36=`$bTKA> z+UR-f7!sgbeRL$OQaA{L&qXW{1jXe9Rp4So${?euuRos(otNU_$>7IaX)&p)Z{!6hUk8DyxcY&(0EK@AJjLrrEi0cYI{E?)@^Bu^RX z%?36&I%Im<59ubUA3K1(AfPHHnz*7Fv=^P>GiWbz8!_TLT&hVI0w(v~1QLr$*MhC1 zNgQoi^V2H<*c{DXf6@D%I=3EQ(kL;DDETGm3;PA7#q=;nP^Jg?#7b_CoEXXGPKHbeN2QyTQS=6!~a>tV}ww zki!a3yi8wvnQ__1Rb?!rOC5g+qJWI3Pxk^{UZ2TbSkhUfR!(EN+(@BiMqN0{;0iPU zQ#s1zo0m)>-6(?PGAphqml4ETRN88 zu7}%VE7T5#iOf)Qh0R-e6T(GR#}ZF_AYNuKeTsQSV3z{2AsB++LR=dR#cc*T-vTma zP-q%oa9t7x3LP&l94BkoCd`wBbP-!YKc2@*uXg{3MEnHN-Q=_Eqsd4NR8tCZKgA## zysVS}!6Jxpo27ZM{*B*vu97ha^2ZJZ*RK8($So`P6eQhCwE4HS3jQ_r``*kyf96zj z>L03VrY}0(WRXm%iWHoEyU8~1WAaqzMe1xCMl;tY7ZxZg$Z-ur)Qwbh)rI8BF5k-( z0<`zI#~Uqf{>yE;r}&p}Y9J$+quiuh&Yfb)+cXpH!Fh$`h7>tn?*73FeNs@YY(e2w zkAhd<1}On60%L(}7=UmQ&xH!@o`*Q#T>ob4Bs3*j|EGxAS}2%guq-uVEsf@C^$T^p zY~J^&iG!)I;l0&OM_VvEv3Oad?Xmr(DY%R)C)us8jL`*Xs-$)r4U617nEX%N-2ICi zZ+z6ndvuAc-PMDkPn!NGrLr#kCoi!7B{a59(_Us^+a-(4Q78nB#ME&Qb1+UB-bIp@ z^aG2O)JmcwH>>W^w#Hz*MB5tUlEVm?;J6$o1Qd+zAuTS^dq2QoimlD&L(H_1Y4z^a3T- zfT_JZ6~OaL3Y~aCp&V?&*d^5%!&m6&X|l@Upc+<{CM~n^kcY-y26``eEyp#ICiMg7 zl+=>Y2DaWY0u^|$q?j@=cu-!FFnE#H<|k@^M&6)x_hu+B9g0YbJk&sXl8^#BrQd+( zEQ0+{)_c#_I85BL+d9QLVy`6j$^CgKAZb+o&{8lC6Hoyg3mXl?Momia#^#T$p<(iQ z->l_-qG5|fE7PpGS{g4CKv=)c21id;ocj#Mo|LX?yr{8BllQY6wslK#5?!D6d9p7M z#6zKxCL9EXgir@WMZ5JphE1EKCwqE94mfKwJ6BMTAd7#;s%W6a4AUg>uFdK= zj<>dIe~ZqH7mtNWypiBoHCiWoMt#u6hh4l@lRFExu91bg%WL*uSi0pim4 zWW1v{B+*=Za4k%Z*ICY3WC6K9tacEEleQG9zC^*x%R=L%Midi4J`Es>u?4)k>M#M6 zYhe{o1779f*>q!t{o5HmZtZt6$Ea6P5ZNZHU5>P&elFZar+cX7eQ)_zZOD3z9vx+{ zlEyyT=Zl5tgn*%sjHl>~7?c2g{dDZ=glS8YQY^X76|Kfe3h@&oqZ_(yMlP^)K_|Lg1pe`ANS z;6=8BVC$usfo!eY;jtO<%k$6OcivK_)1&99R{1)gn@#f2EgiQIvr)cT*}lt#Xq1_Yh|;J% zb7%d#dhk*V=T z=qtOHtKUaV{B3{mQBdmBaEd?}Wiq>sSh=&L8&@+8rlE+-0n3HQ@A{xc<|V(N@CVe$ zz0Vhaa`k%Zkqx^Io$qB1m{~A*BIjCT`U2}jx^Z;|p zHD}DXn(!Ke6A_HgCtY*ce?I)*$t{H05Jo3W*Bs^_V}QBFx<7?Q(yf$SNl2zXrnYygxJ{*B2@N$`zI>=_YP{`vW5V1I8}gy}7k_qDeyE|x!@d=+_gfVpoo}2DVUU2?XkH61aNPO-p}(RqIajWv z=(P)8cuVJujWj}x(9|)G)0f&veq#K{z)USSU$!yhkd`JVqSnbiu91bkOH@%aiU(;n zJ^2uGpXfh=`_QCKqI7?yO;Ue@2(mC7z%4(EPENGFYJ!XuT^TaY*)ulX$@iS;WeP(> zslW;&fGXI+ZiujT#l%-xB-tYX=T<@a6Rn6tONFAcwSs3VTdew*&O6PXPdK!0GhNv} ze2Hc7UBP^uPQZzhRoJ@5qNfX}G0M5WPB9l0Pg4yEGO`x7Of1_w1Dv_~&dlnYGCA{Z zyW#4@&X{Glp~_tsN1YB-l;j$G7(NXj9j>(oaf7IvF1P|y-NfZm6Pvr9mRNJ^)YkRqSyy3TY_p;^lcXi1@jJ)g-AdfhfQr--* z;wXYwg3v z9NZ$NH6vH%hA%L{G6K9%XW>^4WK6t!*@)Leu*7gfd!A;NZVrk<>8&P(MRK0L`Ey>* z?fPHiI!m-A(3m3UTHCv95jc^;QEgZkHWBgfuIZOb-g1O8tcOX?cNDa`LnnR)pSSoPk|rZE3|wXE@6f#83-{Ta^oR0bm;l~arzS6JlV G{Qm*sZ_U*J diff --git a/summary_results.json b/summary_results.json index e087dc556..da7802a5c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -264,14 +264,14 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "Nodes": 1, - "Processor": "AMD Ryzen 9 7950X 16-Core Processor", + "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, - "host_processor_core_count": 16, + "host_processor_core_count": 24, "Accelerator": "", "a#": "", "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.383226, + "Performance_Result": 0.278282, "Performance_Units": "Tokens/s" }, { @@ -300,14 +300,14 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "Nodes": 1, - "Processor": "AMD Ryzen 9 7950X 16-Core Processor", + "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, - "host_processor_core_count": 16, + "host_processor_core_count": 24, "Accelerator": "", "a#": "", "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, From 2b11c0c8c22cba9dd2fddd749257997093cd2b16 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 15:23:20 +0000 Subject: [PATCH 16/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14417 -> 14417 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1f31dacf3..e761c5668 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-01 12:23:02,615 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3270 INFO] --- -[2025-02-01 12:23:02,615 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3270 INFO] --- +[2025-02-02 15:23:15,180 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3286 INFO] --- -[2025-02-01 12:23:02,615 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-01 12:23:02,615 submission_checker1.py:3307 INFO] --- -[2025-02-01 12:23:02,615 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-01 12:23:03,659 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 15:23:15,180 submission_checker1.py:3286 INFO] --- +[2025-02-02 15:23:15,180 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 15:23:15,180 submission_checker1.py:3307 INFO] --- +[2025-02-02 15:23:15,180 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 15:23:16,244 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -67,7 +67,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425185 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0ee983639..2e9c1e130 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -557 +558 diff --git a/summary.csv b/summary.csv index 6c7f293f5..494518f6e 100644 --- a/summary.csv +++ b/summary.csv @@ -45,7 +45,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425185,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 568a07746..e1a389bfc 100644 --- a/summary.json +++ b/summary.json @@ -1389,7 +1389,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425185, + "Result":0.425199, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ec256c5a088a1676a68c4826cf239ffce55d4514..46540764455e0cc026c746f10eef0305e1e23ce2 100644 GIT binary patch delta 379 zcmV->0fhe1aM5tEW)1{0P3qvYY7WT@1a-V^G_yAv_!ocYW=riP0RR9{0ssIJ0001F zZ(~q$Z*X%jV{dY0E_iKhos!FH!!Qs<_x%c^y)8d*DZ+N3Em<{Cs7p(CLYg?D)&ohc zlfSR*Bu)Ze6||A=oild^Rja-1qhGL2XjP*uNl^rfX)aV(qxW_lKcL8YsyOYnf*Kvb zp;dENF++ctw(x4T0qX_0D1ekB%+zS-y&)JowgX9B5*#XsTWuxv;c7c?;!^%Rf5*vv$9LUJ}E&5!LT0B0C_kf6N7S(4$4*MpUAv0zfJoKpCMfko?CoZWjT zX5%mn$xuwzkUztpo0qp~)L5u-RSZy5aYmR0>a|C0Dtvtji|eWLo3P@o0RDIg`oi03 zkYgS2CvynDE0R-n}OYM`=H-9)-v1tGR delta 379 zcmV->0fhe1aM5tEW)1{tXzHG`Y7WT@1aaK&Ftaxr_!oa79ACX80RR9{0ssIJ0001F zZ(~q$Z*X%jV{dY0E_iKhos!LJ!!Qtp?|llRyDizKr3l-Bw&c`6p)M`G32BpvT0cl? z9lw1QCvg(+DaX~!emlFUTI^Jp{DJe*=o;l2K?!JXgw&fFeYDH;5hXq_Em&tX)Mx;Y z7R_D7E#-g4!J9J{T#(?C2-2Q%Tcd3VmSXJr78LVYbm%CqjZ-Yd(rvJ1{D*A-7lb@u z1%U|`7~TU@I}y Date: Sun, 2 Feb 2025 15:25:45 +0000 Subject: [PATCH 17/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14417 -> 14417 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e761c5668..297962b2e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 15:23:15,180 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3270 INFO] --- -[2025-02-02 15:23:15,180 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 15:25:39,955 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 15:25:39,955 submission_checker1.py:3270 INFO] --- +[2025-02-02 15:25:39,956 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 15:25:39,956 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 15:25:39,956 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3286 INFO] --- -[2025-02-02 15:23:15,180 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 15:23:15,180 submission_checker1.py:3307 INFO] --- -[2025-02-02 15:23:15,180 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 15:23:16,244 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 15:25:39,956 submission_checker1.py:3286 INFO] --- +[2025-02-02 15:25:39,956 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 15:25:39,956 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 15:25:39,956 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 15:25:39,956 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 15:25:39,956 submission_checker1.py:3307 INFO] --- +[2025-02-02 15:25:39,956 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 15:25:41,044 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.027 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2428 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1692 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2e9c1e130..30ce11587 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -558 +559 diff --git a/summary.csv b/summary.csv index 494518f6e..ca8429ee6 100644 --- a/summary.csv +++ b/summary.csv @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.027,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2428,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1692,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e1a389bfc..0a61ce606 100644 --- a/summary.json +++ b/summary.json @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.027, - "Accuracy":"acc: 76.200", + "Result":10.2428, + "Accuracy":"acc: 74.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 46540764455e0cc026c746f10eef0305e1e23ce2..4f962e7ad100f63d12f1fd5d2b5f3c4267825ffc 100644 GIT binary patch delta 1066 zcmV+_1l9Y|aM5tEWg`X9LI=l0vuY#S1Aii*bU-B{(UC5|jPfw?YP$3x&3H|18&CM( zHRAFfCBQu@B}gfU)&uCd zfUgPgH37aRz*jEmr5xaE0emfhuLbb6B)&8l(|Lzc+);TVUaz&Xf2i~@d>SihlFSBE ztYTLs+qM}igjEl-K!_uME*bcf$s0<6kcs{xjHwI&RM0wvx(0AuZKHL?&oMDIpw{la zFFx?TTfEKefw6dPYTEYq`ySTzN`J^SOX3g#GdNJd48AB}>;w5^weSA6$DW_E^_Q|8 z%gy%ntPbpb^$aMTh_{4{(V~8~<(EL94crfHjaC!ox*@>b%J*)FM3=MwpcLx?8>P6p zwjB=4s`*55K(J3Ud-p-&fL%-Nhn_>$lux65!jFfqVIRRYU&IZ;RAiYJb&neb!;iUneX-bxDI2NKqoxOv9?3?neW& z8vXkd`)~n(QZA|+#`RwRrGE)>k>gtW6Ter5b;}}n# z_Afh2H18}o?7RgyDsxt{e}u}A0W>o5UgJIc^HaiN*eFj4_p-)l!WvW6<(cK%xT~$# zO~MwVI)kXhYOjszY}K}HXEmuKA|TqjjnHNHc5vOj6%t$edc8@voDR9kMqJ>zc`Lcb z7(e|Fv(71D2!GD_p5i3|002+|000pH003lfV^DH$aC0wXZ*pZWcx`N*lFe$vFc5_A zeF~$yE!(L}5w-(u$*F-tQ(AHp(#8?Bevs5UdHX81<0RlyK^JLeznxuFE)TMee!)7S zRfV!7MG+{bxlm1oJ~pfP5k<~Z#c8V*ROko}E$h3I8GpjGg*U4WSTDdu0i+yZrb2t~ z4Z+y4JxJ=3;7~!_X)CD@SKDAi**9$f=V|(YC3woIr}zYn%|t{alCueEer($VIK$9_ z1mzvhk_=zG9;|e;1-)|Rl)@hk%v#UlRz9ag*(z@LsmpLrVx8R~#XApjZTo|VtV^UG$1>O9ShIL*V^ zWCh6!a-VKbEpO^CKsh$~H}MP@a^HS}`VX^oGn@kj&_V~tM3WUaR{_nFZ8tgrKa;38 kCL7N9p5i3|002+|000pH0000000000004l2Ad~4gXtrGNr2qf` delta 1044 zcmV+v1nc|JaM5tEWg`W6!z$lJvuY#S15~yuntw>8JJJQ1Q645=CB2jbd@X>l1@N^1 zzLvz724gz!Fp4`WPsHoBR`w5-9)?e2B~6mqV2V}js$|PP zDG)NzUxYE00e}ixhfvo5j;n37uJ}17rUum7z4yfj-gkeCx0yXK7OzcB+x~vv!`fa6 znPy2GB47pw3Yftc1&n3zMj>Ay|11Dr4#X%kTF`+&$j#$ z2(*Fwp{>zsqFgrwxLf(&4Uy<__8*jDJz%30H`lhqfmt=5C=LkrX=d*}NF1Hd@$fb5Be>>^SR;td`n35n)(B#=B%3W{JD@dC$#z0JI5$2jfUr}?tI;Sa zfohU`N1&H53DOrePQbyb%qVT)0nK~!S3*G6@=YTLH6n$!^y z5N#mcM(DD8JGk!N3W+U!z22l-&fH`pF7VvEm0V+tpZ*84%_(6Bf9GaP?IZyJ08jz| z01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%WA_g5JmU>3ZuO(KX56+cAza;HBhKaOLsz= zIHJ}ANv)H=uk0jF0$vrgk?x%{cLr6fz3ii3uuf=Iqbx~L1d3@cR9B<-b{#*U$a$(b z?X`j$9l)Vgb5}7#f0(xLYPA9D1-K}Hlq1a4Xy?5l7(2EDNnH{gDu`QcCH3KIJ8UTX zrXAosP4BS;PdW7zkHFZ>M07%OHY3fC?I!?d7sg%Ldnab&Fbv61OxBP;!=Ia%w`tT^sBu*cP*ZV6m<8&!M{O#6eF=-}sq>q#;;jJw zcnJE!+i8$v9q=dwAji08<)e6dZr7;E)4Yt+Je)QwNnVhw{2E){)L($|)a2j9GvJu} z_7gOJvvf0@0|j`)D&Iwu6*pG_&Xa97IsrhFs5d4X=VnXoBmn>bPyzq|5dZ)H00000 O00000fPo>C={IN+7U`(~ diff --git a/summary_results.json b/summary_results.json index 4d0185967..be1e14456 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 74.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.027, + "Performance_Result": 10.2428, "Performance_Units": "Samples/s" }, { From f03045c7690033c671003eb6859f5a4afb85fa03 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 15:29:23 +0000 Subject: [PATCH 18/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14417 -> 14418 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 297962b2e..807853846 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 15:25:39,955 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 15:25:39,955 submission_checker1.py:3270 INFO] --- -[2025-02-02 15:25:39,956 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3270 INFO] --- +[2025-02-02 15:29:18,737 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3286 INFO] --- -[2025-02-02 15:25:39,956 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 15:25:39,956 submission_checker1.py:3307 INFO] --- -[2025-02-02 15:25:39,956 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 15:25:41,044 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 15:29:18,737 submission_checker1.py:3286 INFO] --- +[2025-02-02 15:29:18,737 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 15:29:18,737 submission_checker1.py:3307 INFO] --- +[2025-02-02 15:29:18,737 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 15:29:19,767 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2428 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1086 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1692 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 30ce11587..c7884c020 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -559 +560 diff --git a/summary.csv b/summary.csv index ca8429ee6..2c15329ae 100644 --- a/summary.csv +++ b/summary.csv @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2428,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1086,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1692,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0a61ce606..a9e4b8e7d 100644 --- a/summary.json +++ b/summary.json @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2428, - "Accuracy":"acc: 74.000", + "Result":10.1086, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4f962e7ad100f63d12f1fd5d2b5f3c4267825ffc..d10aa16dca2380ecd661d686de46077705665d47 100644 GIT binary patch delta 1080 zcmV-81jqZ)aMEzFWg`WM%W7{$vuY#S1Amrk>VQf_q9a{^8RcQ()pY4Yn(><2HlFam zYsBR}N`QM*N{~_xtq1x84D<&W=npW^A7G$Az(9Y10lq}?bSVe;5`ZrO_!59G0r(Pt zF9G-xfG+{~8USAd;A;SU4S=r!@HGIw2Ef+<_!tfUjKAOF6*T0{B`0Ukl)CNqlKArt=P?xTEq!yk2W%|4`{+_%v41B$*AS zSjDbNwrw+32&*1ufe=UjTr%({lQ)zCArt*Y7*iPlsGxNSbq(OS+D7Y&pJQTbK&{<- zUwq(ww|JY`17q>p)U@sI_dTrbm4A?Fmc$_fW^kZ@8GKQ|*az~-YTx~Bk3BzS>n~+H zmYePCSsmE>>KRZv5pM|@qecB}%P)aI8@M0Z8m%VEbwhx=mG9jUi7sdVK`GV)HcD}G zZ95#8Rr87BfMB0y_U?nk0lSvk4?TyhDW69BgdY!I!#;v*zKAt~*sM>RFMnf=AT~>~ z*+RAhS_74AC$xieXHU4kfKDWnTAz6-H!%l zHTw4__Td5mrCd}ujO)GrOMesOBFDA#Cxq<}26A~K=!OZErh#xb5e z?O%45Xx>?F*m(K188LAy~ca?=ck0luu+~8?q!Y9gf*tB%QMTjaaUVc zb$JrD7}Xg>C02WFRA;NUZ9A(;9T5T1)@_6?ySIbu?yZp6(%0)vx)0^dO*Y~J&&^xO zHOBbqf3wXgVF-W4_!LDY0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH!!Qs< z_x%c^y)DZjr4hCRZON*ELQ`706VfCRwH`=no&0?jzmh;!1#P5z=ggf!zB7~)Ku!zsyLv! z6r6GgEEq^vm-zA$7uPT!)^WvK3F7e>^&4-)AblN(BnF^;+)??MJw0z$s4ntip5{e7 z8#1TGf|AA8spVDu8K{OP|0a%rKKJz}sQ&=7bTga-1&GUPZ$*<8H&+49lWjLT0YH1ljx2aM5tEWg`X9LI=l0vuY#S15_fQbU-B{(UC5|jPfw?YP$3x&3H|18&CM( zHRAFfCBQu@B}gfU)&uCdl1@N^bzBCxqd52NlQF$U>ueGv&sPr&=8Y^j%%m!1eVpk>GwizpgRS&a3h$DY4 z8TgaQ8%lwYiT)ytsSE&A&^m;=25?+$qjkm4F)=lu*6x43FFx?TTfEKefw6dPYTEYq z`ySTzO2{-z;t&BdI8eY0z9?Yq1NmgN@BX&Oo}aSym$Dtp&Gz-I4(xsP3@Dw5w}gz* zqJFmJmq4Hm+z)MyRukpAA;8_r_il(pm$Uz%6zc&SrMS7a9S+Q@`9yI*uun64_d()- zT}$nUofFsm=Q2X=*Qpela| z*;r<8i_~0df7Nk))?v$ECoDg8NrM$gQ6kh#!>WIs?neW&8vXkd`)~n(QZA|+#`RwR zr3rG8<68O?!uAIPxjYec!-Psx!9_ET%5$yi*N%SU7*C$|FFQ*#?<_a$yahNab5^o{ zgvyWsG&1sD<30QHQ^I1{C{GFZvc_n_8dKHfndRHKtF6~f!WN@CgQ&!6uZ`+#)wXSC zHK`;cA|TqjjnHNHc5vOj6%t$edc8@voVm$HT;RERE4juPKm8B0&M9FCf6n-x;w1q9 z08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A&1%Ci5QOi23ZuI%+o?+twgYX+sewXM zT5=Q8#u2rCkkmSP`zp5MB;Zp)7ing{on2He53-GZ!8)N;g|Z|?5h$j)P)&tCHmmp% zMb1;jX{!}f=m-ui>${Q}f5NndH>(X;FTh0sq#R+ULVNEG!Pv1qNa~W{P(j>jE2$4x z+h9Z4H*EmtY5IUAc*?1#_ymm2L_{N!vk7T_Y}*4k!_a~RjK0 z!XFLHTF>I-J~%NMyRJ*RMQ;uHGyJuFeeXw&g*vT@0jf*R2(v)FNVau}uPh! zR=g9ypN>JFc^d~A>VQWf02$(*mCwcV%Vve@Jk5(Z&BNJb1<4C?pKebrZ|W~VIX3w> z@eCMp-+qGn53_YMoC5{WLI=l0lNL8u0nL+cH#z}7lc_f*8_xKi;w1q908jz|01*HH g00000000000DyrYlj=7<0d$ioI3orxH2?qr0MT*l*#H0l diff --git a/summary_results.json b/summary_results.json index be1e14456..6a92db746 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2428, + "Performance_Result": 10.1086, "Performance_Units": "Samples/s" }, { From 99ff17591f5694f256efd8fa3c37f181529460ac Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 15:31:29 +0000 Subject: [PATCH 19/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14418 -> 14418 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 807853846..891eedab7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 15:29:18,737 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3270 INFO] --- -[2025-02-02 15:29:18,737 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 15:31:24,190 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 15:31:24,190 submission_checker1.py:3270 INFO] --- +[2025-02-02 15:31:24,191 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 15:31:24,191 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 15:31:24,191 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3286 INFO] --- -[2025-02-02 15:29:18,737 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 15:29:18,737 submission_checker1.py:3307 INFO] --- -[2025-02-02 15:29:18,737 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 15:29:19,767 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 15:31:24,191 submission_checker1.py:3286 INFO] --- +[2025-02-02 15:31:24,191 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 15:31:24,191 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 15:31:24,191 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 15:31:24,191 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 15:31:24,191 submission_checker1.py:3307 INFO] --- +[2025-02-02 15:31:24,191 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 15:31:25,247 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,7 +50,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432666 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432193 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index c7884c020..faea12cbc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -560 +561 diff --git a/summary.csv b/summary.csv index 2c15329ae..6e467779e 100644 --- a/summary.csv +++ b/summary.csv @@ -28,7 +28,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432666,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432193,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index a9e4b8e7d..ce6ac8ec0 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432666, + "Result":0.432193, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d10aa16dca2380ecd661d686de46077705665d47..a6d6bed7afc6d69ae0aa98a1528b4c8a97636e92 100644 GIT binary patch delta 4225 zcmX9?dpOhm8{Uke7ILP{rp;-4m66I}<`6rGh&9I?b7&+<2pf_^$~H_6Q6XnGr-~Gz zEajX+2Stu~lTf7keZ9XwzMp@-*Y#ZY{oMEcd7kSt6gm{jJ1rqR2(eXY2$J{%VQ62l z2^BBHc04;-Xzyhic<%a82kf|&8V4uM&hkRZJH^=C^ zL%wbf97EO30^vWP(fHuj-A$Aej3>}St!H}OKPcV`qKfvMHi^?9J85<#C#6efAJc!K zLna?mSUBT84>;41EyLO)8QNRc2S5?nzmxM_7&*_Zx<{3LpHuD znw2RH_uuPy%JyeKIQE=k332ds?7k>pC2^Dh#J;C=1*HYD+1wHC3?_YMSbA7UVtntPoPt0C;Ry*6rpzb4+b8-$veZE&%sQvI}rlJ6II?EPar)cg$f<~g1Cgt>c zs@A`eL%hLtg+lIBP~=s_>jre~DLP*v%bAvWO(_)EEICk-O5{ez36XJ=mB8U54?PMCiBGKmy$| zrL13AjUgvxXBVF?OEB(`^=?hNSXu9M>i6NRGrt;W-L!=FIo#}9YZpFH7h3*ZWF^RM zYOd7ug9Vx!vU4n{CyEFtAJ$GBfinloplqSYh{eoA8rYNX2I>pp6DSmUGq_q!XeBB` z-ZtAw`zScF86KwITB8$qDRp5R=TA} z&~m_+rN|NtW2w6_pzzIyY7U@z8>Vf&>kTcxsz>AAXXZLILn;hbJ+j+wzp|-tVK2Iu zcJ6g3zggm-qNe0mVq|IFR(y7d zqKc=jD-2P>ZuRNY96wY}wMBdBdZ1-v!t~I|+f}92=fzxc%?k5Ld(9f)f$FC9Khv^p zkCxYAMw6MnFD3KSSGMj%+V&}IFczlNNxCQSH(rpf6bUn~?%cGr+i23;>mh&6--sLc zQV7(yo9U5VTf&0(I6Y4F5idt z9K}NJb0mDXZvb+(nj^3fkSE4_XQrG~t$*N*a5VrGVmeO#@uY!qsrKc5=%H&V3hEr;Jmv|AL* zTO;Zf9M0X#5SH=p4jbs4#H#05-#5=r*A4iDZ8h)SX_r@W-8Fyw^Gsu)lggo%L%;nL;Un&{fM@P@cX99ejLXZQ&e$vID;t0 zhlMg|P@eCCgy(!FBS55mNaJrn;G4bM0*RzMD<@p!`sS)nN*AcQ^S{gVG5og5F-@+O z;I_ehWmd>NnJKAVjTVq0&1)qN5WvNyP{750Oeo-Dv@25V{%RDVQ7Cj?CcZH|T*8ZD zwJ!8trE!5x?LlDWN-*-5;7=xfW}g0YA@+R$Ka_P%ZIPC0Z{(|bZIh_&rwa7&IUaQx z2JOLPAux_a(=}hlfd4Pn4xVjJ9pNOUbVjP#w=qAAZ5%MV@8xofgsT#BZ^hP2I6SYa#SxA zEln<3>)Zj#!p~bLu0mXK5pM{a@!KNZhOy(qyNCK2(5w%gn-#uHGdFIHYR#UCbH}(( zy?!ftXIMJ?@;=)e?VO>XC_@c+ zbx3Xx{)5UBGE_0_d62=$UG;|4imBk@bCD)jdJt^cOSB#`Klw8235cKU$26~}OC6v4 z9?hVEZw-{KDGo$?Ql3ieDZe;SZJie%{fwL#?XMjXDfQy)zwe!JT~a^#vg8V>77CIN z&%}jm8eb7ZvBM*QipfC57ocRIqRk6LE))-pyp~Th$%f&k!$-4f;EZ79arE2V`r+asBbU@Y(xi&yByX)Xmei0kVvPqI zAag~9Tw-k{F`&lHEI5SxA8890XR!b@1=Jy4`C^ZOO+Hy7J#Xey`TbL5d?EsV~r;SD(j!uom_2aaMM)`LZfd1P3(-y zteJy+Ki!?z5BV5VFndCPNSt|DKYQXuer2|yPJiB;4)tFV+VdN{<5&|_X>$XJu_};8 zfs1MTUAymIBFtk&pMBa_;{Q=HqW7p{XD#wRV#RGUD&EX*=psU@#(WH3!p}(03Ju-; zj9^Ei*^ZY0Y>XfPTq-yS0GI0Ptcn=)U*`!C?$~|NMXa}T)4vGE7)1nh+}nrrm3at1 z!GC>THUAXmtrPAM0(+<%zNa)b%w06dh=;mDLA`{_r%=k9&Xc0N)l`#e zkmvmOXjr5P5ABn@E5>L$f#6!nFOSb^N5sMz?VpP;x00VKbd|DVvm$m}R6iF=s%8_% z@mN{P1R!oNXaW$I@@WB-gC7Ib)rqao0FQ|_`}%53PMB4}1CwA+7EDg(cNKm`{D%uF zBk-JPscpF{0Zm1zJ)Zc+a`MIcJC04N2c1Hy=6wuoR9~e$$1I+WQ@GTn)zoFhKD9rt-8^jY z0kDjfV)tqJDUq}UpwdzQO`Z_P;Z%IB$S?r$3r|p5pRuhi1mzq^7z(+$Wx!8KlPV=3)lbEI9b1W=z*)VK+M86D0u9bbF4EdhRrvr>H14!4lXMpi_tZ2gm1IlOM*JjrW+K-%2BqmNF%9SN_P<(eG!zCOhKpBeW0s!HXxXjy!tVYv{{>Q>vxV zesFotNhx3kIzoGlu@J|4kq9Qa=p_I!8-MdzLf`+X?=!auT~Ou)ZF>hZ8fDIp8ChbH zp$eBvX|>j6|GX}xEiOl5&mGHJjieEEdOskMz^y&yrPR);X@<}?E#WR;-fFI~ybb&f zhuzk`Yj?n_eT*6@GpQ(gx0>DIgp6FJdrp&l??AZUc&bHdgGBr@>3B<0d~%YD!z4iD z0AbQgiNLCIREMVevm0eJ!Ef<4C4jM}U0mTt>2PZmx1 zbSwcep4jOz7k~5;QLiL3)=Xi*nLqLUxS?v)Z-2_JqSwuBB>!*J;R_jE$e*-N|7vRe z!bXx5E?1Fika==F=F`aTbud_7cgNh ziXd-{cmr$`x4b}Swpgc$ekrRy7;c<-MNEM=uVyquZq%;YMPgCo`r&NdLF>lGI2sXu z4A|Hl>(fhsjV-&{Vr8cbgOL$CE?I-9Saq6vI{6#8ah62hU#n+rJ?(^xgUcIP!Ut<< zA}!!Gqlcn3z5&P^PyJ-d59<=urXppOF`{=ZEe3~#0g(FfL^0i62jXea1AcHx&jr1B zVDgXn3w3)6b!^w{3v&kk|2)0lMeM*k=_7L!H^-^CI|#!=j7QHX2RSSLrnPf)oxYWr z4LWb{m#?-f8-ecy{rw6}AFCq;EkRMOP1L>oS3iRBQFT;1-i-UhHy`MR%5{%KB18Xp z8x+B~z282uE2n7Ts7BTabDE6>HI!P(55;*aKKc>PNC`hR*7Q3i01p+{%2+WhfMeTD=Zd$ieBoZKPWgB z5@*yaS{(Ce^X14HjfSz6nVeV14gLJTZTMTi{z8sZAi}Z@-N+y*VVA+t$C-LK3}YgY zyt^3^EU1h(cXz3m8A9!Df)j&oZpF$y<}go+1eaWzeR0#QdqFdQPN7=HC0mo?ARF(U zx3g*gpmp1IRr_gUrvd{PvsBy3fyx%l4Wx~0WWzXfO~_4_@O<}aa`^IlsdN=1W{Hlt zgk0CnX3l14(N}v1FpElVKy@nA ziDKT-&C1-4lrC#P8b<94-dby@`%E;GTIZ^fj4O`~>b|}!=O3SS(j_=`D0#`3zi!rE zUPEJRZ}**jl6Fz(=djkC-q}t>tje;cFe3J?{f3+ICtp93GI4;vR4x+-#xTCMzA qdhq$!ZxPL8^}p)ZnfM<|8s&c)!u6CP|9@uS7?^~6BjQ;`5rJfG} delta 4226 zcmXAtc{Ei0AIE15W6hSq)WaCYIzx}qE*fiN?1Vv&EK?y0A=zfgTFT5|DhZD@*~XBi zRK& zoHjF|{Uu#_YpX*?BSlT~qwU$pzz(djov9)fozj}uv7F40_PfyvkK*`DDtDXA(kCuu zJtWLxRr2;FI+G+e+wZD8+(g=`Kt6F)Awl8cq32{Pv{cjQTl!oDNTBraIA6;2b|40g3mSmKn4w{qlHu`Ax@|%RB<0W z$GbEi`_-*=L0mS><+k7PS5d2Y$y@)u@;Y^`TxIBc>*}P<>f)j`#Le2eh9IGXTkC_v z2^0d!gbI|_zKha;-JjLsp?_2Fy~0N_ne&nmu9=O%(j;a#u;6*i&8_A$ekFCT{7Zqr zvM2$%Ly9tCq4CGa#sqQH*Vv4+wx&|KH`H$*Uw2DV=?*i6T`~(XYDcmFl?YIyC-}~w zz8lLDH8p6T+bIO79Q%Nj#<-b*!!{-#zMjt*Fhf-AT#maA@e<6uyfMG$>l%m*UFo=) zt?v+Kjjp0P1tmJ@ztKg+HBv=ZaOfuFRkG8}3`AI=GMncgO-Df3hEzKiIGgd;F9fB@ zQ$Q7O$GxieTlKesZYbXs%Ji`m{1nEl2)WebkVclWCfbOkxA@q4t0OecP}P<-2--JM z42lJ)LV#L5!O0JzTG7`EH7Oexzfpyf%8D_jLR0aDPrE+FEBnQVp)*X~)-^Z$%~vCv z8Unb)?{&0ho*Ykctg!b;?y@UOf}12^|-%ujwV0UC4}S z`-m%c{Fn&YHB4&`s2w>2D?>(FxqNo1D=J>E z5ay0}K*P32nr-%FsUFt0;$>pg1Va642Y;F~O90d!S=wo5t6hUg9`EZWiuPOUE-5~k zPj0M_<_=!;y>cVNc5l)@t>`8-jyQUu|Gqb$`@mNjpKXabqY-M=8?4N1Qo`&5Ny%7F zIwY2J(FrB->8`(zM%3PiRt>0Ed-{vcAj-odK4o%2l+bpQBh15h_=ta`82co~i4OxB zlCA)p*znfDYchR8q*Ci+ul8r;DuTF5V(S^=Q}^IoDCp9=%SW2tJ?hpxkv!?`A?aiP zw)l-;aSMX2_N0CIGl**In8^(>0Q}e(1qz^w!n5=Va_iSE;7c+qg#EavQ*fYWoTHrO)jiqrG3FJ@^2f!FPIFTo#3Cr6s!!k{}e zSXW6%uK0ud**L!)0^9{S@^jR%j@amBe}gsRMz$Q~ruhY>?#ea#MUNOT&>WI`>Qxy4 zQcci(F!pdabLgT-_yPl5>jeg%iOo4?IhMJ$q zJr!;bVt)_y{3|ae)Ejf8kaE!%gUBx%AYla>M0!bha6JLZ3w)$6#&uIGN|8)^_z~{M zYIkdTeb-TsEd-4#j16Xb_87@Vc1_e5R`eXzlon;!66BEDzKKiuwlK58B_nhPZZt8D z+zc2^yh1Wy|65BDA@2CW?0`d@HOM#Ll7YY8(^5k1wW$2v;RR5)VGksT6|xr0r1s1u zPIphwop|&)lG%tel?gT}F57=@I$mpzbh%;3p+_IDx}aFHPbX)LaV^LIj&Uu>AdKc3 z8*-|n@jwz=b>EvD?QY&CdK7!Umm3m19nMaz!0(I_Nox`K^BE$4Cv`G5@?T(t6oI21wk6cct<-z1CM$trBpMQpINLp(xJ zZem0vKx(H(9^@kC#~SefZHQ)WV0CPk@jf=o7EDpG*zN95uW<>o)00ma@B4kfO~>5s zP8E&+ARdFDigGJ*R>tiNkFrR2?^g2%0J~aGd_yuI3k)rwr-3C=CFKOrkDv+lE zU07=vOlc3tgOB8x!}h{({)omRg@NS+L50)J8N16*?`34e?#|QB>fN*$=X_Xqgz(u)Sq~bx5|%DDbGkZKZt5>g(BkoVu!sn zHp72-tzVCSjcb#Mn=2ae%0F&fv54{Kr?mzo7mT(TRvnMP3RH>w9f1ea5yE=_ntoLx z1b~WiO9fW)*AkZKF*h%8ClKAV5wrixl3CFMBGyNy2RN~|V(XK~VQyx=4W8CAlfK>Mot9snB{;TVc0}ma@bfsp~zcR8BU)zxm*oHnJ`k zcfi%^t_byW9d;NVXjY&Y&)z-?;|BVP8z>kw!434_K65Jq4|MPSx+xnFK|C0WrtXMgwzu3$#>hM1i)s?`>KVp05%6|8D*h1B#)Rw|Ob2=`SZbl3_Bd4&PkF z4@ft8PN96TR?oQoTtol;wj95=eLVZ&)g0+dj=xoIBau*#cC&KLi1Ve#kY<)XFUAS> zx+92JNz`dEVb*BjpPtz|=oyYUgQG*_e!lvN#s5wGxBAf1hB)Fjl`U|_^}@dpfN{>C zSMvgQMn;7vBt7(ETon;<-mwE-NLqNKcl7stG7cLUJ22?L#aJ zo0e61^Ipm4#ao zsCqr_7)LY<>(RZ%d=i2g$G-8LMTSPmRvOZROii4W;ef5KKnipn1A*aj- zlcrW~aYH!=;)T(G=SI(-Ar<$ma4D^Gro9YS|4@G8JI;IaIEc385l6 zE|=QO7vnNSs`0Jy8or%2a!6RM0hCd%w)RYilRq`O@_Uz{MP}F%s}x zU#vZrpZ8#!I4H%Kqg-U96i`F3(FTjl#479Ee<6sBw?dy}8({}qRMTr)Y3k>&5hXYvLzTm$C*kQe{CcF3M(xARVBL5?{)Y2K_Gx~EjT^R`=cd2m5+sPVqc?o-PPGlO*iv5%L#=F^&k z&SYl~N&hZ;dd^>uvsak!Q1yki*tyt<8;i;N^5>* zG_r9oi1_kXTHe^)wmT7hbieV(z+CWN&z1kyty^+jtfPCG3 zdwP%4yC>n_2`61A_ZVjpIak-;4f6;8Cfb;Tc%=Z?L|S}Q8A8u{sj?VyPg@$t>tFU%49tk)KY^Fi|u_sK8;vY`+G2S8@3MWp7g|x*AOGZi>jM?8n)`&IRrRaUW>#2WVRE}95 zcrQLIMAr}c`>Jk}N%m(iK4OfZw5&eRURR=zBo%2m1{)3rG;1Hv#qg(^nXZf;ZBi#z z=h|y!;S%!WpAOk(P4FL6^%}2r)?F;qKHr?P_^9s2z{Q3B7pHSyNo57`qhy^PZLLYl z4F3%q3S_zpPOr*t9KG415G9v{0Ov+c*ls`8)4S>TTx-oUd*0;SQM7C^GnUF6E}3)K mhdxhG`K|Jhr2lI}_4)7WV4WkN|Nm#8p40>W-?08u#Qy=lN68!j diff --git a/summary_results.json b/summary_results.json index 6a92db746..2d252f663 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432666, + "Performance_Result": 0.432193, "Performance_Units": "Samples/s" }, { From 537c9f2c6edf50d74503e91279e2fab39b9f0355 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 17:05:16 +0000 Subject: [PATCH 20/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14418 -> 14414 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 891eedab7..1bb9c8a6d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 15:31:24,190 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 15:31:24,190 submission_checker1.py:3270 INFO] --- -[2025-02-02 15:31:24,191 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3270 INFO] --- +[2025-02-02 17:05:11,390 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3286 INFO] --- -[2025-02-02 15:31:24,191 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 15:31:24,191 submission_checker1.py:3307 INFO] --- -[2025-02-02 15:31:24,191 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 15:31:25,247 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 17:05:11,390 submission_checker1.py:3286 INFO] --- +[2025-02-02 17:05:11,390 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 17:05:11,390 submission_checker1.py:3307 INFO] --- +[2025-02-02 17:05:11,390 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 17:05:12,437 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.9895 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.3742 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -63,7 +63,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1086 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1692 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9578 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index faea12cbc..3bfc51a6e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -561 +562 diff --git a/summary.csv b/summary.csv index 6e467779e..8ff6f728e 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.9895,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.3742,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -41,7 +41,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1086,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1692,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9578,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ce6ac8ec0..37070ae43 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.9895, + "Result":22.3742, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1269,7 +1269,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1692, + "Result":22.9578, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a6d6bed7afc6d69ae0aa98a1528b4c8a97636e92..f4ca126efc3651bfd506c7cbf488e4870c1c0b6b 100644 GIT binary patch delta 4982 zcmZ8lX*iT^*q*T^X-J{5&M+e~rkaE>jD3*3CWI^_lqIrnGs?c3VNkZLudKr$C5@0h z8H9RCmV~@^8A{*O_x<{w<9P1lc%C2Uecjh}p67KPH{<@h`}KD~oMMsJX-M)tkP@4= zGr0sL%hqB`9s_m1OSkc-n|H2an@b)NWV*t}%JrO�)h~EgQ^ImB(n;+To zyzx9xNg`!yJgRmgHalt2b8BL}a4aCIs{C2NG)`}&IpF0}&oRd4%I3ab!R;0ghus%5 zooe8otuN9KxF$bIp+}4rjP5}obVNwNJ z7Yud9n8<|qp7nIyu92aO#oZ%lsZ+lGVP`Kn-*(7|pUl=+#;WTRNbz5WTZ9PhO&c*r zVw3F2vI+&<;v|oRFY3~{x?-P}az^IpgjbHPU7Npu$>=zo`Ay%WPL5_5e4T#sdV0_z z&QIrUp!v>ynCNc3g|RxQ4m;gE`SNJGpz!Q$>*hjka zx{<#wwCx93=B=I4v|rdJ&u{L)3Wb+EnVYEFh&17BNtyZXl^r+VpO?IKhZfa`CkW?k zhsK%L8?9*Vd8ygA6SvFtd_@}F9SVoPpK&w3850;QoUJyHM$Vt|KVa^c-2Ep|XWgzR zHLh{FZ=ZH_FCz28vH5MiU)j@NkLNc2u6DlGoqNyW`^N)9>aWmGn3CGMlXGs1ty536$p6s%uWB(}3Kz2TTN-gHmrzSp)E}BG^ z{39yU06KG{%f`+}_+sKx53o9ekH^% z))&&}a*Jbw_hPO*Z6!N!;kQJW(OUt`+~KPD%d7XUP)#j+!gRU`b#tID+(&n_gUj$rc0%Wu@sM-jmj$Hu~uk(%>BBsrh3|^<}T7 zRUOk#IIigsYn%~KtEJsk`{Dc={Y(n-g3ngJ4;pu(#=8HX-bZKU~v zasg}JFC4}^UpVeH7fv3#{jLUYu64Oa?x|?qWD`i3fbj*-R!X%@x)>twWQG?6V!8X^ zU`RHeXg3IH{a}Ffs_^^3lL?XBP7;?G%mEL{1_X!nvU;8PK_~sW`S0e$Gl}pm_#qMB ze}R1>SRm}!1`JHUOlOk1cDi5YKj@0MZs=0R2J{U7p#g z8Z(eHC$ZZPsPeL#O+mqiBP5>orjn#?t8xQZd4_X#DzrEEJLQG$`gI$b2E)!A5)aL= zLPy5ZJ-?J8BL#LXHPzL|x)II0P#8C&WofwSDLJD5LA4YP@NFkVuN1(@WoOGTCG!bE z3A8d*T~t3dC_Xy98xR)nO4Mu6NuXRj;FYq4W+@)Pq~PL(I+G--1HCNf4w}KM7>Ik} zwHR(qy`Tu@bxX?M?s)NLiMCr4@X6?{uvAl3gWj08^^>XG1obNf!&kvfBwf_6n}~e>W%<_Tv}|EzSMeJ138Op`aaMPu1YC z^NyUvXD$hMKk#esy(y_|n3TKKfybfV08pP-8m?dGRWO-{y&~e&<-jU3haX18k5CixTmBadxHRei{0kx(-?@5HMWR;v*E=bp?ZT2Gg^? z^Utj`Cb@Ic$18-Vk2XO!Tj4IRAhigN=g!<3#$3OC7_;yHXlN~y84*>uCgs)tgQon& zKSx&tewuj@hC%fk5yS*;tS6W)Z3zj#_v2H(-#xQK}zDa>H0~!?PXUXVAm}CWxAj< z*zwP*7VUX}huQ)scc!#k2yHG_-hI{=ddofkU{W#(H3u)8$Cu)W&&zn;i`wIei8d$? zgM|6^@<0{LV}z%)ID}+@Q^mldVO%O01Un2Ba}2vYaO3GS7 zThO>P6mJ#yc|tHu<3~NS3F!szJn;rK(}ha?NL|e@>-;q$OQw|_>afoJBQBbM0PrJ@ z;0iD<{|`XXV8-D9h={E;O6(BSlNF)kAcMcH5jLO*%%1)EEiu3FK62i`zH-{~_ot1KgKU#-JZJefcZ4I zF*sysQTg_zI?@QI{fOK>&1V^tx-Vjve9yq%)kNUS_H?wduEae;Ckg|%QIgFv zKd6?(0g84EHAPA#S$gZ_V(=-hX58U+jo&Ger$j0h{gn7%250A;Yw-N#KJ6|#f%g#m zkOtku!kQ$78Y~Z?&{>izEE^%+3IoqF_E~5czh(LNS0KOWlw@(GS%Js9*vl$TF}MgT z>X+?lXf>V)t*9p4c`y#Cav2Y7^HK!W_3(pXBxk-8HgbFE{FY}Sd-c5LSjtH@o(Km% zdBB+ingqKopJ(5s6ZN(068 z@JGQ&AQs^AMmj$UJiAfPo1OrMjaF>bvti1IXIlxs=EZXzH|jgNxcE-?tyaT0DjKT! z^ZF@oX<7A=r>iPRTCsWW9yg4anCQHWRuB094#r7Shl32-Ft_))QO_Y*sLC# z-0{$;2GlP%hVxo$)WcokPEHY`7sIia^WYuJmju<&np|C8SHcF%r(8@{>@Xi`VlaU^ z80ju6G|M$86m|)-<^+ahAXeF72ZOB8usaE8t}a$+8K(3WmpGwtH34m>9cB%rGnRJG zZiG4-zXgfz2P$%`BiQXEzrFT39~dD)Uo44!$#VFvvh;||K%O9(0J5{#IYQ~_jkTg1 zX*6vvr(KCg6X)R|{{PN{Z}UH@#9yS=R@HhPd!#Fn@!2b!u1)n1V_7oq{Vbtba!`4ffA5JkFw!Vg|DNU`C(qL^s#>5t-D$5hI47MmA~rKby*EL9#7osL=j=P zQhJPJB4+q%AezXzs`pKhBbJnQ*_06;SkrtKQ)z_vgwHs{QMibiL8I9TWEFw@GAk?j z;zH>M%B#Zh>@1pz$Fom(E44}3i+Iu;--*7dvcH7b|3io=D1OxK=sgd;-0W8?O9oGh zj_Ms|3%LKPzSwj8!rI!0NBhr3rC5mEy04BGd*~r6dQ|fyj57t``4U2;>^*`SEE>Y4 z!s0)q!8H(|TR5g84~W;O>UlWM;y>fWzXDESY%RK2?h%g49vosfc;x5DLueK+GQQs= zx93}zY)?ywP28YMf?#p!V5jyLk27ns>j&ht&8UBk4vn1zO);sV+T zc6A8~5HWREjaX*3@Kg|BT-fK*vA`~m}7r>dfPub z>V?*($uRZ_2Sc~uwI=5(x6@eZ4CoD;#>h{&8DE)<;&r$`CznS-`YI5I# zha!$ZDlf6WopJ)R>%r>3#FcF#7a|zL3X*&ra#h$fZb6uKBymWSwAHd)u6m!+amY&X zD%l1C2TGF(5Cy_6n_DX)I)^K~QCNW^$2&o6XU)`09&Ev%FZy}DOvHoPD0$OgN!Tr< zJzU8ev=?-a`BQOyVQDGp<9%Ek-oB}A+EzYwa^+B|u%6qU7C%C#j|T@T^~By)tSuZWx< zdt{JwP`f}O_|4A_flnZ@8-^SnX00{vksQXC_QH=PG7+l&XIkFM3C)dPnM558iO}^5 zKD%&69Tz&Q0kBdN7&$JtA5YV)6$dcb#sCncMT2wz3N)W2=nABNCGCZMpR3{695aZR zTT|?%f^-Gg-@E+VxOFuw(&mDN{X)0;P*U)Rg{ZF&jC#Mm-zq<1yIvvlTc!Zm^KVOG z(Gh!Nz-2J||J{#~m2Lss$m>e#Kr1prSqs=t_EuH}s*@{~WkmmfzF8^%`w*kV$t%hl QKwq+)iZmDE%AcSA0|t0ZdjJ3c delta 4948 zcmXX~c|4T+|D7=zlI&S3V`j|QW+IcwHpY@|iYSbI&%R_G>ob&4ZkVy$tQC@dtd*t6 zh)Iz>q?Dc9EBjV{)BXOQ*X#NG@%-_A&Uv46K3iYlU?BY_5BmVbLF+V8*Q56$x0h0*qa&NJJhrXgxV(IY(W6nRneU|KNI%?etCh;uMCnh@h$cBZ z^)spZd&`6|pU&Q{AmzQ4Wz*&6c8lvH&Hfe0&`C8vnKO+ct&tk9@)pMYa;&24ch<(8 zR6l*q@5hhKuV`ZBp%n)Dngh+3u$?2osFqc(Rg8~A06C`qv1N*D=qCQ3+^P9)i*GB8 zGvl*WpVv(M$&M2rvHQW3vLDn-9`g>|lyGo;bI-x$$vI?z5$a1yt7Q7<%~?(T_frp) z^?i0dPYPXgY51EjpBxAUHrsy#gV@!~2OCl(w(_X&bF)WalM25DZ_;1a& zppdZFx&r(LDxFEf20KZ5jDb~WM$)TRB>zA*emtF=@*Bn9Sr~BcceKy96PcIj9HM46 z8vJUgA%V^aeVpGd-$GaY`;Z+VK9j%uXeXCD!t4+-+5MzcLGwe43efVfh-dC@O6av8 z-wc(Por|)_XC2a;%NFgAsu!dWfWzlUhoReb;kz6AcRbDV`12oRdt^(T?i?>xRaN+% zFXW-#Wh%0FI7`;hWG-~6+QzoN7wMhpnoDGoYt?>Pp9NCRm<6l$Z)_z4mcgO;ksm@; z3Z*l1+232e%<|r1bH)}fsCSJIT0GAG)-~6+KI7)0oF9w-{p_@y-(vpCr=Nk!nqq~c zKvnAcoKy6xXGe9D8i?TSLW`8LOTi!RS63Y^ERL;I#FkYUJQ+}4?NJ1tt48|inQDyj zo`2Fnj$3uozi-v=fkGD80sTgL@dj#;=|;_I=O|GBQ0rvYH1nj`k`vOg)x8K7A;jR1 zMNe){YW)z4$Ox^TB);>RNWCvB9H;Ih)pF%f)Dcx#tOvjZinrM}!4Gq=LbOeT~K30+_5YBLjw(gtntj|wvHX9xna}cgs z1V>vcW`OWt(P+G13x6Z=`WtA0Z1;t(9ioc}OLS@*&rOi2KsuS4#1NFEcr14a=0XYz z=IsaVzCVmy0~qt?j25oWhQ%_JDaZ(l`A_KMLVxvpa-v;wcLm~IZ1l}kuDFTIlqc1{ z_#*El)Sn8Wno)~*j)NayB_rI#j-!~M^ygxCQL>;b`&=Q`V52#Ga57pJWcTH!e}`L3 zrbqS1a>wPKBh94D6f}jgGbY}m>T|KyU;4q$-FuFX08qXb8TkZ#|8FyDD$jAmeG~*s z<)$ZzxtZ;eP~rTP%cgPi7!X!Y;y&pS`m1hT3cSVucRrx5o43!I*fyo+lzPHxRhLrb zf<;_k2<@(3=(ChGHqOpgiI{s{P$1WuPZMQgZf2OEYeaP%jG$z+*z~CjJr&zOjs0D~ zH3a~hy(Ne|f_JC|tsCRoD|W&@U#uxMLEaiULfTb`5Fti}V2s#}k5_z*7vEySsyf-P*#S^;dC$3R67{|qy0BDSKqtru+JhE@i(D|Q z-byEahQPuR}z58M!?4dxEaj3kSIvL6cY&g)bQT{*cU4#rP#xmg;2RAh>C=@M8CDvc;+z^z{r7i?da7m74;^7b1d)qG-Fus5_ zjG$}Wbc!ML=cX6IS1+lw{}}QVpQ&kFiV#7gL&&8iA-ufK!tvLo`|}6&q%KnYm+Y*} zM_Zv^Eti}BSxJi*+Lu|c0{X#BnLVM!G`=^YaA+-eyWud5GEfRlXCsHL&~B5!PPKNR zKL1(*ktnPULskWrqf&)UGA-mTgUL;>AgPuT`2lr{tYqI7=PO3+6U9RaLD>cq0WZPy z6r8y<5>b4CAmbwT=mJ639ABCwL{N*d6!q zbb4GUnhab!a_O2aQ8Wh1UzKKfIz}L!`k)`-%lzcfUXpNAbC6pp%u=l}2w(Y0B!~NM zeOKR}ll`~g!Tr%@KlIFr^a+b`r16g8s*ff6bp14!e$Qd-G+n_ZF@VZOxc zOEX$hmP}FIr#I}g?uQH;x~`_(Z_t$=lqu-JaQ>kpZLE&E^8(s)84GzqATaqk6 zb}^9ys}lbPX8s|<$V>oPad&tyosNDDf_}I8`j;7P>Ytz)=B{h5UcDqPm9Ed!d-WTZ zmQpteppb|=7ur!SViZ~k=eNx=y#}`SrQl@$BQi(nXKHPmq3H@|&X(bYyh{(y9uxHF z3i{YNjg`tac%hS-qUiY@+oIET!ZfG&fo<-@H!iOR!&%!LRs$@Jlp%JAEbyOk6yY`` zePV#C{b|2&^RF54+#urH?RejJf7J8b_Gf>iC%uNlK93Fa za2V1O{E8uZxOc`JNq5p1AdVz=ir&*TP%{2Fi?l$DKw4I8AIlv*4uo=d&Gen>NYowQ zcT+o|T?PG`zOhwATjp9kJw>H9rAR5Ptv=+1)@gU@jb<%WQDq2V$XUxB2mmA~-EEo2 zX^94SvbPUPA9;FxI{QZ<_7v;W@!?r+52dCB@+9nQHw63K51(NgZOg%J1G%Tu{GSWX z@SSYXg$&C4RcsDnnYaYXGO;@a$}%z97Qy*qGlI~-7PurB*ANoI<4n}wX8R)Eu$*q$ z1IL~!#t28j-b}j!^KZ|E#(eRt4NUtBxk5@dQ+Jd2Yu`}LU4qrbcX-qtZfFl43*n~n zH2&rG(c>VEi*Ax#L$D&2%=0z5HRJ}RYj}=IepqYfu@y6{rj=r{qUE)O0)!Q6ULMY4 z<}Gl|k1%TPs|FV*n(S=`f&`5dw~CTA1&me`6Do&e4LN}_?vJq~=rdF=6fIy}xYfx8 z%D^uf+}?!P;=(==_T&Dj^q4z6Hl%B?ADZ#CbHB`uqHUK|B~f*z?AFz+x6Z#rT~hVT z51bL0MhO8nd29V3CXPy^g603}-*;w5WxryxhYa$v66_3-e{7wqbX(BUb42CT8T?oA zH^xvvH0YhTven5tHOsOYzoJ`Y&AUDDbfG9xk8y3{owPTg+C+DXPMt5`wZ)A{68O=_ z(k;=Ckxs<7JZGMTf2=gfiHqzuz8&cy7e?lLAMno?3mk)Qr!V7Ffu*hpZ>vvih>XTv zPE>jbnN={6Rq#D1kyY@@d-y|qLgFb94v2yFvx3ue>~^BK@_DkSp@Rn+aVgV1Vfsds z($lItTe&37Om5t4$Y@3t%NP3IT1}g1eQk>pw&q@(j4vc)nYGP*ut!fdn zakDF@AM!0aZ{a!b^Zv*ah93Z`k7|sb!y*_7vg%72~eTA7DNL3v7c==iupN zboz}b7B+7dwq&q33tO_AwFG>?W1G%Kcw+iP5x&jB_TyNzdYEU&b4i4oU@Po;?O3ox zt|R8N1@09btI`EKQ<5BH&*803N8Kf&KEQBoJK9`{Q+hTv!aOeXi(Ei}70#MoC&sMi^JP)GeChS!uvcdU{g3 z0_K&#{bpJ7W-g=PU-*B2g$gpACY!69pYlMHQOMVmyOac%rR_96oapp6o%=j%6fK-& z8U8l@c{gS9Kf}+~jT5d9OY`ad-z)RM~tOP45>n>&_%LJ8-dhFBLsOFAw)$BtYO>RHv%#ym) zKgvN+)?S1`|NKMU^ymz`Ys^nNX$|QFb4)k0!h#2tto5XLyHiIXjE zscYixH-2CpV{PPZe0@qPccMdT;Btl#Kr-x;XDv?35XK)zuokDQg?qXfP%eIJVLLV4 zH-5i!n{7w$4^&+~ch*k+xh?Pa8ULgkxHoXVEvclZr(Wj4RstijX2uyIkt9Qh3EM!5y>h(ZN3QKE~M$D_|%7D&n>2FUA$oK z6Un+I^*nLi0&#jqaft~w=F==5=Lyr!VuZA+NfljM%fEWhvqS zc@N71JB;5`Z?7<2eSC^yRXFL|u^M+yZ1RM|V%+5@!wbc<7;TY{*0qxx*VH5;et8g2 z7LGLmZAKnHEr&u=8Hiod_kUz$cj;--T`W<)i@|s4+|$faCRiipE(;}jUr6G_z5h@W z6dobSMPhUYFXj%z(wA+D_+xCBJ(F9WbGQy#Y*8fM@+z0%Pa-hIpE1M-2H9L5XEjg^ zIkivHDI_St-LaSb(Y_Mg229wB;2;QVz&}xIY3y)SkxJe56E;9<%mi;0ylm5A@$K|4 z*3_^{vw)+)NLm#99I(pX7i7(;cyjlqf(_^SDFM~R$@~dR+!MH(%BybTMe4eTR`sH< zd>`vt!k2#b$G;})dtP6d{rw`1-zv$>X77lWM$5CcBRhWhh#E^%`n-MXr?0+hr)yrZ zlLLRdC>L^P_5R#!XxjCdn)O;ST2K(Lx_8pl!LeYWaW4CQbPZ4dypn-sk2tL~4lK~J zi`pW81_dcf^v|;n!4++VLL>7190#=V#f8vP`O+CcexndL{d+iaeTcr*TXXRj(XYmm z`Z6jT?u>qMpMtU#k1NOFdMy6g)&$3cp9LcKW`| zyXqgS-pf%G{YrTM!?2HZ{rLKP_6TB6Dfg%i|LEWdQs0UQ;5%kBF#4LNfRg}vy*tnI zUEcY{Rn&HF(#x2?3F5~rkJGWYsS_MT@i)Q@z4H%aPQ9j5tl0gEqZZ!hYj-Wn)lTpiJB?7qnTDj)msb{s<7AzDA z79C>F!q3K< Date: Sun, 2 Feb 2025 17:08:04 +0000 Subject: [PATCH 21/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14414 -> 14414 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1bb9c8a6d..9f0f18f29 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 17:05:11,390 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3270 INFO] --- -[2025-02-02 17:05:11,390 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3270 INFO] --- +[2025-02-02 17:07:59,038 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3286 INFO] --- -[2025-02-02 17:05:11,390 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 17:05:11,390 submission_checker1.py:3307 INFO] --- -[2025-02-02 17:05:11,390 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 17:05:12,437 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 17:07:59,038 submission_checker1.py:3286 INFO] --- +[2025-02-02 17:07:59,038 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 17:07:59,038 submission_checker1.py:3307 INFO] --- +[2025-02-02 17:07:59,038 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 17:08:00,080 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1086 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1857 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9578 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3bfc51a6e..19a3842fb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -562 +563 diff --git a/summary.csv b/summary.csv index 8ff6f728e..4370ab57b 100644 --- a/summary.csv +++ b/summary.csv @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1086,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1857,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9578,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 37070ae43..da928e231 100644 --- a/summary.json +++ b/summary.json @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1086, - "Accuracy":"acc: 75.400", + "Result":10.1857, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f4ca126efc3651bfd506c7cbf488e4870c1c0b6b..80d8df813d353301ece6b25d71cc5be3cc479bad 100644 GIT binary patch delta 997 zcmVvucFOfW5$^pIv;7b6$1mH^mz69V) z0KNp^O8~wGz}Eoy8USAd;A;SU4S=r!@HGIw2Ef+<_?iG;6X0tCd`*C_3Gg)mle#Ah ze=C>tQV#I70KOK$*8=!j5?>mO>Ab@z?x;Kwuh&}HKU8`cK8=+$NoIp7RS_{jZ)lP+YSe2)qJ8j zAlRpwz55_>z^3ZuO(+aXOWYzNwsRRe{lG;}AVi6d$~kkmT)`zn4V zfvgJJNcYZ}JA;blK{nAZSSPe9QI@1A0>v~JsxHyTW)(l6$a$(bZM1?C9l@bxbyqM$ zn6~g{wE^n|xF~>>Bg~X&@4X=yJGKW&T@oBBh&yd1_2FuNYiua{rZwO^P4BS;PdW7z zpMbHMh-g4^HX+TAZ8`vF7#fhEyu(?N;j`C+mF{Lir(8It@J9nTt)n=(4^B+Rwr!Jk z)>%XT3~$%3@7<`eP^VQfKvlsRVHT*@wkq)DB`mI?&acCYcLMm+G3YnmhC%u|;86%b z`naR=IeUI3*{o2Nr};ci^KdrVf}{&FfBZVNysAF~<Ey@tWE;p76hG z#N|CofO}L*kWvn<2l@jH^amK|4=~UlV4y$1K!1P%zC`kLDF^rxfG+{~5`ZrO_!59G z0r(PtF9G-(0AB;(YXE!=fUg1YH2}T_z}Eoy8USAd;A;YWO@OZn@HGLxCcxJOfB2dJ zUlZVK0(?z?uUyhgIl$Kf_*wv83*c)>d}%PI^A4l9qw++&UTbClQ0Za#G*;3inGL2` z#jZ-WZ8KH~s~%>75J&!8GVmvpHL36mgwVfEGd013AXUK6pu5wF?6-xjKY zsNWu{VOC#q59|urKvn(_va!tG7OA<^{;K2ptizVSPFQ~Gk_Ib~qC}{fhE+S=j|OHn z`u8XH;Q|1qTvRuV>%IO)$F=k)gzXOoa(N=?h6$CXf{SJvmFHU3uO0oyF`hi_ zUv`#g-dS$gc?)n<=B#A@2$dlNXk_HQ#(VbXr-a3@QJxa+WsT8(FrN|h_pYh9RPx3XhDGv9+QmVi#LE%ezIUtuAEAS-om8yEROEI zm!q-kx~!WG)|j8*uhr}OFls9GVO1PZT?$UQ0~U;{OMHEai))z=tGME=1o6~Ied6sj z$gvJY5(Cg<+_UmIdwwNZFHv3O#XQZ6c-G{B=5tEO=Fsw{{sL5|CjTa$0mt08pP>E& avvD(=0|bc6YHyPfH@XD&Jvc&>&o_Uu_TpRs diff --git a/summary_results.json b/summary_results.json index e488d3fe0..40ae72b31 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1086, + "Performance_Result": 10.1857, "Performance_Units": "Samples/s" }, { From 8ac4fd8c88e4409da1de4eb759c6b75950a22ed0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 19:33:43 +0000 Subject: [PATCH 22/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14414 -> 14415 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 9f0f18f29..73b4d1ce2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 17:07:59,038 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3270 INFO] --- -[2025-02-02 17:07:59,038 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3270 INFO] --- +[2025-02-02 19:33:37,898 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3286 INFO] --- -[2025-02-02 17:07:59,038 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 17:07:59,038 submission_checker1.py:3307 INFO] --- -[2025-02-02 17:07:59,038 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 17:08:00,080 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 19:33:37,898 submission_checker1.py:3286 INFO] --- +[2025-02-02 19:33:37,898 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 19:33:37,898 submission_checker1.py:3307 INFO] --- +[2025-02-02 19:33:37,898 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 19:33:39,181 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.278282 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.428593 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 19a3842fb..897ddea45 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -563 +564 diff --git a/summary.csv b/summary.csv index 4370ab57b..593770220 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.278282,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.428593,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index da928e231..804a86730 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.278282, + "Result":0.428593, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 80d8df813d353301ece6b25d71cc5be3cc479bad..5bde6f576cc981fc6998a961cbc359e5289b9a75 100644 GIT binary patch delta 919 zcmV;I18DrtaL;hCtOo^TvU2lL?RvpqEZJUu~~pXtZ^_<;7-9cY8eupU=X3zhzboZLJ5tjKoFI`5yWF8 zb;vq&9oTjus>rIo@BL4qH;+yfVsFTRBw~%;f+S)MCXvt}5(y0=k%-5FB&u~J5`PYC zNMenH!S?1GiLhlzbYf_bL})dltPqJR5UumkC@fi6WFh`Ep zfmL&)smqGdROHP+Fof=R#Fnh8b`x%5Nbk;yL@b@NJ_|!GxJh2mD*tV3b%189v?t|R z6WFe{@2evoZ*KB9lGqE1HH;HhCuXuBo(ueEbr#2H_rI%h5uWRhS0NL!*MBAJitJ^X zKiZ;VqTs#w)hQfdnFdK3WUo~|#WA2&==7q*`&b6Mii4jk_VI)J!DxSQjZdEi^^?#Q z^7?J0dDdkw#RU~sud{UK?8ZswLCl4a`FR24rYQ<(b*`tp&XQ$$rHjHW$+9Hcd|EKa zV`=dzFS?&3+2%u0lI{7Z-UWY9O9KRxfHoSlb`G^81g4z~L6a{sD}O&|Nrmk|Te51P z(3FM1?}V>1!afaGjKnjhQt0M0P9AVGPDvn0b8uLmpLY(cMF zIi>JN1GCmqoZJT|CS%uiNmumNkUzs;>(}>w)L5v~su-ZU1#3Q{b#roztYHzow8oeV*f&o>qU9+T!bJ^^x*B{(AnEj0iD008&&s|5f1`UsRNPNEI=UEI2b5!r{Eg342e!Kh|p?8g$5U)gvL}Lh|1px;xUps zWF5K=Y`YLuWL4ky{-@BJM<)ugH)KE(u|{t}60ru8NN5m=ga(mF#A87c)jARh2Y)ss zvBtq*d-IJ%*fJzKF*Habv>H)Xh(xS0RY;^J~8 z3@Ha_S|qk`LD@%?#)Nocjj2IAwSPYW&yvgdDewZP+?aS^F;CL*!*AetjC>B5BggB& zsyWisWyNSJ^5!2HLiamjOIB682{$pMcV|T+md;t9g&`N*B(Gd_lk%(y zY**X&)e(<3H+dXM>;=Ud#tEwvGg%PN1^%--i(|C=-&MH?&-KTvkO|rAl7DqY_Oi?$ zZBa2%@Lv4t6ppY=gCq^I*D9al7|<$odQswiEQ4Le!Os=@_`&^Pv_H7Ur_X}=N$3iB z{Wj7(>#~>Pf(onGSvqreBVD{Keal2rqR zrZjXXq=_SHJ&@Eo`THtE0&@;Q5c*{o2Nr};ci^KdrV9fG6_GJpI!wY;i71Le@< z-^3Bn=f3^~)gMqx0|b)+I2yB%GqnN*HZ`gdgp*1(TLM@PlVLU$le{)B0lbs!Har4T zBa?6=7L!0XDgnxqYd1OpJd>t3CImTYq{5TXHx>aMljk=+0dbQiI3oruH2?qr05fHz A;Q#;t diff --git a/summary_results.json b/summary_results.json index 40ae72b31..fd8d8ac03 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.278282, + "Performance_Result": 0.428593, "Performance_Units": "Tokens/s" }, { From 803c4a3d9229dde30707f0a17d40d81239d49e2f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 19:52:00 +0000 Subject: [PATCH 23/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14415 -> 14414 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 73b4d1ce2..15b56adee 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 19:33:37,898 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3270 INFO] --- -[2025-02-02 19:33:37,898 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3270 INFO] --- +[2025-02-02 19:51:54,812 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3286 INFO] --- -[2025-02-02 19:33:37,898 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 19:33:37,898 submission_checker1.py:3307 INFO] --- -[2025-02-02 19:33:37,898 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 19:33:39,181 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 19:51:54,812 submission_checker1.py:3286 INFO] --- +[2025-02-02 19:51:54,812 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 19:51:54,812 submission_checker1.py:3307 INFO] --- +[2025-02-02 19:51:54,812 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 19:51:55,866 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -35,8 +35,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.08857 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3379 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74722 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 897ddea45..066ce6048 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -564 +565 diff --git a/summary.csv b/summary.csv index 593770220..387b660b2 100644 --- a/summary.csv +++ b/summary.csv @@ -13,8 +13,8 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.08857,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3379,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74722,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 804a86730..53d6679b4 100644 --- a/summary.json +++ b/summary.json @@ -429,7 +429,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":47.3379, + "Result":56.2054, "Accuracy":"GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", @@ -439,7 +439,7 @@ "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -469,7 +469,7 @@ "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 5bde6f576cc981fc6998a961cbc359e5289b9a75..c865faec5865105d0e994c33ec70cc995c530c16 100644 GIT binary patch delta 3478 zcmXAsdpy(a8^`xG=FBKGHH^t&TOuKcnUQl2DP|52nxa+?Ng;+Nkun?Wq)0(It zo<>2U5%nNPW#>4LM|ZzJSWsjibsxzMUNVM%q2U)jWoJGrZi?yS-snK?}voqiWbnb_{#M z-|QK*Dx?TgmdJZ!o)}vNS1)ij*`Mb5A)QqC%kXeEHQj?*5L}x#O<#n@R;6mHuyvlS z9-cjJG%tQ2Po&#I(~BC_<4?P&|HT!YRGFL46(OeLS9m+`*|t&)?=V*OKig<@Bfcn9 z-dRUifza0dua%cYL`7>F4Dl&Sue@y*bgz8LX>LrznYpLsvK{d~QRyKaW^IP!2-0w+ z#*kdw{Q(ofH@&`Wj{cDrSB>dQ7pJeNs%8oc)iQ;L0?;b&E}V=uPW{#0VGKLcQ~01i zzTm3nBEe=_cacW9Y@TcIP3}3+Yq*{xP?AMF1mactK@UmU(14NF) z1~xAh7l~+7b6gHIn{E9|v#s?SAG0^-eN?z=wBNJJ7THQ3Qeb$a=k+9*?(2Wk2}zBG zTz5i)W9e|bRjRQH8x*?{TsC34R@&1qC{WZV!*f%_OXVbl)SCDbNZW1 zz}w&Eb_=}wQ8)rWC!MO`uo{hi9;{wZZ_^a&U)4J00Aj{yAV{6S1SN0 zx(K9c8xR50DKN6dm3d2)siixWNV3XHPRj(D3FKWLk-eu|si70YKk)TXDf!bv|y2H7++loPjr((Si-U1JLWiX@>97p`oYO`2jDgNeK=C34L$!k0ULH z9-^$3=3Ta=An9+EqGS9@{8XDc>RkJPaY!wIDb|K1qZJR@5xlYc`#m3R$`5_`%dkCx^uoD_YwksU(P%EgDRUfb^~! z+17w)!ae6s{d3M`=Qdk$xb2RP->gxx?Nw-$WMyr+=}G;X$&R@y5)DyVN2wr-qFxQ| z`tY*ww${kAdqs_ig2H5iA@iWJEP@RjRPG0{Nl7}TBQvWU30$?e(6n^5jhP3xU=?Mh z``1cBi>Lbj{H%y5pXn}rQtmad-7hiOU%qMkBC)BTAYI&HlfxR};kot6t}-7KILQRa z6cPmjim_GVl31mBP=-7h`fQ;1~Zk(y<_2y7UBD}1q#R9 zVv{{Py$14uxFEh(UGy4(VXZ;4z)0tGpmJDmcoaPMP>QRJn}SaY0pg16-vH%c>%`tv zQzP;LuGkuD%ktvbU8%{dRN8@Mu~g_lDmjfE8_@eBlk;|K3VGP8=tQRH4uUFApMF$X zzEiM(K!`b62#2-N(aT`MMu(O^h_jkx2Wi$CqjPzXXK=-Kn|{!1>5ZjZxgnGi>DYC{ z4-e^IrQ(c8+U2*O-*$rstDZp88#cw(5Qu5BwzY8B6ur^9t=xMpTq@n8WZ?*IEdv7+ zN}jig8qs6Z90)DlWX9MEoch$gOQR#q80J|s|KP+#yH0`A^x5s&93S*MEmb@O{8ozz zY~G(vrx1X!!I`yfl-QIok1nNK!4^L+T?{*=Fqld}G>_JuY06(-_VO=$FzxI2x#1@E zC~wvG%KhfrKVQyrU5XESkhiXjK1!pJ8n@?ZP@g_C`axzvAzi`ecxr_*O2;H*McPshsi0a9|{KGf_b7_;dmNrEXO}d|fY`YO=b@H*=W&4t@S3?W0GW+Ky zr%y%d>nZH3dK9^IOcbeq((mS+#rUw2Mb54=)(;*&K2f;YGCF@YTP9IveD#KSG*UQ1 zKZ+QY?w$ATYcv`SBsM2QreToCBju?`2m8N^u3||E-qCq)O*}xn(Oe7{`RpPPJ*qt_viYngRVUDZjVHaxNx|Zgu ze?h)1-tUX4wQ1kilcA#�nuI%g|_rLu<|)Lm*gYxG&+bQaXkMKD0sBA_DhUg|_Mm zo2?T%byY^Oz3N2U1G_=Wq<_r{HDdk#1fob`my8EDIRDh+;A;_V-Y0IyUSGkdaKsZd zC7UW(hWsZ-BoOsH}KxD*IMy9stOhiuUz|>tekVDDyBMu+L zy*MvrG6x+nU~bXtZF>|TfQ6>8$L7W*MJYMLqCB%%WAgZ-ubbYfr!PtHqxt#~Ih(BX zF6k~&XwYw(VQUPwM|Q=$R*vH`x8t4f!R2todhVo{zCE~QF?|d+1AJ~nL7NCHuNq<1 zw`y;u5NA?~ICl@m=}<;dJN(HK?*GPhA*rPh0uBu>t^ND6giop_lnpA~u+@>*f^K?@+SaCN&w?EkuahzHHt{jmuG~hh z79GhFZK-DRHgfd5B0X1CFl=pgx;H~f7AQjNRgqwB`V)wCDK@3G8@rh8Pdl_>oN#10 zO^i%(#@TylWb0Rs3m#;QK{{MP`DdKM{Z?9XK*5MI6>;N|#$F179;^4%uD%STA)k8l z48nc=13IdBPq3#B6Y#vz4ZgnNsY3*Ii*DddujBruCUBQLq5_y)*e?%M1nEkrKJ}Y$ zci!qWL+Sc{uF9GCXQ5l|Hor6Qj!B6k^TO|UZdiN1&HDO!nMJXhNnijb=J9B*Uc<&S zzQNMSoa|`xWsTqbihEz@g3_gDOpdPfhCSKg?S0}hW%R(EhC2y=1VazTmZwFZlR|^i zp7ea5Tc5f7F)mQCA#{5xHfi6RY7G+ujg_xtY-k$g$=q+&{lp~fSjh`#`@hf5#asWG zG5cq|@`E?qz+N$^^%wC~>+ss~&cT?n&e_qHnzXQ|k6Keqt@!znl~-RAd~F82b2FT; zd|cc(e?B>JWhFk5`^V_vYwQbV|M>Km$93m-_svvKdbT8*-NMAA1;Xjxb?cM!;HH1< zeLTRDTkD=9jzcADhY3E6-DcVp{&C)JD0L}S?%q?(M0>ef1NdZbRR-RV7{5i(rSd3^ zHf){r)A;Hla>~y^Bl>9T?T|yelxwe~wOijkoK=h5uVPs7#OFX3)}s0cu>YlBXLPmN zp;;Lv%=}Y2?0J2bWwj!~q-{UI+7VuXPgzoIALH7&&e5lQ9NtEws+0pL6Jd_ymM1|o z(aHJx=lZHDe@N!>ot(h*I@#zn@&wq5H<&5;8UB*4@K>`|O+uYiOZ?};=h})dh%RyQ z+QrxNLSE~?D|>wL7wiwQ_1X7(UHIGnouCo@W7NRX(21sw*Vj0WOzT!9)O*<}@9xrp zX2lD|^XV=K(7tYX(Lm>lQUB4bIa~f^D?V?Wl+WsM^(A{_V&+V6`^?uZ<0OkuPo`yV zC~qc?0a$6)(f0h5`0p>ppjiX~&5Q*LTM0isMy#GX17~Rr^f69YS_8I>3d^1UrFhU% W9e^`tEU`cwW2coK1icp|KK}>h9Z@g< delta 3470 zcmXAsdpy(oAICTM5IZ_(WHz=+Mq0wL%xub#YtEWmxlHaQl@i*BMk=#qE^$a3$tAWa zY1JetmtyYvDT(8jlapJ9`h9i2kH`1(=j-u)e_pTW`~CgCjd>SS8>%M1W ztw`fLHs`UFg^uOKot|MuT**!YXb9KUoODod{YuTWzS%%B#e1D)w1at_rO%sTLSo)vSk9bmKfa!f zX}z0asU|Y_nXibTAN8x z*p9-cUY9N!_<*-Btse5ZY`+S5NUhSIPTA4T6nPK`W<`A9owP0gAxHP@k?8E=sAGfoDLs7OK!wAI719;OQs5s{fWNb!|87438E7uDRn5;SL#}pOwW~;S11FhI(+8 z2bpG#N2Vebtng@ABqmLfwCIQz$}p$cbm8JzwLs^?m%tUAxeU>TK0^fcXxzsE%>>*^ z?P|SgG^$zU1?Iy`%+2At*&<%^M)z*Ekhewz`d(qALfSzqj1Z7>**_o-Zk0g>p_9I4 zajteEqJRPeIwTO00NHL1&K+H+OHGxjci?CsRhSf|v${x}&^zy;f zrq+%dzB5u$chD!e?xFrVjk|HLaXL%c4{s%r4tpD|F(H`k@U>c1Ga4yw_db?g@f>lF z*xLV@jK4`v0g&15kz1MLb^=)V)q(SaZTQ%tgZ_2nXIJlB6kLq&oUlBx?*Xe@-oM)@ z()ND#p#77YDXS`dmoM3*2wOCjw@f6ELg>rQ7(B|`Vt02)#2&`I5aCr z)XitJPb!6^FW#jYb1qwyti$@YWR-+EW|PiS+B3f_gq6DMp+@q zT(se1By?2r9gxGvP8)X}fNzL7*Yzw$H42tU`>g`g$Vo@*J}rzOs;G2CdDY3YeelZQ zzBo~|x!QwP2o%W*QdjXdT4q8#x5JlfRozLPtui#_q~q$wpx>oJn5>U_)SJOT*(J42 zIrp!WaQClATGZ52Ry#+YI9=U7_937Dq9_Szw$T7+UE3;p=ocEsZ+wpY13f67>%z;7Fk`TMKXgx zr0y5b1VZWTDv(@^lnWz+Mi>9Ep(zx?etF{h%M+{9e9feQ4^@hm{pgCa(bu}hnGXQVRQLGKwMbmy@dP%dtWp$Rfg z;%9+}6dzK#+vev={>%~*6atrE0+*n77a;%afu4X|si2zG)l}9*D_*hJ$o?3eC??dM zO8|zpYD@w!w1M;bySYGlx?S=6V48R?*#wG>iXHl5sFU(|AATh;ZQkcy|1|DJGuGSV zKFh%+=Y51UZ)=akfzoen(%TO95=GWBbje&=e41iVo@$;tb$rxONWv6PRi zqH}wXl|(iz{wG}V#V`yQh#*I@4pZp6>!&+XAEoVuc?dy9BbmLlwlm)%a%3IHvUXQ& z)H3Z$h=g?NCcAW`?~PWcok<_mtKqh$W0hU?@E-teOEQ+|aFS~gkY-^Q?IRieBT>iA zLExrhT1jI+$;r8HHqPzmJT30a%@MkU-+xa%iXB8YrnAP>Ro)EoensZ{W@LsC)Sl2E zUCK0ZncK!DDz{61)E*!GOX=5SD2?jdJ2%&V<$Hs!0U)Vl|8z@f_#xBsR_j8T^DPZj z5loAfyNpGq!WFClLg4(6IFNmDFVPs3j;m}<=>rInsQgbOFhS;#JxQrsuA-HUUb@)5 zKD>s=Q8i6(bIEU=dbzvxQM@;e(Gi@XS<3BbCVW#40^Du`>aZh zY2*BLJq-d{_8^P{Mh`_`={&dbJDMQp;y1xf}GRsx>@@&#K9T(Qm(j}U}6MJ8H4R`V9LN>llwDJDYz@ZUXe>u&XP|9 zvRI`px<@=%OmV%AlpQ$k782g@4ytq_hBb1q+Ih%lPumLE3I+_Nf2pQ%?>mluP;3CZ@HtM{E z>;c-t-ud`wwu^PqgTW z;LEK1hZ}t6`m1Kx7VteGQ|>?wMZOWdgcT`PQ^OE>uR^xnv)V^8{CshnjG4I&+c#fTo(97TpXYV%x-t=a(5Bkthqqtsw^nFGVk|j!%R=I4&ym>u43MURDz{=%b%DtZZ+e;e9ubZj#M*8M{muhGYcuo!kwqU{Q)7uyG z9|bbc>do(#gHJZx{pNk}IVy^L?`_XI&UGQiV#v@)*%SS+oMXA9U9zKK{kE0N+sCJE z?KTtzXZAy7|NDnxXIc1ypR#8y4$62LvQHaov288)fZN#-meyb!HqUbBe^>j?QVR@Y T|80o_$FfmYh6*MJK)~~VS{`BH diff --git a/summary_results.json b/summary_results.json index fd8d8ac03..957e0cbf0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -811,7 +811,7 @@ "a#": 2, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -824,7 +824,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 47.3379, + "Performance_Result": 56.2054, "Performance_Units": "Tokens/s" }, { @@ -847,7 +847,7 @@ "a#": 2, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, From d8a177fc9579c87a236ec83b426ba9ce6ef8ead0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 23:36:47 +0000 Subject: [PATCH 24/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14414 -> 14416 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 15b56adee..0b7b0d08f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 19:51:54,812 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3270 INFO] --- -[2025-02-02 19:51:54,812 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:36:42,018 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3286 INFO] --- -[2025-02-02 19:51:54,812 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 19:51:54,812 submission_checker1.py:3307 INFO] --- -[2025-02-02 19:51:54,812 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 19:51:55,866 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:36:42,018 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:36:42,018 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 23:36:42,018 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:36:42,019 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:36:42,019 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:36:43,150 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -63,12 +63,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1857 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9578 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1795 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9511 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 066ce6048..a13be20ef 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -565 +566 diff --git a/summary.csv b/summary.csv index 387b660b2..4e3f5c477 100644 --- a/summary.csv +++ b/summary.csv @@ -41,12 +41,12 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1857,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9578,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1795,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9511,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 53d6679b4..06df1ef4d 100644 --- a/summary.json +++ b/summary.json @@ -1269,7 +1269,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9578, + "Result":23.1795, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1419,7 +1419,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9348, + "Result":22.9511, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c865faec5865105d0e994c33ec70cc995c530c16..17912712e57208ee272cf1c9fe5eeddb6a2a1238 100644 GIT binary patch delta 5191 zcmXX~c{r5s`<=0crcAPzee8p>XBkV<3|W$8jD62iS+j&0g&9$pVF)2Y_BCZ!l4a~; z36XuTgt5zS`h0(Xyzh0r|J=_x=f2PVp6luN8T6@izepce#+%lCaFTn)L}Vh6N9^hui4QzM!N8TR-8aZoTp6rc# zep(otdoyu5p4xqD;of!ocz1Ghf#EpU=B7d+WizU9ql>ijrVm5*K4Ew2P|-PVrt6(?uAYyEND&Yz{+ z`qn3MGUyLH#?E$`zZA1e+^032N4R{Ok8 z@Yt{U%CnG#w<^4Lw$LBFurs^(f&SDb$2dzvy_&d=qt+MQAJvHs6KSBh|mpGZT!Bo{vU8RMNAITk_ssp;U|*Ms5@o`;#qht(OOmnJTTX22(=An#CVZ)>b^R z{XG{q=l8KyEj#>S?>JfmDuXSt)YBk0s6+Ec$22W6ERV-pCbpIz-^{z(xxN}csoeO} zxa6~P?1$=Zwfec{?^|!G-`M^dm^SigxyQ37IQbo`oL1>1dp!x)rN$boEc_eswubu< zbIJe5(AWDH_{$<`^y4`$JBiJ}(t`IL=1-r+vb22p3YX?Dirzz^reulkCnx*wp46;Ie)bsSzSH!y^mVy-K!6=n95bLGi zIhGKimthIIUp9i^Li4N3TtoA#d`UQ!hVXcqP(8rv zU;RcKFD^a9Xp2X$;pUnIUg=()vcI0!5&5LM2e)uG^&sZt?g{Tyr<1XLNc#RmbdcI@ z-ku9LrnHCJle|Yd^?3iNHi-txR|P#g*3g_jJN@y@{>Ng)5mvLp7{{^1Jv0#<5<#g_ zOgEe)#BH|{w!I5a)~eI8`4S2XJDSwprCz3XIE6HQJW}CC%UTp_#w{58N_onz+^T{N zc~;}o$ID@Eh3eJ~GPUvzO_Np;>6^U;WZBMaGXq!YX8+4- ze0>Fl3pLA{8DbrNDlJF_^u&W~{Eg!T)vsn*^&Cs(WoH9b_K(W<4QyLk%f3L8oc|iB zgfiud&1{@QbKOOqME-Qri)+oID%?~bA##a{Z-N^j`v-&l2~;=Uu@m}4xZ+p+tmtGx z`U<7HMBwF?*MwW*NoCOD-I-<&S&7DyF!hEe8$w;KcZ8xXF;zMNTl!LvUTaUONQ>ir z>5!!V9UqB8Xver^)?G(k58s!=9Uv1uA-o)GQnDi&!eJg^ppoe4FMM%L%6Ny*Lq-TY zx;!5M5xXtN{xboo7hh>?r{G&_Pmp`{?X@;axX^v2vwdj&v5qTDT}q<)q#?8K_KI3| zKuKwhe1qg!?zt_iP7|7guxyKiT7;Coa=$P~t zi?0)7OKESh`*Ms=(Z|o9`fADP`;CYQwk5ej5}?A75N?5Onf62oJqBSP$tNXH^j%z_ ziuy%r?Bz`>wGLTkE@MdHJ_0!IHjyG|1K;FymIHV(f_|+o&%`T~K0J;J~ z+vLO$G77Wnz6qTjy_eI;45*mwl;Y5`LoftWU@sfzQbrIW%$C%x`{kJP9{8$8et z!O+zK!h3};vhD-ADE|b|diWmDTf3O{xzCmdA4ENQNF*9;+c>=nTV{A9?aU-J_Hf3s zx<@6eO&m+k9W;fj;gN0`_Yu$`UJMcsbV4xTIzX&fj&W$$etn@NirH!sGp@8y%ZPdQ zMgCOp=?f9>=wP`Ny_ub|%_-viAIjA{P$dS=@MqUU%Uc&$I6x=F4mLK9%w*G2|HS4r{W+!BThTJSC879Gapc9w`h}ZoItbrORAK zFq_nc*imW0k2^dI5sWSdpU%F^YZyrjMm;4 zG zUHY+1%m%alaQ}y~2VUIvxRLT=j=N*kN-ubAJh8Xl4jClz-%}Cb7SYrn6%m}pyFyZ- z!u2j{bT*#Zdb(0@nQ-<{RF5jQQpCL(F1yH>PC>{YCUuOL28im=+Sb76Y#v5@TYM`n zgC^}ShTScNJ;W2`m-4zxU88_5EoAeecUMvAUl9Svn%K2M-p5ILJGwJAn*Yx07!wh6 zQD~EK;&w>h1k0_H?#*p&CZk8jbn|R}nt7t6)}ugkprm^@B#yO{fWG|G2cuy1O;1$|l2;mlfiogNJ{Tke+=ZDX zag~6%PcJYe2J~24x%!!yxl3azJ$3nTv)EXtt6(!C$7?T)&td0xL+yKzEF&rSGD)S! zX*?yr{d7p2;4*wF$nBbcd{c*t@3>tKBXTbYV32)sQwl(Y9o{x*Uk2AvS);GHc`%kk zyX6eqL{u~z+*ZUgNe74^&kts_p(VU@DM@ z>XlUQXo1<`72ptaTfQ;f@bJ)$DCrw(^i68_N5#q#wULtWap<0U%%{TM7uM@XO`}e8 z0#|O0Rk7wh?t<}9lJ4}t%&4rB!{LnFpu}(_0~bUsnX5ZzU{8?U;kAr?WEN4sPtd!2 z?P1TY?VNu?9;p7t*%!oHv6Ooxz_X8~C{5IdYUGWqr(A?pZejE;2(C%B0QSPi zD7WxokP-6DXxoVQL0Ax@7q>%~M9GAWV%Iv-^(9GfVJj4rq%9<|SDBdN#KI&P4Rm55 z0is+nY>CSzyD#TrMygGp!APyZV9CQ_}*ONiWB_xx=2#} zHBrVsxU=J}uedroh7pn!5RB~7$6})rdjVLiE9OqGz$;iZmK`7$&M+`7`@5U&8J1V+ zTJU}NjJt@SN{VartXZ90L0J?f1F7*IB)1qaW>&`c05vJ{{Z%nJulflp_6jSa2pjDr zK}iG;^9WEXL4*3lb2O-MBNiz@?NKrxtv-u)_k`n36xxdMzK8+f6I1yn+QHKzGy;@v zy}sL|XkO_4KBsZ0JijqMuno~ffTJ{^Yz8IRC~!ZZ1nUYgs-^+Atr0?zKn!_hm&+&A z!Jz-*6%~!aQseZwrf!Sf$dDShzLg{&Sl>>OYohj!+vtwOieZY2jFR)YjLoT{w=3;B zRl8wmIO7CpD-6l-4dQhQ$bj`xX)A>j6M&?1b<{QrW>jXfc*N8|R5S|Q3RgNYAIsbl zCKR5OB3_?|XS{r-JD;L!`qS|ZvRDHW1z~un0i}iDUBfZD*%R%|fLK^nC*~Wiwcjw) z!Q;)rr;l#ybcqI2D^;urDxH=4SdtFVPb}p<9Y=kAuZX-V1N-vzBH|>Jyt3|63 zR6jz-;E)IW9MAKA-qh)G#r$lSYZLJe#2936QlX{7COR>jwAMoZVH+$qMhQNv3{992 zty)oI40<0gi~D3Imxdg-9q%Q)yDEn6znEd8bc~B}cnBH{M+!EL*90~sY;74?CKoRv z?k&}&!`#|;@0Pu5sQ$YgRCPPk)WDnVkTQP`3gLXcfd`xP$l* za*3Lhgt&7~D?tL?77JB)ME3qosjh^$1Ib+|?w^&K+h`4V@b|=+K1oRLYE~-7X=Znl z2@@;ysiUpkLHYoHs2(t^zS&hU zaPo#7W0-y9zmFz8QPMtjY;Kdkbbm~&53D=kX+jNP=l*!G=z4NHf^ngjb;+?dMhSr` z8bomseN_!8ohwCk8Fy!+d9barBIG4#Z zq0DOiPwrt*Hg-hrqkyEcD_`E_ysJN!U3ujaL@aPPouoZw@$OSK7AG2qUNt4nKCNUUQC3SA8rZ#f z`4U9RWAS{{a;}GskSHWWusW0$Sv~rgx}v-OThWBhr``XC_Pjbc(jt7A`4U|cx^oP= z9VL8_{L0swo==Lnd{-2~n;lShF|w02QWp`4jaD_C6%^FHrZwxxt(90-df9+3oPnhb zz<9*Qcb=JQb}QN7d)Q!DRVEjL;a4-he{068&tgFI-4BC16|TwJL5?L46ZBEO%c_LH zGz}t7M4xH{NNZe^flikfW&%weUMXwebHS7%jar(mb^j;Oj~f4aJ^O+WS&LP;sXDYbAqmpN{`)bGe=sJYvByI|^Wifup*#`}F+0(-2o* zkEYeQ^zS;Q9^ty9mOgj>k@X8>EeWZoeFb6*qaOe)vUdH;ggL((Sy_r4XI9D~Ga+NA z_imRs<&l*wId3^1QmVgsaoEd{#$6S^xh*V`*8W|Q9Ih<-!Ls@~NfqPG*~C$FxlGJ4 zVWN9r<8?#&%QJuTlN0i4VSi1w-fR266N9Y^dpt{nkMfpQ$C^tc92(!ZKDnVT^)hCs z=1;)TM{y%kq+Z#!UpNe=)_NLpqtWJ!P5BDHJ88+=@3>;eDcH2v@=UJp>za36(`n~= zVpvw+8H#`K^o+*DfR>I8001!7bO^eE=|24V!9|dj<42P`+pmFd?VmE|HA`zwnneB7 zvaa^b8B8W_Mk)B}mYu_+#L)H~Rj=uqB<~APtGWLINv58g&}(&gJy@(`h0kSVBipHJfBfc|T>0@@+i8b~{5#W|^I zZ|w@38~IzeX=}#rW_~{>Ch9yMh_KupOVn()-x%MI7(u=x7jxw_EXH3uKjLMbdT_+b zF>bmt?%1IQJXzz~ef7aH(VurqoK9fnW8SuvdaH=H#k+p;iJ;Mrk5Y*CbNc#$4~0Z! z3TMvM^pgw~n*RDnn%bx5YJBCnY5)K7FG0x~*h1J)Ql~K&COi`6AZRK}0(%Lb%BnyZ gAzxYaf6@*pa{&1X%gQjIKS4xAm_bI7sy*QU03T=g2&u#7)h*((>y<^x)oZL+j$1BgsfmysYnxJz94_mb_k^i0R(6G6xJeLZd4@mw-%`QfJMK8o2{Zru z{^4;KuSala6*q>q^kv8SykGEP@w_AR*wLWN_$#ON$u)V2Ix?z8HsT##iI?{{ z?1sJan_T4pu7)?bk3sfb?+;Ld77qgb6a_cN9`+0y!JUxYuQxZN-H8`3>kfMhJG<2Q zcrgwsWlCdaXWM4exX;eS_sai0pAGL4+Qjpe;2&2OguNR5gZzy-@~gK^vl$mhIafU| z(x^^3!hQTsK04i<6e9mTLEG^+22G<>)SRpS67Kc&5~ohTq7H(qy|)|8zKtKDt(yat z1XGTtqdv^W<-GikKAN4zPX$C(mc9>Iw9(sc3}`DrPfZ+dAO7^qb8hlNp0+Oactn49 z!90gwSiP+dML1%ZZPJoM3Ilu*ag_wj2c_yWLU70G>1dq2LhAJRMYB)fTOr%n>Z?8| zT$5jqTFYvUn%xh3_l4*O3^5NMMsV(&Cet2R^hCSTk%3i}YDRGR2!bIp*pZ$C7Juyb4`{8;@ zaeR{n@RWk^(7JeP6Ujr_{q%&OM=((iUlTkoLGzq8kpU4Q!2ui=f{5g4$s6hNljRZ< z`uEdyN#mn83VX-ZML2x@L$4{gIU_R?=5pX$hA@2$Heq$Fi3ii!a1eW!e~vm?5}L;# zfb~jT#lS>g>+&ybz8YWa?r?D@9sd0-u7l+I+kHlyo6O1ESbW>HIAR{}r_&dB|Kus; z`e}{1F^sj^aItZ&?ayK!e)sR;@}J|``QfG2v7M`hdcOMps>nR22v24Y+{SwQ&*v7o zdt4e$>&LinhbIs`?9R>FRCnru~PS>KtP zn&X^wT&m~GSLcbukL_}K7~94M#_{H;4X5GW&HJB|e-=F|57gOr>`#ra+Zy~?|L1Xd zrab+(W4+%wiyMru>;6``ne@JXjNJWtj!FF;65imvy`A9+5l`ele*S6Pm|gwQQKLIy zVhuyv7!86V)$th#L4N8I9E^hVAb|W~Wc(fwW_Ii?@)37ku{OFQ>@YheWZj(I;Aa+k z(kx=`=8@!%V^dD%q77US=~!jtV6N1aEBCSq$`)Uh1g6zI6PMEBxm`8>e(%+ag{fUq zS~aZb6XiPol4wwVvM$6OA|lmeDEs~wCND&|XG5QVg3@cJ2}fPl;nChLL;X~g^j9Hb z@-^2qnCF^jmx%jUx98_xvAKD90&H=W!;B^5gZliFwP8Ocp}f7Ysg%OcpATw-^w?JF zk_u=J$a2s_FPwg6W@!uyY=M7FsneO$iQh%n=uGt$5N4#iW+cLCmJG;y!j^}b<3BjL zwU6E49BZUELoZ?dmvp^14baq*So(74lJLpGLTVwCJ~62vCVx(@sQkKkEi0Eb$;_xIN_ z_1igt>wg7Fcl$Us*DiJhFl~1p-zJ(`^oQ#7R#$6S2dS%1WA#QP2Myw5()F-F9wRyg ziMh5sOcvY5&|O%jnZpg>$x73c{es;G+v%szih^D#&wr!etSRYORCP(a;aYDWEs3=4 zAx!>uz!mF@pkAs0nL$AH(t_`7J65%VWc~N$Qwpfoaph93^Aj2GURES|bh3B5) z=HHAXvi=q1`^IeMLLcMx6ByG z`LabinnlUU0M@gwbekBbN;&>h0uBzyiD7rLU>I*$NcmSs?!ZPdTi9)HBS5S&`5SAp zp9+)-8e1=x=-!x^P2yYxCQuNwQIUudVInaNu~CJAaN3#0%2h-TgfO9s0Z2sa;6F}? zPc{h>mR{?(mnS-(NaeAq@P42aAJpd#n~WPD3XtBtAIX&@b|iKYi5ilpo({YC7D!6D@=0gTv?`F5^?pxW3CpoYP^vbCUC}wEyK(drL1|>xU zL^!oosbfU4fvVJ*BHy1S83KzZJ-CVTRGMXDG9t9;pSgZO7C)ItIh;=OIOwqH`x-YC z{P%vBh4Aq1bduw~w7^EX_Fgd*cbi8HlYB}g2fekE@i3oQH}OPfvnMIMV9-UyVK}dbuNPsm8i~sMC21WW6KR~C;aHozj@@H_ zS`p62K@e^^vHtN$YKQ+*cJir6N4luzjOggP0gp3r;t9w0PSPoO#qYRBya?8Xk<7y# z>vf(t8`+O!vWeGWIq1^KHaR7QZk}Fz662#7)=4w9U6GWH#P;S0O~tzrNdz730nbP$ zl47^yJQy`q#yif`T|Op1aU?nvUnBOQ!;p79A#Oc|lk>$^CijaiD_dEP#powq(&4}Q zNz$MP1DWrK{W4Pd_7H08B75nUFK;DD95H>On4g?D!O95!0QBP+27Expv)kgGbY%b^ z;2l5C(w;XDK|wpTzV6^|Zk+==#UEYHeBE6+qtM%lOP;QrUY4C@9gJl?kO|4x1GBhF zUIEO4#?nWrCR7#gmY0B5ouF6TRS8_Q%UTtJHRM`u@A?sgZ8=tf?q06Dt>I3bQB`2g z*3;Nq_K+j!D1}NTVWmM;Wb{ z1DK)tBv`Ya>gm?isCs{Y3aEYvaG%xw)KCV+p3OAD{ZgIH^+4g-x9nE9Pd!3qKtCg( zic~I8o2DOL40X4@Bsm0!#4ythz-6hKu>yCKQC~bR(wS;ZbE-9MjY~Gz8&Uqs-$jDI zR2jTk4G?HlOm~zpvy~+~Mo2EnB9YI?qqZZJD1R+m|DmsGk046pA7pT^LDJ%=-I}Nq z>d}y5kTyu<_m89!Jd0#k{Hf(2Ai7!BLBn48Yu{}bw~^sU4ptSH5b-B zI#L@y+rtEig%kQCXQ&+%1Lrh5qA$^=)NJ^s)GUcT(Jm3s{`-Ih$2C7MtvaaZYYkd) zIxf;#Rlw*9Cl>cm5LIE27!;ifOqLquMmKmO1*-z2Z`};?{u^WgMs!RgMTtY&aVW(Yzu+k%Y|OaNN2 z4Fd?MVB!4f&664`%}TQ^h5VE5>|_C@;w(&Tq)jaid+y56pNmqrlFk17mJ-vk>z08b zHdk7&@7cEwBj|HUYKg_uJ=(idT`EFN6Gb$&;+xG(bEuUfV!j`WpfiLFp92!%brfox;cjU_B86J<)q}mc0@~%&nDl!G z6eQ$WI68vx)OPSE=0(jvT%qVG)2zoYO!yI;J!060Y*%_GcEMnZ#&=yo-B{TJYQ#B7 z&<0R*60AD-T_nW0cr$+qM}XUB>H=ER4h~FhGT*P8TYhhR z1Ao|aVxCgoUi)X`Inf~3!Whj(K3$`ih48-HNyo7X@Tl0|2zN2Q#F4&WcynraQxls- z2icn$)ZiL47^KZghSC^aXM{BP<+j^@*Tnu5z57T*7Q*o(9p@y#i}8mhT-v0go5h$9 zYbn9_Q3`sp+{FD-1K&3}zB~D1czb!>+g(hQ`A$WH`@YAiyVw)UH`_|{Q>;pfa)re| zd$j!mw=QQb{O5AkfeiB}uPJ#{vt@~)NW5RXT%(G&S%T`)vM(eZPK`s`PKDwB#u-u& zsmy&|U_9+JhPQ~UW&NBbx<+O3vFjzjDVrbxXiCv`0{XQ8U|iz#3lpGlY@6tEO40TS zRg>;qUN@A3Q~Q?IHj&#{9eJzkEW%JM`bwdn<%WW)!qDLp_gsa0TUKp*7T2=V(EDVEM@wjK^Pa4ud5kS5g^RK2|*Ley{!ovusm!9OFhVUjSz@ z%P|-e9n-v<9CO?VJ>Q6=+vXGBT){1}nD`jVen^XV2N*YgWp&xQ8th5iJ>hYLxDo6^ zNy@0}B46Gc*n?jPAI&A{?jD8k)G3 zm8|$f8a=;fZY1j6_8BGcFip%Y@lxc2`y%`4>_>bqXEn?RLpJ{ak>QWn2!QtUe*~|1 zQ{0D;4=SMN*#nQJo1S54oisgqy~lL6!q!jm(A#0i1r+Y{8liMYFj41#NLn!ZqYgS}7P1E(oJhfWxP&OSL2An9gM*4dH6%8f7?KT|YSDOi_9yLS z3#V%7imgQxotE#pDa+=cuv)F9dU^qg_l@l}Au;)M7=)%Qb$)I2cdGm0eCz~3{y9we ziXi(XNzZHDd>X|d3rf(>QG))yv{2bKbmi%T*wd?(**)ipqNeU9Cmc|M5D(T2$I1xI zk`=8$fsDi!z+_rNS3i6&-X`7ZT-yJ^K?KDfAyLnZWFcLLH3r%MX(&W1MaCw|Tk|)D zbze&cWY19QJ5M8R$~-ZkvJ}%d6eTr&zy4MfqFn5rFsHT&wTW;x(BySL^{aCeVSC_> z)7P2`uQdPHHFk1Wz@@|qU^i$UM&cnh&0*{aV0*Qa5KFq>LvWMr_H=63g zZVYW!E}L`&D!zsKMA!*n&FaX@JJhXzHR?LSKI}BLQ;7RG1w|pu=%bz+>@oY{Ohs+b zg^wlNjnq#DE26Y5?ftG!^l>yqq)({^Ur49!K~^lac)cEWy>qgXW2;Sx_^4W79a{_0 zIVZWbf*mq`cxmQT*-ov@74nr^ShR6hwig{oFl3m zZev&OYyIOTQ^977S#Z&rSAGK6^a@E63mU2@`$>~Ls;f(ltz-0xKrc|M^n7_i^IhOG zmf5*ou*$!Vw!`mx<|iSR5qkq)KcU=L$xgFki-k>bzd(QNyE^|X>eGH_(+iju^N6v_ zM=rFIW^-zqvsJrTWtJFM%=+`k+F7D=cOnhIfIIyHS!2D&V#5z(w_&3%RYrT}QE9~k zhURE9sL!^%=ABz)WnY@yXUh>U&e`go-2_qp|7%``k^`_Ex2FW7vJ}Pnit^yJl%;_~ lIJB}V5QZyL7XQE2jw|y31#mx<;Xq%Ugo-Gg>}>$$`yX01tX2R3 diff --git a/summary_results.json b/summary_results.json index 957e0cbf0..a43525381 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9348, + "Performance_Result": 22.9511, "Performance_Units": "Samples/s" }, { @@ -1436,7 +1436,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9578, + "Performance_Result": 23.1795, "Performance_Units": "Samples/s" }, { From 17cb04dca6dff6a10fd9bf20529ee30c377846aa Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 23:37:56 +0000 Subject: [PATCH 25/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14416 -> 14415 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 0b7b0d08f..ef40512ef 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:36:42,018 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:36:42,018 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:37:51,521 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:36:42,018 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 23:36:42,018 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:36:42,019 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:36:42,019 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:36:43,150 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:37:51,521 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:37:51,521 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:37:51,521 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:37:51,521 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:37:52,581 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.3742 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.1547 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74722 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1857 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1863 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1795 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a13be20ef..97b925a42 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -566 +567 diff --git a/summary.csv b/summary.csv index 4e3f5c477..f0de1bbd4 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.3742,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.1547,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -15,7 +15,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74722,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72993,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1857,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1863,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1795,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 06df1ef4d..51df02191 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.3742, + "Result":19.1547, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74722, + "Result":1.72993, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1857, - "Accuracy":"acc: 75.600", + "Result":10.1863, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 17912712e57208ee272cf1c9fe5eeddb6a2a1238..e45d8a9e8b4936162afc8ab0e627f3a4d0676ee8 100644 GIT binary patch delta 5875 zcmZ8lWmwbi_ud#SB@%*e)J8}n-7#S>6%nP&3DSss0VlOdNDGV@Jvs!W8v&(DIu#_P zB?d^S{HMQn|IdqaU7zR0eVudP=RVK5o)Pz9_X>Lk@*PLEuy|(%LCkVDTgbQJ@~k3d z=wrzuaWcn#Z>PaQXpr5TK~b+v3%XueRuIEt zVJ4TFF6YHkn;QQSZPxZ~-xl)lskes|Hgj_50~J_{cSv!|6%FgxBkmf_Gl?Mt8RAyyhOrI&Gql(S8cU@8CIJFhfdgDn-b@W~UHHv2 zvGA~-y;ykX=#WHC*Qg5b)fb+=*CkQbR&il7c-2jWvNi@6wlrMNhH0(c57E0h1Bwz9 z&7=n79K)9|%6utx@^!MTwdIJ(<1oi_3qQkLUEO$W;slAN-o3No2cEa zjU|Na)|1>M^i8X{rsA3AKl7P{oy&u7e~8nw0}D&T+t+gLywLVkw92#$aGkg^tuVaIdvuXJr4VeDW0l5~R)ttk&r3z!W}o;|8I ztZ7Yui?@y-7TkHkS?yp&7~WyAM;eECzhTEK4aCG}%zB=mo#r|f)nvbX|FoxL_Q=-H z?6Ow1A0KpZmh)%8jTk&Qi!qWcSRRxS_jdiQ))Vldbxz#t7$H4Z09N?EC#zE)y%D~C z4%ALy!-!Xa2OSQ!&Z6L*@!JaFi2dHr0JJM3dIr)?l-{d@p*^&;y(Qwr7pe?mlxdf# z+iXnb~UJBXLT$yYMR(__Rzjst;#55U9%yL+< zm2Uw3j3g5Ps^4HRRoCXLwMfHHl$k<3x4V8Kb0SHq_gPWh#tMWZly%|D;aA&lA1`9U zhk*L}L@`IPgS#qd6fr4}$nT+~hPt61r>tI#4jR#~2j5|hLa}PmXh@3Y7Tw^lVqkgP zX=?GDT_R$=2iParXAl|;&`F9@u8JbZ*LsL8ubWwY`C=Q4X;yHSsdN)~`}0kNwKeA5zK>51EHc>u7sEiq6A1;0 z;OHvCIBX5hOrS5XV(;j`783@D+~SHf$%1{;s*2(+*O8KVUe&|+`3cgu*{dDG-aZ!! zFz+(aEcm<)Zq8IPj}{cNv|3nA{O05HZr}Sp3R<3QpqSW#qJ9PASWL9VV2ua4%jbH8 z8^bADd}uvQfs2nX&|PG`!hL}9g)c9rTX2fuosTag-OErS8pM)?bZb{(w=M>&v!U@q z+C5!3wN^}?9p_5+=3b;CHm^DBPD9N8m+6h5?t^h~<884P zgWQc%I$|#vu@?UbSWUXpei!*x$l_~0&nqu{Fq7p&>e1RS(Lc1r_Kl>HxKX>`y5P$p#3qAXkwmz`g`)%J&D* z6NW`yDXcw~0xlI+s1HK%X)62kGZkaeKD4%ZfMuEjrfs%`;(jCl;}wzAyo4FRF z_6uZf_Rn`a^kvWVCL=8NZh^nYYVJZ4x0py^^E}L&Y@i*UBp_QKLv3yFs6EptZFt_O zz*V8m)0>}?qK?VLM5VXmo@G^5j}<2ppLDg3*h-{}dVi+crX|Er)laz!b6AH>yzghj z7|d13hJ!7fLhs~_BICFW<+(XvYI>7AJc~FxC|-oG(N>Av!ik^_7e-qQ59tW+hpa0{5RO;~5|`p=`a*O*IAHzMWmqpgwDCxFzq zt*S?l=tOmwy*-J9j-zpgW<>5QUh(%BuGlCX0ks{=B*kgNUP&jR=S|IVym_aFOk#@| zS@SFGkjJIZM0|`wA&2KxZg8-TqFsoM;v_2){?o=hRgddZvQn@6x>WvFl|^2zHm7r_ zMIP8APWu3ErSY#4gE=0aP8oGE6i0jF4%6~hb)*AD`(z>e{GS5`<3`w1%m?oGBIFGy zYE>lV@1ID}>CdXheE!iJqIY>+`+wHREqNxxIbqk%&VALT`t{<&f|))8;=}9!_p7w1 z3-w_+tbqFjq~}GClKdKpc3zb-K8Gp3A@^ll52m~6tOFX1LDk6?kR{s3Qr%hM)jPDT z;>x$`-iMDb(K^;3R?Sl}Ae@1bt}MucB<9~(r`21a&z8fAy+;uJ3oH82FI9Ms-XkY zK>Xsbjff0_yV#13W%2w&`iAj z-205E$;A92Ol1Izb2zssI74d&7iZPp9Au<2M#~YLAL!YI8rHh*H~6(7|NZa{Q8tDu z16#1Lq*Lflt^t>B{?6YVXGs(ABVabv4y#e6<;ur#G^?6NRyw{WJTm#PSRq)HWBy(v z0`k`tXv@$tI_PxYI4e^1CY~!oE)qX&5kO=m-r}$vVa=#I$>ns-Xfh&>#iw#`N<$M= zw?kP0g8pMMl$%s_wmTKCk3d5{c^sM?yuUXh@(J{UZK218eSn|_5py*&xiJ%kAMqO; zFpMWXLXBjVQ2K`8zv|sL$$tFpk;|#yOKM>bn_i)kSqquoEtCAW_0io+$Jcp-;1746 z@CbK7m^EoZosc9TZC{m!diL=46AG**Vd|;+_G?iNC6}xPjcSM^5O3CP7MpZjvCSB!g_vSs+G$-Xjj> zdQ1!7H9Cm*O#5Os&#R2Y;KfHttbvBJW65>QQ_oSykED8I$4w3Ed}9CKRlB`N9YLuO> zGQNWavrq3WTd2>~+=MchLvc=wSGpeV^6)?<)pwt8aL0*kgP`YwB>oUb;VP7!5K^1o zZ?AtE%I2rUtSS3(TawjR3dQ~ehFun}zbE-0* zJXR^{lAp=0%)YHb?#6wmf!FUsgIuQzYwvE^#3FbPo1+;`1e4qtX?TKwZj1u}Qj9rh zB*`hU@t-eU!#_FiiKY3&wIt*yH&jcRthQ$r=XKC%0b#wz7Y2rFb*ZY)I*tsYi(0Dw ze1BP~ll~Ct!g5A)yi6qmWzX1-jb{Y2$(u>|+aS9c(kyOW{JS{Rafy`B-Kjn`mOh<6 zio_^Ua@`v=&{~i57PB>V+$o=)lbQ(h>PBq5C72!KDH`_IN*~(kk(@X~0h*6OqDSBA zStHd5-Q%M>O_Q!Ux;uqnSMoKf#rZC(x`_dIAQPBj84xxc{zM5H%uOjz(vZ8#&Km$HOlS!M!c%W)Mvr%X8_BpYNA*F{|7_OTV8nVjTN!m6Ofl?&)6J1_| zv&QU+Bed%I`spvHDs0;y?0iAhKB+Izx)M|ZCKdzGOv`OI>rd|SZ451F} zeJafM>eE%f!NGj14r;VWqAx*?m_Ld?oWrE55C@5&!O4lB}TZ<~Ozc#T4D=?Po&nl^~tJkT$ z@nTCc-YZ!pDGa8Usv-hyi)*rYp616mU&yCoG17vW}zB}Uq4ivii<(hJod;f?;%k^s>-eB~Fl4p%+GEX$1ST`sq_AvX98eDdW4JZCy z)^aCzc+|>0htiwL%=CAV*TO=~#>jbjI;GpmNjpx$D`jl)Fga;0F(lBi1ksS2OHEx# zU#nt5aqbUU)iBvK!8}Ilm1@gu;ToQ9eld6_=-P0A#ptX>I_cG8)<$9GTgEDJyd|DM z5O?Y@J-~TQ9(p@t%H&Zcdtdwxt;#ZOhMLi%-&__Owd z93<4h`s}sDUwPLZCIlDW5BC(>%c+Umvu_iYeRexs8q@E&1t<+r<@&Xr4^21=RL>rM z$4`vaYm9oL!E4oR2JZ3ak7l#I5iRWh;z1lMw!o>=T1Tc={U z*%jXBtMRyCBw`1yX~^9zl4iA#9xqa!eiyL2A@}xR)$y{b8czJ|(ec?Fn=54xyU;13 z<51`0+ml(SR#StCyoo+ca2I!u;xCnaY?*1=8r?kSGY@rWD=(CaQ7jL6@ zMP9O;vM1?X8I* zt%39Rwr?a{mxi1`Q&-##075#+#Ul&8S`ywajJPJK{3E9NIlZlp$sZNjJ83@v>yAD# zFm!)$AsW#Aj6a7DSIyxV;G#q(-!T$Mz6Sn`qa%EPlzPft(O&x`6tYMEwI;K3*54f^ z>urhm>@s(tKfC>_Ho%`tCpv`Fs0%O&vkx}Svij20iN{woBq_Er2rtks>ids8Zvj)K zY+-eH04d9v3B${0`KH;5`oufw-83 z2kTf_%>O(a&mw0G9Ef+3Qv|BS7s~PfkE&5QHXtN^LrxXw5icOmM=2@;AbtM_V$=Jv delta 5910 zcmZ8lXHZjJ*G(u95T$pih8{qXCM8rc0R=>a&^t(#DxCxc69fek2nY&Dm0qPcMWlpY z4G7XZia?MmUwqzw-<`Q<=H4IsthM%D`|LSGKEpnhu2-qzc&c)69#_SX(=hHAzy6pR zja&7blxapVVvXzS9c16o;%RYX<2H>ncrSPQVBp&BFSB{v(OmtEmmzaiGe^zqv%@L( zPfL>v?}_JAsl9iW9^SN#cO7djaKAI4dy{v-;OkgT>*{HcOwg!{W7RbS!xtPiZdFrf z7LfFd>5J~4Ys9mPu8;>1B#(2{hC@Y2o0SdJqn;yo-$B-@)+M#8_H1Co6Urq`Kn@N~ z#}Ptqs+udNTh#`#22SG^f~Q-`cRl%x9iN6O^RdaeX!^3%UJq@gkr>WZe?$4VLJ$d4!8CAe0X+Lv{L zr+%$G&qJ2-lz45dAwT+|7dG)jgQ+X_ac1{*YvQ_2+F$iHsN$Q5Y4q*e^7m_ghlE$F z+Om1SO)xZgIT7WpplcSl;<>j*su;f?wN+5;KW2JovpjIpSYdNLm=qe`D&{?1TM=gc zYawpIuc2KfE5f^f3aJK>MwgiBs*N?NLUJc2)lD(KEA?V7Db^K&(7i?ItNV){$saz{ydhU?aNn~6n|07p#?IYm8FPt<9TclgUFoY zXRyPEHX$DER~trI-NVqC?F9gfUJIcaEHuY01Jk)SW((I5ILGQxLE4irsFfgQreE1+ zV4>IH2|8c4f?z_6>#Lk2i|c$z7^S9&c+M)B3b@WT0TlqY$yJAa+5KaQZvUZtgx0(E zjWS+LYL3Pli`>90vxi({aGq)=$thW!dbTQ-g;_Sg0?@YIYp>0U|A8%xk%3a>R zE4ODfM>>-{$GdfTe=E0$1m3C&dVZ>=K6`Qg1sJcqNPVI6CX=yl7;zG)p7OKZB8Tv}P%dFk0f{wUX z3>Hq$R+ACQ6Icb01PXHnrB zC@5U2Syj&v?ebGgDUZf7q00#0)LYoHWL zn{-t{25B%W0UEkBrRb(0ZqE(up{YO%uA<#x*=(}&*vsBgq~Pk z3;>JXm1WyWKq;Kb*;N|Wb>((GgMVly!EWCe5zF;cSl2b8Z-E-u4Kb(vhP_5&{uoieL$Uy=r+Ms`C z#K)FW-eL1)pPHeLUp)8Kkk#`W7Z&VDat0?ru1A5n1oosm6T#FdxNQ`lq(ISkF@Y-b z7ooYIH?7n%WR?sL%_E>?41J?B_ijJJUDh##q9yivRDkK6M6@qJx^tozDoGGI` zY@nKig1#w#WtI(`+x|B2o#=MQZRK!Jq~dI3%s(c<-}rh+SN!OkWj5^|$zA}m0!-QB z01BCa+VpBe<|iIzchdnXrn@ECHEiIZU=s9o^FqovT)6z?f@X>wRIYt<;CSNcqQUlj zin?k<`R*Ghcj1a<+LG|FAUI8kFxPyF6UwY#pyKO)A=IctO98u^;)ytl9?9E3-6W#A z7!4}($gj(3Y{bFNcccY~O>QimuIcwn>n$H3Ybr2Pau1`1Uv|*#{o|GzooIyx`oTdx zUG!Lwh-KzKfF9C60W`cn0{Uy0(>@Pab7O!ddFcmDqCF3z1GQx{OV1W*B5T=WsndcOPbRN_bN+Ox8C(&U_3pEVr=3nK{ z^q;>H_KXRZP0^j(FWa8MFCJt37s%wJU9Gxqd2y9l6@I=-4SSCex^Uu8yhj@9xRUv0 z4yXzT?b27GV4&XzZW^9Cn8G0{AvQ)s!>G5c8mgaI9cDkL6nn}WsSoBH|AkE6#Ak1f z7BeGj+kVaTTK$ROHBQvxzhiT9EioSZBubiAFdpc@NC3p^>;O3<+!@7DuJzE{7&ZWV z3HAIo`{jUIC#Ge=QMy9CJ@*9@^Fm`*l$bk zyQC&wKtB{K)~1U;67{owq}(qUGYuD+5X3&vZPEruLqOlOA(Rj-uU>8Rh%4g=T~XeM zD3kcc(Y4R`wG$C#7_$r_wLB=xj#^>nFWDxNI@w<;Y_8 z2b0wbuXwH8(RcDj^pp4>DhY52tLse&3(jMm!KskzjZP|5R_fG)Z?~2RUb@{%^moNnt@JxejRuF1zGyM%m4=xX09X<7*sl z$4o2Q->@|{d6)fkwD&Y=X+H(@(R~Ui_P}HKI_hLopy!SbALDO2=}kOpgxxKJOkQn- zsVobqPXXFi>2mQL$AGES>B^E`+&a*y8rf( z8Egv=FMV3M;S(cg0p3GmS5Z#MkFWy(GmJCp-@3zCy@87%vmbN4JbS*EdB2e?wm?R; zd6rzKxp5u$ditX*eu_oBgvOJ=M?eYJUT_?9H;!e;2PJ3zO;=e1oLd@zg3-X~eNYGx z%!!U6aUF+xOf4`X3Uu38`&SzW6vk3BEXUi$hFU$yg6+s`kHc_2yZv8HwIAtaXb8b9 z1f@QQsg(TA^ARze)9AS%mvjE9wiX%RX{Rho_+b!0KkH0e5`c#u-_`G2WvnC9zTx6V zQx55sz270Mq~7GRCi<3e1P^krH=ztIQIr>W2X_W2YG|n-`rYG8%JbpI6cw!SPr0gp;9LXf}XY#ApI&<5J!7c4gE7<-kcEmll`M z`FeVO))ldKH0dD$@VueaLfxXi8gV=G87G?Lh*4vP@1SE!$0K1h4NhFRQ!Xat0X_FQF;T!`-zTm&!g}l8V@eJ9`O>Qm3+@8!ud5pcWF12K1owZ{ID`H#esoV zFb3$rKmf!z5e2e`*GQEV8t)CaZf27@@Nan)jDG#&N=bU10wN8LZ|@4E{rKKm;*1ge z`L0Mp^&MW?Hn_Vh&sR(p`GN+V6cCK)(Ls69#Mqts*Y>?PW3TOiG89t-OOKGBwMBkkNx zLnGt2|0-!=;Al3oX5-2T>t4E(2>xKG1>vBIiWSeV2)S@wNLYPSOgFmom0z zit^SvwWP>wF-6CC$NoK{SPdKv27OS2P=c|}5h$H3VkaFS7FyMf`bKH# zH%fc-bbI*Ole=0yBEjTH6>EYD7v(-?gd^m$%I~aUp2_)rf&5~=c5wPsSo%MvHDsm) zKnTyQxbjo%J`v#))v$o05QlZ3d)HbCh@;_|D#fBDw&nrU{ zh$2;MDl|bK<7F_POk~p#Q`S@cp9GS0mV`LZ0BK{Xh;`G1sMFX1x0`E-rs^&Rm ze%6WqA3~Ud=DB@q$1SGElcEERIz)FP^1OHHw;O|wJC{8pCX_{I3>j->kPxV(h8Gjo zQ&xk3pX`uFBTagrPy|TBiLed8K#S0(0LBQ@1nLrUZRK$UgFhcq@x>-G=Fl1^l$me- z&N&XsLXXQf2uLV8^X0PSUi-1?%qyE9Y>L6kGFN8_y#J1$Ay{%-eHmB$k!3=u<_27|H-NC#-yz z>BwF;mkEA|4u)2}<%ENNwqgf&=R5{XheSU7(7#vVoU9pSU*eshXW_f5j0;Rt!()W? z$Swf4$F=Bd^?0C&I-%COGh>iov7@W<30 zV}(Zuqi}NNg#ueqFXpvw*~Rjk(!`b#sX`OKwJ&k}%ZX75ccN(-_-TY<>}(}!G|VVg zB`2Tyix#gNFp2RQw;d2!MUU!h!(Y}QxoOl=pf>kWkYJ5}AhYcO9%qRzFlWqxqX#}x zt0+9(p|Wf;fo;;TlzhsFzE(=PIdZqQHWC4AyWa$+GO3QwBTru`p6F$c+Q00lwfq*E z#v3KWai31fY2KnP*ENSlF4<>A157Mn&$y>bDQ2IY@RlPrJ-94r3VbK1Bn;_Cdz7TL z99KG$AB$O}Ye2@6M92qj0Z@e8QwsW*fcGLq7|Q@@C-<^B?+N^yu;wADF^prVCxV&dii&zc5z}nG2abe|WdV zA$LsCjN^{uF{%2S2fM8_Vai$lo6FKNVdLMG%<}+hkaIhvz_nqxu;&8j%A@|C#@sr$@^~u)KNW12b?P0f7C11bTulXG? z(jaC)h|(?F^NWB&Roc%(ZZ}(9uqyKKyAoDB{Z1?P9fFPf%`RjIzHWHdwVZcvCWdDY zTv+fApI=bavMwxu90)q0q(8XCxz2E7nBNX1lcZ|h2W+F7*9ZLn1=KMGIN}D_ zQqFq-!mpqzfV=X5w*P*n18iC5Jt)HOqbg{3<$<_1(aT85)C1$IyB`5<>>X1SM;6Ss z-0>SNonk*fs4`VdSrFAtLOG$YQsm$+&&5rvO0Zrt@ob-q(dGhQlGIb_&w7`y*Xk2mg?ob2;olv?$1Xot{Q8c!UW%r3?FF zP+Zr<>_A9c)*FXW;nmYY$Qj2WL*j`(@%4(!X{hO?!?sXRvcVTit*?(CsL$-_3k-vP zqU&@^V*O?r=e<|L6m@b4M5NJ666%%lcdMDJms}>obCd+W6~1Xb74YpS&p*^267=XR z9Dy!$cTQC%%%{sZyb@5@8e1hGu44wUo^^>ks7a)T@C@zF{P&{|fX@cJ6}q{3RK@T; z-YA>TV3;i;A`q(adFR=b&i0cp9tyklG42AJ^=S9BiPZ0u%~}dZDeBRlked2C?$RII zHyfZ^l`9o2_^hV9Pjq==Ozs8t69#{Bqs(9ZJ&%oO0BDGFOk7zCt+*JaYV7fqI;XWi zeIA9h%A{$s*UB%N&v~P}(r8NZxZCaDCng_RXGTlu27(^MPT25F5R1KL2HlPC>S)vk z^1LVf+?j6*lLXftkV0|ZxBYRi8an*h%~8Aq3RRleWOP(ba=S=-TO`fZ9s zfsVJBMo`P1TIu{Vhk(HwwE~(U=o)Y*N5v({Sx?OhtK0dzcPVQoAH4m3NmRsfDiCgV zU^$tn-f6ov^(S&1@nNi(GrMUy{>J5T56je}6K3`)diz9w-d!;& zfwhL*J#*D|Pty;BV`qW}`#uUGnlGsvhw2OQiX@KgYw2ehN)&^QPinkx@lyWpPCG#X u`9HVfSQO2G{WuRrWgrw+peXV`sfHEVf&92NMJUi8C!%y6BrOjhKmP}}{Wqxq diff --git a/summary_results.json b/summary_results.json index a43525381..febf86334 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1857, + "Performance_Result": 10.1863, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.3742, + "Performance_Result": 19.1547, "Performance_Units": "Samples/s" }, { @@ -1220,7 +1220,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74722, + "Performance_Result": 1.72993, "Performance_Units": "Samples/s" }, { From 5870613e082ec6a5decbc887ed9d8911b68c5ba6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 23:40:07 +0000 Subject: [PATCH 26/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14415 -> 14411 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ef40512ef..26d47cd57 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:37:51,521 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:37:51,521 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:40:01,968 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:37:51,521 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:37:51,521 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:37:51,521 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:37:52,581 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:40:01,968 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:40:01,968 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:40:01,968 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:40:01,968 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:40:02,995 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,13 +26,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.428593 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.379074 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.304628 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.1547 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.08857 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.15441 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1863 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2051 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1795 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 97b925a42..5fffd3d00 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -567 +568 diff --git a/summary.csv b/summary.csv index f0de1bbd4..779b8b7b6 100644 --- a/summary.csv +++ b/summary.csv @@ -4,13 +4,13 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.428593,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.379074,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.304628,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.1547,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.08857,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.15441,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1863,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2051,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1795,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 51df02191..34eb9586a 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.379074, + "Result":0.304628, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,7 +339,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.08857, + "Result":8.15441, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1863, - "Accuracy":"acc: 75.000", + "Result":10.2051, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e45d8a9e8b4936162afc8ab0e627f3a4d0676ee8..c04578b69cb52266ed9ea79f26c7c7c5c54aa44c 100644 GIT binary patch delta 6944 zcmZvBcTf{=*lhyRBOR0)y7VT}q)APvg7hZ6NB}{BP&X(@ZwW=E_uiW*O(6mzARt{3 z5u{29L^^))_uZL$|F}Ez?jJj|XU=<`bDrm&-Lc2Rj~m>nDe^y1(hD%sa(OUIj+>(9sEj<0rB7FSH=Kt4_aXJ*m24pUj1F_;5MK$gwk5Vwh|8dOq+iY6>R z%t?}9#$PUuzd<7IJ8qezL^HI-lieUYn4BY4!rRyK&@}S6p-l zlp%1QmBApmrVyf=(=Xmo6^0c%d(r?i?4d+24Hx*I7|G!)QW)uGW+7+Gt``_=r(yImsGze&da`>j_QdjVfTe4z0 zn{?arq8Q$}ikKwp6(AU?wN>V{cNP>H(e!r!I3IiHE)g1|T5lA~jAa?$LDSSGAwTyDFeuMYQLTER%NDY(7^swE zVDzyZ|8ToFOyOp}#8%IatkGc3r8|?e@Tl#f)6DeWCp$CS)t4p~W17X@^J&gkK2MJ3 zWAl);(0?+JF1$}<-pd<taK#)cnk z=1SfdnJWXnUpdwf7S}vkv^NQ~IYlD<8U?(y<}Uo-imJ3Ju{R07ULxr%r*=b0)2F4U#g;&L8!K?3E0K_FHb2)*Cv*y;ayQC( zSrpK59zegeQINxaj12E8iQ7-U*N-MxtZ;F`9gNH^E8| zX!2&+_m5=u(7))%zCOPt>kC)(6yX|UNvv_V`G(1HA$-J#0-;yS>}@UWHe@?~ z;F@sq2D3zdm#xmF#_f*gsaW*H+M0p=X>EBqrSCeQHZGo}^udtGi&P6V*d9)b#6oaY z;hpGg%iij8wJau(eQSz^D5?_;+288o2pSQo#K3IvJ>)aOAL(U!wnk2;+JF`Fo#l-0 zO5s)ec`i=;^{dp?VZlK#@?d_JVij{_piYWB(`Z)-sNTSBR6M1vv|Mno>Zu>EqF$Iy`@=~G zu@JOZ_$vJ|Xb}HT06@*B1N6CN_2mbw9V^-*2u?oEA5ClbZgE3pmB^RhnSy89$KImp zll*S4V9aFE7ZaE@+R`oC11>twfwDw{od{LJLg?AfV_ipeZFsgTzwk5BS{X$qGWJsG z$-R0qWgrTb%!Ak%|V%}-l*B_3Qg5N^YDMWGK&?-WgPGYGX9 z(t9$bB}$vz;^8j2ttRNiQgB;M$V8<)mQhVH+?-AX^z^Kg#{{&gk20JP`<&in_RzcW z&XEISyD3{p{imfbkIGDI2l15?j?_Y54NWSk30S}c2M@Q4R4=C!i$mJZeH*=-;iPo6 zsW8@X`ZRR%7&l_bQet^Ry_wUKl=ZGiuC)Txe22<_v~SL7J*=T_U%@lnYq1S|5DoiX zg`g|F?uD6sUm9qs$=bzudMBC~ZH%in<)9}Lx6<|>l1A&bY4n!<@ziX2wmM-^SY2B4 zgdDJuo=!MYPcE=Zk-}N1-;&=M@(LDUB{X5oo7AcyG(lYo zEB4{3XZzO|zVrT0rCnhyU~Gge>jskh;T5!6h?ci7@jlSbJ~JPJNUBzRgbi(09ZCIcgH}U<0@6VRkQmqXKG^81eAw6>UJ;Gi3NoLc4$9rGQ@h2 zpeEgE7gdmC<=@ba_9eLi*G580SPXOYDW1hOJMnKgK4!)Y^Fj)BKi-t_%e-E=l2k4~ z9$=6|tpO{(%cD9-Alanb#Nt5S2x<6t=TMgokEg8#AtlSAz+Y`zCRyvcmi>>a$Q?_%6Vfz9*kAI}SJZa%5r3r7~Nw79!! zMMjr}OeoF#v*OZ>C*(HL6I2x~2%}YwYEBC7?56YXy4l^1n34KB9wIFEw3!D2nSSXX z>nJ81FFZ_PCY~4n08hBN`=xtN99eqN6n9ajMyG`U6G$%~S6piG4{@3b*(7rnD81DZEXHYl`qPcbWl~4 z*tRJk5~Y`asUZT&Qn}R8yS+{*ddZE3-vwqVE$XC>kz96-qldJl(A#o8l|K%(6V{#~ z07k1Bp=ynTalxb#lLKxR*U}XY^+5;JN}qHu*IZ#+8sv-}Y63Q2$1!a@5}IKDkeRq! zX3I+RnjmCnUEee@y87p4{8ZjB4K@Zl>?PiFsu-M+D83PIX!XZ9sm;*N3+WlNT^x6t z9zGWnU6URTBd>&{hFg+W@FkG&)9V8H+kKRvbd@Qc=Uc=!(t)#T_((V92}mKtIQ%KL zYK%1fdmp~i1$ECh(+*n8=2^oYbT6?FFV)WySF+QmZvIJy@0x8>;wsveFN=@Zmh~ST zOgxff9{7s((dsH z{b@+eaH65$js`o?41=Ow8o3I-tm7Mq)C!HZr3&!9nTZ^-w7dU;O8g@^gY?I!>CQga z%kSUbRJ+7-f6NsUQL23*o`)QWf&aXZgwVlwHunjL)C9y=U}^&5tC#nr!zpP*KrUyH zJc1_&uRV|90AwYR9x!`%es1{;*XU)_Hkq>8D15f1at1_t_33naEvGl@JpZ>>O_RkF zcdg^t=;4(y(N046RN4tvE_?&zi!L(>k#)Es0k-8Vos(FX1~D}2s9|kI?!iy>x0j-B zr^i>=JR}10+GgvhQ_8j6y*2UOcpb^R#DHFSCk6&sD~mGfRmVipj)JOVTtS9SyEMI< zm=H@aYI6PP=Hn0ty^qvv^2%RobYB1H9GX6e3~qKBS&w|IGJ06)l6vpZZSGL$m%)^w z9pWRk?3}L&e2lRfYXU#SnJAPv^MCotI?_QWAnx$TK!`rp za4ZGvXEVat?D@wMBcGKdD@GRAFA`TW->2Sa8VVzaL+*!KlEU~BPC;=h%`r?M`f#Td za3dMP+zk7_XA4QbCr70k>=Tge$HEa2wT~@^PM$Sg<6IhwH<^m~YT8j-mNF+9ZptfMU*2;{ zwL@3_m7Y!Os^SDpHB|abk?&f1Q({TFD_CLtZVgmf_L|CbJd8^EB-vwwL@ zbtwhI-v!`kv~b4$_xPQu?oDVD!E4!X)tf7%@?FKE=V_?m|E4*N)PvA!U`)l5;#0&(I+B`1CA{(+nSEW>s6&oY7Jb4RaKTZkGq9{_{_`I-SQ9Z!PUT+F%5h$4LHJ$}5(3v*ku7PfpkQ z(mhIVo()>0C><=F5ouY$Pr^yBevSAVXD0aC+$Y6G%`fizUno%pMn4)0_6dCp|LjIU z>^j46T$;=ByvQb`zRMheM?5?0>OTT>wOO7rwe9?=2;?=ib8n(<9-wd3gg3`TuS%r; zI|pBvBxDciDzXczS80tg2ib@Fq>!`|vR8AX^9ymppHw&4t#d2!f@Y`o^}?qrxU;cL zR+W11P|WB`aoIL!oW_1*|Dv(Uz9aW(@^`}cyREiQ7=M^r3^DQy0<{Lf`$s}5%#8o+ zXs;$wz7jX#HAE z^eb9`mV&zsEI>O9D*OBstRF5wIGDUV(=WKG&jtKv9?`65+DX*IKSj}?NwKu&-HVmp z{JfM!PkshJkPvo_PO}KKK97p08dY&KNXg+mmuuU#6Z^f!^3mDlDzTRwlm?4n3Gn`VrSdT5R z8j8O-)H>*_A+@PbQpxXR|2>t^j!8I0%0eZ2Z1E8$rtg%XJ4SBB`7?I|6gd7qdG+^V za=&~UlmDXjVMOib!$+a5P8)LLPEWV(&Hp;QYwZb@k{?MSJ)AsvYJ7LhC_YjKp)58Tkd&(`;P*1vPhb zmx4(sp~kPYF&@0!fZl_j^0eK1_4i#RZj-6<4F~FjADEIe-szN$6#J9zkKjiRq}1?} z%|7grrhqqmcq)Xfw9zc9Bsbp~tUZFbmRgpd6 zbB`E=vM|6Aw3AO0`%@;EDVunhbi@YUi;tCDPg=9_(DJrJX$sKPj4VKum-p}`Ji zEp-ssArr+P3RZvGn}qFGH4Gr>5~^g}m0aUn#{4&vtB%}c;)~PGumGcI%1t@O{_cGJ z6-6x8GYN|Zmls*r)X|+*V1mk(1U~<&i z7pfbgmAdt%^-S5mhoTNS?w_tO-t>1}@|UUq~OtzkjkYKH|C!>_O|YDab0a<5rhE=QO-UdA||aQiXKK(8JI=7q!P6$_=7oe^okI z3-c}ukcVRllL-t@M@O5K94dm2Q5Bo7rd)lML~znWG$)#8zbp8gUuMmXE2;=-3N&X& zdz?ID2Kw)>QyPfnZnPy=Eex~rusm{v6JH2(2h{7g zdr-y8yi4@Q>W)B(ls9e2K|wKp)%*TVobt?E<*09@tDjzttci2^68mFWZY!9NKIZQo z7Ve~9n*Q?C4ilxZK+&3$WGP;)6ncGsC_#UE*$(g0ASz*|iRyz1 zlPvW7(_$C++l#fumv_k8&`#BHrVr}fuZXEmDXmHc>~f2Ce?R0{eVZK3UKP-k!+o-q zt3WmaGzBJ2+gE3-cLc6i1#WMnHZpgGX5JgNQ=0i^yyZd});$UKySowO6S7DgRcbLW zTPhG}+it)?Q`np^DJRy5SUf;rtlij#rK_27OCy)TH=tZ(+tN2B)$57;QKA;d>@(R^ z;-$TlVr1{g7Z)#f?2pOX7nceE|A5drv3yy;?0BZ8S#aDS>Z6k9Uo@TUyLWsPlcLz6 z{op1ZGug1_sXGCPcOq>jH4Fn@65gdadv5B=)(lu_YvXlcvC4FT`?;@Cy4pq6wkQP# zDfdf35C4C+){?O`&_lLmFXAw7RDJEf z>yMWsiix;nVUyDk@6YEkx-`6Hahau?hYs-pJDV<(vd(c0JB~}VX!)0L()TX7coUx0 zrk+^(G9nNY&_Z2U2WGce@OVH)sb2y9I-h%t4P6R)-u{Y1a>+jXb=&1-_jzj9upzzLLUr}t=furkfQs=B-Ol9arSQ~Q=NXgpLihYT zYm*-N`#-O|W_2Eq+-TIM&Bl8ky4KM3yTN5`2HVuU3eeusZ9zUxrbLeI$ri=N(qvU| zoLFRDk<8~jWO%@m#s}}`GnzZ~STEUqx#&k+TXHdvUQT6wBVJIKy`P~Pg%E9)(|RP{ z+OzdW1=w!D)-y2YzfFHnS@M$itA1Zq?ye4TGwbC!!}(({O8TLxT;g`u#jwdL_e}E$ zSyV0!slS%+rnb<7vS+>{GOzQ!%zFHfyate4Ex@={k(GOg+xqq1aV2-Aa~%&(X_U6s zl4s|7ScI-6HQSWX55s9o151s72f0?=$?xXNeB`Sz4eiFcT2GzYEwlrYcjc4c<$R@M zq4l1;x6v7@{AR~qFRnPO%f8&^^EXr8(f!YzrQeoq+5eI?4+}YxlT?r=hVx;hq&WdQ zy*^3C7kigX{KwuC=343tk)P_=$qgCz)cxB?vT4vB?SSuDqtA0YiVpbFvC_J7jA=sG zBh16c_N;crO1y&Lk)+fRo8n`h>-9T|D4m2~WheyfOuNH@aSAdxLL8eRC#>q+q_e8(Enf=b=ui!uvbL ziww1V)B1kF^MT;ZpBOpRgR4jH3vv`PAY@b)wK{%*dtEt6b~CPa3YN}S>_DCU(hK7U zmFH%DY6krBYt)vKb(Q<4{Ni=4Dm5$da`gQ_xwkBq%I-<*HYRpDrt{GuT=X68v{gDl|wyoI=wH?aSSrReLdo97_dBRbL#DLqW|Z&Rl2MN zxC65z%ftTPbS4FXgatt$M$rGgP?7g}F@C(v7&SRz@F2!ZP7$n#DU%cUKLr?dJ{SRcUQ(eu|6ctc@ZlT` delta 6934 zcmYjWXHZjJv`q-TcLXK0gx;n1CLIyb(2EhIqwpajEfIorfzWH{NHcU0kluT*f>i0! zm7?;Z^Jd<=KhDfOcV^ArYwdORo^yv@j=ijOq$D|TW(*gh0|0CjUQ)skI$-Yl7QY6I z;|aBECm+O(f-E+0csuIEt;y6h=7Y^pao2*Ag5q*noBs`*r(}+CsKZ-A24isc`8p-Q z=+^yKxf%2Ptc}U>O=~#=K~dD_>jkejCj|>uGVwLSM+u12_3!&i?-5Zd$92{A1`t6x3TA4Lpn;I^p~PTig~ z`bdFePG{bJoO_oNwdHm?H-nje^|q|2^wl!#!Cw8V#Ya*uyS^8`0kUJ*B?+vb_0d{Q zhgW5rJSMPoWKK~~n6OMsQ*ok6MzCvUR@pSe^k%kY<@ox!dc~c-y`8`>(#<2WWu0>o z($xc&pVqRz9JkiyBFqoQ4j($Vny~!fSo{(;ms$xI5u0F)Z-CIg&|HbvcD;WJ=KA(A zKW2hO$pS=>ct?o4`(eu`>60HT-Oc`VM}ZMB@fJ$PWQTQk__%4sCjIGN92vB9p`4>~ zuOr>aC5zMpS=4X5Yr;VKyeEQ3apAr$3S2av`j~`pm1}|sVuV=`4-IgL4Z@kNDx#xt z?@Bl#yzeMZqkCdRG_Pkuk(0*L+y9Oj+}0*Od_F~KS6e|7fev3AYhXmQ)*XlH-kS$S z^9p4Xv!b0N))ctX>SgN3#PWOI`gaV5Z`|G(yldh8^Y-vrRk4WTNV7DeIW3=Z>s|C= z&d!>))ZWwlWQ6ZMoA{=Z`Q|^%*_eadldpfy<`#xl*2ebf@*a3y!plLQbyo{tS7Xp@tU_FtMWbVM!4m_utBWL7!U}MP6NKODq<#%6>bN(!$2< zL~LIJH4_;{Dp`k}PWG;%HM$e_Wh1nY`$qx3?o__>;Px|#qk5>Xm%65pXuQZuwNb1B z={j*6+8uFy$o3;zVNh=F5bDs76dT6G zaW|q&l2 zi`9q`prPTJu(R-qzM?PuEV!iSO9%7uXkL!+$IKJij#k(|QoUcnmwOjGUq{KL;>o;P!r3tYOh@>6VnDW!Ht_(InLWiG zX*JCLX;w?vlL~3|mjDD{GnuSiU++Di)t5q!H?LAq)Yitt6Lm{5-_?bWVdBZ`@WXGf zpk>h;l+j)?LZm{W1<(kG`iSIi+GSuk4l>yuX*SGWRTXRYt-KamolRM;vGjvn&mrpQm)M?9X-1@O`9ZR-614kX$V zemsHvpj_1vM>--2G{nP;4MA`&HHAQK>Jp@{AE|u-V4b9hdY^HnWDpj3u_2g#GzMpQ zM&f5{0Slg;3$!fRZ^EjR8t>%yHz*N7@;6O#)YXWeA@6jH1GtR*0h$;V4bs(x3PVab z0kX#nDASDS!4ncP$A1%E+gXBEu#kBIE1@S?UGVEqJ4kNQIC^Z(9cf9y+9NE-Dju)4 z4~dte8Q|r5#Yw9s2SRZs1LZ)8oLm1zY6AQtJZdUd%!OAjb2p6kv;S{?DTev4B(ge2`z5AY(mmN{tE7(oXd$w0;d%DOt#gd_C? zX+*&U4-Mb$<-N}2=nFhyA&LF?=Q&k1lO;)KPkUO&?L{+%d`5}(Nihkt4YTe7%(mgv z9|swY7AvJ9Sgl;b9u!RI#j}~nurouIb!Rv@R?!ZS6hW>}_VV~vE*MP>0d3(xh9-+a zxiCZ}^x0=^kxkTUQRvtm~_KDj}J^y*N_M5Uv>qMrVcG$zj zI|1Tf_o^R1CKu9K_whc%be&I8Hf!g<;S~AE78i}i!1v>*#aZl_s>nrkeTbPab|2K@ ziSC_6)&2@U418DuO`Zgpnay8Y5C6yrRgVJBJ3Xp z@f+d9N_vF9f9ioQe^!4gz8e3iv$XK=2_Iy5v0V^F?-ol1Mut)V%7vBJ4BOit#@``aw``gWqqO;A7ihskYg|3VUBwv)&A| zUL%%fZtbv}d?Rk1!W(rQ;Lk7=^$_4^m;*qs=sN(BtaKFy0|*A5#wNcT`KOQF@FaCh z4P;0Hn6mn7Co&83Qg}nlx@6gyVkc&08tL7fmH4sS&1-iYDWS_%GL6*MG%o?hQ+qMz z0ISGf4#0@(4k6oZ~p3R|RfRLY!H&>6!SKeP2XLss>Vd=~etCfL~|<-Qi3 zKLou|JC53zIp=pJ`6C@fGKDTAQE5Mn0l*tL8B4fJRBwM!`Su(%;#a_|#?1M9 zC$bPj!P^#gVKM*+X%w_nwve7Qll>liC+y-`xFW62VOVJSbdWZ2qdoHgbVyrRhppW&f=Zv%Fi*ifRqmnQ)I)fMPGJ zWPVLmwC$6>=>WC*7kLwyuLXs*!ts0Qgx=iaJDhPElWRCcRve-@=pGJHTr9-?njQ=T z^q(^;G$2|KTy*4MpZWIHT#h$cNvPGw$qd0J3zMn!w6ibZ7mvmJiue+ol;&p z4nB<4J#vD2XhRCYjFE-Y>T24q$muR|6#;WFvD1aDVmV-Z&%}?MSMgug&JG+oNrFIp zp2q6i))KSbD={aJlBu)a#H{R@;P0Bv$=}6i5kQ`khn&tAm5&4~a6!s|%5g!;43Me- ziVgRk65tFu^{jpOU)rnZD!8=YopD=E5Pchol*ABqUdfCVqg4qCBCzp7%E zpSS<|5UR+Ai*H+9t^YZ4yGy*C+kBsB{@huq98kGi6F_dJ^e?^z)K&mO`+g?0V^@qp zUQwB_+g;LsxM<;-!Mc^oV(mQ$Ef#`yp(5)sKIGtlh^ZVtWoC~T+y_Cfhj9dIpGPPX zc7t)%{jj~^Sr}u0Jgu5k$G#YYzv9oTtGZ5W=muG*m&rkuQ9AG3tDJg}9=uUw^5AK% zS-K(wWk#5)C3dYJhxQb4j2whuW6vJ}Uu(U8yK>&U!%t#TV^Qnh~u5N3mZ zc_QA*E*yH0Uu)8MKO#gA9-}e&0|&OK1VLF5&MOImtY}S%8qWuCNxs}o2qf>vCCQbA zd_h#SCQp5W2*{YM3zLA2}iWb7xxl016^U7?o*iu1Xa_PD#3qoji;;1?Q zxtDzawo!%F!TeMg=#P!M|^Qe=2QOvx)y6I>dQyGa`qFvmuv_cR7=yMTs>&ljJ!+5H9YM%LMBs(n7PqjOdgf z@(>g|p$txE_G$+ofCgq-T>ucSRMrfdxIs1vD2`Jo$?Yv`&ZOhnIH^jZ;dWKd zS^M8X;kAYn@k476?C|ZMn|gQ_5B-qVGG&SVOI5STEkEx?FYCxU%Gf^IISrjy99;D; zR$7FbAV>jcIl%=Nd1RsCW-bG+90#6tY~n1pOW+Z%QU-PoB@XU?CcyZnlO|wzxUm8+ zmO<}~cS<-A=0v9&*8VI)C9j~pOHGLH7$?jf_Tx%&_I}uG1TBwVFpk4)x2$`J_9@59aZ+Tl;Yn4_ zncDfPHutPX~bPTRV=Z4y`q!{F^ce6)CZx}e0-%F*Xc%VlY z9%?a(&&km((T8 z*HzeSsCs(@#HP4Fw#I_2CKjwRaTh8JcCzzdvo_DBS`siA5tlCW6ZET@QyEH!pMJfTSR*-5A6}vTi!G?g-rc&5>;k3WU4#+RdB~?ns?>n zSYNTDw6cgJ(>`Y1@3715Q>FpfBS?OTC_kX}dSu#_r)J^wTgnVhgN)nHN;x;Tpqft{ zg+K!1WCbyeglAkF2y>GS8S0>~wz-3?!wQ#i0X_7h4?M$A&Vd4$UMgf!@sDLU-~Cj9~$A1w-X zH>Tj*Byyh@TeLZapX3r9$ z{eb>OiGA^!-pjLa5q*n(!!s17*6nV?qiodeX6^kHyE%_6aLw^~DrThs81r5-$gi9s z_*3Q4RQFJ4Ek8nEZM3u#?C_?$Y%;nUr8}uKf|4(tY@XmHrL$nTjh_-M&`+Sgu~CTF zqc`1_WF%Wb?CjkO%-=J6(CN7&NTv4Yb5nJKrt^=4R$aR*QA*jA3Z)Fcr<)XAyql-$ z*yyQ7{?81p$!@J<#u)Kr-5bN86zV{Xro>re%X|Xk&nX(jiTFTS17AvvOtDp5d^hXU zmrtITH0ABfeH;YE`g6LW@+FjYnJH>B_8+qJId|5cAkNs?BFBe_YU(Q>Q=-I*4hjQF zMCrueJ`+K6xC}KW+6@zoWRmActXU_vl4lrl;8#vhPDZ);;_zd4ERW><@|=x-1$xrM zn`RR>N0hOn*wJz5XYKo#yp=9G@mHo;Vawz4{wRntoNlclA-Nw8v)9@`7u+uC_h89c z6R@B~sAt`kkbInH!C&ej;u3E60e{UYk*Q8IO^1*5fz1I;&qP?{J}k*oW+RP>?=bCt zqP`)5WMj+9pI*z#Qb8l}TiUjstP$aRV84Ni2heMGSoagCe+g;j)ob;>@V*tPg@Q39d^V@-94X+<`LVBZTFZ6 z$RG%c^`k^PlFBEgdR3sbOS9EP3qi_TrGS3{hq`a zVv)kaH6NC_-J4amn1u!9v6Ko}6ZtRm7X_3}69B!vvi8evRNMZW+Xr(9woKORoAlLh z)EX>3*je8T$c4TuT>My=@ouFzZ+I}Y<@HbBg&wW`-cn-1=m^UUjymeb z(x=aea>JT2%S(S&TbJ&hk9mB@Hup?5ArAhezj~+hm^D`{YN<^FHF6}Zsy0XFsgP+f zTQK>K%#zi*C%NW%S;P}~^I=j&QKM0Llmt(O=rJcP>n`3fFBKiO zNRMqYdTzS@1a)i8C46Lr_a)yrIMK^Vah%OAN4Z@EfqeT10QwCiyatgUL(#>034uIJ z2*4!^z!tRX3J9P;Vu8n!fL6lxxqq8KH43j+Db5gf^ZkYY!4;n+f^Vtni<|A(Ui*sa7;-YgSFit$ z(4M1Z5DETb`|t5tCrP7^ck#Yd1RTB7_LjY&(yanl8bxNfF0HkfBhn4&iSTbEYF@KJ zDCg^@gLY)?4Xz{>d4Bm`vizX2I(tw~)BQYfF@V_>zsejv!dT5Z&ho)tK9@D~E*pc& ze96X)Q(~fuq)sc>xLzv$E?F6Nlt{bL`ZzuOcZ}}{;h*|P6dJ#OJff(%M*G{D_7j$# zhQ-A={HWI+Dg0r|I)`v%t^f6^^VB3o!t5#MB3cxzeD+9qIQco6fh@H5zB5UR|S>$b4}} z=#AC~a%L4St|Q5n8f9gu;JcX9#Frwj(cI-?Ppq@eTVt9R{g$O6ZepeP!*$DlEPfb; zxi9&^6684X@+y{yt$kh)%`IDUR25?9GQ5YXK=ZVg8V8U_F2ts3Xce$7ZRiJ0+#EFR{YT)8Um3o5`k%1Ms=tY9Jnl1t{@)*~I4M(L zYr>8c=l}L(DrpAblLV-=DsU*lQCbeDn2;yUefJ;9Kmgzo9socC_`e7F7M!pltpxN+ O;FRGa6q5Y=<$nObHv!}T diff --git a/summary_results.json b/summary_results.json index febf86334..49af204f6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1863, + "Performance_Result": 10.2051, "Performance_Units": "Samples/s" }, { @@ -932,7 +932,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.08857, + "Performance_Result": 8.15441, "Performance_Units": "Samples/s" }, { @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.379074, + "Performance_Result": 0.304628, "Performance_Units": "Samples/s" }, { From f710fb53d53177e7247f6ef9996bc4ce94eb8156 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 23:53:08 +0000 Subject: [PATCH 27/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14411 -> 14414 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 26d47cd57..1ff769b18 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:40:01,968 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:40:01,968 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:53:03,470 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:40:01,968 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:40:01,968 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:40:01,968 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:40:02,995 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:53:03,470 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:53:03,470 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:53:03,470 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:53:03,470 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:53:04,551 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,7 +50,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432193 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422753 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -60,7 +60,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2051 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3048 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1795 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -68,7 +68,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9511 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1396 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5fffd3d00..696290484 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -568 +569 diff --git a/summary.csv b/summary.csv index 779b8b7b6..e2ac88fb5 100644 --- a/summary.csv +++ b/summary.csv @@ -28,7 +28,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432193,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422753,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -38,7 +38,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2051,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3048,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1795,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -46,7 +46,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9511,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1396,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 34eb9586a..16218bc17 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432193, + "Result":0.422753, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1179,8 +1179,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2051, - "Accuracy":"acc: 74.400", + "Result":10.3048, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1419,7 +1419,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9511, + "Result":23.1396, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c04578b69cb52266ed9ea79f26c7c7c5c54aa44c..5f495f9feab8abb920e1c9280bf990c9ada97419 100644 GIT binary patch delta 6917 zcmZ9RcQBmY8~1mWU7{0&Xv?lnbP=pxRu3Ypvsxq~fx+(};A z*%|vqLTxqao}nnjF)bPl-L<;9jNjGYuCAXuwrcOum0&9J@u5IkXmestYKO)SX1d(U z-}k!&ycmz%tag)0jpeHxYdUYJz2lAbfBKnqu(R#pOn9~k=t(;IaeQ5Q;o5u};Fsi7 zWMN}h^Svmu#EjW6bzo&(IE@-eesiWF)5j^7awo6t!PVN&s?UvR$F{xtk;BG<>`&wG zsrf3;AB6F}j$=K*;d=N3WH_x2DyKIKySe2#OZOM=4DmAeonQU-#VVQg=ht)_9B&$q z<9^$g7s{UaOa`@NI%X#20+Z1Nf$xT{MX!A2=8Lp6s>r9Ax9lmIsy)2{iADw6rVQYm zC+&Sk$^IM@v%coR!OD3{^#=S$a=lF&wbIVGe$%n5giShQ*b^7r-d!PYuO^v#tJ2l6 zmQoJmNz&5P!ig7_j0Lk3>ERXkb-Owa_Er=c+{YR>UnqWcX&d(de8jI!EsF!tHk7Yj zG!;kg{Se86ux20H{Z?sY$e?1dE3QkxHB?i zR>c*`PvsCP)3wXWU=~Bq@RZI^mwV57nLVmXhG$0{^ZVY+yOPC9X(Q#RX^bl6&w-|> z+7sUSxisG$UAO*k`zaQW@Ev;M@sTO>X}$r^ezNoGS?Y~HbaW^b2humkZq4|}1l(;6 zrN{4!MnxufO48F^9LhzJ<=DnW2Cw+g)0O>E;r_{Zh*fXT*olwyy`n~`Z_nN2vDlvD zp{EO|jiux1+>cgoM?-RW_Sh~_+~FFgBZsd%nxK& z43qqCQX!t>>m6AZU_b8^S`*u$ZR?+U;?B`vo=v?HowNXN?H=8~5SGA#&s^@+7`Y*>D>d}$=mTSBhn+Je)iXmL5v{MxD3PjzBj zspy@_%jMJ+HOuQE@zTkMOU4g})U%(?#5-KN|8qDKnc-W)X{YYG2bt}NVcN+U(pGP* z%fLmF#8N_9fhPg2ZO5RDd_TO%7u;=QiB|Pyzdp^Nec?#OKRPumRlthV{-s8* zSGxxU74|+OgPlgJV5n#$ab7MJmVHvL7qdQTy9~*b)IJcK>%k=H6k_| zqrL;Bu~QHuxS^3&pzoF7!0Ecxk|Z-=6zW`M|S?g$jZ8TQM;S?SYqsbY_=#l z_^mLHpCb!7yayCt%MuP30O_6II7EX{E1KXagaF8D-r4<&<6JM#!$;EmRLj>K?K-RU z%0Dl*Lgv`2Q_-~yT`p=JLwKbJzDUIpETGZP-=mcr8mEB;Goc{pcFKxRfx_4#vevA8 z%Z!pbK5+akRT~t<7s0Z?DA`?fyr(0I0pl}>6wN@s<4C}!9JH?^ z(OPu!{JS(h$AH^&{F>XU=$-EHd7dUE?j06$nWUpdO1;RQNFUy4KBQ63ttoolUiLXR<*7*^%vjqm;8T(QcQ<+ zp6IN1k~T~%67h=?D?C`H4-f`=I%2T+F;Q~ZXE^aPmNM=&3w-jt8}S<)w#})RWF`Xz&j>-T7}%F&e!;u8&?G{@0S_D zqO;^d<#6J5EF~?0D058L_dc;keZu6UmLOL}29bb3Fkh0pgbyEQQW@jV9#dAo!CI*r zk&Xj!*VhOA2(qEN7?)8oi_g?JIYw@Pr+v(YDE>xd{(s?R*xe|1U&IxH`VskIJP2G2aK7ylVOpQwL|IG~81 zc`(hrDOId*)m-9@R%dOxc_tl3*faY@rq=XCqUM^mRjq|YO;Q(xp#aEKsaKHPwP=wd;$$`C6)ye zcV?EHo0ansv$cOw)g~~yp4M%$3591b`%KouLmeS^HH3*Bv0zvVvLMn7BoSm5pJc0t z{O?PwTv}4_JF#)J!dblgcO>4e>w_t6Bva(3ODd&U3P6p*tRlmc|nlKYLY!g~AtZI}Tre zkO~3rq*-p#G>v@a9L)Jm(7U0CHcY}xsERS=_ zzpRA>0_$~E<=YSDnNn?GI02P1z{9MVM3HhZ(98?z^K z_0n2~rVvtXhLRzsge8Ihjxt=1WBFtwH(K+C=<0)Z-|BxxI5lylRs5wsFs@-;k0#CJ zB>I^Yt@O;x{7+Q$4z5}UKci32z8cG0TN%Gf)59+`z)7!4Puvf1CV}Zw>WJaz-#9J6 z+J^;778Y_5We7x5uuh*QHy(Y2BAZT_qw(tS9#V9BS)yXubNFvrjBI&Kw!yXq7kg){ zW>^l-8%}~iRwZ~S8gffXm>4Z#4=pKe`T%^WQr0m6I6;$2ZaRd zW?rfZfs&DzTDp>Jv^6<7%w@kmJjkq4g)Bl;>M`kSFiNEJ)9=P@Yj2opDJt$eT1OO9 zcW7lxjmAg+#W50a5aOEz9Hb|BMpI%P_$N@Cv`#)5uWYCrze*r_z1yGg{cG4oKXz{= zE++d@&S2MNzJ3EMvMq)-h)Rlk*(OOrrr9O(?8znz9zjx`n%n{xWpRlWG~}>E za0N{tC~?_YB`S1u?YxW#lxui2^5MKSDdCrlQPAJgtMRLua$1*fR)SX;PCSAHQ|xGy zohG%?!X?fS`Z zVB@^~rfo}H9TJr@)uw6|tq2dj35iq`CccSHBL#;c@ez!m8$lMa;1Xy}j$3Vr2Xx>( zhlR|iSc=Ri4-*0X@@KWv8eu{y{#(y{(N)^(s$Cwb#~Se!aEMfDvc1x?Od3XIEls$HSMk@ zv&f6gVJXEf9KMTA&)_8fS(B>hJholV>p3qoQ^f|S$HTFY|AzVt27`+$|FUP{ivQjZ z)jmuiNQ+x%2%wB4NJFFtKkM=qIrWBzy8S!pK&bF!nbLdLmltZ>bmepFw z47G67B0Z`yA`Iht97l^`u{$oKv_rXX5u?xBIm|$1hklBXMvngs)67owg|81Z2|ee0 z7j2yh^*G~SEXAIYdI~sl=B9s5N3mJaKY?Rw{-*6123wYBJ+^0Az}DSs%{4r>CnfB$ zZ3kuGI7CEXgCtP?(Kug8oJyTb>@;urnmn0LV*j1UFX|Fg-a(dqUTl=~DD|tXJum@9 z2-8ull@O*KVH8K!5{w54a*qYq{%5?NG$+mx+Tu6hdK#h7HUAoe6FQE0luE&4rN;%| z&BnUi-XRO=5_j@j&M3C&^1smow>7Rloq{QPEC*ho$oQRDPy&70Iw%N#HF7M1F)o2{ zEfao+{A(5cd9Dkx)FYInGByzc|25I0bhAgRhcvNYgamb)9A5pLt$&Irv>Z?Z7PH!NeZ3}7|{Q#X0Z;~2{8dk zedndKd_?kiOV4<(j`OYOMo;?Y4fhxj>?RhQGSlCvWuip0a!%V+pTF3^(pIo=U6oOm zp&+4d=lz79Iz{NIwKVhP^L_u1$SfeWB-a ziW7^6=&S_(jYFU;PNDh-0!AXzt-m1f2|h9b8p4 zl2==?22;@WdUeuTbl7Ske}IGC_5v<>i3Q%JXhFtKUH z?QKvZGA+UkBp7rrmNcDEq+;eeZdEmpbykl_j&*<#`GlWpdEmE|I2~T*(A+xf*xyxF z`IwTnrSf*^`=d+tFh9vF!rd+j%t7dPN!#x7!MTRZ9~nWqnLQ3)mjBv^R5Vh#20my1@S4x)P#T#@f~UdDiEp!@U1ehjh`0sG~w6!YqszU@&ztOEUD0T9V({^a#Hi{F*e! zG49@8`b^m}V%i}Jos~1)rn=AEO$fRX@{Z6F8!P00gYd{|LeMq@XOjMPW-+tE2(j!h z*g%}Yaix{AONaZA+D72=8IUO=9cVLq%$t6ur9UZ6AF<#N)sUUDKp?WW=&DA#1TVBn z1oJf*QZH;s20kKf9m_vO8wLI~5V$EgeBqUItH?_<3|@m_ShvBnbl&+yD9eW=X|591 zUw5Bdi%pjWW95yamDT8}o#!x69xdO_BQg0pv3;QY)Oy2at z9&~a_5ZOz+(Pbul8ESlXx5Lh|gj1+E6J5=oTvTyds2CG&Wv{O>+T2U0w5&qVJax5^Q4-9@B&iT>GI1)g5Hv*0iG#*&Q!<1Q_cs@!4m+YR zNeM?pTAzhO-$DLVFpH49bB|&>{6+m#DCItF`t6pbTvpUs1Ls38nPeC98h;rpsuQ35 z>7PZ8#Vf zZ%_CR&Nf$LG0B0ccdl%9hA*$dm7ig`V3JjlxHmWq4>w?LvUHdBl5fGA{TmjB*d`R`aA-Dcbh?@wiw@H=2P@*a4*)J?W4c| z`W-KyNb#nKI)f6`SI59n|G+w<8Ikt7WD=VM-ou{0;9TC-(K3y8z{U831M(8O;x)bICNOvOEV>O3LNlw}Zr2)RgkCRK zNhaG(x~Fqg?jF9&oX-Kjc`NJ*n@nga{dN|q(8-=LNn-ABdnJDIg^-Z>*>`k=BSI>1}8r7?&`U;$; z>oKlb$L>swS}wT+ndWc5Ox4D}ysw6j8LJa)Vr))wYaTMrlt|LKFnpOz8sw!>EfVIo@ zYtLaa&xchgLrUtCpM^=)?89!jo z9zEC_!VlsHhY?NMl9^faZ444G9^yqUcZ<7U z!<8@$i?z6xV@l3wl5;{kD{}X_kdxq5GZ3k#?U*79S-rS8JxA z1lTofw&^-XBBHm<5npp1 zpKtW*d?xOm4YqpTY4tW&(u!g?>jploR{R^g7T=fQ3SvYyPimB);SZv~eL?Ivz@Z03 zJuXDsHyi3Xt&0~v^>h}?BC{D<>o5_z3y|jPy%zhDSUf7zsXS(vhI#y~_{&3jp;k9K z%4@!G6?z?p?kFz6kk&%++>b(8UiTlMEV+Jo>2=6bWAW5hBF}^f~dJ z?$GM)EF^=(45tON!P@Ry&wA>9siv(fw0D4hjA+Q5o4s0Sp1plIH((VS7J$C-NQ)z$4?Qe zu5M3X>FM%^ktfYefD0D98>10YOVZU(6&8WX$i9FC(8;F+u3RiW!Ogs_lmAVA&PjeAj3?c#1~oIaPGV0-?ex~zH9fUfqI{bs zdapi?w(YtW=b$8R>y<}aC%Vl#7t;b}iv2{ew|_o6E*SDY<0n<+J%25hCjl~2*`nQK zZS|~vGr_3T)&4EmqT!_ZwDrP$Po@&u`Lnvfb@siMwgTdN zA8R)G6TJt?E=kXpN^43t_u+2hg_4mv#m6(B2F;$%dYj?yaT8^g%#vAqKZ8!rS;ES5 z@snv@eJN>f?W7rLwb&;ydt=w2%5DLBGE&2ycQpu{s00Vl_qa`rR^#OIMs|~kQmS$2 zFJ;S$#~+N6leR;pJk?8}SPMSDcG}=QNrm7eo;i7!-;{#M0n-{ScIm@~s^-`%vYTU8 zpK4S&2LxZL#~+w_&bX2?-V0XiJ1=z%7kj!m@R)dl!s9+jK@L+(Nn711gTMxf$ zN?SXlv)MKd)Jz#2>sz+^r6TRfGoEh>a0Hj9M6E|IyJ@C!&b|Q+ig|IV_(fF0+g$Ka zp}B#7?|=5tS)_ruTNwV_o|hNDebH=qj;RX&!*XTO(j1%JsY+FZ3Esp?eib&tKehhZ zr;E1IO1F(*d`7yPuQvRwksX?{8LR~ZrTolD82WjJ$Tk0X&+tIw{pzBWp9&KxYDZSB zujpLTh$HP!nlF*wXS|ZiCA89Al7;yH{-uhMu>?2Z)@8W=cScjoGJ%b82w4?yFV02w s9#{dFCChvJzx;whAR{6Whz|7sp5PF6+>$I3?1|&P%L5UX{=4#j0GuNOWdHyG delta 6903 zcmZvBS5#A77i|L4BORoN7Nj?kCcP$9=^g1s0tgC(W}<@hmQYlB@4bmsg-}GgR0R=1 zstAEd#~Z)@KHYK87<)gQhdJh+YtFUyIA{FP=%WTtYLfg9l=Q+FIJF$o`(yCKPoMZ- zxWD`)7L;cX8kS`ure5v1yx==9+-rTe{KtM^L|>Y@I;5HeELM`@ane44Ia(Tu=wX>C z3lEx&-D!w>fO|OS7mXDLZ+H#hp>e@=E%vWmOITH-5Nd=VYi;{r$r z<&ruo`@0skyKTaz?X@#6n@seFsyBa(hCW>`3!U8!qshOS`QmE%Y|qTMXWuzM%3Ar( zPTU;UnYFI%x?`zp3EBP=|8icA{UKJ~Qup_do<-lM_DnrSU1^3ZA8-C%e|b`RbIb+l z7ovBRJKaDt^;4y5Fp4QWF0L)M3zCMogG<)3%}zUEkqY7TO`JQJViPPF@RH&QUK{I< zYd;ZKSxPFkaBzZhBblZ!f7{0}dm?Yb^P=xkQnal6< zknpEXe}|A46AwHk!ednHjboXytW(>F(BQK|sT{*Mn4`5`%o@D>bY5FURe{Uc7#A8a zo(YyX!r~ACXU@`&HAYD0A@C2K&-UX-5s^i+_PNH$yZ5wrN4F7YrSkr^jO9`ZKk;Xu zi~`!8Sa^9N0y>&6R?0Cn9xTT{*e#AwV9A%*>D`kx9?rS) zWO5ZAw?A^3pZoiGZ+^G>%G7E?tJrrb&Gp*v@$ph@9-7R$a}JJ$~v*F0W!G!3>p zLm&bg1$?y^F9Y9+ss|B~tR~B|J{2_CQI~ zr=_RGmVo#gD{$d!&p_Gi0ni6wZ$e;?=@gKna<|I)SQXH5UdRDymndaiNnG(&d^!xf ze~h72P*%Rj1+vOw&^!9>t-g)T+JLJleL1UO*UruT?tYle)#u5iNwIq5V7fNb#%|In z1A(u;4pcI`?BahX-xrJC3z8&8pkoIxTKst|KqWJpyakSfW0~LRpY#)-U)+}UhanaH zdE1Zp+JKBB4C(PSZjhi*T2n=JY^WHI0srG1KAdGYv3qt!uuG60YhVlOdQ1j~md}>*L z^#SX^hPDcUkxxOQX&pM3w^Y`N{2{IsymNkzR!tw}_xl86W{M_Fp|)sik8CfP=oBZ) z8Vz(IR0#{BXTOMbAJ?^u_S#fnbUEldI=zJ`Fx$* z5@0QQ3)O5VIysHjPK|+z9Q zjOe`?(h{XjZ}ak$+))#BVJ*0$CSl-0(f%X#cK-KHb5Coi+xIOvUuRz zc=yf(Jfgjg(lO)@bJPCS|)Pl548~w%nsKB<){x*^FqYJ5cbB^jU60A4Wre zRl(^>Z~9;s-j@bjYq51Rp52WmMw{TO%{l3b#BFo}N2JmEZJK?hf4sF?p07_^6;_wl zJSMl3o=Z5^NG`BRk-6Z+Jo3n{Rf5P=WEl)MrG8+btY zK-@>Q;`-V!P3+vpiFb@y$&_4L)@K^1&dBdNLO@4AkK(OKhxzYCxt*8r+5(4FA=ruIRMrhiEFR4{iXqvhd zTI|PL&yM^T{cn7~Qfb%N3K$zft9rrY0eA)N7NXT1OuQestKY(J#9AU&gbG;4^4vdF z{qNN@;JO>`SE+ieLxqH9<8t>5K(2cRM5>}%e1B&O!xMml~mh&TK%qSnIP;Zb$CLr@>;aXC; z{8W%(4z(t<_#Ut7Fo9%~UK6Vmc_XOd-_EHa8yQbq3xJ<<6n-K5y^u~2nyN`C8xOsc z197kJsk1&fV5eS@sx*WK)UbLnY9m1aSv5$d{)Q=(9g#ems+4hR-;G`<3@CF!dV);>(KgynTSXGqR zzA5M#NECKEYN^ zkfwj5@5fiVq15d&9l*8h-ZdN{>WP@|tbE9Kq~_oSbYnlM{eS7aT{BtbFZ?I0=U!9> zNOC=W7GB5X7F1(8G^F$6APkhI&MWk%AvME=hJq&==t46Jh<0n_E|}cJHxQ{68t+OK z;QKNYIb~`0|3#7bdvXTp_i^*R1MXMfzP_z?i{%;26%tXZeJP%Y7>a@YxQ{@B=%BpY z2LwoJ0_1BTH39PVD|P8eN*WP>+c_wY;JV=(FJd@lB@tdwM^6Zke1?1Us(G7CS#1

x!s@$^00ICgqiWlXe-5I&W5nvEOZ0GiZeCLywoG$Nq3 zoTqaU>(RhR7MwM0kv8OB5URg@6m@&O{=$}FPeE_&c8mI>sGf)U;32w~Yz22BKy*k69jtIq45lv&$<_*66H9R07IZi~vpVmII8aYKi zZ}`d>4o?^O7{poes}Id0o+cA+Etu&+s75BxgHVl3Hyd!@^zA06aay=++pAyHtGT?RK_A{Idt?vf{pS8^A>G4G$}r;e1@9|{QDosq z$6jjxAZCs){qkR|z7S&Ng!Fw>Q)Mbbq3*Rv~ebZ>JV2d^=TasY4p%{@xtax3--|7rDm2|Hcq^#c6pQuq=m@qFAz?3K6x`E(mov>$iyl~^yOmYgI zH{AGz_);#3HxI&9OVIC0Oge;)Se9i!9m@f=yZL<>kyiWZ%vyjJ zYalj1hjZBiD~`97n4fSF+Kl6BGWpH-l|Fq{$^_$B)*K?Xr#T$9yr$1VSU2X zw)dwZn9s<;vx&NSh`vz^)*KVPE|L20E_+jwkUfOE$UdN6r8UM9;27zbLefsiUd^qp zN#cY*sUFbV7dGMr%`WYmg-=v)=M$N1D)qj|aLo8xaoH|coaRB}z_N+yfiurp@;AbO zyQ6kM7;u}?(w@Sx)5d}sDQ2Ly+4IH^=W!OJ&Q}9;<Ymo{_XT=V~Eia>=UnchhuR2v|!-F@XI%;?_73w%6)3}$q}AXBxPT*1?riGmD} z*o~RDa3ZGLq%b!QFjB*Hqc7b{(Dsd*=x4M5Ed@^*P=Iz6Q1&KPaP@d~ZcxBt z!2O?jM6;r4XHXCR6h#AO#L`~$ELZwM_$Z4W{|Md_xh6W1H93wl@tNF~W!9KaX zycMguD`0F#_yn`a_9s0M4nYj1)Ii7<9`s66z#6{tP*8^#ZDr!|^E-a6%LhnI3geKK&G4G0CI^_U)Jb5EOcZ}4SpD($4D_I?VF*E&P$lE3g)F0 z0y6))Nl;o#8SwI$R~wf-WxVJYOpZFAq`DOPu+Y{4 zf|DMhIn_GqBQ5HYWGl zsl1HzWLMS0xSww{$BwE#@`o{ZfPH>>hE%-DyFw3EcLYnMylq1a3yKA*-Vb!)l4s^F zM|~w-|M+@rL!8^6I1tNvN5OLZ5u|TaxQl*eZt{r(CQ9=QMQcuywRp8s_|1cn1cSNN z0Hh>Dakg>r1?s-<)?*L&BM-?UkfSQ?hME3bCyH{zQOWliHPUl~YBNr|E)P9x-doX& zyysvA{k^>~?wB(xE0!4)l`!8#^}&=$7JTt>xf}TP<;L=>yJT%>m+ClkvwF{KVyZJr zn^FOX+@k$o4>;G~B}a2q1vTaHobKc*kc}amf|KSPt1~t`f;X#zcXv@+nfpTX?~U3i zE&Mayaifgt9)||p+Y0duTPBVwwOW!b6$rL(H{_%#Y)+Vw6KjMoAHp%V9_*vi)y%k+ zv8&KqU~aNqX%YUlwU` zGGEgyIAs_$sO0?@O()yg$xksOiXAx!ZQ`|%jcA^|8wBqZX)~>181j+uEyX$VP*=8Q zKucR2ZwiZ5rVHH9eS^}|DXO+dDKMDq%s<)I%uIN*B%2r8>oaf^o(?#vKyyDlO6{z;9AyLn+0mN58(o=fu4Dzu;Ult&O;{V;|iB4?+52#jsfDvM-dXj@yKo4`@ z|6WoalmZ_7|F+hWi4DL5_EjI^2wzlv?ScFES7VBaxMX3|voPOJ7cqJ?d}VQ&rQ1hN z@j-jrZZop3aSg~l=M`GC{3{sgdpBIXDernyZ!CQo5rCZ2tE--i)^2*~G4z6NALtLGN(7EfapoBhttUvXf!2z)7g zG_W~TxtciqfwJbH08Wf&v!*8B7Z0KJA`a2CE1cEp@I#V6>cn+(@=DF_eG|ck_2~%J z)%#u$H~XlV?9uJbd|HW2U2vT@y(sj|zq>Kxm4EQ#+Gj!c(b%m<9olTX_mO)I-GB#7 z)^4~>&8GnE8{HP-=VDIe+@5SzY$8oo1;dFw%PW%kl!u56TG9OA`(j>euO90odmtD6 zkb6fi<{|RcY}QxeFB-DVo0o5fCN)z+5hIxBGI`be(6ud5kP7mxeS@TX@P6z+;~w z#7+xx%BIN1v%_QaX5W;OC)0(l7nd|jM|;J)Yct~hQ$1^H_F19tMswDN)|x?Pxi&q? zol9kYRhWi$lU(g5F6~x2LCO2_$(=c$=~!ufXVka4!j<3dIqJt1M|3-u+kN_K&NqJW zsjKwssy)YFvgVIM&g3K&~DCj2a0%MARnI?GBW4n2U~@HXzKof5yHFZCyaEdESX@d=~cpb+!gEYi2p z{W9AZcq{6wE6FnoW|sYDmAP?V8k8!0zfye2P^&Dq_X}POg=YT1$f3-xAHFZhQOE$1 zQCZdM1_b}^&Pj5ZcXv>*cD?36);X@cG%>5Zun15ygvf7DTT9ke9-KkM>)cgp*5c*p z2Y&GESgn+)4=U2aKx#v8bvKG5!uTB}Ow3Z(#39;ByG14KQRua5|4jtR()`6;IY^us{p`ACKC0tnCl0gyTvVgLXD diff --git a/summary_results.json b/summary_results.json index 49af204f6..90dfb8d05 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2051, + "Performance_Result": 10.3048, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432193, + "Performance_Result": 0.422753, "Performance_Units": "Samples/s" }, { @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9511, + "Performance_Result": 23.1396, "Performance_Units": "Samples/s" }, { From a432a8436fd5cff367b5797ad37a25797d0e5cd0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 23:54:16 +0000 Subject: [PATCH 28/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 14414 -> 14412 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1ff769b18..9cb4f2148 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:53:03,470 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:53:03,470 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:54:11,849 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:53:03,470 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:53:03,470 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:53:03,470 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:53:04,551 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:54:11,849 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:54:11,849 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:54:11,849 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:54:11,849 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:54:12,957 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.304628 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.1547 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4667 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -50,7 +50,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422753 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433279 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -63,7 +63,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3048 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1795 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 696290484..5629a9584 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -569 +570 diff --git a/summary.csv b/summary.csv index e2ac88fb5..6ffc4dca7 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.304628,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.1547,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4667,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -28,7 +28,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422753,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433279,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -41,7 +41,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3048,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1795,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1931,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 16218bc17..7979230c3 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.1547, + "Result":16.4667, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -879,7 +879,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422753, + "Result":0.433279, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1269,7 +1269,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1795, + "Result":23.1931, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5f495f9feab8abb920e1c9280bf990c9ada97419..617688f1b21cb8699de5010c9e9a8a9da8b980c8 100644 GIT binary patch delta 5258 zcmXX~c|4Tu_ntA?GFc{+Z5YO2EM*C=Wvnrl60(h*v8Gbl2ICn^cEt=s)(Vv^yCNb? znj}SbiiB)$c9G>bz2D#S$NksyIrq8FxvuM+&;2p*Ltu>$C$o~IQ5l*Nz^MwrJl==+ zfB1lm^@{%>A6)1JeW%I?;hb(h_#?4)X{E7cYR9RYZl=Uv5gN*YL6+S2-hI(e*&OeQ ze0{e6Nks5a;!HDKKJdy>~wEux)>K&bJy~_4v}e*Y;e+I@M{w2IbA%-p2OI zpy=N3tpJCWrm!Krx~6O8v149p3bAatkv_fYGWF$J(c9v=vGTyx04>3}3K8e(Q|Hd~ zj{T^vNca*QUGW6ne4|1Mw6)&)y?r?4v~ zs_mk_e$5LDz4^WET&<#YuaEzuHhyKAA=*3q=J(aMq|lu6@`^UD8EHV-+n{m_Pv37} zS7tz`O}>Ag9}LU^e8=XCh9j6!;THsq(r+RBQybI((3DyUYi5`!VbJ~5c)b39S7jw?Vr=Gg96^IntuCWv^Ot34t z;h6dyP}`s!>Ksh~M;Z}hng;};euMPK#LGRs^}SqaGXVqn`Dcy`czrZ!?Ft)v^!2cC za(~L5rh~8t=cqj?yS+!t z)%`vp$@Qfb`kGxSy#;eU^p61=hmi-C7p}f~6#Cq=r)z8rABj6|#JtarUEh0r)O%wy zPPqQK(&IAi!8I4?^+TPWmoBbpX3pBD)2MzI@6Cp$6p!d{bzeIKNtmXnjdFT7gzZKe ziGQbU=Nlr!@W6SHOD_c8#JZ-{_e!3PkUQ8zAtUG^l!0CniGWqYF^4f-wbTHwCl-TcBr5w7@(UiQ-289%*6TJ~H3Mr&g4nb5_8f*G&%d+-h)K5O>rO=sNq z!6re=iXiN0M}6OV!0dwLXByZj6JH%G&_OXo79z#hyV<74cBYg1z!nyp3VsUf0TFB6 z^GQI-ChB%XWVb3woDehq9irdfwkloj##@nN%9gyUl-%MjzN~9`F`u>_0xk|C3a6)% zg@dtdfp`JBL@!p%ge9@H?>y23>^5Ncu=gYut#%SCX9AwyP`$%}TDC20)~|^26Fc4& zvsKCl!M?e2t;|;rb+=?`V1&?7#KW>qV0p2OD(7Kjg3&<6$&sNoSfvhe@AQjvo*9!D zLHydxEtQ~}XOVIq(S}j+cmX|^`}paX^gKy zw8XWvw$f$Wx3{j&tmc2gx+oS0Ic_8)eq`ZEc4V-NVr;9wX!Q>i|JstwCoh1RWVbrY z$01gYgtUg=>~Y>ST`Gau9iHoKOm03Q;<`P*K7_gj(+rzavnWEhNkkg?>|YzwYf=C z6kkxu1<9aUzE{IDOwtMvEvU^8{s=ibEtOn7$Vn;WH}or$upsTSeGeD zgkd(Xe$Z-2@t<^#)~pYbiT1Au&UEzb6=-IcpQW!%NfMGehQh(jf>_0f5H$FwC^mO9s>#f$Vj2 zAtPB}q^!mOY$Z*7_hJP?mT5obG;6XV(m}z4`av9tcy5a#mTvF3uzhneEAaXj?>j8G z`?NV?SU%D!t7?C#$3f*=5e+>L+iGs8JW*Bs2X%cU*foK- zQcUy$89m+srD855@^;ia772Hs1f|R|1p*-{%rJR}6NJQuP6J6o)7lk>QY&7P&qliP zn0#>3*RncuUa3T zr~R_x%;_I_ePNJD1 zB1s|cz%rsBPnB5nmd#IpnPot@lbX6M-j*)$_l6&K{I0bF*LjwfleZDmw>NuH#aivs zE|krSulHL_aKbX8=H8OZP_fM_1+$BO2DIV1UB3kDlFw=x*7#pT;`BH0|3hEr5@@A+cC!Vw7d#5(;z?jeA$!eF-pr$NFT^}1($ zQ2e1=-fNn1PjJ;$D^46u@?@{rFVPrJAWe%^b>rNoLXt|SMroYj|u{5=D7-#xM5dCW(;R3z~? zYTkb-s(RJK{o;*H#>%T@`6Hv)5&=i_QR`=~btxRNNKU5BU(F;aM<-X#N1{)ZY-yXT zgN#G6~$v0G(24NU1jt$~Jy+nnxZM&aNrYMSBK+NBKcI+<6$mXlWtBSsH! zu}$PfW_N*`9sE(&9XwmEcSIkze$Lytf1}ZA*1p_O^M`0!Io9#L z4g+*1fMsijrUUjYFY(Jv;B3sF1Y3{-asNKm`y(QNjy>l#p;rO-y9@1P(rfW3SU^BA z^P?|I=MLLscAxYJK9@& z0Qv}ONerTu_p@Yv>)I~E$LKjxokgz;zHbouMpNM&>EZT4lIyrQ9Ay5aa|hCu$Y#_G zCo(k9FY-B;EJIv;KLxK4-OS#G2nDkDg_j8A*XUQgE*o9m`jfn}3{a!1&+NHg*S-^ElX@PSSearePhtWEfU7tw*Nec=>W zK>@%-uqEufh$t$}U`EkUEUBccOW#&Q3mJ!JQP+}#;5}IATQXXep!7xMup6SS5~rc? zriqT}M+ibeR^0SeDqoEC*g$?0Z1|?f_EjZ%`k;e#Q+V>A!z%1w^RY$CRZU!y5X_`3 z;jV|wb~^WfpxAZ|^phz+GZT(@nM)Sb0PJVNA*Ngb^wApzI0rUgNsi+V_U9|@H_^7? zQP5n+rL9&)ucr(C0-+-@&TloQkzY-aRdD{~(jfzVHXqVZml8bQf@=RIWh+_Ekh}L8Ds^7#2CvQF856d_bo-rbGZP`DRg(Pg7fm zrThAYh!0Q3JTEOdsg5H#VM%r{lST&8vYm8p^4W(wHPCb{|0Wnl^kyJM?svl>9t@yZ zsp*MYXO7n*?1;SSZ6GJPt?&;+Z8Oiwd(7_5GCQNs3eY3L52+9{kC%o6EAEf%bG4Ci z_AEKtau9p67ZGGuG|HeF6;s_q8u^cE9WROzzwrXo`?|+O>)Pc_kbq>WO$n8Mwa z;s)!N%4M-;9%+UHJB(Q47-XCi%P2+;;smo}eYpkDF_oaeMG6u64pSp3Q9!!MS`}Wx zr^I$`C+9jW;&p7G9sMBYthr>Y{Bq0Ec*uDyl4mdFEvp=_*^7j@^f#av30@?|+4Fl6 zZVNiQ)vV1j3IAIy)R|t)V}!aY%TG)tx6AUF#*}ebn3Sxk&9Y^}`|B5oAL18_1uY4H z_FZtUFF^$U}t8ififq0ndg)EhmkdYL4_+{T*)K=GON%m8t+kOW%byc-l zcjrG-B_U-`VaKZ|fHe6&nFIa6k&Ubh!}`yu6B^$~BiD^HzeL$%#XlhshDy4B+y!*W zRc}2dDVSdgw&|{C)@^>`xB)kSVbak;R23vw0rJGthEKpZTe+i1K?~!03l4<}fbuFU z>%;jUCBxLVq9KdShP2z0JhkI*P@gYm@&fi)xHDt_3t&lPIBNhIjL5@mGCWI_xYWNI z8yXO^-xVGmA^xe-I2VyvXVb1SUd6h8OU)4dz@0lW{Ins(P~mA0ppg$Z%Y;c5!1>A9 zJDSKiF&1@3qs8jHLO_N^UkqeHiI|F>m?1`^2RJ5S?h?_C;>bAPm@l5P)bLi{sMW)C zQ`8y5xhEeygLIrm8xHLnUoOA+pt_?^{K5pD$Vr@{YQ+h{54(N1>CJ4E$(&h;L^~C zuuwxYj>Nk#KX3+jjG!C17D}S5i=GBN+a9`ut{tR%W4#i$7>$k6FN@6gsHqyh&o^vQ z2}{^MZJhr;=cfb?rz3}SMTx+e&<;jcjA>!2Dz;Z&+1`U$WpMiHK03&AE$T1GXp3Sw zB(P9ptPeuVBlXf`ZKH4K9x$v(ibJ-Ib@0(}`e@J<~Q_x63X%xB@(8uH#l@-(%j@M{#$HVvYlwJRA zrfXkmDM#LepTo8%+JV^3GxdA;8mD_tpElBTmQfZIse<;P2DGXcfk-GEp}Hz;w1pq- zJq~lRrBsa=6HsOo z1J3!JSq5m^;|XK8|H+#?2&Qrw?L}^9myKNZuAl1@t#ID>`!LmJuzSo}{|XVveZmfn ze?vK^BnwRVnesG2uG7n-^__8brw5Qcjc}}g!2yx4olLT0zFrcs@S#ADwj`bR^GxMw zT$Z$%6>qdhE=S|J&&F-{L@j@4+2pI@^pY-5e=2X0%|VzuhSiUgwwauGb>H)Nn+C0Y z0B7pdI)#_hU&uXtM*eL4S zyb*Bce>T#DUfY(Gd;aL-&CcXdrr(1yKb3EGi4jHCq)&(uCoT7!wA#O1yE^ChI(1Mh zU&B=(eNLOcvD!R$>5y4E0d;hC~EcbUEWjPjWFjL83%N?;?Z* zVe}F;;Uap8s4=2^<9^>--&yONf8KR|=h=Hd?|%0>ecl7!m3SsvR%MnIWRfQn3ZRm4 zRaS6wBYu+AT28%XA_Z~DiUZUAvO7CX{AILR*EDlz*FiFrZhvI%W436$i zcX?L5?8ApV9*bG8^F(FFi`0y^9Dit#_9q5Ee9trRrG1Z#{8j>xQV;$;Jg@oV@$o1m zDAl#Z#=)`vdr5AY6_-h7|I#Wl3t&onLDoU_LX|S4-?ZO3TUn@m-;8x>-)S1&Z!XS% zJNA-Eq~`ceq{y=bp1q_bl4uADYHwUKwf?SKP#IdjJ1;#b%+-5*_A`*EX4O|z-)(%j zZZeki^Ip|E#oqxFVXe6?xv7P~L|k!b`rx_LS)kHviJnd^!xYzs6C-Dx58yA`ta7g< z2RIhWdYMzUJHyFkq`SAbblh6E2LCs$$svnLZF|h9<GQPpOKU7dS7ODZ3{Mw{0ktA56}k9h+DGUt}Ir6E`c z#%Fk4)!|$JO6KD>ayyrCoZe%_i+FPpDXFjbG8!f7%t z^tZ$|{1%>_$qL*y^z8e(n_=T#(Ro`YF(zX+D=_5I0-x}0+uLJzU7X(mGTG~+*QNtd zAvfD1*eZ6VVq?<2$g#2h*;k6CFSwTwLs$x6W34>V5M1EcCu(=(Y$wJ9o-tvxH)q&| zY&K_v*jPgv;#q~h?8a$#U?BxUJG`eD!Kl_*o0Eg_&x+zz&dfLRY6D?@m^X+wd(T~_ zasQ&bo^vYyq?HglJl|FXxOd+i(bdOy>fZ~_{O!g6!8)I5DJuDj>ApAo`azYMp$c<_ z{>(quX6C#WMg9OM0l!WjA6ElMd&dO#3@B^1u@ycI%9Bs7RZ<;1SMHDZ-v#Vlo+bSt zZNhCAyF90M#BIy}NlSyiNpkf$s!=(+rkJ;UKhbrAy3EZ-Wo&a9D3it~jisG5dSpPq zAhRaA zf;7iB)k@MWo-AfAY1y6+%9Kys{b7E0P&@zObfWXA*I)b7F*$){P)BW#9mu!N=bYO) zgZkRdjX6m%H1UiOpzT3OTl*m>rzogG@>9}vbJ;fSk3qe1K#tO}EZ&!+u5Vf@%up)P*Mr&DhCQ}K7j7n*6DwK|IO;)ubURKlO-V4%>WG(EJ zG2oI@=Z@9y(bqPQO2k@{Rd>{*`d>^IN~`vV6ugh&P0e}+v?nHSlC*2paNIKO=%A5~ z*(W+ZejVNrOytX)96n~f&b5e<;J{3a;2l?md(Q{_(d)Vnc)VXmYd+mC>rP9mqtLYE zquMRquTqi>*8}};=J1^~!<%ll#Sk|i%TNarV>*(w;W^j9D$4vSRZ4Sf9At zBj!snfd4@X1-WoDz)7IQ2JR@h80hx#rF|?Iv!n}-MTmjyX5G9#xycE89NuWD@ivTI)0}(araxCZPnAePj>c;KR+PGrZf{JN_UrL z5U|A*7s6d4*g@uLGY_-^3ko`5U#+5QD=AVpxFk^F(kc>5!>QHSM$~L<%iIz<{8aRp zM26*%!2^TUFSH-V=cz=cs8#MPvV{OhQs;AG#i$em>^+=%k;s_vj2k|2+>Q7NPS_9| za;7DG_XT4;Na)C<5AyKApG1~C!%MIx{}01q-3+so#A=t$s8qP5_}GP;F%{{&E=;I} zSM{NWiP$`4P!*iIgUHB2A)*dhdtav1YmZyJ))N<~&Y=<$BZ#DW$p!!-sg)cHBuk#4 zfd=_{$<94+*XKKZ2>QXs=lGEuHt(4t`7R}BR%w59g(lo4Kb+$+!EJ+b;M9Q-tUCN@ z;IX^_N~mKLPnCEnrs%)ea~v;KdCe!i^HeZZ8^|M{Au08FHCX71a1Wbp81u>g{a9b) zQW!gVc!l7S9LfOrVd$M0S)O7cj_~`;ZL1bf^}#Al2PBPBP$gChq}PCbRfu>dz?^8+ zXi3ChPU{g*$pfj_GAj{h%+pr0#e21VN?zwr>J(`So4aSjM}!5rT?>r0HTCfX(+*j6 zJi)vm)R4)S%da~ z@F+{nRQalwPI6^3>Fq5IkiG5a{~9?SZ+eK>yPPW0^lzo2P}HtPn}r zP{3}9r9B&!l(nueNtS4c5|}gpBQI#9;k#BcG-^Js6d@_Lp%V<#8!9!)Bvl|k;tG`o z<%c*nwAz-I(~Q1Y`zN+i&fn6bCJeD6nOV4o$PGy#Tcs7|6}-l6?Ea~37aLj4>b6*? zgXb>>Ofm$aSekD>RuVj72qcq6Da z?ZS*%%Rr6C1$p1-w!Kqq&Bre<^k@PD&s(?Pf!HRlEnLzbD$cQ+`6+!*Hj)k>O}A1r?*I#K}oe021U&XO9fv%Q35Wb zMfAg$+JHBMtW9CR{xN!waBUIDs=lD!F|X%Yjib$lQZ1y$sXg+uK8cOnPO3Agm^NbL zTaFiQs7YLACW%V)L)kRhsQaL)V781#YWVS2s0~pe>>KTU5K#gnm3*Yi_taHi0>KI zqxrW2YS|RwM04iLo#yK>Sb>82x)b-Sl<(~H?gvV$rJx~T-6EJ#RJ9V}`r(R)D~z&q zM91XqdQNRoHL~YC^xeE@<%O8CDp{jkX8Livlrx_<+cunNMd6K5coA}K6dobz`)|6r zeGG-yw)R&ruQ;fetv5-l4z!4Y~mdPvu@;pXDsZ*IfF;iYzsCi*@0e3lZZa+OP zI|oX=P@k#lHo93Q>@$m+uH}WZRlte&FGAgd!Qe9LTTa{piQoI^^!GC;(h^p$hA_rZ zq#?2ikGlLNkNn{go__-q&Xj8Ybl!Uq`N3eBD`M%*q;tM;p503LG!rs*o()qI9f|We zOklxrJ04atI%2#wsIkW#{8pgK{UB9HGe5-~GmwvbS3ZgKf+pyWq3LnIUB=k$MyhDtiMRxAq=)C+ji*0Jx6T3)dm!r96Hfbw%ew&`eC@F*cwf! zviq1wS%O9*K7LBLYDJkoAf-<_=99MUq<@%guOBZX8%FyqZwE|4QNnb&(MAc=k@A#5 zH&8qV3G<2vH~i-@QkE-pn5FauxQSUJVkNlV_%|#6EJmaFzS{lb?^dH-p6Td!tU$_k zQ7ifKCTr1Gw(B})Z;!y2Nw)ouG4!IY+!(Q5eFF?cv<^KQ&5@8ynck4;=nJP9%?kXX z&pg1$sS{Hm@U3x@8aqjkM4QqlNrSo00I&1qZMIobCmsnI+*d=wSyzcHg50l8D;Why zKfgF5lF-}HRtfjGcJ8cNvjBi2rD26%PL+qJ8MFPZWwQ!6N^k;bBex&qB1GC)D`~99 z!0no^>4V-`lbx#wJ`0<5)YMldRIF59!BM-W@8dNhOEtH^StVm71`^?U+(+rIla%h- zz&u;Uu+F?eTOO-UId-E2daRy~xHs^3blI~MyF_UStQ)ZyN%#Uvs^jzf*t}%Xwf)Kj zmAZcsFdCV*H79OR;*1-eI>pqrRD{)Bx8a3v=%bEXF8qe2EC^SAP9mP=13bbK(yfA|wyeIs2}(g{MO%Ty!*0dX zW>bn($^|qEYHE3}@_0}1Uqn70q*)dEbt%C>m@A^N(LVlXtz8kWto=t-hl1J24<}fV z+!LqB8Ncz9Koh3MfKC_1Q^)!}I z#d9<)o4v6=E*}#9n<>1DEnFKD9vx|8J_o>H+BekaoVe8!W~@b0>5m!tZNM!1Vk(^M z7uI(8Rlu|pihF;9$oMh@kv;!@BCagQ$h0*r-GPLBI`BDlhJVbfqx_M&ZS<6LEH^Ed+FDn>%jljoP!4?NT~yp(gG!X01|W45C+Ge2TC*SbfI-rxt3XVNMy( zo_+d5Da-~f`bBlZNp?G$0sOW$fkvSmOWOoKPzFLFWiIz&>0ey0imKhv7yNO=!XrWV zMq6umga-FIRX$vwUgpY;a4h78D?hX2x?DIV?yq>NPDiRwsmTXadhd}D>5!9ZE=ihn zuUOvwEvAPO8D=cm*IR!S@?gjxpm+UHX?W}UV3ggJ-vLEa3-TLoD=WRz?m4Zl_I1rO zHbC!h?2bg1$!y62?RfLEmD7Iw56sTD*7iNFy-vSI8zZ-(x+S#@VG~=ECSdKYDC$;F zSWwQG)jg@|mf%COAm9*38h`L(P(->b?`B zB7c7oI>D7RT9XA-EOTuDod?s=FAhJhZUyjG6g=R{hIcyJ| zo`Y-N!@OB&Oi1bVNs_V>fLf+`#Fn~k_5D|-X{nnL@;=&SbVM5w;NFz+OPXqNccB?& z{7**lw2&#CR>$n2cbeA3JbLy~ySMe4(0=hJ+KGFXKGPnw9JdHsy~pJ)Q8ymW_ur=; zzwB)W64OgB9vTSIy5a9hM{I)Mv}9?RHdt>TtDiJIG_q|A%0xRd06D(vV*G@vjM&wf zMNi#K=(iW3fg64T8bQ%D@OFF!oy1K4-)8T<4d$64UN$BRo3qL?*Z+JpImXpSop7Jo zw0?}w|Dt)N1V>mW%6&$fRy?#{=rzRJ>1E%G=6FQ=t4M3;Q8OQ1#yUX{2Fh5-NgiAv z7h1o*WwNL9a(P}ph+Bh`7PGCW)mw5bXUd;-k`+jG`+bF)#wlIRBROvB|NF!E0%Z&S zkhF$U0h=cYD{6pylH3( Date: Sun, 2 Feb 2025 23:55:01 +0000 Subject: [PATCH 29/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14412 -> 14412 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9cb4f2148..0cc965313 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:54:11,849 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:54:11,849 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:54:56,861 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:54:11,849 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:54:11,849 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:54:11,849 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:54:12,957 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:54:56,861 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:54:56,861 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:54:56,861 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:54:56,861 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:54:57,898 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 5629a9584..6aedc8e1c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -570 +571 diff --git a/summary.xlsx b/summary.xlsx index 617688f1b21cb8699de5010c9e9a8a9da8b980c8..b9000803cac44309bff105cf5f4f003864400ac0 100644 GIT binary patch delta 324 zcmV-K0lWUpaLjP9DKZ4oI?LOWEixZ}UrWO<6vf~D6eaI%+RmwjwgZ(tIS{NUzJ=s& z3pRg9qFukeS=)6R^hwA|a?btTb1x}Y2iZiwV4cvaL|Kxe2o%#?s9lNP>vjBqBIl{% zw9yJm)PX~*>ZV|ZFm2)0Y6I2_a8Up$N0=$m-g`qZc5Dxlx+FMM5VzV&>ciE4?y#Zk zo9+PTX?lkxc*?1#_y~;6L_{N!vk7T_Y|{fc!_a^Pj7{!gmH{ zttWAEADozsZQCa8qPK?p8UEb7y!E5TLLFDd096HNgjt|o+p55qm$0~oI==}k-U{H4 zUC?LV#zBTU;86%bhPWr?WAXGQT(41;r};8Y^Kj~XL6)~3ZuO(+i6M>wgYX+s)0gXTCx+; zBoVb9NNSz@eHFiwfL8@=qdu%BC zrhCA7nm%9&o^t9bJ_BPj5z&C;Y(knJ+jIcVFf<@Rd55zk!xygyE8T2Cr(8Lu@FxSa z)=`|?M<*s@+qOx&=&T`shQBti@7<`eQ0G-KKvlsRVHT*@wkq)TB`mI?&TqnscLMm+ zDd;nA!ytVf@F)ZzecVy`Ts*%e)N54bX}*lpJe)dTkmWte^6k0hP5lKZhbI3fj(|S* W?I)=I0JCp1S_A}FDrCWv%{PA^0hlBJ From 240525e78d4009308fefc86be3874eb12d285832 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Feb 2025 23:57:12 +0000 Subject: [PATCH 30/86] Updated results summary [skip ci] --- README.md | 134 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 4 +- summary.json | 168 +++++++++++++++++++++++++++++-------------- summary.xlsx | Bin 14412 -> 14574 bytes summary_results.json | 122 ++++++++++++++++++++++++------- 6 files changed, 283 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 0cc965313..e704fe483 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:54:56,861 submission_checker1.py:3263 INFO] Results=57, NoResults=0, Power Results=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:54:56,861 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3270 INFO] --- +[2025-02-02 23:57:07,634 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3276 INFO] Open Results=55, Open Power Results=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:54:56,861 submission_checker1.py:3288 INFO] Systems=43, Power Systems=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3297 INFO] Open Systems=43, Open Power Systems=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:54:56,861 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:54:56,861 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:54:57,898 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-02 23:57:07,634 submission_checker1.py:3286 INFO] --- +[2025-02-02 23:57:07,634 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-02 23:57:07,634 submission_checker1.py:3307 INFO] --- +[2025-02-02 23:57:07,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-02 23:57:08,666 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,56 +25,58 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.428593 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.304628 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4667 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.15441 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412747 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44853 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433279 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | -| 32 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 34 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 36 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 37 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 38 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 39 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3048 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1396 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 50 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 51 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21828 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 54 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 55 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 56 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436692 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.304628 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4667 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.15441 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412747 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44853 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433279 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | +| 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 34 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 35 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 37 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1501 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1396 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21828 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 57 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | +| 58 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6aedc8e1c..0751c1b1f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -571 +572 diff --git a/summary.csv b/summary.csv index 6ffc4dca7..27d242328 100644 --- a/summary.csv +++ b/summary.csv @@ -3,6 +3,7 @@ Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2 MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.428593,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436692,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.304628,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4667,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -38,7 +39,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3048,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1501,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1931,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7979230c3..c3503740e 100644 --- a/summary.json +++ b/summary.json @@ -120,6 +120,36 @@ "weight_data_types":"fp32" }, "4":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.436692, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +239,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +779,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +809,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +839,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -899,7 +929,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,7 +959,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "31":{ + "32":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -959,7 +989,7 @@ "Units":"Samples\/s", "weight_data_types":"int8" }, - "32":{ + "33":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -989,7 +1019,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "33":{ + "34":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1019,7 +1049,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "34":{ + "35":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1049,7 +1079,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "35":{ + "36":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1079,7 +1109,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "36":{ + "37":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1109,7 +1139,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "37":{ + "38":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1139,7 +1169,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "38":{ + "39":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1169,7 +1199,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "39":{ + "40":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.41445, + "Accuracy":null, + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "41":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1179,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3048, - "Accuracy":"acc: 75.800", + "Result":10.1501, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1199,7 +1259,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "40":{ + "42":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1229,7 +1289,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "41":{ + "43":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1259,7 +1319,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "42":{ + "44":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1289,7 +1349,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "43":{ + "45":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1319,7 +1379,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "44":{ + "46":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1349,7 +1409,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "45":{ + "47":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1379,7 +1439,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "46":{ + "48":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1409,7 +1469,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "47":{ + "49":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1439,7 +1499,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "48":{ + "50":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1469,7 +1529,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "49":{ + "51":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1499,7 +1559,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "50":{ + "52":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1529,7 +1589,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "51":{ + "53":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1559,7 +1619,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "52":{ + "54":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1589,7 +1649,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "53":{ + "55":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1619,7 +1679,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "54":{ + "56":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -1649,7 +1709,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "55":{ + "57":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", @@ -1679,7 +1739,7 @@ "Units":"Queries\/s", "weight_data_types":"int8" }, - "56":{ + "58":{ "Organization":"MLCommons", "Availability":"available", "Division":"closed", diff --git a/summary.xlsx b/summary.xlsx index b9000803cac44309bff105cf5f4f003864400ac0..7eab498a6581d1631aa559e5d0a3bd278ccf7279 100644 GIT binary patch delta 7161 zcmZXZcTkgC*zFTS6A?l;pwt8e0(j`sqyz}^m8=Q6D1nHlcU?pzS>Rq|$#Z$i4cG zwtB?itoZj*^dQR-dgpq0J8kx2r$;p#E#j&gzK*I4tG2XmbCP7~iVqPr6@OxKR(Fb@ zbyI-A<2#0iGviAG6>gWy>Ii4b#9>$L?`KbW3bdw2Po!sMB}AW<&h7?bbFT3;3hjh+cg9<5w$2mMa`c`=xGxH{*;pYx;^-C=?b zl$|S8FLh$5shl2P8+OgVaPtcND3=|2#>0_g1CQ@wu56}uOp8B4yW+gP;l6E>kdTGa z%%H8ofVx&wrV6W<>iHqQxtsMrj!+LWFMJPpd@YuWN;N(Hlpq&QJfu<*==^R^l9_I#^DAEdrs9YbfP*SAp5v&%u==P=pRJjT1SO0R8tWQI^Z39)fNm3so z5v>u6*cj=3D|))h?_9Al`I&Zz`21i5H(~AgKC0VtF*zn(D~B3to&Qjyd-ufR^LNi1 zS1K@?-$1n#W)Zt+dvn59^nRQ}cgwQ>7bou}>snEijOVuKFOHsvRcJWPg@x|P_QLS? zkbZOx7Y~DWPqj2QD`&=yuOeosZ!msS zH{pld*jr2H>nuhS*`Az={j=HifS}ajc-MvtpEn|@#l%l=dnjM@S*`Tt9jUyPr`fBU zH_KNxf=XujgKTvB20eRx$Mc$Anu&W>I{L+x7G^5Tc9m{sEwAE-++;3;&dnqYTi*G5 zJTC55ux}wsCwDBumXADcx~2`5hpm^nm%`qeieB4`xmW+R{DfgXVq5+ z{_*+K%7@*-mjE73bUd0>Ha$7n1j@Z=*@258P1GjubPeq6x&mYu2RFh#;ch|R*a}>Q zKOmO=%Lp90h*!9r(hXK;cgUK*!q)s5DDEmPm2eguSeYJo*sES%Pco`50QCQB4(rd0 zl2B`$ds~z8D3!1&So!{b9+czm7k~qlo1ceH1EPk~U)Diygj2V$c_Y6EHT!G$(>;Lu zv!wAr9$-J5@)2U}=l0KITb&3_9giIlMLEX(>t{XdwoT1hdNLf@EGgEZ7DFpt@ma&o z`V+GdYe9F=X&H(i7!rom3YjzJaj9qQtNps?_Hz~XPd9PO3yLBjoj3$E8_K=cO}R9C zycD|tMC(UCwob_^i~wrjAw9(js&QuCd<{lB*ofe`u|X?U>O0tp=8Zvn1ZMIm~P^o z@Q&DtZH?XXYr(^fG~WGd_HNB7dPPFteCz_+sy@TC(V)*!rDvzI!!Y2X{3KTO%e}Ly ztt4;_QnZsB4~S1!Vt$>dy3YtSJ!D*~A?eK9uZ|(34+JGSkOs+U@2`bBl@`68L z0)nXQ3jl;0MPN2&g%1PPp+^Vs431$G(YSL3YEcHe!&$S^%mflbXlB(bUR=W>XkA0M!-<^d&T2K-c= zVu1s%f2TGhI1J5r<8V>0P%K$AZ;cA>vG6}MH zSb&K97oB5*pyJHpAJK?>?s2QWLPU&Fl5W)Za{-^%J|Q&kVEOsTH0I_bq}0fa*}9~h z+J_mI4C<)bo}p!1s~@fzA)X5j*rON7tr9mnbUoWtQFVmYR#l965dS{S5)gmy>?n=q z1~3iI^U)rHhh>7jK7_K;l>#h@{Q4B%|M)S1>`IO7>%5bf{$)NEVK5DoGFo6-ri~n|SJRM5;0@Mg!?OhA)Oph37pSN1J;LVIjTnV+KQfMzi|CNk*@ zpg6_>qb9p_`Jra~4C3ykf@!+qLSLz4J8#M+6`D(ILG=)gn6z+s(4GRHPXnq;CiD)$(^mlYqjwv1HM=^m-atwX0 zyC^IjwA9prmKv1aJz#W51a(FM`V>&?hF`n)A@Y&mX0k^B!Cn*mR$9>bc{aJD z_tU^AE<-$P#J_JHRFJS+na~P`;x6#m+N*w%0gfah70k_Lj~43$qae0K?Cc@Gav4d@ z`m;zPwzKFjwzKH>6wksgoIm);x99n=F3?-6Cx^j`8J#zIf~Vh_`JyKKg+r&IHhF~> ze^2l$2fZ>q=nF?Gkbd@wm|~vpr5#feBIi86@bxoFAqa%b&|H$?j4MxFxxeU6d!=AQ zFuUCWI)}CHHC%*pFo0oD+NH08p`hz@^nFr4-xvkaMDZ{rWi|EvFjMj>d$wyps={)# zks>^FATibYD_5Pp592z%1{_uc%@hNs*`BY(>Ew6`|8!i)Sui+TMR*Q8Y@XCicw9=q zm4p^A;oh&s6T>CAOq3}M`bCr}Jln2_?kpU>BI?x2xymnm^FKP>WY;En+LDxJIFTl0 z2XZeyt-fVzoY;LZifZ9xQ=Rg~Z|H@HYx3;#&-~>v5qabwMk0?KADN_-^Rr8UGeQBc zu7z6xn~RE41;~w*st0jJEbXnu$!B|!;~cXn8V$D|jAptdN@$=Bf(DxKxhh+u2)fvU z`7UMSW_f5Nn7m0I4g!lds>k`wtsW;)RjFoIDw;&hfrKELyuWpCECdy>Em}IbB!0^? zFey8>L^o#(S9*TB*v>REu4&GlFL9_=moRT$@Qab}P`tHLXQA$<9T`R|XYz69#(%_L z27j~Z)kD>GUWBq)?NezgtQm^sAC&2lD1C(<-1y1+g2ti`Yy!nL)7E z2gwDCA>R#85n;+XWzUG~hDvG5F8WrP$r>?~ok0$TpMr#hY_&Qsw&z@7TGXS~IoT~T#WZa-9%1Hy00Bz`bmJ$vBfQ9Ars zGBkT~$G~eV!x2UC_IH}D#=GbjXcYfp68JNO^RSA#Q=?%YC>^RmD+us9_hARg(AL@i zt_nA$_&eL#Qg6seSHH%EX7hbMlbfx;s*-0(4H^$`Nb0p5-a6BfWT41I@W&QR66>uC zLQ>9BR|T1p&p`j20x}go5bKo;{WlR<{9n0AIx{8FK3%m?N;z~9DtvVn(^SryI6Dzg zrjZrgn3)=ZAk-qpe%ie=4U&UKLdgT<;2FuHd#45!1VS0#Bgpd}P_&F&B z!MN-hy4fF9&m7#&yq*Xuw+~e^rK;s~DIW0UxaVuD!I*|Dk%vS52iKE`n@#R<6}7E~ z-wqHS`h*rO2rxbGW(T4Er`E#(J3YU!T)fURP*Z2#YZl4G{UhG z+~~qR;7tu08EB*>`HTV_WJzyA(R7GcCk!pJaAt zbJ(^S*R*qz&tpHO`hN4Rr*j;k*FapMhju~CSkGBv!-;_NK!^>86Ay@(T0O2J!7AB= zSCBOl9TTNQR{mYYz=)Aj&&HvjXYEt2u+5uxBWll;T<@M3mp|o18Zy8n*qO08Xrd}g zawno=f5)lfy($t+uI|t3+&x9DUYg8rO$RYAmiAL3m;3~Upq`EBxpG`+Bvo+3bHn;< z)%DaMg+9N>kJu+;2K@IV`6p>KV@2``R=Sk;Yf6ccbH=O?BS##c<-AKFcT#94S@^@D zphuNKE9>@DYnbQn$^dH^vnG9e`gN5V?|3Hvx=e^tLnvutVDD^H=7L?T3=F#@$~3XN zORo$CU9h85fzVlNxy}OJT|1^FL5p@OtD8~8xgc(>)K%bY8NBE3=Yr-B!oC}q9fUx1 z)oVp=UWF#_a8n!Be9m>9s~N04H?nfize5Wv?^>xw@am7P*u6u{5o6;@R3p!PliRXvTE>R3@K8B*|5F%Gbf?9tR0KX?k45I$h8|dLivULpSsZ0aP zhVKrsoj@9mrm8b*Ug;O1U&}NAs%gyn=uHqBX3;Nxm3I)uJqOkh#XWc9$ZkN4IBr%^ z#pLQvSLbg26&IcXYDvHdw!f_>7ZyF!QZ zcT_h{VOR6wju7mx%x>bj%@xz#CE6)CUqcaaWJZ*^kKPy$fIAU&j_NO|(FVvBu#G|X zH=EpgT!tU5hpb%Y2r-4cau3E05%PO%(2rILzg_Q!Ywm`v-P<3(cN2~O-Za6(MJj+poi{s9MMc=j&zz7uD?ClNbAJwZ94<|x(i2Afa+C+WYZ|p$n=h~d+!23sU6#q$A{Leyxv^Rz zs{S#%cjRDg5!NzrvT-(DO7JxkYOrPtl@BKVY{W86Fge(BQ?L$efn1io_X~Y6lk-Kb zqk_*@{_gdv%=J;)50nT#-X;p~d0n#p-6gmlweCNP^4-)oM%#|heQ#rZyQ&^>b{5e5 z-fG3MM>$`u)H*Y9SY-RE^{rL;HRH5v&Xd7qx2GqK@gesIR!cjWC{ElYEfy~we39*6 zR?wBSYggC6+K*g?K&p{1o`*){0DuHP&WYy^TIj{arJjR`7{z?eh2}lmII8fx zsA=A+o@|3x<)k~O$@Ra$$J3QBvnWz!p`Yw@85e5Wugn+98)W+)uIR@+5qD6c&#Cia zY6K^=b<+~?8k>7Rl${&FKRtf8*wK&UrUV_zIf^^(iuEQ3h(F;gcN{8K&**J+Wc{eE zgDw(h&hW;n3A0=mF(l|b+BpCH#(@0=>!+D1ZmAfHeyy7HoI1DSab>eR7r0RmE;?89 z;y$>v{7SJ@tZvuigdBF)kGWtPIo1~EB*W!I?@lwLnPT|OK%)!;=0_SbxgWgLje?}u zqm{Hv2@gdx$Wxr@37rW=c3Fgt)?U;nHnde%zNcZS>61@GmP2j|8JFbfE)mZC&yNPL!iWUt<*gxaE!{{zI zsImy#(0l{^Fv&x&*Ld+q>`w6X6As3lS5|bYDk8CcpAaaMCYgdkp;AG`p8e4hLY9sJ z=MDBBjaUAM2%PlwAHu!v-=?})sq0pm4jGlSLv9WExN}Kv zxCnL4;zvempWYUGysujwLW;_t!Ct04+tR<|?Y;-r&nT?mTSh>n$vLHfAIbU)#RQjW zI6Xm6@sJmmX+vH@od*J~%!|F0vNXI`{WYPm!p9kL0=nNxLe2%SlfmpmF=&^P=Ff78 z=>z=^hVz6kwg|e`ioSzlW3NT3f8Q+I<(brI&ff;|Y2j7~W8MPevDjKS6A*VlCH^%XQ7w}aAWLch^ zZ-hy&Egl7{BU!ASEsA1&=e_5O7L?Wq8zJvtQH67NZ)TEvZ03C#c+rYJ378#EOe!zs zzU>paNcxKs@@kH5!HvI(`S~lCndXR*u7%;nE!%+ao3s>;QYX`G7tU~WOUIKeJ9;J_ zaXbA#X)#rVZF=bap`KAE1uL8;o3NX2Rf)w=MtM;MeapD@tV>+*iek}DwO7N!!oD*PDKgFpzz~<(XD=wG_PsiBjkJ?>oV!^MYLC@K_bnFT09Bv5P!RHSI6lWc#0a z_K((|r)&s*aH;v}TA8Yo$?;+zfkgoWwS)87R2$at2JeS;XbE$jl+7rBzu@;4E_%4* zwG#gnNK(`LnY-+rbfW`}pBP)4{<0Oe(F)t0_S8F7swJ0HzWpS}&ULz{d%I)@JWVTt zPk3U!p4Xja%u=Fip_%0wvbhp^lK#qnG(B`!hFHZFhQfV{X+Dgm!^2%P2VNJLQzHU8rTgUWqi7ET{cAcv^)1G}juYc;rV>HW-=O)IC-RJ#`RkXNC@)F!5g@YTr zHW$m2X2zl~befk>^(6_)B3~@<61}JE=-PgMJ4&)$!e}iNCoIa29(3ut4^}<7q{?p- zn&)x{H-OynCE?0e=FJRaP`Y|{q)XOC=&~bye@SkF&(7PAV((2k92hR2`zn{xJ7~ml z>uEoGcMJ=vl-i?G9eUK`oX?c9MoLp&+VULox$cEY=`B-{PE}yR9lIg~#m=oyGu0sE z26jnL>6^3Q2U?RBbYE1BXEVjy;&u7dCH~@U0?qBrH!*5ZN3pSsnIa0WsCU2{1F~;n z5QSiQ({&H#!+{r%sL-Nb<1(?ef7ZI)+~y^nS=uCi4ezcVFPC#jFqc>`fee}6(me?e zaxW#ZvHnb8-MEk}3*f^%%zhsB)BTS9dl35zfPIWCKSzA=Dl{mfMDA;vcSiK1)tfaT zoa-BA3s{We+!j?!DEo;K0hN39!cG|%8 zFsJR+)9U(9uUEsqHGErYEX);jy+^U=dREki^Ft*}_VoC^R!YBe^ZJpepE3Gv7CoM@ zb=74tP9q5Lk-7o19C$B1sDH4oKZ^TmTTvu)=rbPF!o8Zsnh%Vd7#)eFpEVsX0?QA} zV5K>CyPM7EbCt>xc5q*ok9PJqM!1ju?qSR~YJEr1vUhAtE-cgI4DYr0|PzO;^KqrYNv#_IUv zH>OP=wfVSov9N+mYk1%N>$mKaAxmB?Ya<88l)(t+IBM}|nL~lu*>qo@r$3u+R)`z*xwyS+VwEJQBb@>Nr$0BF zVtihUiHaNBCM6VhdKT$BKAfA~SO78UH1^(s^VzTuqSQA=R^fb!!76-Y_&II$)*Z^2 zXwxBGC}TjgF5l0iJ8;GdAIbQK6+W*>kIuj}xz&lmJ94o@O~}#&I3KNMSs=I83MYGdhJ#cRA$UR2cXy^J<1O zmu@*S{>uG5cR0s>xdGqXDAT+rmhm3}nH}7j()l!SE*CI)=hQDqwOa6N^u4aWsy_OJ z1m8UOwP))x2r^^ z(v|hVqDdhOTat~N1iz6S*#vW`vOyQ`*sA4U&(DYxVYK8sSn{gIdiF4!5c zlwMDAQQ259t-RZ^-)@tnWxm?bl($j4v%U4x;qFvy<iROrhH6v&Eo#QR;?~NoEg@6o7r@$bp23{D0BTtB>Zy zr^k&X#c%XHs8(`d6K{w44R_22YxGtV+>c!>y%BqK_=@qt0Jka8+3&q%LC=<)@E6 zaA;&^E;wz!gtfY`7*WsfT3h@asbx1P_C!szRt;=p7FQA92w=@*bXgc%D^4B$7S6Zb zK{Y>lFdqX9fc5ovZ@Avr#sqA2ti*kwMt^2Iu+)aVExNIlWL+?r63i{yr3sHbr#ffEZjGY&UG0-FF#WqVY> z1A51ISw=q#*?2w8qDQ#C9cequ1y!}trtn3u&9g{$7jJLJk|&z#hu#ndHTqvh=Ye)~ z4g%ij*olWzni*2jmph-t1Z^$<&=Ft(Jo9cLJQrC$k1dc1-hjv(FR=!Q^bq2)Wg~Rd zy!CL%5|%bGjtxG3+>Q7RP9*;4;*V!w*q*IjM~9eU!4BeK9i55}X;s4jw%Z@+sZFjv zX2io!zSTB-cjb2*v{qFHWvFzNA*i3S>`lpe2lnM=)O z5s)j;z542sOQmHefWYH%FWV^QydkY5?jSmpW>2beR5(K?NIcpJq?vO7idVc9&D>gL znZ?<085Fln=I$Lw4iz=ObeEkX+;f5TS zM3O2Io+8cIBgpT?2UV4Wg!AFSa*$-c<>#>|aNdXuO9dIA{C-fFk38`!^Dp>JTasbz zda-D2BrN_X-c9X~@g#=@myj{EUeQ>tpEqJk+pi3L) zxd{$<-q6)>RU?coAG$^v<$aS$O8rhx3ico1BdL6QXx(GYD7xVgRf!-&#!?XaxP(u% z<#iyHrNmgAl=91ms96{Fic|bN3afKALt`mIl6dID2D}Pn)Sf*WS;BeGu=*q6)STpz@5Ml z7)E>iCsknUkJEGAvv)EUf7Pav|Dd0?t!+g~6(M%c4nxq%Dd!B-ycytghCWaRTAAv- zJ}Km&ds!;~jAi^YmgGIgC`C+i_MJ!TFmTEWo4kNgxOlkxh4Vh>;S8h8a8g4duB5|1 zLf*GC4F7uBO)o(S&n>|M7>aAEB9MPOM!CB{pD~9euUOgQ}7AuvlZ(}43$Ek{}5w0fwut@rxszg*fnx{wxE1_w5~({ft7o^Y<#;rFLQmqYUPkG>5T|1Z_#&Dme*xwm|uTG~Qh*g%zi^gy1JFlM97%rL>soz=RPAEcI_9~G;Cj%&|dfG(5jN%g- zAd-Po6;Gl}KsSPpU*Vm<@?v^WH|?fXieau1v~6T64Gx*BytIld6QA-U2RTIXG?fNu z>-Qye1g%`DVSFrokDxFMH&m4Z!pw~aQ}l#bXiEHg9*810%j`4FF3v|iCy_`_uU-oF zUvjLYN-5!{T_Y^T3aQ+&^l_=cxK;43*A6nxdWa>nawC^rbmqzD46-O@-_aTbc@l~SDU#>f z_Y2W4RIKpU+>hCNtGVU~Qgq{83@&hNB|VL~&`BbLdyfQG7oTK>r3Ktw13%IWlZuPw z7yJDnJKaui4tMGS|2(#ga@FGI!U^KhZw3>IFNsDbRrMx`qZMv<6s6gt#vP++q1qKB z_m~4vR6?{20cI3}B=^7(ie*tOpvyrPufV00wP;LTs5|99Yg9fZQdB+#4pEd}PFFgu z5k|D)zxB))v;p=8-=AFE0tsph;_5sxdF}10CUWxda5ROiya)vCK+&@&BwqqbT$A*D z!rjQvL<_5(Zk>O}>zA1lKK}sE^1^6xIHMLi_S9kjfkaQ@h`CX%f9#0)CiJ)7u}Ik^%cCPg|dQ#|1&o!tgFHFIuv7*PhJ|SF#1c97}PVfF}0L6)8lxC$JSWk(f3<#XL3` z;>(&i%N+$JijK#(0{ROk9q+DNNl(G)pfRRU?HUr&!h=Kx?6vHJa@s@`>n<1?e zKkk4*97sS>*XO4ztZACLsCm=p_g@3CO>a*~6}a#bdqi$uFLW^RArm?7m5qn!I($+e z-mrgX_C^67ZbtE1K?GukbqWIMW=)XzhgU4?3K{>8f30=7ynoSb^T)~;K}HqCUC<3~ z#1CVM7M>ds8@bg+pCk*zbR81ZhYv`$rWE1fRumJ8A`mO6DdnfV9zd}K!Iu5~SdZI8*>GKW zsK)NN6(7GTU9WubpVEfFg|DEdN^yW7_7#=}nPy3am4>3+mSz2FdPCvcTFKvo%}{(p zyu9iX$|v@0%GlCQM+8psPn1cb?bU7LUlt62oi5PnAIgE3g41WhFTJlG0_IFa&I8%? zwW{=KZWIH!&nYSWVoP~r_@=B;{p9L|XFnJ#roIS#(@16p%uq0E(v*myh$hJ`1`Fm* zSKKA(m~_zk;AWT)=EZTle`o;TP>I$X-sno>7OAOH%54uBb!3`7L$p7WI!^t@`z|>k z1E!k{MP$NQv8e}l;Nd(JvZO@wR62Qs)U&>;34@{)i@RQok`g^k^8;$nA8N@54}bXL zyQ47Pzwu%4=2@bSz*Y6-f__KOKEbX-%@VbhijK<$hAZ~Ge&iKuswk(B{&opvgryzj zhrLPpYO?!lBYq;k1nNzGyBrTcsxK7}TjD)2tZq4)w5^dse9q zrR|Fq9k&dI7wviLuK}dVs1Nii$_b=fJ4I)=#^m>qymi=R$4t!6j?`R2IrxT(v@CdE z@a#;)YP@y7#Qhxo!z8pTHs3GVHKkd&Iwo6fzCxYHz;NFFo%&^x%1}`NR3w?9s!Z{6 zh{+zDf5!q+$!c>{SqS|%#N1qG7`%?B)Wn89D$A2>Jg}%3t)gB(gpLTOISm<4R8Hqw zBuwmIc}Nrs5Mk6!mI^RKos;>IXgLn5BGJ;c_9xxex>3MG^7+x#12gvklDPVRp;0J* zNbefyS(&v=CMWC-1i*Aw`O7Xag$uh=wbE-Mif#?RlVmS(>;B+EkaM$Q|NJG5*7A@` zr0ya=DR(>M{ooeKOZ}{$jw;*#dAUc%{(}AuSogGC6@?)P3^jEGC)nQaS$d%?`bgR0 zmW9VsgMFnvINv85>Z7hp6CD9{1t&4QV7CKT(1!E^!SOW8`>x8>WFhOjAtJuY{}PjJ z4|;T(=!|lT-XrBXShX-P!=b!%>=nzHehD;cpGhmD_w_G+3kxL?xUB#elniOEX-MiQ zI?_BxRV6K5ikl>gKR8BuI&KC3!yKxgD*^{r%e7QNNKv>-g0M;G6Xi3yEEVB23xKM? zWAYO69_?_qtYUa)Z>Z3>`_)pRW`aoqq=BaZEI>+lLH{CHFGzr)6!iL7FZ-Gv52?Uy z=0#E?7)C;D4{{@@kGALaPknBYpsHM8_>ULLFyrh5p1^69TqZ9n=e z2^Cqtr9Ze{^=*)-Vtudc!fHk7Lh4k=Z)Zg8XnFk2d48>_a6vGWsB_(KJ*Rzo>5{XV zxG%23Qm?aKzK@&o3pUh~a%1KmUtJujN(XMbTqOd8{i&$QXSdUToO zqyq7av`(1O2ta)|+&S5~qKnWnY37KF_Xh_QBzGlgdd`eF+|;w^*53`!s>X2C-G3W? zu~aRYYB&BQ6IQjg_d07f51jDwRrST2%AMDp}-3@0}7G3(5xtG8HbW`SSpoI$zkW*TiCzeMO&&zl-5 zahk3NxMm%@BQd($silhP{`N~(+69+()XL&VJ|UY}n(>(CLBlKwyv~V!a4JJkmZsG1 zr^X2!>){9^ur1M3Tql&8Ap71bF{y*g-rd-_upx&i_pul9Du53Yo-3uK!*-=Qop_JE z%eAd~8`$OHjveZwtg0=AkBCtzILY^_tr2(a~uVr6{5KSn_ten|)CJ8UovKy9L zCEamwB?qR3Ivl>+!33>E+!I+X?0&+Yy&!A>1=KP*q;5UPu|O6*x#<>d_uGp`O|?uym*Lh+Vpa+tZRSrcmKl z3?uEu^@&-VjB#lM)GK_uu9BwhjRZ#eAn$Vk%_q^m&(crnM1%-qOoatDgJ+biJGUwg z?>5}lnmK8a>-NAvCJTnOC1dA{eSULF4FXeFEFR^WWUO{(7EfiCL2}rOgsmw9W{?Ge z#ox)np3gFg#V0B)&&P={UXjM(T0Nt#+QsU-(;B~it_QwDMCMO@%ukP;?ab-#jcD{a z^qnMVb$1p~QN$0Whj8+?8(2?!=zghT_*7)?Nck@6bJooC`F!&<$KK$R-({Z(BMrdD z`5UiD?FU@%#N(!0)WZk<+)z;+r}2=_xI30N97$uzWlg|WITyzmJAB?uDJ}S98hzL8 zF+$Z9^EgaTS1^JaKQ#tUiB{KY07>??cNDJ;M@cQ-s(CEG0E|cX1tfzGKP29!C!MnNXv7O0t`bsY4U9b^lce*_A=5>f`< z`rpG#f8T&?ndTpn1>OxQqu#m&ZWd@A|L?FqFZEeS8TnQXsF}5O9Q$|JPH&Y<(?e@H z#&cUC=_<<3pEZT9({q0U%%+8TM?_-OF7?kVgC+mw zgjCQk;YOQJreOiWy<@h$-Xb#7<5hCyhoh3&o_w=V3C+0UNxUstsIN~kGdEI9IG|R7HAey{?-X)lgF?QY#?U(aTq-xM;ToK$bI%N#0FH^=5s zF^^b%u#TA}dYy*wtrq^Q{&rdCd#GCRdVyV}*yDv!H^>-`y8%c@>$T9g*#I>$Zwx)Q z0WSN1p>9fNy?qq;UH)z({WoaU)jta1+gtelFID`2x;j6$irO_8GXjxq9}2m&%=HyZ zo7Ds>@vI_mdbW_<=0o9Jdfnq0cgE}Z$I}9vIdzezZ0A238~1ZNRk=(FAY*cuaPw@N z>7v%u)bdY}axK)tvzO;zSr56)cB(|m%^OfDM}E4#?6|#$uqrc1d4D8vZ`XZ2Pz7Iy z_Pg-H_!$Efy%jwzG1prYPmY-$09A}qY>@x|XP1 Date: Mon, 3 Feb 2025 00:00:37 +0000 Subject: [PATCH 31/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14574 -> 14574 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e704fe483..468b95937 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-02 23:57:07,634 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3270 INFO] --- -[2025-02-02 23:57:07,634 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:00:32,085 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3286 INFO] --- -[2025-02-02 23:57:07,634 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-02 23:57:07,634 submission_checker1.py:3307 INFO] --- -[2025-02-02 23:57:07,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-02 23:57:08,666 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:00:32,085 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:00:32,085 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:00:32,085 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:00:32,086 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:00:32,086 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:00:33,130 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74391 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44853 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.4711 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1501 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1258 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0751c1b1f..81790547b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -572 +573 diff --git a/summary.csv b/summary.csv index 27d242328..531edb1f9 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72993,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74391,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0219,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.44853,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.4711,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1501,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1258,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1931,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c3503740e..2ca1e81a9 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72993, + "Result":1.74391, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.44853, + "Result":8.4711, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1501, - "Accuracy":"acc: 76.200", + "Result":10.1258, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 7eab498a6581d1631aa559e5d0a3bd278ccf7279..8e600d4fabcd8f37405d87c34637e6869c25deb3 100644 GIT binary patch delta 3777 zcmZXXc{J4R|HfymgUL)|Da14k#f+tCEML1(VTdBzSVCl{DNAJf43Q-jsu<6_Qbk@cQk>C2QUw4(U~ z3!YlK2oH7F*+{~rbi83(clAO7{$t1YvBnC*8Mi6kE52Tr3Tu(I3EZ3?-;dq8_ ziE-b$ERtY)&dA#Ym$4y}n?!?W8+xO{1V==&43k?)nt<-_M%UExQ*+v-edkh=#8>UK z5~WvF67Z99{@%hVi8aJe;x59~(3u0}Lp{|kjI-8vq?o_^2}52>RTPutz9x(65CIG~ zxN%2YE|XdTHej{UC!l59)aw%6-TaQqA98WO4Wan{|K%Wz{$&2eE1k zRz}7|_I32QI~2d(HV>NEkGHve!t~h{Wr3;m!OBvFBb-kAtq2bV8>(Zeybl8 zue%j%t&gDmlgzYygL>xfF0A4`VKx^VC&h=#y1vu=)NU;-8t4ZU25f3x9mmd|$Icc@ zHk#7}KN{s5NF$Arw8R?A9Tchqr@!MW)=FR9=d{~PMB0F>x5e~GKB}<;^mi8vA@QomWElTRC2~%^66ISm)9qiMwADP@m-i`=VbdwG;!5 zdYz9Mz1c2imJ=-X-m-S#Okk$YD1bIulAo2kBUo!BWQeBwzy$0|zR}=>qj}$|p!(gz zQcDxwxs!$pt%70A98QO>!k)SNWmf6P9*q9Y8>tyO*JFuJ)e|>L|YUbW*T(`FG z`5YBc#%z)r@vwQ98_wy> zhObmb2JlXY7f9&$5LrSntLm{@@O<*ds)D@>B(13YtJlzlqM+CyhH32Rl{f)}No8z| z+tODW{JPl8*DB~WlRAD5B?<|K0XgoKOL5x#A)?X_rsxjFW^qJaWM5NHs$DTP*xby1 zRQ{DP#iSeXQ|bgUAmKgthxN? zEkBngLcP=y&nO0SXw=sm?7;C)=$JFc-YxE%Pw!Ln!2yL}ooKv=wb65AH@epd?IbCh zP*`S2FuqDHW8w7WCO-3aZFI;uh0Q1FE+Izj~0(MmrK_GFiNHW2so^QJPWfp(Ca#L+` z8?UrZM=}#7)e2PId69^A0mp$bvrf&oZw+oZP^4;?w#sj(M)?kD3u(=`x7;G~>i}3` z(?_uf<$e^)E_V$B*1EJ6W(TGYJ=r=s5SAx~W1io0To?!*3RP zLQ=7~#aH%q;WqV=ldpqssgbKp9?*?RI&5HBY&5&pWwGp9*W8CyEiuCFz%D}z+efd3 z{{c=!KMXfk*EWr6y#gqR-n=asH&yy2{rvr|%;A^8jo$w;M(diQb)?w2b+L1Eqjs@# zlb&Gl8gF}RvvXUly$Q9`9-yZf=!bj6iWb-+2xi9jzu$^z?5R(8RozOu^#ZdHn6e)e5Xwi09>#gsF1 zl>k)F1+~jurNLFYadZP_3X~32@uCF4FvrEVMHWfdZsp0!+@WssH-#4*QGqn=g)D35 ztot_E)HK6+{Lx3BjgkXL6nLY~s9v<3bw%y&J~1T3xlmuTa8$Nlp@_4q0NKM!35frx zLJvhN*$$#)VFC%LA0%vslmXM46Gp{Vo)}~6W%~yKB?~wF)YaxM-YHJ z@QB*eeiw$els)({gDG_nEXj~Yegsd|avhHVgOzI|QDhG8sv{|zHjM8wN?a}O6Xpzg zv<3jR4n&FHibr-Z7^AbSn85(l>*?`O7P{y}g>5fM&m7RYnke(d>o0TC+iZ(u{Qa=E z`0E*&X5Lu*Ec(8q(y#L;#IuTqzfPPk)aF+$U#=5cb5%5s7WUdg3rfj zl@2z3(SW*GX>gt7yZGRjuf~B? zQ5gLcK(^{BtLIC*ukpl@meS$mdd>Q3NyFpihKs5DN}*@RAx7TL=_;Rn)yJmM3G{St z#)7LF;!)b61%HQl&q(BpUiVCS7+&$zEdtYaF_28FtiG%=S&ljRIVIs}~fj0`Yl9GO=cSVoNnTN&zyI{WOAAk ztG}KpEg6gX-AZUaXH#mTzEB+c+xEzRlKiE!PEMl>ARuK1N#q=f(5F73 z&4N`vrIJgUvRpBbvqZ z3B^8451)bq9yUsIKO4LHa>&tsh9Ye)$G7ELjk<$|lNPU`Q`&$4n zYu_2r&}e+wvNmtAT(j>xn!#13Ne~i!4#Yz+Yf1)$u^VK3ro~vn;9UU+NhejTXTObG z@%Y*^+d2ff(nVryG9hO0{UlEY^8&vRHg%CoJo6ZkaiYr48?1~5Ey-kVW=^VQY_|Az z%g9Z=0ArBsxo-z{|az$*el!gfHOjoTFJff9Mmk@I>xa>LR|iA;q@+ME|esC zZ;V{AqOPC=%RNg(bZve7>%#q{gnEBd8%SA>XCyqv86_4zv?M`Hq9xo$#}z|Cm)>he zH{2y#`Zor5_Cd|?d1~uTmE;*p{muOe&MRLsN1m;&f7dBrO{ICCV|Ji8uHMX}irv20 zI?lV<#{OginG~$T z-uNM{KyXrTcIpZ;ALkCbpZIF5VCdJ6R(x(_FXvEG`Khd`yTuLK&6KPk&x5VgVx#Vr z6vkWoJs&G#*7;1;)rync4#WMY&n?!3YQ0*U=8spTbblytNJ4tZ2q-1Cze;tyl9>C8 zx7Js|6~c9I;E57r0EYzneOM~+=yeKzexu?Roo{Clq}ls z)TucCx-ILps3UEL0U6A>?vU>UVY;pC3JP!82Q8iqNVLLslmSz2eLJuz4kbufD?f)2 zR^>gXmDb^Ral#RV$;)Gcpte)b^7MI=x3FEFfuJWp{MzyyHBzCx0=1(IjhEk@&0jxV zu6cT_{1lMXVpF-!DlQrRS*np&gvh?mi#yNliFw2_0k+HLVeRQ3nJ1UA;8p%Lmbpyik0;X~3@zRvQ-sT|J-;j?KRlN(znQ z?Apxa+gj-Rm~p5jZuMPjCf)bgw|#9ZPOc2)y~dULPtAys2qIdRCgjK;LB7WJe)p6% zm8E@6HvS9gJD}8yzIyRlv`NrLs!uS(Ss$9CBx!yx5A5oH>uxNg{k1ZdmaD1N*`#jaJtI{uI_@XxTBNzRm7<7io1Jk2ZH1d}^B!r{X=z0d^k5DTa8TQ~SL*;rzMAYK; zYNBeMADHZ#%i3W2`LE$@k7Riq$(Y#NNH5U(Jgm)~P(0hAY2KuYjD~_6RWTr_OoLux z(9FtS3VW4KUZpxdY6c|5pDp&u=+qpsSa9CP#UuG$p)sDiXG3brldkk_JNlAsW?s`& zFk9wQttW5QG#3=F+J)FVClaqc@`-?ud<%!MNLcC`(`C~1(-L$(CT)`vRH+fsBlrSO z+D2r?KNl3XJ{MRg2Y@bGx(71|5_?32f+OyBTNDT9`;qi|xVJ!W`j-@+TWdo^PJa!N zwQp6|F)i@lhQ5L`4W-*@&`OLp2qz;9^2eQM>wkzJjDKg-uamvykm6{+(yi54QZtZ% z?o;lRFMmMlTVE23fmnA#@yN6$2uG*!Bmz-3=G z*{Cv7wh$2OEWl+OC7?vmAesE>_ zd72J5SFz80a9!bg^X};bLj@kzY;ANx@i?>JIwF>BnzM>%0iHts_zuWcd&cZnA?)vD zVCkQBQw*od;sQEqk!)8;6vxPwm59a)f#m72U@9&*q#-*aN}ExuJ+ef2VnMuuj7EZk zuV6sPG{evc?fk9|c0hvK{qBfPW)%5P6yt;3G>9mKGtV2iN%hp%bJ@Sfh!xIZ8W!xe z;vS{FfkGDp9dKtdv1O_l?bZBF zmr>t~-2OpvoL)I)qt^EaXTv_L=U)2FV+JCfATL5H*#>gpAj-&S8}O7G24n-nvo`M1 z)u;f!HF6kdkJ?aLs)jl-#!hvFgFn6H@MiX#JRye`^O{y+@N8OJM&P@^dT!St7*6{q zXZzf$5kG8@BTI5Cfxz%6%IKmJV7c zJW(4QgdyE~8lD&S%`0vRV@-Kb@^HSid=gU?8`zEM*zY<~x?RN*QDC%GU9hcg&;`kU z*Z4I)W^pGydND{+3hCQGnyDa%MYD&zxMy0Qr?ZwpRO=3MGJ%iB_uko7K#xQ86D}1N zf9%lQsVQeh&OPEIGjhy&JO4>K*iEgKW$uTN+I99UE)}m^27{u%Cj*Kz7f5+`>+K33 z-St5AAytYECzUeRyL~XMd;~Y3LQ%&|nRvnlm?jc*1jl6nk<)fpXvIcb1l~nS>sEGK zj#%biFppNwDrlZE-S^WyNvprY7tE<&L-~#LYA8m|7v-K`I z&xInZz%79k@9l=@YXxNBZQ8NF`DL zZv!-`it*Mx1~rfJ>%2-wH7(B{k%xT{+@4mY z-54d+kEMzUMjcK``Ebvd1yINpfo76>is^E)Kh4r?suaU<*DTAJ&jhDj@XSxojHn2Y z)3{d7xGIwcPIrefXh{@8E@S;wmt&itZElgTY3@_35+v6poc0=z41wYn!mbmEjbMbmrM+(g;IC=Hn$F ziM!tv-q&!iq;trmHA-?m@5S&+o^xuE*th+XSjGKdM0d1qIftn}*#dWDuAJXXP?i{I^#}-8w7*yoZxvz zu{-+{#Bm^C`Alpsr)d7z-u8pii%2h%H8Hf#*0{A2E3aj7&VP0YUfAGb zbyeJ-Y&mkrG{3%?OMvl;$`Nk-$c(RIe1Rb^4|EQ@so9eC1*N?LtIDhgvMMOgVa=m< z(;kT-F;m48<|64`6AC?b(Z-J)$_uOGOs7bx(zNs);ULeZU2em>hThrN4i@AYJ8%?D zRRV=Y$4!8aBO$x2%0oh{>ZF4I`=MoG^Q4yn_AFbd|5reZo`BHP5tYC#c|a?eHVODH z0I>hFXc$p>YD*s2Dn^?;@jVM*&$H;s5=#2B`p|%gX8mpju;*Cx1cpjQRG!?bN4AdT z4;6~iH|sEbl}lFBzNSRD+WmCogO!fQ&(|+J^qS3}Ld{hNSrF_knlihTQN{H02;O5X z^aelRX=5N;;OV8}nINamz5quLb?^BQ`(xS`4Ml9ft}@t+)3dMS%bFAk!yAn=L{{J$}Hdut%g5m zEIN0t`~9b0E?%FyH#0V3?mZi1u4N#?Qkdi&EgjO(u`yqfGBpx+WZ1O){e}XAUGq}E zHC?{z`x>d%Q-Z*DVv8|cOCFLDBK3K6T!!lGAF~&=NX;U=p)Ww*^s-1zTdO9X5hQm# z9P5#L6t?UN>nSTp3fRnfDf`qy$c5+lUZ7?<%mtSya{b!vCwnMjrQ$Zb&VWg$dl7H? z>Iq0idGkH~_PQ8+`Du$wG#y~^c>+b7b@O!FR5b{@p0;>hnqOkhKJ^`NURmnl|tA(gqtoiWm_Qsg7lVtoxl@n@z ztiJK`npJQ=+amvAaBF36xk5;uugsbkWXhYveKV5iUCv@}*Uk`FKT@Cui$`3|yBEIX zect&g2p$7)j#oho-7j4cB}SE9d7J5<6=$--Q4=b>wqEXe*Ng=2t){ Date: Mon, 3 Feb 2025 00:01:37 +0000 Subject: [PATCH 32/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14574 -> 14571 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 468b95937..eb06a60b2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:00:32,085 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:00:32,085 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:01:32,232 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:01:32,232 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:01:32,232 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:01:32,232 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:01:32,232 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:00:32,085 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:00:32,085 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:00:32,086 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:00:32,086 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:00:33,130 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:01:32,232 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:01:32,233 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:01:32,233 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:01:32,233 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:01:32,233 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:01:32,233 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:01:32,233 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:01:33,278 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.15441 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.21321 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1258 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.83777 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 81790547b..f27ce70ac 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -573 +574 diff --git a/summary.csv b/summary.csv index 531edb1f9..4243c068b 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.15441,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.21321,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1258,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.83777,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1931,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2ca1e81a9..a317f94fd 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.15441, + "Result":8.21321, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1258, - "Accuracy":"acc: 75.400", + "Result":9.83777, + "Accuracy":"acc: 76.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8e600d4fabcd8f37405d87c34637e6869c25deb3..ac3474c26fbcd3a470aedc6ce5ec16c15bca0473 100644 GIT binary patch delta 4899 zcmX|_2T&7hw8s+y7>ZN@;YtaFUN1)EQbGubNN<3Mw_BSAhhG2ntFSkrF~j z=^c@dAR>rz0YN&_@dfXDZ)f(*?l-f)`R_U3|D2s&_gnR=bY-DSoD>RwOYvlh1miqh zsuK-j%?eDf6O1QVbagN#mk4Y`-WQk1%&E7;qOeELgi*l0-CH-T!sHLxtQ1xB^oe~@ zf`|T5QS(>ynELL&DxbO%L_0ZpxN|snu@w6lvSezjl;A(^*7M2nl6$Oa!HAN(etsM_ z{Wix(Kz^JH;5-_F{q`hpyrEgZ{kWg7jyu4YdiE_#;H4QL<(DJmQ`-(sM`_N2r$3v8 zE^9M|>Sk|0|81zuf0U|2DUjWR5KU1(w4hPG^vhZ@AYUQ_kPmx_=Ayr6YPiT zhaf7J5ebd*qz5BqK%P`=He8E}RSHcQredegv2;`Evq1%a<%qgeG?!0p97+m#m+arq z8~9>eeWZfP;}4#8i#^Ei^HwrDf$+K%9%vzTo$v=8iU{GEyY#xdPc`1q5SOpws?+bW zyc8A5$KS#Y+!5B$-qi(9r+4fJbln>g@jNpRuO%+hIr92A@!tG76%gSM*5JFyafmp= zW79L*Ss5TCU_0vofaEzR*gEQZQpp%sICL;#R{BpADJfa_8iu>wn1uBao9ECLv?VWd zesDQ?b4;=RarSK4aQ|kVRf*r4MIQfpijKK7)f~&rC|396c7v2RG@F;?hK`RMfImHSG&+dOCF~Y^I*0{0xy^V4h{+JXUS8X_* z>N)N!73YBeO2r91Wm8->4?)vmhnEiABuB3x^>uWBk1qu6>B?i-U;?%{ayRD(hwV2z z^Q0_w@u0Cave+#>W8Uk${+a4omxT!y!RTJSn%F3|c0f(63jkU4OLxfVcwE2?_@H-( zYuQ5~_=TwRSkzC+m)0r%9oePt*9fW>KN1FWR*gT7E$+!?&V^R=>!Eo3t!viyUg->E z)qW$y`o*cofQ1P#DjTOqiH*YRkq8!5Nd-i#6Cuz!#EDf}=;bcgGT&wp6&`!hUv+s> zWEP-@;>&rIneZ;>q699=$Fh$`s~+Z3b~8Ihu~4-j+eW2YK+7goH%MLPl72;ORO$er zBGv`3Uv-L+;{4)GV&XI{jKV# z{?^Nm$-g_f;tR?VLhm(@{m4=tmV1WmzYrNS0UCQ)p7NPGBWw$?y)aEs)}ryA8`o<5is7IY zFekU}+(&#yJ?`=JP@63HP&Q*YpeekC5nkD%=1pCI1IYnW6CeRO`UK#JJmYWFv}xyd zIkzrQ)5c6QDy>>A^p40@#o&}uf?f+u8pL_LLsc~k$wW_c)|f9gvZkw4LKF0%69U}C z>_vZ^E||=05{#y=gl6vpNN`Fl7eFS|DFsNrV5RtA3K8T2Il=M@9k(8Ynv>UdXytNU z{@~=If96+@1zq#H@T8NE&@ys>kA_~L)>IDUDM$j!#g~F?jjB_~vGWFGYQeB}*A~07a zVN{n}ud(jGRB5f~fl7mZ8 z|FA({#}oNjnfit_sH$i|FlxrmI8js_F^sCSw33AZLIj3U8!ORMUcg~g{GgI?Geg!sx}2_qS__o~e|j|7`1yC9 z2W?zsS6Y0eCj*<4iu0$R(EV=ieaKEPg>f9d<~6;DIV~!$_4A^?%deCosu)jHVr3E? z{TBkkUTyu?k=G&ZXBWux$X@^joZdZD}>>F=z22UL#k3m zMzsv=)$n2>G|H_VrB>?{(3AMDMko~1w-E8_(qCK4B-7*yZ%i!3Io>i4S3v1N#`{x& zCCt7L(Dbjcs#|~zxH}an6*`$rclXb$he-4xAFc2SZBw_QHMy5FVUd!fi>j&@lIhx* zAN|>R2i>BsVKfgz2arkpap*8EEKND|Eu^rdCW zE|RwGBbf_#tnw8MUwU6(R+T)P9+0G1HcqCC3+PoL)eGzOtH?loPO06GojH}-{n)U_ zo7JKbaBHeK+R(;iI@@z`jGxCL*?XPaow|y?8yW5zz8f`8;jWzHf_t;6dMI9hp4vBC zuTUZnypRspp*g$ZmJ3U6l2j`Ti&ACU;XqKTPP=K6dTZ&>f+JU|Wp1~#<~OR&WSZ+i zt=}NCsUK43x?QAE!8v+MS2uopmdxMxoXPg*7QDtkq`MIJjh8dvrhZFoG{?$*D%*UY zB8M@F@TiIbm_s(+bmRgU+edpd$FNpo{V$d9RI-H(Lm@&WZenLQS0W;X;}~ar5~0V< zL*wz5{4~X*dZBl#6^ddpiC4`lr0Yb=x&sMp5qu6nGBdtkjP}J5Ob7+k+zUkf{RlBk zB%8J>;LksDN|*^@3kij}CQ`YDmo2Dek&52qd?K%My=0t4DjuF!xYt2Nou;yx@j9GV zGMn5IOV=Ul<^Ix=2H-N_8%t-gtQHabS~m%~IvCxv?m__u$Sa7HiMwGRM{iK?m= zG8hmu@pDYo>E43ClvoVnWAX|a&h^m{*qrzrXvoYC{mg&0j=R!~Fyc#wXcO9KBhFK3 z+3=p0#sKu|=w&ouOqLMY{H^dOsUOj^)!hw$O(5-G#MQr~G}QO*ES9$2t87v2TUrVf zZ+UAo9z$8keUqKDUskgaR~9zCo0#IUM}HUInYeXlY~&HfAlUaXNGX?@G_bY%w)SAC zabV#4jU{>hvV@C6$rbWDpSEXiH(L(1*l|@1c#!u#Ew4_`jM%-a=WfYLaoyS4o*7(r zy-~9|(1tt1Djfx=m#nTFXnZ`ZXko_PvO7^U2s^ek{EeC^<PoAw$ z%&Teey&visyYW*|Lzv53Pv~+NVgJMe8(c~mxvs0rs{;5&cGC6 zGQRnVwsy+ApPVtzuTe$1(tdW7;b(_lPo<4jH}xMlI=UNB_i6pg@60attkI-s_j{wG zw?89cTVp~7-Qs0Eu}ocFknVTT%{YXH!Ew^?ftD$Gc$*{+VL$F_& zSw?QcuzXmey4=}dH(xdmJI*w9wNNb?7C|oSvdgSa&Ha}_!+qMQDS8`vB3}I5On9=| zCs@bH zLVFad3-=wyBsm7{Z00e}Q|X!reB2dry;0@n+$$9n!S{;FKx8ToS5dxJV48JWP{DzQ zCux0fDIF@ZyBye|z}RpyEd32uuICe7A)38G881#M96d{$6V;w8W*=0OlmAuFb?V<~ z|JGpphgZLg$OlOtEY$DjWt$9)6iT5x=(fP4&zz1TR|*eJ``up5+B_UpHsUOLoO|KW zB+3q+*-%R~=n&e)rEftk1>A%)_XX1N$!hF3KpKWRU(rsMjVgHqvZd09uI-W63(pPB zgct-y>b^YO)gCK2I9`zG^t3i6UjO}8rA0;S$*%BVNJ0>+ac`%Wjcx5bhQMuVpyNeo^cdUDT6K$%|XI~C|15{ps3XN*QqxgGN?tMq0rgA60R5=!2Y>Fsa50Z?uzoa8zL z2b14;=|O*hzxlZEL|t>4@?5+1aG%@Pn)I;DlSM|dYm&31$nMD*i0P0IQzAC|EPwsf zp8vaYZMdiTN&)YU%?WFj3mKRu&AVRX2|J(b@853P2|(a1w7m2`w%OGsNX|(GXab(w zEFJv_8{YW5*LYKx$Ief7th4c_a=CjQdt<$dy<5Cv$%C2pqHtT=bWB6){hI&XB_M})EC=oZW zy`6its}{m1?M{48Z*Ds=BQ;$si-Eie%ArGEIV{A^D_jE{HcDL)E81^wW0Sb|#xm^r!fX9Bs;^lYrmw~T9!k$O zK{#Yw)2>Kb;H2>W#VjQwna@BI>FNC8cI{AId-SQTFMjt6T}8aXOx#D$EH9=BM6ws_ zghHu&L%@S*&v}*uwkq=bFWe;!<7cnG9lhxwX^f-8y3+0#q&g*=T)q6}*M#68%iY}# z9~W+ zHD}Z59`C6@CSTYPrtgeBE0*~Mfv}(D|0!26ll#79phM%6YdV~hnNp;lI-fKRo>H^` zqg)Y*-ZXx(+)&nQqSnjxS~XNy$i`E|fBGH2%|oB)P7?gP&!}uqN)1h@oc`(>VaK$QMiIE+7&kL21wx^wC$2z>~71x%59T6p9UVyxo4g4 z1tvMI9dcjV!T2vrls`6?=^XW{bUBIj0`NqsInaeOa9{ocGFN3EVVA@mL{+zx{{@V2 zUPgqo2rOZehrR%e4yR*G8+@gB>6s{#Y03=`?ToscHz>_l%KpYBx1cIEzP&CPKQ{M6 z7>X({vY-{pOEz8DSwH?Qh}!DB%dp@ccl3V7kDw(O2c>@zTQ9yy2?=w5@=8+IFPA7R z>Y@)8{YN%&j2 Kec!e7=l%yTsvh|O delta 4860 zcmZvgc{o%J*#BpYEo3y7EHQ?$k0p)eu^X}sWsB?~`#xn4XNU;dCSwhS$da+7BD=B_ z*$1T*vhS27yyJO(zw7zwn)=lY)ezQ6ZwKd}e^s?~wiz#rgs6>VU3?mD9EIB&V|EX?Y+*na9|3wNl!wl>oKVCdZ7*%X2E_A9d1_xz{X}lLXde63|n^J|1jo zNwwIY+zxCtOfzTk9O9)~NxZoRU>k9L=okw3&yeKerSTU+bg^q~(#UeQ-m-23q;qY` zp-G|3+TEmB4%U(TEjqJYQ*1o`U8?z zD^yfe47)yArQ7)9A%RQN|7Ya{eHeG%``&r3Xk#HSCM6t^29bgGPJaVj&tulqD^e0@ z6#+CTSk3sD)P9g=ybJ!CEW&xjjlPDV=ef~^Gi>e|7L;3SlM>KbSG+n#WbjfP$J94$9uN^@w50l^);F?`7b!$(#xWV)xlVahInEl@L)qZN*=*1!l zj);!8NdL>EJTV z59Ir5TDPF>m!mq4)L#;t6~510r^~8^O!tH`TD|z4%7Wo6{7qug=o#f(n)E1`(h_fD z4sD^bJMRjuclOR9sR{+Gr1^&2&B#{06Nj~{nY??v6xR5zg=VK|!`!NOIoh{^*vvQP zZZ#65vMs}0)6JrZOm~Gc0#+UrxN)y+I+mDrBi#@s>z8D->PUlpOo2n}DezTNlrPyu8CVY&}Tlc+;bl>ejX4PKro0Kza7sA)31(;bz0g>Wx2yO zvqxg`*Kc5;l8Uf~N`Pz+EBPQ6{}K_IXI*q(!eS8^Q(!+*Qs?WC%m}=0Xf-bO(kjr_ z0IHprt6Wk@*}V7Tn1E5fq!H$q>=NECFIWbbfLKS2L!sWmfifXvVze zTcRD8kT^f?6yKOp( zn8YhzEayxIhjlR84u%@OSNi-}^R7(iefW(8h2*y zr*RZ>*Ak#@OFN;KVEjnw4(DKKq0sHP^Ib}dYmzx+0EZ>)J<9(h?!e66`Rtu*(Vfhy6>OjCm5K zr=VgG-F91?BP5bJ{!95**0o37Iiqg_n>_x}L#i1dRrx4+>!#$*3Exdgo4{lzo!+j8 z3MFlebsw7WTBmHA5H1jJP_Wv`0LoOG z?8dJ`B)Pb#i*GKD2{xx{PFDCe=&v4t$GwDB07fU0B$u~Ux7|%19Ia%U_FOfSx&z{X zxR%Ny(<7U%Nlar(XOeM`@P$h7bM3M(TKnHC6cW6T-(_eHE50a0-+UOfqLRPi)ofAQ z!b0nV&A9~JD@hLbO$#S} zH|w2>P93GvuewwKuj)!y|e)kV*EWaXuN4e*iu32H6?6m9pDDHBy{blxIg_eM z9olU{@fuCnf-0fXIaZTD;Yo-g>}|c$icB!dMB_0!*i!H0*(jxJ;=#Wt10mj9f)6xQ zx=v!CfA)F0W@eqDR41x5-CKC`MtTg(wTXmoi^}zop}(amNC_)jmJBCBzG>OMtue_J z{~**B7I+*GWGR|9Kmlfe2PnV{ZxO!2VUaZwgI>w!sTzm1e_tIcWJND+2hrxEw4nq? zD5fTSoV6FrLFwv!UfaeeiUki8srd}3VL~)~9N6MEO2u99ZIp@&eTkJpP@CS- zhea(sroH0#*E@)Ed}E$nFR#(BKIkH-ZDRjkk`y|V*+sDn;Dzs^_(kGNEnUc-6-DRv z@<<5EouYHdWJU)XqEgH=%(eaHjrV)c&t^;Ezm5we*~624?hyul7CoQk(!YdLx&3V2 zed!O$Sm2lsMx|n<^RrhLW+abjyYl;-S|!!YQpMn-_D#wNOcvOq;)sUkAgQ~CG3gU; zoGvVyI02^*ZyERg#ArNHkl!(RWO3;qzO1f3usBY)z8-H`CM3xZ!x%$Q{6w(vWDMv= z0WIZYtV|6mN(-g8mX&AVq#dMuyR6vxI=1_ivS^o(B7bgQT@cQg(i9q^ zpdu-#DmV-y4yl9_!Q#C!!+NGmBa!YjK;iJh@z>MMDC}nBqUiRBC}Z#UH=}|GYB_Bh z@v0Z5#z33fAeP7x<1tAsF2yw5qo@}1;X{9(Wm5Icp+7F<@;7fKZs|l1sMM{` zzXjfLbDUZQhi1Q9{996FUh`veYtO~rCbFsT&+-Ae;%A_hWP?3Vgz7!zsfvr_ zpEq~5yw&U@o4AfE%Yq{Q0VWDQ>9oES2~_1hT~IpR%lskTn|SL%uit6;(s$2$ubh^G zhnxaeOe_G0k*V*^+<6PS_YV$28q4f{xxL2}=~UR~rhJxOKY5pDS9Qup_9}4wIPZPu z&+Ei+nZAXJR(85S-tsozj$H!vT0Z>(s-cS~r(k0P2sIlB1fr*$bvFgF!8D|cY18b|h zZuMU#@vdkWO;|xaler&)``$x%7lAN7+Zj9Ti})}-5!{Jd-~&szODpzN9?xaI!o zy7lKgqZIU`(9h^Q**S(DDD*7yk*(CpH4$zg_xb4F|k%A%TU?7kz$(?dF0U2jjQJ>T~gU0*alTWK`OIFbs!Izg}NVV@}SMkb_Zc@P#H1f%Yk@z0)ScFj5n)JWL! z4$l+E*bAKw+|9QJgvC;+Q#Nf*mBXIt=bc92*bJ4%33L7G)dST7gF?-ErYX-(I#~t6 zbgQLoHYoYVUuU2LhDxlVgtsc%{R!hoKv>VZ&F=G`@uOE{k`7L%Nkmm>4p zT#CZ{BKWUVbHmvgcj!`IgWt|2VSM%6-f+x>vfDHI-7Qv5PCia^Il^MQ(d_2=Ej0VP z=E#aaDtV3>We+-3psX%)vgJMi&Q<0ay*+vMz)V8XkEZm~w#;R5s}i52sORnS??=B` zc2*;a_a_B7PmD;-&6cC`LTrICTd!*wJ78S$Pu>hqH`9D2&f7`kKn$=od)nnM$g!tw zm=+z2a(#t`vf$I37yn825y%ytMwZaGgfT7@3V12FS3R=E$^W0-zqG+d4Mok$)L4x)RSSg{k!kxTRpMm@<2`@-U4 zAvJg0)cr9Z7vrtP-tNG^E8M2r6EK>*=xWgZiY4I2i2S7(zV*o8zI2^O_JE>d)7#ed zd83uuBd_sn>Ix+cE5|DUct|g?E~SYf-^HPGjL5~q4;XED?c`8y{T5DT6YKRi8(_F< z2OjcNLhSI@DcWqJF#{_qV-b(lekN##7n|2y9S>L%%-PPFlF#05_3D~eO^CX81Dk5f zTt#;Xfh3$9{e-8(-%89z!+cnMarzS)q(*5wWLR*U5c+#Cqfp#+yo&=NkOJby{+HnZNY*l2H#yJ?oll zO(9R$4Q?P+$~yQ)zL$Ax&@PWi&NdWIj6jJWPMlZv11I<8Wvs%BWSv2elHQSvM^65> zp$nS&AS2C{mvc!E${JN#B69z}3N+7*i%xt|^3dG-75O=_!ShQ)9k;d91s0!aoyFQ< z<#+4T3=btEdcGIiB*R?mOdO)V{n_c^ODVEY)W;7S%zu!^>l!6V|kPxpR z@c-6gwE{cUtRUe;kc045K^*Kwpi@)=PZKbT!aQf0LxMpdV=55n0%e>2{{tV@Lb$J} R2EIV(P!y#7BzN}we*hVr@A?1$ diff --git a/summary_results.json b/summary_results.json index 145378804..7c9e8c3e7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 76.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1258, + "Performance_Result": 9.83777, "Performance_Units": "Samples/s" }, { @@ -932,7 +932,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.15441, + "Performance_Result": 8.21321, "Performance_Units": "Samples/s" }, { From 858aef2e4cacdfb9fcfdcab285ccde6233be2f40 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:05:15 +0000 Subject: [PATCH 33/86] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 10 +++++----- summary.xlsx | Bin 14571 -> 14574 bytes summary_results.json | 10 +++++----- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index eb06a60b2..a765c0a88 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:01:32,232 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:01:32,232 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:01:32,232 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:05:10,725 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:01:32,232 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:01:32,232 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:01:32,232 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:01:32,233 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:01:32,233 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:01:32,233 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:01:32,233 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:01:32,233 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:01:32,233 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:01:33,278 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:05:10,725 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:05:10,725 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:05:10,725 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:05:10,725 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:05:11,747 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,9 +25,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.428593 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436692 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.378087 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.304628 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412693 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4667 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.4711 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.58565 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -70,11 +70,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1396 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0862 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21828 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29161 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f27ce70ac..db08d2acf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -574 +575 diff --git a/summary.csv b/summary.csv index 4243c068b..e4672ea1d 100644 --- a/summary.csv +++ b/summary.csv @@ -3,9 +3,9 @@ Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2 MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.428593,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436692,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.378087,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.304628,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412693,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4667,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0219,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.4711,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.58565,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -48,11 +48,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1396,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0862,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21828,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29161,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a317f94fd..330e4f189 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436692, + "Result":0.378087, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.304628, + "Result":0.412693, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.4711, + "Result":9.58565, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1479,7 +1479,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1396, + "Result":23.0862, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1599,7 +1599,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.21828, + "Result":6.29161, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ac3474c26fbcd3a470aedc6ce5ec16c15bca0473..ae2f326d47026da51fc652fd537f2813a6530649 100644 GIT binary patch delta 5427 zcmYj#XH=7GuysPDBPG(6YUmw7si7)_-VI3aO#}`l)C8pphmz2dB1J$5B@jA-QUgep z2nr|&N&t~2(v&Zr`>l2F`(ysRYwc&so|(O7J7_nk$%~P8rK8=q?-~H$g$ZO-KpHGM zvSDDtaN)FKcVDMYn%sxJ3<%)l>ZeDm6(=k0A0L{6o8Nck=d^zM^G*se`|lC|qs#N{ z_457Be3{`El8Gh0+_NG--BaWXY0>0x;QajqpO$7gW3|XY>SF7HPpky{TjY;FQ^$FW z7e9s~%q6dT#F1R9BVO4%zVdG8s6ZaN%G(|F6r^U$*B*i;}LRdox2L1 z-rJ%EA26$k4?VSeuDlj*VUa4lY>l23buc-H71ja{jsXwli=oq#;F+amC|91wgSXXk zGA7}LGV`gsPR%}T;q8qXzk}?fZLR(+!HqZO{EvTa$viR*E1VJkkhrtG?Sf?Z`Y&cJ z1QFV-_AKVKg0r+(W_*slu`A7!BX#@ow(yq&`z=p5`HQYOg}1|b`-FpN>y98OR9VbM zq-&AWK`OU};B-gBqMb6J>*ab|;Dg3<5cJ=GK>M6TbwH!%=L^z=LNzshRn z+Lq7!L@82X|If@|lHN7Gm=t`09n_NkH$GoX*g|ej@7fPO}kER;tR_d zE}cvN-!xJAW@`>BLM;14#9hI)XMaotDM;@jg3b{-f)u$L`no+&R?x{xhw5OK#!t|c z#DB_)1nX=Q!$ye$PE5I(hX+dwky~@I{irvjz1uQ7Oka(u9z~ zq7An3*Sy`CP(!N8Rzk5P@MNasuVvy=9c-QMofWvux*PCx`?G}Nh-S$wu2 z#?m;TYH{bE80;(_O zJmfUgc}-1IBRah#(({H~BYHZ{6u4w;K9&ycf9qhw=N*=Jta#&*_s?s?+IM0Ud1U%> z&R^W^{MWYi(XU$%HUda5riq&ezJ8|>IK?89K3=4aR1W@#_;3fIP*jpWRe0%J!aBZs z;yQEjusgEW(^l5g`!~#vh*IPUk~xWBnsk+(V7T0JsVhN?eV{zca;Ymbt-W7SS;q>2x;@ z4rYcJ%l}g^xrf!z%BW!h4MhHjNZ=`Q{GY^4KU^Jw`LJh$XeqW22 z8C;AV8B3nv$7ap79DZ5#4pn}R<1NX_c8q;6`L*H@JJn@PT;z5Tdk8wIcM}Ij$LdQ& ze=w2Q3Sk|&^#$R15&^s6%5>AfVs41xGfWU{3eG~fUfG*aL%4?b&&~Dfbs4JNVk%is zXIW<2MGF4fzIhB}CD3Q5VB|+{^W0@ibLFw5)sm|#uX!MhV`Q}+wCxGtkbU}VgqI*H z#r{s37ML!1Gs2yV7D51I{o+c5hyjew=#Jn(*oGZ2MMw-_zv8v~+C3>p_5P+}4a54S zdNO`O2sN0|4(k4hFNB9%jCkn`H8%EHA=K58^XR)1qNl{~Xl9(ge7_(8keaW>o|ADm zm;(quV%=<}=r2ELPNoD$05d(VM8u%uLNUFuO-b0aurDZe={US8ur0kaUb2`B?x2dX z$+nS}X*?Wzq~2?VP}Orh`goH|=u@!+Slbasg%INU%yw(E?s(rIz!lh7LZNO0KqBU$ zj?I7r1Nr{CUtvp9jg=06W^hKOmtg6Ezuz?xw*)!<- z3<8Z1P!F16?%{0U-+VzjPWGlLD_s7@Jd0f-R#ralLTGptgK)4itfyIr~<;xvi% zu+Wmg)|OpwT);DOXa;e8t`clVF1W#@J+tM|_T_AVP6%in&5)VO1$loqDzppC+!mX5 zqeLzF0^vRc<%HC3$t(B6VKg%y2>^%g#ZBl2#UrVIXa$K4PWN%vXktpJk`$d(fMP+n8gGjm}Eb|fh76ST52)e z%tb6~q+4ir-#_#XbzZ335G%u z_`uqFOEJ3sTM=U>VLA@wOfSW+4Z#fcno~`Zj4G#9U{%Jt@&>+ZAiu9yib{|FT8}obe}zEj`hCT4VLcLoX|LW}_liFB@ep^?d>}IC z!r`M}x_-4y9c^y&;VvZJooeB(B*+~N6exfqdy}m-&$CgEvazI!TD;A<;y+b{YCU#1)u>E-{DI0%8+V=&q{dj)A-lKLR~$iUB2!I zt49>|10>lL1q0y%g%tgJwm#r2<~Ll<;AVz?7*%}WnU$aoaP_ED_m@UA%Cc?!NVGQI zGHe?Y&=DxFOv64|xKUR9+E!Y6Dloyfz-{`CZD@kkEDpA2r^M5cc`sEr)+3hOwK)yU zv;sxI2Ig@3 zct|fPSn>6{ja z%Y0yMNzf8sE>LpuCJJC!dNCl*p9?+JGaT^zk8C~@IObR!B*TY7`t|9IqN>KK{qR&5 zwEH>tbe7|LR7f?3`#ttr5>=%+cUGzKF*%pDz%1rqt}8i~eKpu(wp&nh|OX)@?$6;stsUG|jyz9e=*_3YG?M(<^Xs->8#y7T>emfS<% zrYApUu$UB|`5Lg8I&Y#IEi?T!Gh6$YtT!mrcsJ)odNZ|TBSYTaGgW#* zXd%A7niHMgni;L)lihV(%}CI@W(z^*GSSb(VeRFKO zV!@YO%rcjV6xUv4nUGiY)d|&2qR2A9z=nGlkzd$bq9}@tkGwyAMW;i-8mm`C-WFlH z+ZZ`+PUOKm{?D`3Kbt7}tERT(oPTN>PMTZ~7V9QA-9oc=V_fax`_p~qfSUS$++X5M zPCLy<#;SU`ldx{d5@47^Qi5ykb|$k0XW4cWgIO|H<+Rs!6NTn;Q)1`kDky<}yJ?P! zC=o(ttaC4J*16Y4xJke{HvBt2S1i%H@x|5_5+YSxJBWXTeW@ZI#g7r?%L)mFi{>^k z(h3v+8yLp{1?#mMnlDV}T_{mTrL}JT@8WWw72lrTgwq6sWikhN=&bWtw)7lw)_=N- zt@A)@eO!FmYZ>5$ACFWdYDw%H3i|wLGL!ty1JR0384e4Ie?G zx7PgjCG}}^%Uq)W z8=WZqJN80apXC(k)ikIFjpq>)UBhn1ThULuNlN+o_tD{5pwUQ;mQ z8B^xw90CsHpYht~Y(kU?iBJPcImLFFjxd_VE6pSCB#+gGziQx-j(!DB%(-uNQgBwv z6TBFeCdgAm$UHAQdg?6+&FsomcHG3^3@iBpf}c(B+5no4Qq-vidCBzj0g-YN_Anl( z9nc70exXOHNZ_gDHa_>1GN?1)99~sStRvj82m9&%3I3waj=m8Bqv>R5ih)ygKpGeW zNthTo@bjz)@|-g>=6}3ni~I?baW^~W&Nl`}#iD-NjbA+GoPa#x+3wi#y*XJXgigNr zz8WP7`Gj)-Q&8F7lOL5wYoUQu*s1R>l#nN{DAef#m|EA@`T2Xb8=)jh1~Pq@l`_7a z+cn}RWa721(^6qW?H}OB;`q&tv)>`$`mukIxU5|W~LHOWLlChFs6QR|6* z&~|<3_>WqT04;srxJsM!sSM)kFSu)hDCt*I=2Ai^j3$d=FA7eT={~gv6pne!R0nWc zJh0xhm5nYFs@BVz1kkU}!eYZYcDy(CsoOk@zl|dz<Ko^EAN=g9x9A+}(xSS807@7zrLm>aN5MCSL;?b!Zdo;PyB=hx$ zu)pJkSvND>|Lwt~rDpVsI>FA9^U7LRr<)PDvsGnVHeAwNGg~1XuK#C}tds?kfnUkZ zo4jML1}HaOc41@IE$w!Dy-aT28$aZ}FaSbce z+6<4SfN!xuQj1&4{qo9|0qvBKFnyX@hQKg5RgJsp8c;lDmaz_Hbym+qPh@N*FU5lw z5c*Z3qUL(s7bgF=f~KOmQ(ZUz!Zoq>r5k#s90nxod~Vj|H@Eog@?4ir4H6HhqEV9s z!Ew~ceLybsNs_guWDH#fWdilz_0=Il`*$q}(!5H?F21ML+f$bK@?uOS%z-F37Gg1y zSBTXiHV?9yTM%6YB1nyf+R4tS`T=$LyyH<)BQBN3cNIupiT*Io9NcazOsz7*IS&ueB8guem8jmI7vE zO`HofTSNh@N~y?7pa#k$O2tHD7aYba%TV^9G(<;Sr?RxelO&Qo<|RgjHUg< zR7=x~j?XxT4(0=umzGOd4_nF~*xO5TSJE6T|G6p$WkXVcxqxohGn(&xyu<*V{%%6flWrosu$(U*q)^k;s$ zPce*pye=lX**SgM;Erx=z%vXj1cMdUc~^j5E*>AwZgOvMY0-KI z#J~H<-P>Mw1QdeGO2NGp-gyHua^E@(No@?juDiuGg}h;pu&MB*LCNVk@e6v zWK%RC%~Eo3_F=QfMbSebNBoi-WgIP%of8VMRJ}ICB6gD2s?l&A_}FbzGmeq>n{E2k z3c%#F=xRmqs+Bwgdw`wk+RjRp^7ZpqQ)j`g!xzeb?E8BDt$Sg?XEFp-%=w(H>ioLn z#hbjlYW5@1@R4heAju;FTX$B{Q;lNy(VQrgAk#6u)1##ZGs{fZEVQ7Ihars+JAL6d zr=+a=zrD&-7*Fr)U&#-4LO%Z5a#evF^nY%evUlzSO)$xK)F?J?V*cD@$1L3u2l`>C ul~jPs7;Pot>wohJ0RjMK6aWAV;D0|<+$+ozB`A;u^G4|=)r8{TKmQL*J1jE* delta 5391 zcmYkAXH*m0*Txf?fPkSJy!3$5EQp|#&>=JfsG%c-M0!(DdIBO{N`e7JKq)Fl2%#t- z0!ku~5bA%p_x2xtyh(+DUK|d91&(`{GIJBhjHTek9VWKNIvSfF7w`~-Fd%{{2a5- zKkP13h@B(T5e+B1UxWt_B+OMus6Tz!+PD5;Cw_}^uQ$w>`j5Hs{mx;PpJZI8@Fs5g zuzM+McyGDIFz=kVVe~AT5=C`%{@|f1+?0)P&3%t-^-@kRw`1gc6 zd9#0tG3>lw9QKu=pt?DAuoSkC_It0hU}Jj3OD_LbWupx)RAYo}O!nZfpnU6_>Be~% z?)d~n)@tTOGNi@xZz?7?3sLI0+_RE5ao)thK*it>s?dlra!%M{XGm4Otzel`ig95? z@U!{qUz_MFIeWqD(!mbnFUY2TyRQ*rTYhSp5BWqX&T0Nx4Bz~zu2=KW1`)+$yYJVB zvqmDcK23O)hhh7VmnJtWJ60dmdf}YAN<&8jBEDHF3{us@^0&JD*4JOJ_)Zwy4Lx@X z-X51$9CzzN{;r89T#oxrg6ecY3Wvh2#6(g;{ppO5i)+yagsyhulxl`V%)Km9ol^ksM!z($!u=Un44_7h!eNm$zZ$$OtY0>_;zmC@?{ zOUi%5TeQn@iadJ`$gQQZ?xinQ@fDKN{N^px%Y@vpAvs=3vxfCMC7~afoyqgahrfJ!pE?R1<_g$owB(nqF@|SD!bsi8-f#DU9-kw< z1Rf!-qJtTgm-qisE10^SH$8v6bZRc_^{`ynO=MfAe@k#*K~0LCia*8uZW6gTM^B@f zJfAx`P521)$_d(L8$v6>>?2HT>nA8u)Mq&a>nmJpO zcW6`d0VwZf_&b@8L~iI~7x{zeE5O$O+<-*>pu_hEk>SQ-*xbJtjOF-LM8P@=@)xjFwrK znz~#|b@4pmOy@|4FH?X$bGz%T`iG&;3ck$SZ_sk!h$w_v#0W~-t6HF~vTN1n_q5Cf z8j$j$7x$q}L<3kJOlp-Yv* z?kxHmTd@|}wpi;~`ea%4bMxCBOOl^ZEE{D_nb;51Q8rEH_rk@;f{aNSniVN51Ps1? zmrI`@0FsSlt$Twnx?PB;0e5GYFK)S1xK2J&*qF+6_v^jHDSsq|R_5e-u&j8S#ovO*B`7fF`Y`dfu{G1V zB0ih_o3Y-4-W=+Ub6x9XiwW4>l@9T|hU?qr);|<*ni5cK)gM9qb_F`<=ruH3=q0!_ z6}0w$JY;l=fQ~ag(BGRwF>ZyWlk+3TppP~c*^ui%FNpb4^&-}<9{o+-5!8akXC-N( z(`0J1ty&Sc6H%Z)1g>;5Wu3FP(jDievucj%l)co+5(M4uWF0p*OO^Zn9V=ObdvB(f za8`k3W@OQ5OHM6B!Ojdqy}RB(P$E{DylM8d@{lV^gpr12QG8}W$0SG?ye$EYzHpz` z3P(RqFMw6#aXMLpGd2a<5#V}iP*!LSHTDAp`J%TyyrH<(S1u1(BUFY~(hX_FYWBr_ z?Fg1zu*3@_$&>`)5k^osFfIc!VWNz8$FmwJC%i;5qZRZtBV{2a_>eG8Oc7{;;|1iD z5WJ+rIdXGd@righ8VKZ$qdh8Sa4VAdcD{k4&uKkVSk%iY1;rK#8R)UQ;ZM?(LDE*? zAftQ{1J3b*QX~(W72k3jTvOM=lt@yH(KHDARYIt@4|)B@S`NqAbTPE!(E_Q?K>WI$ zY!Wdp6)l|0^BD0wm|&WdfXVEqX~1HG*w_sfnSH98#9|}h`q9UQuZ`&(0qnp?{^%o&Gg-*8X>6VFT|ym;Y75`yD7L# zXGqggL-yn4195-V@# z){|JN$X-(9N~8r(1hMOh4Dd3v7(f$1aq7Whec0Ib6q$VhYz|x>z^Xrm!cSkb9G87!rtY-pJwC*fDJb*G^72eN8M-L611eApv?yO56aQ?=fcgkp=g_4N>F z{C2alj?}0(8)DXq2QPxc5M-EK8DT?9+Ua#(+Lhq68O`JP7ulGr+8DZ`(8O`LUH%@S8MW> zSW-%=)FnHiPdFqA<1ahTXCUrGo#Fq2-FvfJ=`eTjbjD)K_?>-8;KKC+(WNw_YX(uO zSAb4cybBaC7Qfr9+$1w9#9p&O2V~x!t;AIc-1$$Y&#@0-MZ(JLyUepmzIiXtZ$G{B zo5Tq}ZNDIdDQvGhYbP3)X5?-FXVo~u{lWn}7yK`PC-E4fJaa9=mKD=8z3m~_`;VTP zkrDXev9L|VISfQa%!%5{|HWnP&H6a$iPq_`z6GkRi767Q1`W#AdU8@K<9fJlyJ>ZN zEaVfYIvxwsd+`_1eRa2A%o=>ReO++I>wNe#8MnUJMY-n=X+h07C0zHZqSqTGPjpq*1m>HBF1DT>k@Q!R;j_HUnEkyw{N_=d3Cx{Y{#hX>!fCS9; zh1r6~d)+rrmj6er{kKLOsd|)#lA&*mRfu27n>*-^imQQLuYXtU)R;VP9=w2~2__bm zHS1CPgAwd&RNcrDo^KEft+a`Sg^T9YslCWjab*=LK@W zl@)KEU*gu(A5BUTRqdIv#j%UsO@}yl#Ij&G z;kj-2FU(3%4Iha?-FITManlyJz#l^^(fQMf0dVA7GA2WJx~pQKvW~l7wQRI9@xDbt zra|ujIe}V0?3;p_azwG!}#I8}B*6-+iM?GxA z`FLp#7JVrlwdmB9TD6m!T?OK%CB~=6$E+zv0`t+u;5r4!-$2|9@>|@RCV;qcG4(2F zR7rdwKq@%UX(hG`p($NBuMen-1_{Ln!~E6PChJaQs+Yr(%wPjzLWG>jAOt&=%QD=S ztsIuK1tP&K;{`#Ak?1rq^_V?Q`R)2k6T=c|F?blyYy-4-_we1NxGMB~DXedoDAkZ!aIk>K{{OTftG{r zYn**|L6DhNzT11MT$-1`j>A{Cu=SLicft?jlnWmXZt z8NCEF;xnvW@uI4=h5t>v1egLh!moDfPV%41y*$%Fq1YlJgxBQuo`+~unhYv|5CzC2 zm-BE)62MVF*ChCy6LaZ3y^Sm9TzRB)^FbMF^Rc7h8N_DGj?DcdeD|YZ=|7GB3WK2y zDK86$vse!s+VAnQ$=kX1T=E^BwA+7C_BPO$&0AD8O-4D9pnBrCOz)JWB<5n%lq=MQ zxY)w%vlAqS0uOI6G=(C>JbCWp=4{jkkc+~si3H~S#M3a_%4E7m+SmJTQ(hx!zw5N9vpu|2NJoI%6bn_s?4eR2y-#|B>@)e z@d>5*79HG{Jf$58v!hSMyb?IIIzzHbrTE#!SGpzI6ckl9>m>#Oyb`W}yS8kvFdD0% zM{rMoCy$F9Ol9@{ck2;n=z1ZuA`)vr89Gqg`I9Cq^AM2#jlvbpUWd^4xm-bUx6N1~-Ni|Y9(++h-nEyV|{dl?3f$j&v?5GNcioW)mJF#yS{oZYOuVG zXq$;?4>`cfQZj%{XJ!~6(|MSr)kN-|a1~6q$!~Sj{SC-WWx}b2HT{+xOaGGovlXj= z3ePj0zPNnoQ!;+XXEY~BNc@s$H)0~;Hzuk;Nk5N#BD|n@vl&3`r$elvF8k$NmL{^9Mg-IRzxoP5 zu9v3|;!j+U+0r-8$TzY>9JCgTaY(+Bl()orT0Dx`VTRvFRa8lscgk7@x~3bou1cz? z@|Y*a6%=zE>0`=(phaAms9Y``T&J4)-I=~};!C<&l0LG;9L3P4_A?8UIrHD0=MO-A zN*VVXeitv2I&|7rTU+6;Ge}$9HP0(+YueW*$scc*H)_zQr$bLS);spcRZisN-sI$M zl~zwAltvA2B&T_8vU$U4$*Z^ezJ}PDhX*jiRP(t=ovR!5Z@0VaIy?UuPM;GkP2%oO zrJP$ISsT64VAI{`EJ*3}qHd1N%ny%#b^cHz)c7RLeSLLpv}?xQuzJ4p;{ir>Cs?aw zes){?8-vowg)?#9Q#OyS1`K z&eq|H%BxaQ5S9wY*BithWkv7{;=b}p-048R6}h&7&rurhU)e8Q&)SP#@0}5ugz9p5 z_{H|;h(uO(Zt%*$&nwxwtMq$-(uBI4T9hVRKjvH%a!em5viIiQb|~~^C!rS@LIUe{ zK{IE{9$r(V_4<}$_u_p)!m*kztkE1oCx0+s)8PHeE4Lk6)mU~49OE{l6U`$wjk8Pb z{sFqm7>={54UmVj9j_cDUar08e6_V#m=$G^^xS5oZ2>KQJ$<*?`T6$!eNR6VgO5Oo;eXO&;t) l4j;b0zO%VlXc`Xvx>{SU*>EDitw diff --git a/summary_results.json b/summary_results.json index 7c9e8c3e7..0f06568f7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.4711, + "Performance_Result": 9.58565, "Performance_Units": "Samples/s" }, { @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.304628, + "Performance_Result": 0.412693, "Performance_Units": "Samples/s" }, { @@ -1220,7 +1220,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436692, + "Performance_Result": 0.378087, "Performance_Units": "Samples/s" }, { @@ -1292,7 +1292,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1396, + "Performance_Result": 23.0862, "Performance_Units": "Samples/s" }, { @@ -1364,7 +1364,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.21828, + "Performance_Result": 6.29161, "Performance_Units": "Samples/s" }, { From e145b701ae0ded8598efd4f32abf7b151346b4bf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:06:12 +0000 Subject: [PATCH 34/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14574 -> 14576 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index a765c0a88..df7900070 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:05:10,725 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:05:10,725 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:06:07,167 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:05:10,725 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:05:10,725 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:05:10,725 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:05:11,747 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:06:07,167 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:06:07,167 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:06:07,167 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:06:07,167 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:06:08,207 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.58565 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.20426 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.83777 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96539 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index db08d2acf..bb7a7c1b1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -575 +576 diff --git a/summary.csv b/summary.csv index e4672ea1d..809494061 100644 --- a/summary.csv +++ b/summary.csv @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0219,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.58565,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.20426,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.83777,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96539,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1931,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 330e4f189..6ee1a2af8 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.58565, + "Result":9.20426, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.83777, - "Accuracy":"acc: 76.800", + "Result":9.96539, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index ae2f326d47026da51fc652fd537f2813a6530649..42a7f578c6cd15babdedef867561eceb8ad0edf2 100644 GIT binary patch delta 6974 zcmYj$c{o&m*#3;kl5E)`S%$GsD%rA=Wl;8I49UI|3L(ze*DN!P?6POy$zI4-Wb8%B zl3fgv{WrevdtLAE{PA4p{BiE{toQv~pR?_`<5}%MbM-t+y`X~u1bTw?pfLr$Ji-T^ z7&OgmNa91=&i+(kV?V7}cd(2U$vmEjNq=oEot_m}>>%@n_RrI2gK_EzGG>mv?Tg zEDotY_dT&SKX(<+`P3#g=hR_%rv>OOn)xx;A^*3}bJtGq=(&w}ica0k&$7-oxfv(5 zGDo)Bs;bC)ZzQ#2#(quo5458)wXKV$KY>?Xe18)+^!#1M>j7YALwih& z?dd1}p*^EtpN{@56(F-#?QLE(Acy&Yxl_a(bMsm$=6i>b{$YC^u2wH^abjb4E3bKL ztmDD@0mW15)b6>hqfITFpun)+Wsifwk{hVhV!L|hv*#0SH_c8Of9+Jh)f=|){wx0{ zee3Uv(Xa14Nt*-3b>owRp8dUSh#%DU(cqW2N!<2FES z7tBz^Q?xEzD>3m#`@oLstnPqk(@Hto*DqD2$83=)I_mkcOf*HoQ*0C(Sf-AqK%R45 z-=*=_;$HZCh)`(@I)QNyyT@O@J-iK$j;j0K7j@mnPo3Ku4)NmJy?Lg=9ntxd{ya!) zhf~PY`tWXE{mZMZu(9jUa3*k`AGbSg)M6P9(^IZ@`#n#EUI?5ApV@dW{Pb@*+z#?- zQrjAe$fUZb$Y@V&fpfQExkH_|wo{ znNG#1D?Zyi`64U1HH)B=+IzbHQdaKv@8#?YDGTP<<=ut#G21qz8^GL`vuHeSF&w+# zpFd%-Rs3zP_Hg1w+o|z(2J^Cn{bVanm{f9nYe#bTfUzG*Zodug6gxysFA-$f-Zm(>u_H>%?qJRC3KFC%*i_%ta z=w`kQYcYH|x$xidAUl&<@=|^*Yi^th%nrsa*o}afwrR*ETKV#5ubJr_a;%APpG<69 z%V4ng(_#Q_8U!EqgrATXU3V6ray?B6#Z&CC($_t@?xhUa$;k=}wx2d;eo8TPJ9`+Z9uBqSf}&W~F02YH8^*ocLouJ` zx6`wYE-ns8c#-}hN0Tg6zvkW zPc+$DOquN=v5+J6F9+?E2DVv)mr`ueCV#e?%`7ys3>Qi-Vm&HsM3gqlES^4fqvx*) z1x$kW_6t7-U`2!q=EDVGXB1kK@q zBHn72kEA%J5YkCk`5K*$OtYba5Mz{qlB7kA3%dw804iv z*3Zkzb>lvw@*qiT7Q!`UklRNIdOtw|RO2zvDR20iQdDkZ=<d)_C+=vw#^{tD`Z;hE5e{dIMt)1;(j4RPO%(OM@gof>;w^y0Xp%?S zCMVI;U4>@DATP;~im=d^$6{O(JUwF$BuSJ}#l?!+lw8@Clr`NS`Hf~=a)Yu}_ZCgd ziFwS?D%1RW;QbT!{K=s=LWwdk*$|}!%a0omK~Pvx(S;((EMd;HU8@G62=MIO@e-FY z2wBdIp1aMAsJFeo1jSziQZ)(Azw}Lu^%>C)gygKvh!z!2LnFQzEH2*qMixyP;;lDE zGlS^icn`7GFUTlV;gA<(0;+>6j7cPj;t?ThQnYrCi3%lGmf%+nnrp&^o8r1`uGDg@ zz0?yC@};Gc^(a9j`uFli(x0K>-!%9ppcp-T&mRn3EsG^>4M0_mAcL92Ne(*D)ld?K zcg&uKXJg_=YmF=HAkc5nJa= zGmJr~neuv4>&76-JcxE0O=ps0&?CfEDgSQuJd(C~5SPh8>cI=`VS2bjEh=@MD;j)Z zAt5==DUJWgi7)bB1K71A6$fWbYcpZd6VH%zRB zhY(8$faRBqg&`SodblK?}%} zy7@f=#aZ3m6@*D|PKjL#pe@=bj@r5li1}r{76XU@JDn%eVOT8BtEm$EIV2#k<8e-7 zu*Uv71r+MR$yW?EAFVm0#;u^TuR$@B9Lc$wzLl*$eOgTBc6;pFM^TGze2*S;D>EW; zH(w8$G$SCE$w)cM)v)6tVKVUEEi&`d`~v~!%F#UHs^o0oOYYJ*icfmdRhJOq(xBKHY1andW3K^Qr4V6p*pqEgOcw-8sT{D8U^)}9mP<;A%3GS?($ zUzzN~;;49l>@O<55fr(k;noAZ^43~`-uky3>2Z$7j7f+VIk;`%_lL?9R_O2Dq9Ew= zfC9(jKzG`ApoIQCrd221fLvaO?ywaD~9g0hRJk9;cc#Sq2?O z8cm9}<_?t_)u=Z}Puj+taeRXQW#Nue!EahYg}-pnlL(or=?{ovG%?YtzhCc0%axlnis8{|N2kno?id>Fp2IMF&pKpr!Hqho7CFSN6 z>LlMsUkJlC)j>BJDsdWQZVqLCwclq=h-^$C$t!y9L>IJoE7#QfA^-{}>S|vknIrGW zAs7`%pb-YuAJ4KwwrT2@HtV4k(1aZ*bap;u*v5I%FNh{RV||b1*B(p$6P+*jYarTx zhl&fn5|J4F5fs0u==_mn^RNU0SSnDSRv6@Q{DK1_O?wFxj3l2sE?}$7_)HSwU1y)O z673>v zNmtRx+P(RGMXh0Nw%V%+>(Dumlq2qG(<}e_G46`vFLB7oPtw z3*BW~7gHCQFn`p+foo}R0hZ3V!0fK&y7DR1?;AbKEn7AzKBy|b=N=m6*N=o>A0iQ6 zz*k=Al4#doE7o~TvvgOX&R?Q;S~;6GihqgTT{^cq!QY;!G~;3^p|N=co^-jrcg^}P zmddAmv9#>9R45L?P4JEQQM#g=t|YY0K6LB>)q-xSG|NOerP4{w$u@>VE% zpR)b9;Kp5wb~2wu+3Y_q?USON03DedjHQ0k(``bCJVZrYPk+%0dUSLdq$TVdohD)OgIG*e!|&u(R}8x;oyE0*&7lzB=~62N zq~SAg*HOpJ+t0YORQUd&rvYBtl|##xp4W|oS+n2^kzno7Ux)@Uz5} zX6O^w7cR0lGH?i2t@R&pB~GswW3%vp8i_Bj2E}NY`tEf41pA%z zi+!B!ZS1u?ec32=-$&sk-a_od@w^t%m|>c0y!aU?v_Q0L#)>r;BNt6xvacaiPNKh7 z=L1K|Txv>x1_mBxHU&x|k5{M`kClCkl;4QIg!`y}N-vVdJKPGT2?qM;gH<$wBg5$h zcP>SWYv(_a>LH306~})fCCkjn;W#j$H=hl=l#?i~f6|TiPx)^V)g&%MXk*&jv`=76 zgZ-&)KS^0pn<4y$zm?lYp6$eOAx42aU}NN{!C|9ja)^J`^a zWB<2@)=A-4IK?z5q6))dHJ)F%zvlOxGj!(S2Ex%dtK>ct^~rOo*@|A6yrhSH6i;IP zHohK>pirW&MkC1-?I0w&fj5a8N$#<1E{iOeHaRRQT>A@ZS${-e9eMqZ@ya1?)q z=$}|Hg5nLxC2SAC5h5yaazsoNx((mk>j`FpfT#C(2m7P zpNDq2-7mL3$d{%V=dgJ^bO`XYU>q{(UiR<}(EmIVEM;}~bZtY9iB_s5KFnbqv9 zaqR}@#5(n4P*iGP#$%mNLtfL}u~V04QReZ*4zGKG%GNgEk;RJ({AlU)>i*7po+G^T z3A4go0t0Ht;D-yTd%Ldk7D25Kj-5=m)~%am(y~tdD+i7yHKC$IKD2% zwzoTTG?$YqTsO3#(EH_6@r>7ljM=lTHkNG0y@{WL*Z}E_+keh$rFk!p*Q#_@w=ckT z+T4X;(kgRCHDl~Q9uc6^c1BykGKMlZFJg+Xx;I-tzT(QCe@P8XbbqI+;?Vdur#;cngE~op*A<}S<`O)dep#?%f z9Avokqr9DIe;Gcx_KhYU{5V{u6yLflMP-d*x=0!o%YB5UBX<|0-e5FY5of;kV|D#j zVE(#_>POF2_#OZ`Z*Qs12-7&mc4%782vAC;R>)_(S6Yn^xjRpISQR%`BGk^*fphK9 zVMq|gAx?CFR64nY2r=*XZPO{tw3vH|wB)W4HfP5#<0hDLt&bdyc8B={6R#xfm|e*D=V@eK1bQ+)lAgpaeNm_yt{VQz z+m6sr$UE<5;PzLq5;5B;@85$dSR4Fm!s7a^aSrMm`|!o+e)skKQDGGQ!#Osw`$pv` zp;Grd&Onr9$-fUTGnX6G(zFEGBixs)dq>qZiw%s_K4ccVm(;HeBShtnc;ix^Z|U9g zblauVOE0YCUshq2;^X2+&ypIQh)-_4`2m|8qC5SDsITKt)C_nwex zp<7&V(Fcj%@gKJQu=Nx{Gv;_L(RBULvp3W_rm1HgLpUY41qaVAw`G(wUu0M7fHv8*C@|yHs4BT z-wuGL+PRc{O?rC67)q$nzj5X3fOl2Zc4X-GlTP0-s$VIMs!{IK@nd{GFkU}md;xHhO#}wTpu!hu;kk3>r(H3F&@8aul1pU?uFr)#H}!n?V!Kz zh^9EW|C6DI`T5wrubMnO+PdXv}n?xF+ z)1_xwE+#m@P%}}Vrh@z!0$HV{%2l|FKJNnk&jb1nPWH(lxPxUr_J0iJ|J`C01lYjz zSULeF;12lTAa8pz7FN!&K%CllqQb>+AM^%%) zZ|~P<-PiFkO>SDo6lK{C!F1skP*(r3H8#!o#F!vhR0$Qxa8^33CBjqRt<}4Ci0YA8 zuEd}AD$n`Ark5u(5S!ifiTbTShjQgxDTz)v)gCB`{mWc5Ymi>oG&y0w`RAEig83b* zjXRz)xQwk0Q}*eT^)&-&re{oo`F>}vLw8;)esAXdcC&(S(EA~}HRaD7U5yBqP0$D! z4yG!mc{ln79_*|+7_WFoX_Nm>`MHRcqW_B%pmQb#g3ah0~4NslU75V&hX9!jr)8EJ??QTva-Mf^}6nxXD19N9$>V-NNI-F z-=`Oa_tv?JZPs-9s}oL3uB#NMX4YOU0I(_!p_|plroVY-^(D5|2%rkKO zU}nlDXTGQ{=yhD%RKeK!X_r_|>mX#Tz5HHAbwo*vY)4?mX@j>>YGg?4yTWKA*M_Mg zT%+^%#=4s*>+5uGb84%#zA|mwbCl7y1N(m#n8kD2@rcBIY)dh>iOV@B*VTg5xeHlP zkOnAO*YaXVMTLPPr|XRnw}E%T$_6Me20Y*OhM>f8X}^;}?FHDpK|)#DnU9a2_|Gzl zBhr<^QSrh1A@pBoDc&`SA0uz$)sz2q)mv7K>gGdUiapyFau<}O`H@*OA95u}CGl_a z_nTYF)r>YHzPWEk@*b*t+GJWoaqh9V7k|25Yv!_zV7kHwAT688i+o^t4Ej6Qod9zg z(3QTgAtdaw!*#cwwt+JVSn5!(Wkb#0|j_T;3Ei2ML zEM7pTYkCZ_S6iAJ{Mz8=sI-$OvYRA*&lq%_;aRb2d>|Kw1m5~j-{n!jW37T-~?~Hcnx_7w6?|2TXyDr8G`Tu@u-Ip{0KgPyNa^Lz- zDak+}aX}D>5%hl-I(Uf(3x+Xce@Y60U9pr>(%>IhRVlvzO}2*=6Sxx_AteuH!1hY< LkPVApzWV5v9N8bK*( z>A2(f`#tyG^T&G5+2=g(-fOLQ?e(5>4t(}}s-4J)mYNz}JE#EwC!{x-DXb+kt#0{t zN-${R?Fp;b-PO)|!C`ZzaBuCFjzN8aV_{~ZBlnN3Ii2IKt8Xt{YHOfmrQBU_=IUl$ zBKT>(!uD@RPc!DO_j`i%1Rgp>ZrPRwe=@iHU-_qOr_TV{Q`fDkF>2WZE_3iU@8OZT&AX}cVrUiVy_ z`;IRx%F$=YzxZ1ENLVK@TX_1-o>h%YLtrC1>Bh%A%+&C90jj+*>3({)E$jse$R6kW z{(5(3$AKaxqcj(Z#&w-M6X*W#lQAu z91a|X88!LH$vx#Y=5Cpzvk*+L9k6;LuiuF0+46C{0UbXTzF%4Fp1K+ICnT(u$MdSr zD=|d&i%HjpcVMX8)kLNhi;d4y^LEkg=$Q={dX(9nxdgg*=97nHo11g$G4y@=XscuE zO|a4VV;^v9TgE*W`>s9>m~t!Q=uN&jOX}-KDm!+P-BAw;yH!+GmXK;~%ohsm10m}i z&xgArXB^$m-S6GkwQb0xcqJJket0|H9ivLk5*}BdWhQ4pa#Nql%ccKlQk8n2MLVoB zYAd$=i?&%}jZs<}CQLXJc3=zA!-x$@gOm!Wj z@orW&fk)8ti_m~ych1Ixq1eEo{mk*T?si9qZttZ9hkp5ZnDrg>-Fai~UVCfaB1-QH z!^REL>S>ltet$X33!_ALN$WqMre={zxb{0sx?{SI$n5yNe;3luW|k^lPRw7y8+|g@GQNaba*5$%M5|M32t`cPt54C4Q^|?6sH$9p3mz!{MGcJKI6EcY00_X z(T zk&4X?LhswCxEw<4XvIX2;#InHb^@tVR#H9#f|$sIW?4Fq{r3IALH}`FUW5Q+kk>|B zFe*LgGzPnH+3$bV7CLXGA7FJ6;T|ky@2|xRz!d7>Vagodj=xVZ zNcSDo4-4N&B<9dK8t)<1ea?;$rAkS)0n->Q6*2c$w^Ej#ph`oN{Ag2dTmx&E1_rXU zdgLhyz{r6c>frJ2xrn~O*kM*w%4F^Fk5y;?r?3wg=G?SY%ZL{vKMRjhqb)`=a|{-| z9-#9|J3e4ogc^U?cOCw1zx#b+KY|?3gVh;r$vHIiCwoX&)Y+*ZR4GBWxCg^BLAHUN zlauYLEt)c7}NY1r@;FIB&l&o(f-Fw&mJV$G8qslKB_1vd%=YvyKmHO z+6MEa6^TFVV;F7XY{YZya26B;u9!PeZveG$H@yYUvk>} zY#);*ZL%p*M!J3{8(Tlj3GYs71hvlBXF~(^`<#?}(C7}sAX(XI#Is?Zi<#dD3XGa) zC;I^4O{OAETGBILS|Ie~{$>qMXTeboEEbO{7?|vU8yt>^^hdTwRL7ta0)D_{g(B-A zz=p);D1rA_a0_XqajLP9F#34VOSauGNLtnMWS)bbbNRgmm6D}8eh?@9$^)_fiqk_4 zPg@{57f03@5Hu}kSpzuI5bdlu3s?|D7wPc%Oy50gnt7Cw|HlZ8Mn8^2&cZ(PcLZPR zugamps&b}D$jq2o~zOnrnfFL(fG27}fSq{(mS z!QU?XIro6cJG^6dc<=>R7!c|*@HF~Os{pP^pgQ5WLo~pmb#7B`1IJ77A0qadDk~F= z5n?9JUs8}*Fj_pNTXeo#!!~XY(2XFa&=ge=a7><^RTZGtPgR*Q_@K(iipj-!$_uAW z-xDRLO|Pb18!%)D{@N(oaIfc>D0EfoCsmY_d5@R}J<9xrO$<&EOjkg?7^EUPs74eU zMFBwoWk3R~2t}E228tZYwJotoHN5Ug16FQ43W!Mbtj#L&5Oh(rbpn(v3dle5QTM28s@A;GiP0(=lTt6%RuR>3 zUBl*D3KYej4wM3=vTytouLdldbb3&Oqfk1^&yJToa(@#X6P>1i%20N%;dO4p+G|&3 z7NhBhW~fxYIP*L{rk}}5zro0DiUA&%HEgg=NvmUEP<$vDgy8RexFHq`#S3;iN1>G< z>U~ZuX1*c!w~af7GAw9(LFji(PDhEo#+pqpD!P*Nb?BhR)W*^eR6EA=2G`BI!{MuYn1l zz8SUic(^$5St-2Wp0uHL5r?ebYmi0g8GnBUjke9j0jeqj^!GF^yY{c@dLu z%JV}o(wB08rvh9B^rajE{Iw4$*0yWkK=Hb#OKBJH4uB?eh{-}g6*#KNmhh7%ejke= z*TM48&)3;1s%Dug-wnNRD!aBs;>0z$vc%)mo}0Q*rBHmKx2CEg?NrB)^1g%_vKs?e zPnuQE_VhE69uh3rtp|O3Cteq$5ch^Pm*g;b#3CDfVGkoHn ziTa#dzMFA)1aa)cj{v5vk2tXV^_3IqKby=dn8=d3$Qx3~u}|0x`0t1F7ch zoZu>XxZQlLadJdzn(}{%;d?*Xbxv(PCRPqR>(K$oqW0G@Vb0xDG0GuC5!&kp&6XUr z{f4vU4HF*(A1$3=cUS&J7AhqH&;m<3P+*P&4$#cI?&4!jmxEoXYIZsPOE#Bb3`GP6 zoaBNd^!Z|3LRx;+ymz!kuJr@MSc>I0xL+xf;TwuN2ERy;Atg`y6dTNPU; z)O#S80@wUfhSeC@d=jT%CM9(>hCmj=0}0wMi)cj&lZYoI-pE314xYOGc-ZH#sRJ<%4$F6bX85!Lr#XgixTuL5E+xiKx6 zxqMTTRzEDXiE%YL)I%joWc^5w@rjjvO~{@i9t!Q(sEvt&Yowa(c44;U z{Hz~N9&FMdgyeHNmEIec-kjlH1ZWsD>;+|Jx93Xmz@kD-hH#ae#Enp@IoN&@pt1uq zm#Sk{HcwUykrlYFiN?A=1VCfm*F2hHU70BTQ;pMEJi7hJ`j3L&ekZQ zBg^N$T^v$)mR0^^{}H2e(X7`uSlrGKgR+a|r&70wiMB1@Nv6=J%ipOc)s3Yu8gtsI z#!>hHnQ6XT1x2&&R8Qj1@cZo`VV>6QmxjVL_36B?ozW%R+hD=><=ypO zsE<;7p{z(AmJ~mKC{KD784+6+u!?L5khNYeuTTP+w!wpI<(1oYPDQ4_d;fTB6H4eA zkWAs}puEmxP}_D)SGoKQRpEe8oS*yHZs6%uKNKP{qsV`#$?mdWtt;>w#zduL2HIsx z2bv8aBjCn)ZC%6wlw4pRycHz)RN6z$t-mWwjqpiR{&8lue{SBZpCy+v&(CFgoxdF) zYwg03Hw=8dCs?FtW-28~P!w}IP;x*y9H5goiUoQn_r4E4@-GnUj3Ra`eZ9j~3%6dZ zock5b{%#cc+ZC2&Kh|PeA2R4~;LuzDRh!YBsu+wgCMW5c*k)t17wg=%p|?iLj1$bB zsLNIYs<-|LA(fOJL)1>bcK@Ors`kX3Q*q^y1c{71LAUmF&~Qtyo%VKEeCw8^8dy#f zEXzz2U)-mxq-G26>k?I94}M3Uyg3;J1+tDi9nw_?<-<6+34mK^@6D1e)d}ZtYx<@H zPGx$3%GX=EWm!?Af7y-07(N|lMoLwK<2Aw4%ruD6uUiNTTl)OdYEoSM{wcF5!Qz7W`{PrtzPY4*H=7QdgV7+^G~a#OkW z{UJ*OLooWO6E>4phePq3z*S`4TgY%IaAkt~iY_@k|KG(u2>GKfY_EIDkf}`-8Ug=f zHgx@xZW#QEX{Tx1m188I6A^p;do@%5yo|A+!ofFvF4`}OP(%RnQKP?Y@W8J=;mDGB zlGm-Tv$D1;q2;#lNU`)?luY_{WrnVY4GTvpJ|x0N`J2C}AB<$x(@vcVf~rN_Vx45C z%Rj6pWyM-(UN1JAFXi6$z3lxrn@5zn1OmA822?MHP<6>|3$9OePAU0zw}|zpgK3H% zZeK8dua7w@Gd66GZ`3Id(A5Z|=XwuB;8PqnKXX~swc_G?g zoiK&;AQXz9Y(KgN5@@E&O{h!FM6Gjx48-DoaH@CnlQ8j!Kd-GGiozrw^{bkvG^0jd$12)(98ZotYaV7;twOvYaegSHP<)q#LQ2a7` z$QqC@e1fb3Zg^SAL^4BGhi#=tC18@(83~2;$RFhHU)do%>7y+if8na{cO+^$#T|Rb z!jxBPR9N-&S8P705X*q0p!kF`0w5@c;1G~5_bSFnK_Hws2``%9Z_{hyCi>e=*a<%+ zqUOF4sUAG#|4}k16<{&*XwXl;FC!bJJX6#CKu>?hhAnstt*I1i4X^AHg|mi|LAWSS z=A(n4M!r*(!K~Vf4hooJgWjGCa1-$K2oSB3H%kmQ*?+*kr z%oF#vNH2QK8yxF@mHWvZUyi;emWz4(qO|t9Y^zC?N|kXmvrO9TQWhqVs@4$`1%RZd z1Yr`ok@pA>68%yk)ZeB}x!c%kCEnMc{%a-5kMt-b^ z(qVk+nb|!ES zr_85Z1j}o3+~Q6hbmGOY*Aj}=u(|1;*8U}x_Fa*49E%l!#X@0~10Sdos~q?S;1g>4 zTayvac=;2*ZWitiA~g;oHucs^^=J?sKIIT4Lf2Sg!z=?-{w;hozg|m_aPlLOp(p7? zzocALZ*L^;xNe8@E6ABEZ@=g98G&aU0BS$e_UJe!rWYdq0saK zGNV0uHS;{U*SI6_vPZQ(geE<=g484v7GUjj8>Djoqpw%q8gI%--q7L8TE<=aT{h025Zt*nB+0(j9+dq z(|jzUd&#&lvU+|)mzQq-joRlJkBN$-B(yv_>C+uenR#>$A_)=_Hv3g$zTyCg9Ke802MVh!uHv zufz{ABB((YWOD7tYE@I_!;j*H0v`}{5V6(7k0+ku@n52hdtmG0B)pU0?oYIk0;!(q z;I}<5pR`D+E|*Cer+70;4ZTW+!O&H^MS9scMQAFhT!Qk5L&a!`C0%fr2yyfE82=CP zDh;n_G-u9!?Nn((@x(21nNwxCl=7>Lknm|y&y8QVRh!n)w#T3Y#|SU2J=jN;&OFVO z&sd~4#E3a)f!cIlfl%-r|a|1%Fi}eJSdI&aQc4p5! zV@`!9%1}N7$!kAfh1#05NEHkTR`4tI9F2cmd#|m}O2jzA%cKtc zTIz|jYne*Dq2Z$C6vbzlF43299%0wFqDXzI__kS&_*XF+trkYTs|ik)MNMK(JXyK7 zRJ}vZr#y*G4MDLyU&Tfah*3pujm_#n$TVMlz3X&AFsoa@JyFJXC^gJX7v^J~K}YZc12+|f6s#@@HY zI$1O}1tu7cc{-5xE06f{_8NRE^-4hPGm+~&>(J1eN)HY2HsKF12ihnJhgtVs^}sV4 z?-%KTD0eae-kuri@)FC8TM)?wgK0U};=97V8!M)D$%aUGgRLU^X9v?SqzKYXTNT|r zrs9tmn*K3rJ@!|C87g~de*0|M7|*-R=dTEOZ~p6YJ7R2q;ZKXI8m*;|;AD5pZ^4%HrcRkEJ%n#fcWd3t|}Gz+!ryZ>~Yml&vx( z7f*@tqS$@~2I2DJ!BgV8Oc5y&1?c+hP;4{w`qcmKjnCWZo`QdGJNtAQdA7Lr+lf4( z$IW=Mp=0^1y1_)L#FQ$AF!OUHj^n{IpdCv;cSZPFqwMK*DypOVH`KEM?c{I0U*V*% zrP5|(pFs0xGR=x1vcX5bPQpw(b>}r8kbk+AjY_}?$;4|2r=H2=OYdv+Ldw%tK$(e!O49N5;x?+Z1wE!4Je(xz!0fE%?i_w| z?M&u@UeVp@a_e3bf_+ICO4<;YC5@^_y^uSQt|q9~i_ek}uNUy-7>t|{<%e+<6V69A ztWPO?5E5)BazVzL$*Z|Cb%Z~V_Mz%8@0%WukZvtA+cV0zetA5x$*@7MNaXApH8s!B z-dJ%0;5k-Mt{Ye{AZv$>W(| zAkaPGQR8r(;{l|NTCl^5B!bb<$pH+csmCdK&lBq8s~!Sh+HERCk}>}>O&nbU=v?I7 zEA(A86eXqcG=r?|E`>gQc=c)Y(zmYn`suBCN88(q5`7k(9-u_pN~*N==cbY`8P8+kdu}q` Date: Mon, 3 Feb 2025 00:08:06 +0000 Subject: [PATCH 35/86] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 14576 -> 14569 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index df7900070..f6ca3d01e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:06:07,167 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:06:07,167 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:08:01,486 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:06:07,167 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:06:07,167 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:06:07,167 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:06:08,207 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:08:01,486 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:08:01,486 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:08:01,486 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:08:01,487 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:08:01,487 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:08:02,514 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.428593 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.378087 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412693 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4667 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.21321 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.14067 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -62,10 +62,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96539 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.125 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438784 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1931 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3651 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bb7a7c1b1..3fa815046 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -576 +577 diff --git a/summary.csv b/summary.csv index 809494061..183e13b8f 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2 MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.428593,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.378087,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412693,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4667,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.21321,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.14067,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -40,10 +40,10 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96539,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.125,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438784,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1931,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3651,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6ee1a2af8..a9c23bf8f 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.378087, + "Result":0.436081, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,7 +369,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.21321, + "Result":8.14067, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.96539, - "Accuracy":"acc: 74.600", + "Result":10.125, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1299,7 +1299,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438784, + "Result":0.434095, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1329,7 +1329,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1931, + "Result":23.3651, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 42a7f578c6cd15babdedef867561eceb8ad0edf2..3b1bfc32087be54012794e15d8075a1506b4a5af 100644 GIT binary patch delta 6971 zcmZ8mcTf}UyG;m$DoyE94J`x^L_k2K389G;>Ai<0D0~trOH%?!NkS8(C|!E*ib{)s zNN<8vk=~`ljo{41hZNjDEB6|b7-&iB=WHQ+2p~b{hib}Xi7EegUMyG`0C}%W!AKK_3_V` z7zLZrXp*50>vQ|YmgfT=VlM%fW5){D-m+cP4!(LKP|}nc$jFjmG1&{eX)cpyDeZTS zzmH(6Q2NuzUxKbkLSCMOZqKB8?f{l0_BgtCujf-0>&3i%U~S>nR$OOCzwusC-(Gd= ziT1&}Jrj@BzVm6npWI!?Dfd_UYe$R}`<#LDi`{=}OyZ3P1RQP3BL{%2G4x7oglv8-K=3M|Z9u0HpaJZdtLIhah@aYa|cEt^^6)sI)z#@xTG zbv%(iUi+o7TGys)nDnFW5w_`ZhIC>qBWJmF{Nt2k&-7oqoELAX_o`}eRNoG)gHj1eSm?R>)^Hq}$# z>PU!=(2m$v0>M0QEEa_s+e%PtpXj^! zu4CdZ@z#F=SS9QcTS&E6d3th;47@~9-981_-tX{|d>o)648Y-}2HK4Wl!zBeI> zSWNCKHJB4?)cqtGPddl(9~i@v(V-UYw3u$YMxb##&T&zvuHznq1`Pl8{3?2&eImH) zD@Ny|6>q^ZOF{VD>Jt^zT?KKW)|0w7eUU8kS$;W%)wf!VC!yPB9Y!QaXWzwkP!_R; zBL>&|6Bl03FY^y{fYlRo^TXNN@M^G37~f7i#q8_n+wH$X3JTCEQYrrVTnd$FzBKOX z`@rUyh6h9l$0Ecen3v@x9}Q>ifaf!Y*_*5D%jh_A3cbfvHzLQ|5wiR*~?mBq`un516* zkoNd`Nb^)}nrZ!HT>r~x8ylc#`}uPVK$}@2XVIcwRi&Gc0E&S@5cW_rBSmI{6{+oj zVRjD{DCTPyA2V{1OevXR@c;=O)%|nwiK|X(vJ~>C1~jTjHV*&ASmirIyZ+CH4!eG{ zFB*+E30_;t`#g1L_uWr0LFjmMJ%N5#OR`Ob04h&awyupgd2*H7|KJ*ta~n|nXD6C0 z!2}1kZmnPNLi77gGMb#*M7T3Or8KuW_;uojmyP0V*-bAHXPUt$j#1Ge)!9VR9sd_F zHr@mg((;_j*9<&6V#srY2=`>hP&BlIUaX>o9#P?Z8Wps=^g~O4sb(;Zv1upTYRwyb zEd^{1e-Z=SOcS%MQ!1 z?N>eibsXvGcDqQnma!`PejsNxzEhTRTScRbnlmXVl?%uhd3mROhjy45n z4=pocNsulazhTp#wSK$A|BZ zH{Xc3m6~vnnXdvBzQ2xqtAQJQHvh&hDk(OFTC2+GSiOYkz`l) zi)s&wqFlSH%Zg)m6ewSfMV2L^$Yv-CEPWN5Ekz|w1rgNaKOGjikHYbNb3Tz6$TP!|cWss_(^d0`T5XLf8n z*t;H3S#qXr zuw9?MnHd?mlcUc|ihY(#hfiVU1|q#-Iju;&tg9*_j z;K6?WS&l_%osYtMHTgyE*!P$-k0(l!U$u@SBNBQ*$B|AU+J7v0W(Qp^E2z5j^ezVm zd+$WtU|S*ckar!qQJSFauar<`x{Rg@?>FP9AKtz*Fz=%*9bK|;95ea$2yjhQU$m+q zhP*Lc5f(uM6;eb%XmDWOaK*03M@rBjTzs-XAMbSnQhattshA7(f+nmhN92$HXEvS8 zT3WB7$*1pRM!4K@-DsbYw|`vUVblgU?07|{xfoR_AV0X0!4$8jZrqpOcp88t^H<9( zS278oglI@g>-a136UQ5H_V-CDy1UDe)MQ_~d@^+4iw>gx^$aj|;;*J|DlumcFv@Y# zUCkbpZRb)Y#Uj)0A{_H3R8^FSzJZ&m8)!;acDG0JYBWB!d za2ah6==Gx6J*9lT5h6pKtRjcD;eQ!s4^-0sSm)vW@%3f5e7-$;S;F*VjJXK`7G78F z6i_h!%`N4?tt8xdK1FSC#_p}@Br)d1YoO!u2uKZX`4m*Dh>m0enFnpgfuE7bIj-06 zmh#4L^YZ?f4g2N*A4Lb#rcL2(IV6GWgmDz4k~ZxCdCOAfH5uuebBAL8)w#e-kH5`E z`>I9^OVS0Rtfqdne-hBw;zJ!&JVcZdCqf!hLYxR`!qdPI-K^m+RG@Ulg07Gu;ynB_ znV{=$M2C7=45LuUw+~%j4rwT5kDnO!FR9dDHRrdWWsK@))Ql%;y8`Sa(BoKVcPeNS zkX-W+*ohREF$lSDrT|57$>OjNl%6{kFz1>O+bOS0FvZ-BuRPW(9n<>75|MW^;9X7W zb?11Ld<8688#cnljMM8`Nual9%l=bAW2>1t12H}6CBjwgN8N?C6X7a{po1dRbAmmf z3G$Pz^k}CnLBRVMu~v&aCi4;Y;Lgh!E!O0;WOP@%&{}{nL*b5KG{{Q});zJ~JX>n0BSn<9fRh|kE!noX39hgeOsI*HncDaLk%b^fdU zVz_*~DMeiYj5Ngp#fxqn@yb}gVKiRNl>7bwp;bS;cc+enZb?73Fp%jn-T>IhPCVk% za#KXW>a92#RkR>d+{s{K%{ryYlTzB-VfsCDxgBcoJ#ZX_kKb1abG|!BgLTXU6GX=X zy|97FKMe^#!V-o_G)u2%ahHDCs%)pyAKbFzZKNjJhL3IbkVun|*#XfxA>}p(f2dk2 z{$6W))P{Zahc5%n{29o@pLL3j4}85it+UTq?(vsP3o_f@iV0+~%f_)mH&8ud{i`N^ zdFpq(w9F&>9HMmI<@SA1iK;;(DJEzt(5BFF$NqV+bdVffafruBAQ!bFQBGI!-8$G` zQ&RAC%hwUzBg~_?mFdWTi-yE!=!b+rl%;uwU#fhv5D%c0CXavthHjA&^zERhkxn4B zKSErffr8BnDvEdg)0MNEM6UZJ&-f>hvt=+!-n4BYE{n>hh5LJ1Gq@7RC(1_AeMgHn)kCnT z<3iobbxw(eiCQ5&`=W5`rIBP$c5#JMtVS)n&*~(>3#qU{ZXK1Clh;mMl|+9e1^P;Y zzbVw;DSjPn9mGsrl@_9>soDQs74Ns?3S5IUD>TP`mUYBEqR1Ce+&k${rc&&Zp3bd4 zDqLyMVAQzYpIRtyesUL)i2N8C0n$sx`d7Vb$6uk?eial(R}At$f%`h#3?y1G0Q($F z{fS(ck;ERwtLb4`NwF_fdD$*r==g*;yOa2xLqFE|`W>&-_OD>Lk48;Rh-o3eC=a8` zLkNN=8T@b}1OymmLEGUxg^F}SP&$e@#}byCG}**=K~=bFZdQZ+ieTbb7h!0Uju=ME z?!0|TcUd55Qkj2%h!sr4D&=NWfkR4&j)46 zaUXq(G*l?uFjPiqhI!c+A^q7&I-(RqRg^{^0L>)UCy*jG7Gn7Btt*P8@}YK`5-cJW znyQ3=q)L8)-aL4%R6}CBdMniaX6WBX4z&`y6$%;(aXJ6460(xqt^Cu+y*W?vyHCrG zE%%6PN*0H=s>@R!>~fDu_>wK~4APz`rqcnrbji8;bnz#2uvBVRd_l~lQ?Rj)lv~Ye?eWps{APd8 z#K?)BPwO94o#i)yXFzi|c~0Pm6APaw{m#`((>BHhi#;9}hbIx6m)zC5e0gBfl)~3g z{GS`*q*GOAT6kfYEa1yjPaoUbPB%}`T-%vbLDkbQc&#_vF%IEJMVv~x3qEq*>csj> zISZD$d@ImM>uhmhZ`9Gvmts!y!KsU}T;^Ym)iqo_S$}Q7^@9D=_$Z$Y#-dxhDmAOd zqhLtI?D`pgz>Al`-MpX+A$#{%flQR%7l+s^F89W4dRhhcW&o97v=r~tX7*^j06~p@ zlxDzes416SSoQb%bLOM`JW#Pieftu{c)o!W}R2D)597$CN;2^O){>|g=)1%M$^{A^q(H? zgzw1qxW`JS7|78~ck@Sw<8KQH08kzoa3fiF-Yt5 z9=u+%duLLmHN^_i4fl*LYOQ=}qlqA|zJK#9%QhgOvd~jYXq95&u@g(gZNO>VcV6q+ zA+xtd^0QJ`T7+=N=+&ZpYnBTL^)b6yk-+quoSAbQt3}zmP!Gcu0pa`GBNnpcOlHl> z?w-TZdp{bUK5E|cQ!-NBK#qN~u8$V}3HMV5Wmq1btOm>eo;e86K(bh)Eb^oLrhF#z zr&^RfCy%#bcpu*FR>OnV47Fo(st zjW>mV|1#Y_vtjH1bB&JNMdom<^$dk*{^*=x$H;U;%1-|@1yeD<$q2jotz*Dd(F(7{ zA?D#%@!sNFT4{b6WAl*pgj;mLi}Bo1W4^@BQR(iQ2!m9fvuJPqLt|5gIBA?KmA|2YBK@S&R$A%T=_DWLyf zu7s3>w{L?!K-E`2r#pje3K-5&_4-cFE)wUTE zy{y3^|6(Lt-Fp6}t^Zfg$>uvoq4w4_Gfm<(;6>rYT^GxZ+g*y6bH@t0)`vhYl zVL`mrX^+*K!&*;C;nmV7WnE?3>5iuAp(uslgCX=PSR#YyulXr-0f`s(XChVlui zDdzW~*QdDN+Zo$DSzD)?Xye@Gb#!>$Dr)ew==@RFfmx&|P*`~uE!daJb3f|gE!L7h zBiwzo?)zJM>tDzKkXB=%m)Hbwp3+$u{wD54R)QCBGriYdSq4)i#bZ^oS*S4fK6vA| z%mb+ISj1q02Q3&W}S;iqDeB^8?Hrr0img?9qXz z(#P%&Og*ZfI)(>%;FXi^hT0ajo>OL5M58D$(sc|?>(jLC&INz_RM^L3vIqKM6?gy9+$l6V{aqlr88Bw4Fkwt z2H3g@^08yX^pK|ulVBd%m(16G&-!F89nD{vP0aFSO%y4>&AHl*K4%s@{M17j_p=%m zg}V1%DM`8o4nUj#D!1(g-JauoNZ#h$D}uroByP}SEaMsTI68= zYw{z(iDOg+CeFe^(8EeeX-&BA83Q zN}}cxq2BhAI|{?U(qa2VSC#mSj=?CNo`K`0zc$Zaz%nQXvYDL7a-WtQnOXI#K|)6< zX4YDTYH@9>?^KXXtE-}SKzc`VNp(wP#Mx$4#V<-?8+&A}V0yKFRKOezVJAzLk zJR|N2H}8z>0Ar#XzV}DpwDDKxv4(=YxcAu3;XIMuKj|-mHFvp$J*|)K(~r%KwKIc{Zdabw)J1L1~Z?$2E^nlK-UTMQ_eM44}we4DR7nta)D zX0(&Zydr5o)-^xsn%Z8JsbX3%Tv4^MIymJja~^PJ(qu4&H*Kj~8gmVhQE<7a@YjPz zEybzrw`~6vGHJN1w9#4_X%dX`Ialm&rFs6T$lW}~sQ)ET|H=HXHE%lkJ@cQv(fGyO zPonm=%f#$TP5*)Lpx*O_0e59McWV{VtLr~DVY{}Ew8-AXqAY1_OBDQ>RC9gnNAvc_ zeD#TlH*8Tis!{0$J7)=t@e_zmp*j zrpGDE1#zr-@v0Cz2#-)N3|ig+mrt_t<<(j@)jr}}7v(ucP3~CBVz3X=V^KUbFn;W* z03ko7;XEPrX1X$xx5Qz!zjtiIOT|uJPDH5ltU2q8`l@NJcSU7`M4R3>vR#vY1CoDd zNk;z~vyCR!1rr=T?S0_`ZCrCGwVi48Es!AQmLP+GU6PwjtT4+*nPU1-Ya=z?mM69xg5}W-e)FQ1jDHg4 zF}g@-=`(tg+a`|*17h?r43Ou~11h*4;LQQkS{(^ysELGLQxDMEF*+AENF(A~{nbT> zb>K3P&Lv$F5X1GH+^5AwqHd=!VKg$lGc=U`x7wGjKn3E#FgX4brTsP zc)JgOm0*>bx%q19&wzlkU7v)GwH9?TjV{@CNO%W^YF9(7N5nqKczY>zu8+iAo;08m zfZ8n&>aYebr`n>8|7^FKn!~dV7RxW=JgRI&l{YKQEiByV1?s|#gZB@LJ_KS2$X}d; zf(3jvEGp)3GCw}n1*QiB6?=QJq)EowA)F$BjsVK#LI6QyB(RvTmgNH}jwzIM%2lCG zyDQ6dxG>ZRsjn<$UgyFgie41!(NuFs)%9b{J`37F-l>WJyx?R5e5~BJ9>A&(lQri+ z+|&Ab{gi-rlO(lx%nQm}z9tmaI~cnBVM<_|Ps3Qy5a|!(Z+g|rXWiAd6_6^1O%Eg^ zlG2GM#;!RgwU2%t7)?aO*`g7jx$Ei@5&VUMF_k4Mi3Uox@}ypRfV$A?m&`E8C_DMq(#Y{V@X-p`$52P)+H}EM{R$}q>`A%T+IrNuZP}0AupaC zdBc<`gHjBT%8-KiiBK2?%8H6E3_)fIai;BA(+`6I=jKnAxs3pbN@n!@9cEag?agH{ z{zjSx!TFb-Nr@gK+JTUgy%pJ}%4J~4AB)8$Sl`N~Nr%1l#%N>`J)Gzx*7_wGg&Gv} zl1xx-Xq7RU1XeOCY)y*R$~9J{uENSbgYGGs$<9Mkd2f8{+!pN?vMMMrJVN{`U`9Kg~ z6yY9(rxy1yk_r1RC~>NS{pHv^)5c`6d)`#BMMl+$@C-!$9P;u{fR%@s^X^Q9{f18y_rzwFI zJm+~{v7H=Wj~C(rLmu=9S+jCusVD{y;A)j963lu-Y~z_V;lHgo;QSAQm$z@yhpMG8 zW8zAyO*jIsO%0KpoZXF@6zfllFfdN#;b^Ky$fodY>%@w0@TVKZ!XBIOc~a}dBFH?5 zb_PvnoU7j_%w4TewssLk+d72H;w1Ip1NSjK+NBngzQ_{`xipuQn&6Vbf8avliv!kC z99mI|g}QeJ3l&wxhBQ#o;dCPaRJ0vHON={wT!5Hzijq<8w##3jta`o@bvR#Ey=m^t z-UzIaa;?WyCS%(Emq zYgWs%S}Myy&NC3=B!!4scOayQ43HpVju*hRaS^*%K7VNjkL$&W)w2iJSH*&-4ri%T zQ3(faX9cp^6VjZ9`5c7A$A+rGlC!+5c1358R0izOXT4%Qe1W2L9_tM#1nDxqzxi1&-ezo#7yYs)>PJ6A~>}pecXKLFzuf#QY38FJZ z?h|3+qE{gUF>>L6WP|7=<=mxSVeT-6L3K^6Jr~l;i*eU4&p7u$h3x&(m_(r5FDm{~ zB)OEq_CvhN_Ijc2#H(oB?G8d!3YK3P{(~MgzgO8#Nr^MRxhRY4>)SG0c zY!gg5KY{oS&Tf8qc87uL7v7+W(+#PMoV{7~V*P6Xm8UY#`eCgOw3Cxw zMHJM*i35rbHiV)zbEkSK0ci>q{pd;1orkh+14Ua?0`r#rl$~V7_7k0h53 z$*inC<}jY@_on1%P5@WKAl@9zu|?Iy_+eJ)>RvG=uziwBZWR$MLcwH}cy1U&D|@xr z5j9swKjd&oBkT~F-V=#Fo^P5)7PI7cW|UW88($XFPj@wyZ9E}x`JU86F@AeI02|nj`Z^Rrc_`9Zk<(DX-%b0_BDF)g)m}M8+@y& z8V4tHbEx>M^)7o-baN6xUe$Lurm(YDrLNHzM&U$V>x&?BfB%$67_PMJu9tzsc z6?PcqP5a3Wq3NfotkaTGp0)543=Am`Wz?*Q3Qzh1sED#dKPBAevK$vUtR?9w9$mk` zaG<0)qQzc+J#hm(?~!^fq<;dsdv=>n%5draCR?IJ+5YTy@e+6PKWOrSA3_kviK4v@ zjsbGg4g+GA!3s+G+DlfXXw}@0Ha&~P;@`ejXzb&x96+2q82UK%8)o)dGA}0rh-9sd zpllEr(p$L~_++C8BPg0|5Qu;&+$qsiiMB(3>lH)u`3KNiu0IGw0DJHPe_80R`m}_) z(3tsyHV#-%dmClxj0?)?L9H}YP9uNc>RV~swn_Ct*6_dZfLGo;7I}SyfO&o8122p9 z{Iz0Tz_iNr6lwn@I;WYtWux@>f1F#J6zEJ+o^`R5gl`=Kr(Evr->|-irShp0C55vgdWGsyeEm2HKR40_E18A!}k&4352b7&Bg}3fe zbdvcj$>scU>6{YlrXzEMuryA2x=jj`hpI~G>MdD;kB_f{v_yX*21&FwL2R@l0E!cF z&KqpBSH&D8_vNgmbJd1Zh_>KW2woNQ(zEKNwIw1j!&Q?M_tMoDj?`hP)fS}obIoLU=iFUdAKV%aMOiGjQ-GR2-*eP9 z_4YUFE*E()w>SN9RS{&__C zNp9hLOC*g)bjm;KB!*?RfqVW(5SMf-y6DZ{3HRa4_>IcmxjI!KN*6yz%xR_`aU$U+ zdm{@4ao4Nw0iqSWqYRbkf_4%}eTnDMi?LN?@G*%mADm(=TzzjQW0K=;#^nLd_73(& zzMfpPy6=;SQg307kpw>T7|aOG4La%&wGwf1ZxITc30HU@^v>u-v-w8a!89TL^dYM7kf;cHq1`Lh;okjE zb^3^^L&f=@>c}xOaykwU>MrC!uJj?A`=3m+y@kMSqCCWB3UAJMoAe8g!#OORn`un? z>5Wz4CQ%W~Mcty;!J)u*6;3N zsLTTGzR{WmR+%iAkRPfCNfqKu$eu-1Im?xg$3B?W@_D)l=kntVPQcOA-r&<)1cR3) zlop8xCRM(k-0#ptu9AeXG42O-l)H+$CxUa=4<=RkV0qeeHe)iU1dlrYys5}8{O|TR z*X$8OLGcnN{I;i@P5zGK`cbV8&pA;YmMY%e{xkW4!V(++xBMupi zrLs2-oXJyFxyJA(Rz$2|7)2RXZ7_nY6av!Vg%@d`zGXsd?Id|fBIHEIrp&% z?BY|A!gX*dSXjqSt10vZq!2bG*2Mj!f?wwy>OPEj=^q^%J9)9{qcQw7qJ!cgFpB%s z_d*b4_v|96EjJ#7IFj~EIFNj_BvHPqO}m*r39uMAdh#{`FixEu*0nU{^T;l**Khm1 zLV2oDF1yDghd@tr#$n^$6%XG)z0aeu?w+1?j`8*@iY8L>e@xj|SWVBH*Kcu6ZcwA9 zf}_*=GoNaI8ups$jhnu@T{253ad`a^Ro&k4#QfzYeyn_E?O=B!-x1pVj9K=};-=kk zTP6Q@*Q{6aL4oz-(4)n){XJI&^WgS}Cr&2Y8`iC|>Dgxi@}(8){}hcs)^x`Q!_Cjc zG=fi!A6-t(IHl-!$Ke{<6^1rwT@g>(zo+#!ELut{vTNUZ9kl9l3A~ZnRDmc9V|JF`;$M1uz@m}cm7<|%kW)YnAK>n?OXzNG(p#}R#I#Y zc05tQYc*z!kFmSFcdvEZ&T3(KA<`%+!{=bOz!hD-vjfC1X@tkZ*bM4vSfMZ~ z9yC(^LBY;spaP##|3(83d>SEJj&I+Srm{vdT_%r-=RLvFk-LjiZ!#LMN-*E}v9@tL zs9-}??StnUbRTrl*;bz!4nM(mX;{t*Qc9;)DP+D=UP}nQw?KGQ6F*)m+{x61bM4Y* zNEE}tPPI{Kbn=Oj;@Bfjw3z!SPyNA)mg{M+rAy3`lKY-nc^KN4wdHTN>875F z`l`)Ps5_azxwqjWH9ZfB#k_ICu3AabzfGfFbu*1|2;piSvds{p6=(i@N*k-)UmczQ zHmz^TlBZ6@j03-dn`FwfK6W(R8{rp9x|XGi8ij{-RhjQZw?E zuM?)1n19jBz!RWeBWk)^Ij|2=wATOCg2fG3;~dmC51>mi1MV9IVAr0JaSSWU&yV8g7C_IDEaTHty~vYVfbW*$Ror0? zt;`GlOy)P9zbKSVNH|ZnJJrzW7iTie=_37iUuEZk4o2LqjI>^YN_v~@vVmmZakTS^N=En zu7De>c|0ysj#aObo;mEk)y}?MG_}s<9P2VOn?_(lmENsuUkANwYIdT+cAjq!#Q_?|GJa+o|>birQ6?i@2qHV)}OjerC+O%QDI zOuz6H&(NMO#7EvR<#;9mb@*arT{^SVXxvQiS!2mQ{Vg^EKlhC+9j8Od5beB))=06PH4GA>RNYbOJB0m9v)@Z5n6z`!gdI10T1$`GM z_iPB*#j+6h{}`;CAS-J6+CO^qUEu%M4DofQfL`4}*O2Zu0lJubrb+%G=`A+29mkTF zeUZ#@)1FmZQo{1IxoR5p{rkT@>%2~gZE@2yqNvDm2%(EG2eSr@ud{0;B*g|pqD!d& z26HmuZIPaOZtdP5hpC>3=SluSb=G(;1hx2`&Vp?AGA0|h{~XCzZl@+W;rhTN_I`P2 z)?nS98FIp)^Urg)M6`tv1-ddUAWt|D7Er)O78)Z zzfO(X|MqrC_jy}?i`errJ9&X}dJZh&M~Wyt19=t%mioNBl~y*0unt}qfpRYFrX z_68c_tTB|JbXR#x;BMuGsI*eRpU`R5Vz)jQ$uG?j&fNXYQO<8B)8=H_RU|#gUs22= z?4QR?m^q^+J%gsKhPj-f-Ka(Q1Adq;&jb}&MbM#oLoeI;DZ{A;5UnR#o~ilQ;9NMC~-7sGC!19pK70xcIV2r7IsLoNFpm@t%;y2Vu> zW(#;7-!WY{esR_#p4&bI8t<&UpIIAO+9uZ(lzG3pic?8mPosLI3lMOIXSj_!Ju{#lwE3n?_^+ zfP@eLzzF!?2OYAE?Uhmnx?)MCWq?1hO49u7|K%760O*ha05|{lQvf=o8yh060A#@S MNb{17NL(%bAIUU9tpET3 diff --git a/summary_results.json b/summary_results.json index 1d97bf2ab..4a58099bd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.96539, + "Performance_Result": 10.125, "Performance_Units": "Samples/s" }, { @@ -932,7 +932,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.21321, + "Performance_Result": 8.14067, "Performance_Units": "Samples/s" }, { @@ -1220,7 +1220,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.378087, + "Performance_Result": 0.436081, "Performance_Units": "Samples/s" }, { @@ -1472,7 +1472,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1931, + "Performance_Result": 23.3651, "Performance_Units": "Samples/s" }, { @@ -1508,7 +1508,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.438784, + "Performance_Result": 0.434095, "Performance_Units": "Samples/s" }, { From 3808821158a924e0285c61a6bb3b2cb151d5d2c3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:08:48 +0000 Subject: [PATCH 36/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14569 -> 14570 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f6ca3d01e..51e6f3bdf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:08:01,486 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:08:01,486 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:08:43,191 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:08:01,486 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:08:01,486 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:08:01,487 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:08:01,487 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:08:02,514 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:08:43,191 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:08:43,191 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:08:43,191 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:08:43,191 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:08:44,236 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 3fa815046..f61b514d2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -577 +578 diff --git a/summary.xlsx b/summary.xlsx index 3b1bfc32087be54012794e15d8075a1506b4a5af..51a96af387ed9f73bc88f9b274ebb412b761d223 100644 GIT binary patch delta 326 zcmV-M0lEI^aq4lf$1(*$d*oUrlgu(Je{M?A1$Ez%iEd|+Cz)@Z2F0X&*FcNMpk z8wYRBSa3mtPa;Tr%59ByAy|sB=Q~i$XVIafxG_$#5KFhkmho@41zZsFh!q4Te^_9+ z2c~u+q7f|Y>rc@90kaG=S_B0_ Yd*oUrlWRCs0S%MDI3otgH2?qr0O4wu9RL6T delta 325 zcmV-L0lNO`ap`fe$1(*C5ya;tlgu(Jf1xD^+kv*^)IgyrExie86NNSzDePzdS z67Z?V)y#f7yQo|pRGa*Q^U~-FvV|{AD9-bH5w{(1dmqrUCAxw z#=)yI7F>|vlL*qDa$BK&2$o{(`5qMWS#;SV3Tde+7nn zU}`5K8j*reNc-d30U$Vr78K~Gc00000OkbD= From 2aa9d1d517da08b8a10a6df62b9ea8b99dc4f52a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:11:32 +0000 Subject: [PATCH 37/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14570 -> 14568 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 51e6f3bdf..f0f51b728 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:08:43,191 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:08:43,191 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:11:27,157 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:11:27,158 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:08:43,191 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:08:43,191 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:08:43,191 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:08:44,236 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:11:27,158 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:11:27,158 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:11:27,158 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:11:27,158 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:11:28,334 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433279 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432134 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.125 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0676 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3651 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f61b514d2..aa324580c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -578 +579 diff --git a/summary.csv b/summary.csv index 183e13b8f..3d648f33e 100644 --- a/summary.csv +++ b/summary.csv @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433279,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432134,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.125,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0676,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3651,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a9c23bf8f..644a91f14 100644 --- a/summary.json +++ b/summary.json @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433279, + "Result":0.432134, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.125, - "Accuracy":"acc: 75.600", + "Result":10.0676, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 51a96af387ed9f73bc88f9b274ebb412b761d223..a6e3126176a7553b2a9377d731b17a6ba5e13bd1 100644 GIT binary patch delta 4798 zcmZXYXHe5$u=f)py+u%@Lm>2k8d3ODLKPCEsg%$`K)MtG>Ay%31ceYFR0+}%kP=Xe zH0h#(Aiau+^xjdt(dU`DFYbA9Pmz## zZJgteCU2sy70`ng?=#*QM7>$xVkyp@2v@lLU%RLx7YoVN)XEcZbMY>5Gcm=1PfoYP7h zdnP11d(mppay~t1t7Gx0bC~a?dK-Z%<{Y$jj!{-}Si`#bE87dDX*zVVE?hUKuO#~A z6bpTxKdHmrey75M)^e%J@ki5nIdJs~_n`$bG%@e`ufQ>2+dd%)*g(y-kWw+^e-pl*)i0B|1z;lG?nF8J5N$b zC6gEbtoqIw47Pw&J+tGQBPd^@tJ@n_DWmdP9s+7)AF_NyYXyNPS*g;oR3a{a|o+y&0uGzSk^vJgoA2NPXD&u zMZ6kmA+G@QkPgsB4l)KP4ZPAehegb7FU2Znd(k90l}A{-I>=hmtL@H!`&e806)R_m z3nlsnV|#SoMn^*5gWg6vg47oe&|maVr}*!H`4fLi4GiZTv?e}dX3sXD{}hzZWo_x` znh>H{@EW^bf99)fWjz~yJS|{i*1HxiQ%daQA9glNQUHo&S*p6BYH-4RRz@?I13c8Q z7fH||CK?fR5lGKx25WZI@Z@GHD~L5KZ||53WR@jkAAJo+NO_{jUwE&=3GzVzHf4-# zgUaZct$ryyUwcb2S5Jn|JOL(yhBxtZ#1@QdsP{PHM%$6CB7r7QujQd2EO>6vX`iz) zXbuf@>GSwVWKvv?RjHRYmyC@bZ$lrHls&xnp<2JnY1rE=bSi(qZk2OBR`brN-An9> z0zC2p&5{BFa^W70#NN0<8OU%c$!`NVQib&>tkpc;XZV{uwxUcHTTwvoPMXaL8$e z^NQabni8S2FXEh>BAV`PHEbuo1UnPRG_sjHVvOv~S$7|wk$6L%aBSjU%}F^DNvwH< z6(=6B(8J2X2P}P{@`YoRYN^4D9W~B`)M(#2{aBbw(F5()BOHRJldH=CcGF`0vPlt( zZ%&l7j+DBbNAK?HuHlk???_1ORT?6q_}SFuA(8wGL7op2e&ntRrT*`%9F$<|3>CVcHh6ST8OzfdPB~;5|d0on=2zO+$xQZ z#D4;nMmys4c6bGBfg!Q|N~+ev{^hH4dz9{Zal?U~e^?{qno1rCs(n zuJgOOCpVb%5+gL{$(n=TDI|iF;E`~e03`$jPKZ7NJw-i_z6fFq(Z)OEv*WGD zOQ@GqBJq0AVBOAp{r2u>cUNl($JBNbAG5_hq@u^|Cs+W)(|I;bZFJPakPn3{mZn%4 z0l^ZA4naf|nc@!-dM^Q7Me#==yZ1r;s28bv@c8Vs@ZR`OdP`JaP`N{xnkB=ewoB2z z>mAzE4(`O3!6RH0oR*JGT?XPVKm^M?ErnQ~I0%@4>h^k3G!6ptghYMSpePgNK?bg5 zp#0>1W=FPwZIcPIMUduhuWg^g>xPM#YtB#MLV(i|n6dY*dvqjDjhM(({mcRHfKyw2 z7>ot?=2WPThEGx$4AP@3{>m2|G4RUs?dR{u%_D9iyNq@NTH$JLd?fC2A3?McyU2BZ z4gv|Fq)12OOexu4J5sb=2~gp348B+8Z2znM@!V%&j_`E}`nDe?=4k5&wH9KhbQcfY zsY_gy;|8VoL*m!FGy;u;C2cw<^Nke6MVey>8LW)G9;OtkgP>0osucSZr6IvQ5E4S& zaHtZJb3s)GIceQoOjti8i980UZSE5)(6~6TF5u#4f3ohK$YK6JMRwso~ zj}cXh?KUaf2>9}==q+3--4dEIuLdYJIO=G6+*$VWM{MgP`}$+W6F*bbn*#n!QE$o` zMoG26V1hERbx5`LZ4(sruHT$Vc7RPa(8MSW0JA2DoydE~Wr~CpT^YJt+qdd`Gvqg) z$e-iCb&gGhnx8`_zg0s$#u3-rj>L%sLs@N?uLWSKI|hrku!aG1Xe}Vro)q3F&-LK)^5m_CwR1XTtkoX`l}M-!LNK zD2r%g)pMZ@@zhonmDF6SE7ss2VegAdbeNlv8VZ}6aOw%Il`rnVSSZk6aJR2dr z?Es-tvM0^8OMF7<2DnRlEa->M;fWhM2wc(&YjaSF04od;qNi6v^ly zbNt~YdoC(9REEpDVSk?zkpPvNOUEwBQ45}yD~Zr>?p&&rMI(t~mAPxqB6k_z*SJ$% zcRnqMO61EBH5M+tfZ? z!B*KhGnE>sZG(u%Dj?EcV~05hgwuj8N_Bqn0t_TIZUNBs@G zeG4~#Ugj&0W$ur!xV$^DGilsx+TUV*v7*nldVgYRb$s%Rb<nARUU*2( zo!I|j3m|!ZH}HSm_x(j${WWUyqy>ICj7zlTcWEI|x;`9RGJ7*_ygb;v#iahQf zc$hjYag5CT(}a3?O7(}9tSwXkrhZAfT4bEGEer$9?tYhthRxc%{LuVS>4@OTkd~{S zQ~Yh>+?%nYWP7PLMN~DB_x;&PO4UQ#dJQ(-gQvL`BcnE@jl=#wkGhrO2c9{|>Lw+{ zi;8t>k$)EQEp(1D zJ}+NoHb@v>*5k%6VeN{#bztIA5E9&)qtNfzo(*Y!>45%|uu1CuNKs8Z3%YCicIwRY z?%QejhNFZm2iVJ4ocz|j%h&um1<%T~T`g1K=Mqn%7;-*d!zmD2y}6$??2kOZPQ}bf z)p?~{&)=I%PxCR`hn>XAZy5&Ra(~__v?yq#E(m=J2|r;q=rOdaZFX-Xi(V%DiqWs1 z)mKo8b4lcTF+|QPP-?{Bv?`o2On!jpfNpA)`-kFY_ynJ{b+|;>+_vywy zJa|Q%rI(h(Xyi3Z#;sh2im11!J02FDZ9O$q62kU`ZApU}s*sZpNy7mD=Sp{=Cqpw- zfM>8LPjh_!qpc?YyX^khA``DO2K|hScDcjVI^&!Wj|h+0lBTLBHkdLHIpE4Kl5KEs zRhh?~j#WBGPsdA<*EPr7=jk6Ea(Gz~9~HQ>=)yWj*h^kor=CF=kGaiWicV+c&HT2p za+oB>BQ#bDxzx6XEfi=|%o^1Iw+CN$cBkqdS~c$a;T)~}qgHJzEIOf{DTRVK{nx>v zek#1ankL2FD}<$d$jdj@w`^QAD9t?uct)0Kh{lnk zvi@P)bW)o#`s&7v-bxt;qxtWjFj{yRkLU(}GSNt?`k^3Y!5mg$%P2PyTYqPEf+mr} z)Q4Sn{5JSvf~G|s=-T`D9d1j5+Hia3S`VZnAQw{_d;xM;j2BGdHD}$UK%nrT``K}Xym z8_1j+D1;eT0p9v|Xo^@);hl4%3em$=&~IG_H;cB8UN|Dys(*;juN&HlY>AgLZn?c0 zIEEk&!qZuuGRo?aGr${ZGgw(rHnlqMae>2y+w8Cnb3Z0wxu=jV(YEqmhuv~1dWR7B}E!dcE=X@^jM+=`>>QotLSoNjmq zuN_NR0Z%8Zrrd4%Gg@a%`*C;)ptMI^Vx>s9S(eJ9@ha^% zZNV7Rm}KXt+I276`|V=SA>pAg@)J+Z#_*<7bDvg@l5MH4FvhW>FQ{+b?Uz0#%uQ6J zl?%RE9XfEkFBP*+aDVH%fg4dDpQ2xvs;{+@=NTBQ0}ZZ;=9}18Q@M&uC{i^=V(`ra zi;Z)jnv6=*Tcj`I z5!K|x%a}hhdf{f(;=jiumNCyNKnz7lo{ljqdV=F$*=Jo`6UdWv06mX+H<~|m@~gEb ztuYod)KvCmva$>YA?RNS%X57_V)Ic{>Kqy`-|Buh{Xp7asP5A?aV zUxD5{{LPTpBt2fp8o91&QSF*DkWAc&l=apurzMz+0XE~9taI;He0(>v>aP`>uNkHO z3f~=E;{L&t1~OVbVg>Z#fRq+rcQ`D!8Wn;`N}`u zXiY*9onyPw;)~>^f!kA@Mc9}EH=xS#S7t4aQGKH_y!9L-6Km^NeNA6Y1{7>PVO0w^ z$EjeL6VV+={7#m`-3@zVXC#xQ^Q0-tY!#C=rFdD)e)iDJCVnJMrJGcm`qq2cXyuy` zui;&K{dx$;GdIZ)vyOJZkI}|T+gjlPQ`(`6>vcUZSvA=p0~%9FJGW8F74Xzz3AA@6 zK}h6B1~lVmP2C@EE$e;K;}!XZ*$(zO{OJk1EI?+K<%0bCYD$un1OES;{aVfx?3hF& suK*rRQj-_u`Bzdf2xLeF0-Xo_@4`KRx9qYDy= zP8hutHKG$Gx_9!t@Av0BKlWMoIcx3v+U45UTIZMVif^ei10`HA5mfHZa0UT5ru1*1 zJ?~g?KX_34;afBo<0lPCNs9jXN*j*mh`E)j;K}GTIvH(tG(1pmkomIRzP5HW-f6+G zs^uoLE< z#-UqQE#?#lC%ux}t~PObL#8|cDP{ik+>%gV3s@)RV{QazW3o2cd*tmQ80A6EK(3% zUZW;ycWE;Lf;F*5vOvBwh!X_0pB&O#{PTPh7dBy#6={Nfn^|wnW`BVjQ9?BX1a>CM36-^f16THDC2N zbLuCD%B7=$J-@(U+MiDW6k4d9_EV8HTaft&^zD@#pkJ|>SBo+UnW>6!!i{NZsta`? z#jJEp^<;?c_1DyhQLKr^M>Kx)0jfT5M?5`7*=6`jH%Q-tYa<{dw&5+k2oULG*+HpQ zBZnsFzR?+!T#uHo^>a~uQNy3K`wS(LcAxQrKsxw7V;3lW!AheMXS8Zh6>5+@{2ZA0(*CTm6!9HYfq^lYP^JLIn> zGMRGZtd`OlM{|DcbzUu{P=9AMTzftXdCtFDIshFO{~NqrLs`FFLz-}@kX^{r^-8Sh z3zw4IwG}vhRzY*6w?v_mSTKLG2w##P;7^8F%y2>AHMp3e3-o%y>ZW?G(GVG;VRpV_ z;~*KLvS&^ok!f;`UtqHHMZ0pYL*UXi)R&jmmL2ejs&aG?ezeCeeb22Z!eTC6yMNmL z9co+|9$5h$RYpK6uuI1vf?8k{E66%{BOd&eI^JomLVzHUuq7ZcITPLkIEoDihA?DK ztn6^i?r?mtHR_fnLhkH$;Xn2^X%a6zL&9e1n4Y1aGw_7pV8viHMsbkOksmKaiIJyc z)D=Qx$lOE9-cxu@MY;O0MU8ZE7W3BQFS6n8vLUmgH^MQt6O+wfg-!GYUH3|j5d?Dk z$U}0liVfw}lRnyXVPx;_U!)M%NdGaF|N2L)g+%XbpiYVQes< zh-k86Eo8&M5<_CqOCVQbc{`B_%Q3TEN|%V_orA!S{SgPC{o=Hfd42X&l{+fj!e4@Rs#!$5=iUz zRPcz;6t=WpvWiSAR z3^^l*D9DjRP8mPhqOx0?XaG>ST3lOr>%X~Qh)_;I!K(0Z%5)o7fxy}!udKBz=A-4T z9}D&n`qhIwH>$W97fs^wFs$Cmrt3K=2ZH);Y6y6>?PV5CeTW=i8klUekLhr7>O0%4 zzoyT(!c2QHqprybyDH%iRr~3ZoN(X{i4lxf5+?13S;u5JkT^)8N8rikBQ$Lin`un? zH|+&#X~`rLlh=t=`o z&`~GS8~~ONR$?p+^*F-t(yEd3=_2*J3jUYEFu``^Dl*qZ`PDYxItj_rQTPr_UK5VF zXPxDrp`0edub-)mfCFZ3QISl|pvO^YkoLX^FQ^Z{QAz`QPnsgmY>=VuQalwJL(UY# zF9>9AiSXJGKh59T$)3g*IX%)akEPxQex->rSL)%K#)!@0CjDdf@3fW3>@|lflG$rs zc6A$+s^&@_*kZxM@l@_)_ExY5nqi;&ta9=HRzDB6A+}$PT`b=~TFBJ$BU+?p`(5Yi zx#ySy5X{JwO93d6BntBQO9NAo$6wl-hDs=oG-J=oWRU&MQDc@oTb2Q;4ZXkZ^Bfjl z;wZlF|BV^b(^*3(i=RsIebz*pLx#^h%!R3>tMs&OAL?6hLu+3P0m@~<_mL~uAIVBbewJ_ zm+#7WhZj<1o%%iyll=kLOfE}G01_5RpBR9G2A~tx!1sgM$W^H)lbV+6->Ue$qgLVV zr(32w^uJd{mXv%ywVfkU8jV_;{M5(tgS_P$T^3y)QbwM#^^qzf1^Fc^5@eK?6j1i6 zIhh`|^(r`=u@Dq+BKI{jKKA2Jduh~hq~pI*k~(wRdw&6M!iN3go3E=0bBm@i&osm3oQ0BdQVk_w17#P? zJ?O@hd@-=NIEdN%cP_3JL}P6xJY0F#Fcmn0-K~m)c)BZ@wt1Rj?W%yTnqf1j|pUGtGOg)!DNQBoB+D34Hi3mt|X%0^}ZEB{=us zbu(M*ow;faZv(k3Fy%~p=qUYqhh=lVK2ET|A|({50fkzL_|AJDYo zIZ5vG5@2?>#gIqj^Yutfp;@a+068m za`Jt2?>PyU( zOlPm_pVN@}Ddi;=xF+m;^L=z%`|~dnT$_NEA1HQaj|#cwG<~fA}tC$J;XB(vpg4`>O3=S)=r^OQKL* zz2$db!CcmDQ~nFQv(;x$ceRrh&yVLsn%7W9ScUU?r7uca>AQk@s8tT}#I|CW zdz?(VsS@2(yHIR|u0G!EuFltg@LqWpOFc*#)kC3I>5tb^hhfFCC)svELBu>yeUTN|yf>OX@;Z9dZ%+T|A2uJGw5J3Y283wK@WuSx`|M{B+Cwg@ zeBr5oK1`oJuw78B3iB{q78bp=HDsem&1zK#Xt;Y0#_mkkJic4EM5dcQhW z^oN+g1}N*^pQDu!#b48VLAprx`>r;*G5!<2B%JI7PHh1+#hp`4SGKM zao9pIrFB@oy&}>yga0HJoEojI`w3*$88J#&?$?!Dyk6-kvj~htcLk+@{^X_b<9MtY zcd3bLWTgqe%1JD;nBp zP|n=2z~J>)$!-&ItWpCp`QXaH&EP%AW?jn!iI?N$NKvjQ@MY89vn0~69L~Tj-2|cf zzV(u#RV~rT;&{V5XyMrf8z5$aIO*|#w9+_jX_VccA5e(o>|S-Y32~(yZ$0e=hFz|k z6mwHd5-^)Dkfr~eQsYpJqarQcmFqrs7Ts6lAmPCR<(b~vb>R)i){h$5^7q9D1!za{ zeQL99P3+$jW-?1M;`rrP2Y$Lgyb|4w^{8^&2pv`*o4l|tUVGhE(Y^OTk-9%i>7$Ng z8D)W}s4_(ZB{93c^r&tIwA?@o+`ss`in(M=Zi1~KjAw$kz~17)qt!KR^A)}|Orf~qwg4ETO4ET_%ujz!ZpNP7#7aLnV+!T;*c53=q=h-ty7G=dha zg_65uc!6%WCTXU~&0v}jbrNG<2252D&yM^sO*8997yE^oD?<_ol>(1DP`IOuMi>L{l2tru8g%FI4SmF;S#ud4#AL3(E|@#3(9iYFZ3?=cKdQ<$v>c!8x7zd?IksDAaLd)u{IbTuv*;3HkTYDOp4T z_@8PQDOl1!6a#?*Jft4`1$v+#Cz~iOrZz>9N|C?OZ*P`Z)jq diff --git a/summary_results.json b/summary_results.json index 4a58099bd..a10a6e9f7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.125, + "Performance_Result": 10.0676, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433279, + "Performance_Result": 0.432134, "Performance_Units": "Samples/s" }, { From 8b9c43aae85309d9fa9106e91203969b757a1b8f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:12:31 +0000 Subject: [PATCH 38/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14568 -> 14569 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f0f51b728..685c05e12 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:11:27,157 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:11:27,158 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:12:26,258 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:11:27,158 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:11:27,158 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:11:27,158 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:11:28,334 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:12:26,258 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:12:26,258 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:12:26,258 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:12:26,258 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:12:27,282 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -74,7 +74,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29161 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28764 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index aa324580c..4eb9d0d9d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -579 +580 diff --git a/summary.csv b/summary.csv index 3d648f33e..275fffa6f 100644 --- a/summary.csv +++ b/summary.csv @@ -52,7 +52,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29161,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28764,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 644a91f14..c644a96a7 100644 --- a/summary.json +++ b/summary.json @@ -1599,7 +1599,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.29161, + "Result":6.28764, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a6e3126176a7553b2a9377d731b17a6ba5e13bd1..c7fd183cb1c7734d97d073e570dfc58f70c813fc 100644 GIT binary patch delta 2382 zcmV-U39)uLvuqAo41c#aYSprq9$dVCuk;jGZ>n=vWADc90NmWkcj0Q?b1B7_8q#t*c~` zLEG7~y;E#wwaa=n)2NWkk*h#_>d58a6#uW73(}d2hDBud5rLRjM1*A=9$<5C;R17Qs`l!SPDHPgm4B3 zJuCxol1KRfoIan26#zJWI*$RJ2aaUyw1HR$&3dF6uA9sPf_1ZEpXZ5iK2Nr*WRgMa zX4&c~)=iVduFJEDgH588MAGVEp?_FfJtTx|23S2T18kD#_yC(esfQH+HhoTy0jCF! zWbCvdoA9U}(hOHg5O_?*0?*F@aa%PWv#Ie|W24lyO12y%5}zfK9uEt}(&Hf^Br(9_ zVHqHiJj4e`^l3b-0FdajcnnxPa3mw0fb_~vEl95+gfNn9P^(kJ_{0>GtD_J1)T`@oTmbOO>VwCa&o$e?W1H`^rk`~d5-f}Yl>d{{_syE-IEDN@;-R<15g;|KS6r zp!u-VCLif}1kHM+8Gjp~6{4iy!$NWw z)*&H8GQjU)89d}Yg5p=-hLfg>61MLGj4gi9y+ja{&#mHqc^#X8TI z@pLr%vf$qr`7XjhB{K}#)0SPH)vxMGjdI+Sv<{oc5}8Cvn}3Ie5^3|05V9Fy^RNuC z$)3dnZ2EK_Rsq@c`8*bU9ypTGUgT{_Xx1am5JZr7Y{l}f5l6c6IjEmI5h2 z+>?Y>J<L7Bn6>lF?q^LJxR6q#4o~^6R*4#cIz}>3rt(pDh)9j%~V0XYo$phBJ~0mf)AP`1jN!p zlGwOn5$I83|A}8v$>c&njW-_VqZ+L^a@;V9l>=1bj%aBCu~17|KqQ2q23SBW1E8`2 z0jNF=h*dyPeI8H@9uOSKSTA50K(iichE#?~KXJv98_tSgMk*n`X0v-xDOT#l#bwZ(tdOMw>JMyK`@2_z^ZaNqKOavQwafghm}t~!)ob(d`NiFzm$R$& zpW=G$(`>n%P4517kq<`seD$~D@lVCR*vt=S- z5Cm#?h4qulG9Q1I0F)0Q401 ztb8nK+GP8(=+ECe?@NC>yGA@Yu?Sl)RI*6*U$RWiw- z?QGfJDYmoPWxX2fpq2O~k+gSMD3pPWK&wyQVFiFzpSxqg-G6~2 z89QxAYse#fNC-E$p!1lDb)KJt=C=AgM)*9&XxCHoOzeP@{qiX(^srDYg&q<@I0J+p zmH{})qkI5PpU=Yz0GvLZ$AHcQM>2NWK&*piJ<<%>Q$^F%nGC)-sr$)I(! zZ1oiDrb%MgHaBswNtBXET0JZjOMk0}gpkbutA}NPP4XNcVACh{umZrQ&*?GX^uUpf zoi=0>9@Rsd;VKCNkEvMT`8gnNtHxtCH6Ckhl)6^QmV-p%vqaM4VWC)hJS2o926#Ly z10<4%_yCDMjfWKg5`7kr0gDHYWTX?2UfHPy>6L_#O%Qub#bVE6Z2nX0wSSWN1z{=7 zo+xcAhuLG=4lIe{5=pg(h2+z6IwXWx2B;WbCwo z^vXi`$udCpf&TlZV%g^jTt3Tstz>>dTngJqX{YsSQU{mB8j-)!_<)5Z2OSbZE(2^I zmH{q_s{>s6WFJ-lxb(?B27hE9IFgZ0KzfB%J<5h0NUzYU zM_K`c(5i2@&g1#M)@vp63t`Ykf1#oMG*J18Z-h?1HQ3(0L)hlG&Q0L_PGfYJcX=RQLz4bXgi1-kb?e4rFG zA9mX0BR!9xS&uYBB!5|HKyu49pIaoK<@Z!Fw;&|7KvFBm=SS=Cdn~a+l=OR8NbbTq zB!ox?_&qEGNU~?~0Fyq!hgE>DZ22m5Ef_vI!Ve?oblPGEPuzye@Z5|RrHUn%PmH{@| zvv`0_pU%T7Ae%m)$AZrTM>5)rye$dMdZZbG2=b1tSl)RGh|jwxS;-88K-@HUj7cJw zR%adNjwK3+lI9Kz$={UPAt3}Yz}#UO0Fgb42SD_xJFEhN=yP{0xI1tpBfY>~Nj4I= zD@j6#COnGAR)4JSc+vc)wTo9GwK|H!;N2P)|@k*3*c33Ep&JGD7l>yEU z%K(+^NjyNMPugJ>kV>DmW5L>iBN^>Q-jrk^+`0ycJkWpNRxI*_V)?8)l9fy@2utxx zxOAF0?}j)89!mrgB?TT9N~FL;LWpI6z{4_tC3_GLV1MZ|cvuC*(x>oPP)48Yop1!7vodb0tYbH29ivRE?f6o&b;vrFxFSl*IxHkF{dGtP5e$%Z zSOySe&*4cWf<9Y^RX_xNx{d{12aaU47r3ZH9@Z1fg^^_7+AGt3;M!}&0?#I>KkB<7 zmCQ2eDt~NQ<58_foVs?h8xl|Zfmh<^XesfqP$DHB5<*@BBp#LlUfCmifLEWz!zv)J zK8?qM#sfz(+6!Fh0gs0?Lpno#9ha?G?RhGl&%FM#rGn3~O*iQ*K59Ddk;DJxc69@e3-MTnMP~#>0G6qZLPv8z!-GfJ)pEEiE7xYDo)-gb>sK3y5U^R5l<0 z)u#cm3J9vt1B$@|f+HF01q=gd)+5c3$`I)%u2}jBrSe&dK_!z5qEc8tae33ZNu_pB zNq_VaEv+9GYDw#dgpkSr>xX54N;05!B&qaiKePf+>GOW>fA~Npct5lk`Dv7eYd*w_ z{$|tBcv%d4aQS*RpA44E@$|g5_|Lk@%j8ApFCDFl4bP6RFZ0^++f}|-&%a(3uYb#G zvh~{N>)G4UUcK0hU(Ls}`FQ#5ruVa=+V$mNe=-~Wx*pW^bo`&|{MTQGza%ey879B| ztEPXe?bV8@-$jert$SJhfemYa_iAvS9}VW`RcD4Rf zT(5naEtj*&-QOT-1NoVhcoTI^Jp{DSk+=o;l2K?!JXgw$<~-q*|Y0VO^#Em&tX)MyVLEt;E(Tgr`t zS7$7^Ai*aQq&?-fM%xf9#n|&LDCV>1&{5nNr&x%kYk#q2{F}9a3qtO&g1`g|3=hE6 zPDFG>3O*t2kLyN&;21hkpo7PGmgBQGfKz_9VN@=hN`}3KS?fuh+&eEPW8e2#Uyjz8 zKgXY|m$z}%RO;bW98gmUPPqdXjB6@=lx1vokZ*ONdvCL05sASWaN002+|000pH0000000000 Z004l2xRZf6J^>4pz&IlY#x(!{0055ROWyzh diff --git a/summary_results.json b/summary_results.json index a10a6e9f7..30b301563 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1364,7 +1364,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.29161, + "Performance_Result": 6.28764, "Performance_Units": "Samples/s" }, { From 192eda1426863f26862b45f722bcf27dbea5dde1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:45:44 +0000 Subject: [PATCH 39/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14569 -> 14570 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 685c05e12..3a3d8967d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:12:26,258 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:12:26,258 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:45:39,396 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:45:39,396 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:45:39,396 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:45:39,396 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:45:39,396 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:12:26,258 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:12:26,258 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:12:26,258 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:12:27,282 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:45:39,396 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:45:39,396 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:45:39,396 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:45:39,397 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:45:39,397 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:45:39,397 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:45:39,397 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:45:40,491 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -61,11 +61,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 37 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41445 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0676 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3651 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3505 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4eb9d0d9d..ecb9d0c1b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -580 +581 diff --git a/summary.csv b/summary.csv index 275fffa6f..520c09c72 100644 --- a/summary.csv +++ b/summary.csv @@ -39,11 +39,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41445,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0676,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3651,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3505,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c644a96a7..748d3040c 100644 --- a/summary.json +++ b/summary.json @@ -1209,7 +1209,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41445, + "Result":0.411879, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1329,7 +1329,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.3651, + "Result":23.3505, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c7fd183cb1c7734d97d073e570dfc58f70c813fc..6616b8c74ae6dbe2c5c8ce4960b12a2001ffa793 100644 GIT binary patch delta 4826 zcmYLNc|25o`<}678D`RO68j?_v_#M6P=kq&%eE+=8eO>o;-RC~vr7K^q(0zsY)P>rUnhruBZs|cn z$2m+&(rH3Az`%`5HAnw7ztumk{iz)2KeP#~?yf#t6Q6& z&Y{laNG+1~dB-irFXngv4yl^_>3T}VC1rL>;3ql`}z5I ze0?c*2UtTfS{-*Y(dnPtaPEfYVx3X^#ho1 zcJd~0G5yLF^^i6lWaNkP+#4(XVT@LiSk<{Z=EadAMeL`aH_0b*cS6=>LtLgx%Lsv! zPw*ca0(H~Z;rS9}_m_$jf2PReJYF!m?4wTUlu8Viwcq(qR7Cm{L<*vG+Vv>&v_U{d z!sutovPtoD3t_5&@?I-&Y`R;-r`o4~O7)>Vym*CAl?QbF23zgf!j_RFs8WIy)a ziI*Xf$JImD3EIx3Vkbsc#m%eZ788GNLr-Uj?`el>$jVmZ&03qH6@>YukTA;VcS`&U z6l=X6Xc?mb_x7fgrE7{{fk%0@qVDcZ#V-=5k--|A8EFY6cg!$8hZ9d_>XseQ3JB~{ ziemeJlzV+@mj2!~z+CI9{Mj|@E@38PZa1;jZ#v+amycO#A+dL9t9!|#u{qDi8O{Jz}GJA&O^xCc-3Q6mZnLwjE(tI!}sGuy>1J2E@txT0?n;UkYSB{3iTcvnsL|zZY^M=bIr!u`Mj;~PF6Fn+TiQKI> z_=BCGQx3nwx;c7Pn2_p^lUEvAsrqGI4ts|_J9oRrj5xM6wNGHqkPlSv%r9BS{lv^Y zeZfh!J#Fcx8QV&QeIsafsGLuAoS6|5KvfrToFhWQfKAa}JajfQ?`M5h2buQ}iO4HX zs{7N0+qxPkm(K6(Z^ic^th*|t5t7soIVRTBEoFICv-nQhtVkuKR}P8F_NKVLLN$$; zt6zarT&+PZAhTHl9R$laO0MsH(2dQOCJE{)T#FMcr++FP3RB6kMAU??H&fNp85R|t z{&- zb?HFw#dccFC8YxEF#$)-Y!r4ztH8h0Z*GGBq$nzmY@sZP0*MohFtK4&*c8$Ong*L* z8l;TfL0A{^nJJ?R3OJBTl&}~;sjE58`bn*W`yXubd`}$-)Df4RmHBWp|{wt~=DA}#9FR7Jg!p3Ck2s0JSfW+%~-X$0!iNv!A zvq{QeDamlBo+)(b;)mqpsuM(w^v^y_4;E0N1q?g71PB@+yVZtoYFFxJs}3_K#J2L!b+D$D}0Gp*~@vTg3nA%yva6$xNb6 zG@Sbz=*^vN%lKG|WmcMN?VVYwAaCOk?rwquL_JEZ*||?YJ_u9XmT*PNxK7Gne)!Mz zieQ&$ISa=MI5a!gT?aY0<*-(eckKJ{sMP?O7O9n!`3lgWy!zR!Dn?@;Thl+vy%Ln9 zy)FN}egeuAHL%d)%#+E2?w>j+%%Fl;4L`n=r9ALGnfdoc3{1|eUug~IcwTW^XS{k* zGdW~Ym}Duixc=-Yk$Kcq-0mDPDR{`QXCkiLRB7OGE`w%grq!pMc#&e)rA664d4Sl) z?K~MG^vb5ish@_C9S%}Vzhe{?K6KULn9Hfgdw&MFb|c2$+X0Ja0subuJ8jQOPjWZ) z^wScxKWM5gaoZFV8^rU=RFyM`3PtDPZYDSw=|%Cp`vaGr$}rcOuvGq>u`joJ-oJb? zadl+5MgJi28&b2F`m z3D8V;-#t$1Wy$ZOW`LD8!BW9W8@*ZF9O_X&KvdW54QaL4d4CJ-sJ}%t7~n_79^hI^ z3MqJ`@hk7?6-&XwgKAQTyo-DF3!XieOiDTJZ}ySID+^`K05YIUHH2(86y zktA;Kw+dT+{sTu}q}|!zkOJV0FKNVHR2a6IoT8BTeAbCQmG#5`EODhqJ~8i*qvVAY z`6N8%gOqsfzzP~2tI@IIiKL`98Q3ufaSieI6!Y0F{@{Ne8w6pzI%5~knt;~7eaJCq zLRQ+gOobRxWq8gyd$hqL`e(cYQLE)){}Z1Y^M9O27?So9@nc><0!qZ9E4U z58mw3icP=4gn@y3e(9F+BVGvtA{N#0apa}VRK!)})FvGkWKkf-<+{y~B0?HBoD6s> zr1YpYgSTD#fb#|eu+WOPPcAF}2#LnVx+t^r?$=hFU3Lm_B}(fm1&`_9Zj?zI3Lzo`8Fc%*j%#gao>QfnhWnbG&2N6NyU4vJdobl<8MM zQkRa0!0zjVl0FjD))$Y-mBhrlj3dH(>{VB>JnZYXY~>v9x1sd`tD=)mE46NSL*5d0 zsQ8JKs-l(=@v*5RkcfC63h{fS^uu9=RXT@JIqQSw)Yp!*JNY^#|I7;GFtJBm--iF& ztq-tR#@Ae;q+}zdycq1VaQV;1y63snZh?- zMhiD(ktZ*qM)v32ZMJ+J#p6T8*h08W9xHuizp`eh`Ws0hgI#IKue+Yh+Y)5_H|aL2 zl~Vhuu8`VK^|eTfRpC%4KLE+aJ?fzha++&}PNZ6e*~9&Bmt%ObrR19n3zWu6iInl7 zu!AZm+w4 zWQPg$76_INtp~xfyN03aQMGUti0ihuG-oaRC=;)MYBMR9@{fb{5u@s?$N&pY|BH0% z64Rq!zp8m6;kaKSMh5Wx1r=a61b`nKJN_sj?zH{1uzIYBv0eu0UJ=<6? z3Wqns0jxOoZx-YW--maZ&<*9#jUNdDOm<}`J3#vLi-Jl;he}`1W?B)e=0lir$_2Bz z{;=GP3kPUshiwPeJ<(08Us4fv5>s1rm<@Qo_f@1p3sS0$euA?)BW9uVOm1Z9g0MC_ z?R)}8MGzm@v~VqXwvMZiFfSfkB8~?x%sJ~x#-zwOCs1SH0?bRK_*euJ@;u%LV(~*m z%45Ly7oGoGzM6Yn6?Zo{1{jBw1nCI%{8`7)=UK!bpBlt|W8Gg4zFY2(0X%104s&L{ z((pU$k`+#TR$hLIAkD8k9=wRn8p9C2U{y2l`Jf1|A^D&PR{Q9K3&1*^zqizSu?DpP zrsIdqs^#AShKgzPQmMaQz%jiZNFZu;q7b^nl3vu_xnZaHX0JrfqLOA%Q%4%Os2am; zu9J9-yjjywMATQIatdmDR#M=*5ZLQuK`zw0T!EHUL=hirx<;(!q?`8Mk^z$-f5?m$ zCXy1EBLdjw+Uu7{T6T%Z(ImwQB@9y&ybp!ZsbwS=AquxystYA?mkc~wA-xTq!k>;v zyDsO%+fW6Shi`z?bFu#+;n-Q;a_Isu>~_S}Kb*;HFg?nXBFB?R<$(*x4NRfYU0NMe zp2!ZWLE-`Wh7z`dgS>PtOjQpXgpqCQu(AOBhGi=e#)!~>?iXkDVpvYdhT)N#-1>XA zrJ4He!vT`@_vC65sP#euOiMsg#Dt48Kr#!;HGV5>Ld3YRs#6&igLChRsRrFRA>cZM zwq_^Jy})Y)tz?E92CY0L-|{^;B)vJ8N;r=U7OP)gdrXuKYzeJBB5u9ip&}g202pB; z#WZnn(o{pm!N^oE=NNR9ThDlkr8*Y&cs-v1^S8pSzCr0VeSpqKKTGK+F8{!0zVUV2 z8@KCR7^~M^QpH`+|C-h%+;d$1>%A<{L~?fDz(19AA54L&?3yB2 zVNvP7JD^*K$v5NlSPBVn#)}+r4B#*zEE}!~DCfBHK3;n{7-?TDW`6I8e9ct~;k~e$ z8cO_vg+M|qVhAFT-~+MOK|zKeL2LMVpRhQSo^!^?BH7$9PQMY^*QtGoC50-|HXrS~ zQ<5>*mm)io;g#^gCJS1qjm3trTBS4y#Rgs22VsLQBmi0uwW;>o3~6WR@Ub7&Zo18b28oBhK91~IQ@xH>m_ z&9?HR$Lsj8cb@|EC5|O~!qweb^Xbp8o+v0772k8x4bN#^+@Tda)O;Ra+w^gBiEiop z`DJUgd^*I*s^0BLltBb|Rd6U`t$Zs?M;bWx?88Ph(Y~eq=(6?sV)xNo%?0=NuI2A- z_kYWt?LO8y+W-u2Tz=Z-thw3Zr`Zy6tlW_9xw7ge_-4KM`A#JE+>rWu9l6N1*rED$ z&%&YWfZLm@5g>{E)Vgl&s!r4I$y2z)bL+#OlF}n|a@AIMYxU*a&#yllRS4eOgVPpm|m+?e!`~LhumrH-8B@LYE)Ih|pVo%LRkQM&JTY<@hY?{eol8oM~Moc<) z8P*E=f0RS4_57ScVgBD7A5A8#(aK(u|v5MZPnsS282itXZncW zV_Ue?>)hNVm%h8V9DP+AQO^~GOAO}fPfsFLyT{hm6zA&}-`ct!c|=N}O?m+RzjP-d z>by;z{2T{WHL#GgCxzcZm43xC467ueFUgexJ zJ^L~I1hS>7zxgWT<<5y+*Y?)C<)wD;N7IxCgrGi2x`*r8&+9T$RX#Y?nZ!|%E-&d# zmCNy`eng?DhbdoYq%u>!FNHtF7+D!7IT4aFGE__!SMFrN-8*}ecMsY~-rmU#<66Gv o9vw3euY+kLIsgB%s!m;35Bfg|FY4Jt1=Bn9WO;i3fq>8d0pPmiZvX%Q delta 4818 zcmYjVc{r4R_nyU`nCzpBZ6plgOJpZ8Mv)~m#x7z+Uweo~C`(yp#*&nM&Az6jtZAA` zM%HX0Ym05v*!n$sf7kWC&mZT{``qU~=X~ySJu3mr0d-zHocOOx8qfGaASa4H&p!}I z38jSR5^dJ8BT%>_B!voKYI}|;Hz|3 znCVgH{jO$%uuf?li&Vu45g+Gi2xB)RmGK%69nXXa$h@bz>QqnLhNSY6w@K@45ILB2k8Cdg`7`BpZ@=AU#e3prWBZOdqi_@JyYha2GGto3kC}CT zxxabHR`2~)V(r03XcIQoZs4ScOKto>-iQfvY-T4gTxWN~R8Q{}PN+=LSsngCNqF_o z@X5JZ$CU^_)!)yt8jSI*b?@+cdB--9R1@Zk@yNCA)!x7}%+&?cm6nc+HW?Ewmr2k6 zeSC(p$@^H;Pi4@Q|EKuH$c{KdV#Q99?*;Wzn<)6X-C)WSSSqZg_n9U+Ib!tGvD$8J zYYjn)*)bxa7Oy7KgK6W7vz6g&Y$P^+7(lfR^1Q1^C;Kl5zpO98fOV(?M439 z%ICl{;?Cb9hm#9Gd|1O1j(2zo+HlTno@?tV^HVRhGY`$O%3C($QWko%}h_+}s5xy5nqi#9GfnqW7&};b$)5^+)s3&iS^F(r-Qge(H&< zK;;7rM-l;keo~nfl3y!cEk9KHvl8Xzm zVl5)Vf%1UglGqj;9!|mDT3s&b`h0eya2iZK#r*Lt4Yq+%?NFeCY9AOnH>Y9)`HzG# z1eO{g9hKYbNvvsDGT6EO>D={jnUHZl?7mCfHNhZGybEJt$A_YsDE49_r|h)g6eWFx zix+JIR}Am{U%cd0EGqg%1Xr*))U`N*cVm1g80k`)JYvg7#J{5koDjnW#RhJ^d~g%BBslLQJbez1`E(b`*|U%L2OjC8Uw(zVRY*DWPs zdTys3dmUre?$g3rj+-cq*o_u|0l|iXEBShQGzTLlnWMuQ6+NqjwZb4Sl`Mv zD2B`~%cYSZ39`foMpX{Ki{%{{L;a23w{-4ZekZdOM6d_M{B_ZKB*Z0Np!lOH!CWE1 zBb|hYaZ26?yTfRp`^MkFB2hjtx#%{Byc0cKpnG2-b#1+n#7tC-jxqj=wVAV`qkN@p z5@hW{ZUbe~I%_Fbo@N|C|7i8Ca8Qz8uE}DYeo_i9LsVbq;goy~HabQFJqBY)8)wfK zgXqhRNjPx+;G!T2g7!MTgq-Zc$N>z69L`yq1eiA$q(1p}wAv1nVO(RG!4rfo2qj}e z0D@}k+Q>K*+vqtI6h#J81?Q4C;Ko%kFaFsZ#uii-&joRqs*c|aCKTAGszH+`TlVMiw^W$06pyB?Oe6)azp>51*(rrNLOpMbt>V#zTqA&^4zjxC zv#-1{dP2jHQbY+=POVz$QpwNF?<6YOZl5zq^ZjOE_>nRRk)nzoR+b50ij``mc?>HUAZme=Cdp2$>e#iLE^mw8FB zF8B;@J$Hv0Tb%z63&8Qq7Toxkeq^)_g6e(RkF(5oSo*#H7Y_^!^+-^`&@#3W=a}op zaapx5|4t*(m!qQCjek*Z#EpXxY&tLe6%g5;z(ImFz+E}&x$6K_-yH!-JMh@nSt9_| z^)G5i>0i){^y;1G?wyNe(XA!3{fSEV^sTz(9~Y-wYZ`Yu@ZQioPB=)r-$37!ltij@ z?76`uNV>jk=&cI~E2%C-GmP5Pc|I=WZJ<|i|F zq)Tq=tK)X1BKDxTopfU?}+ikFO?Szk)h&Gl!acvZ(bJlTGwDpkMieQa=wo!hC4fY`lLo0m(<|CKRXE4 zb#k7LVNQzM*;Bv8C{2%)5sk#k0GJflatRXb$D?O^`PgAq9J4Bd+%QWr6Uer<6)P>z@b zK#tZMRlckk9bvA+u2)nAxxa0&sP`HQu#cK3{^+(UGMi*ph?ThT(s39v0`p`Zfp zY;s#;T2-J+OO%iw#NP&zZ?wKO-DYqd7LTi~Qp44jaQbpSn0C5Z?U5uoUMbh6Wp?hV zL*nX4ar$Mk481h7^i=e7EtCM!92*~(_6~%N_l&-mCY-ivN7P_^ETzUVfnscPtW+HjibZvf8Ua8X5Ym8hf7{@#xfet#|e z{y za^68=LK@xk78CvTEL#qltzI2lksH29>8YVLCdOVYOgE;l0DyP^(3cGd0Q!;_e}i)L zD&mQPpfgbxso=7|h>|&LEM4-RlpM>JNfu{M+P**d7EF$`eE()2KIVdR!ytLol;2mV0cJ{ihY~3o5>oWgM?gU+b z#b)TTyLFI`63cBBiCv(h!4SGa(+bJ%O}_7t#%CgOdk?hQOlGpGJixFy z3Epck|-VFFm!9EZhzEJ<>dpu}CO4KJRlk7n&`K^|{alLX%9gFo?Kg zr%UG&iL;Ap3qkb%YIX+PizD_7A2)vW_b2ZRsA1v5Qx)rVX?uG>iN?wXfMf zdR34}Pa3x1TIbqrY54c^T2NG+xhyl;sjetv|Ce8{R)5xdu-b0J%(O65smnm-AMWwN zOsCwT+7(Q4;6!>8Pfj0Swy{}G{2e~RX4;{#Y}RqtQFD8tzB|NV#G$n?b%Y`TJqMiU z{mi@pyMh^~-SF+*_+VoYIi*X*ToqxNEq!+70^hSHtySe4C@X_>wHr9}LFp&pcu4>+ z@X$aB->h^%g_e_Uf-vK}wkw^J0M%vIAD9MGhm#qfk=hq4ofh|yzIzM$?K1Y%`IvG6 z-q`^~8={mC1ry(RB@tU#+}LfD7z`ZQ72N)zIJR;R%UQ5|6rHCs+IFW(?8q?B3BXP_ zN|8st467T=-#*xFaVN9+3t;R|Jev}d5;3?K7w<_X^ z_v_1gSbc7|zDLl?de}`ee7W|VQG#o9Rkv!i#7!Pc`D2GqAVg|uFa))vb?A8==fyVa za&b!ck}?<@2!)+7+_#!OTH%pth;^fhruwMQerl#nriG33zTne0C7LG!ZIN3C z4t`OI^*Zjgw}>(5iyaium2d_@M@a2&S7wZO!*q(_boj<GDS02#p*`W)=k_g(k)(2a4t%E-2;Zd2Q?R(ree>zzS|l zF(5)QybqBqJw&<@d9>C2*pij|pG22S)OjO=WDUeZk!ZDM;DmBw2Gn*o#hU=GzhsU# zK*r^>;S5mF{01<%9Jr+~8(fM)T;fK+PEc{DT}2EU5rn(8chUL*y>!b{8U5DW^4X5C zjINzTTF!@y^3AcB!c=dgYyg(vhlD|PyX^zktXT6-!ReR?4`!%2IwsA1(|BM z(HtsAm2czNls)erbzS-Awl{0XCof2^E2I-B8Peu>6#^~=_wN+k0SzjWVqeY;hnjm5 z^XJ0mI`y6mDCuj=-2JdHE7hM>%SP)w_<_vJTYS8?5omS%LlUv!I(|yLTF}dv@wwC` z(za3qw<+norIC~Jn&v(lOB-nfemFR$lr1**5{EB3pFFKaT=u@~8x|TFl|7O2==o(aU{L?4$jw(VizbRCAhrmom0A!pN^FoBQLq`07A) zPbPewrp)~AqQ39oS8I1+UkU&2`Xp_I*B7_jhK=3a^H$auw-j!s?NvSS{^UJ$ZF8O3 zJDHc&Sjotnjrcz3`|Q!h9|xf%UK3^{wpUda)U2_+pA62)Kl+!<8!=K* zFjk+iI}9Y{ovJv|r&}2o_cMyCs7-OKpn&7^?UwmZ1_g48?VNcek|L${odQbM|63=}TN+;f9>v*W*52yfgK5bqg$`r>uy(re?dufv8?-BRjlJ zj>USRS|vJVj}_~FZ86$~p?XI*bx$ocE`79`fS2JCO0LxrJ@<0dqmLM6;tXA5Snu9+ z^^uhlCB;~xcjv~z5}J2~1eRt3a=-qbJvx(~=O>b`P)7dZ?KT`CB>StAMjZ`t8dhYx z_AYO(|2^$AXFhotb{ diff --git a/summary_results.json b/summary_results.json index 30b301563..fced2cb5a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1472,7 +1472,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.3651, + "Performance_Result": 23.3505, "Performance_Units": "Samples/s" }, { @@ -1652,7 +1652,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41445, + "Performance_Result": 0.411879, "Performance_Units": "Samples/s" }, { From b6ebf6a1ec41cff3e1f5453289ce46e8707bb833 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 00:46:58 +0000 Subject: [PATCH 40/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14570 -> 14571 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3a3d8967d..17b20fde7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:45:39,396 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:45:39,396 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:45:39,396 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 00:46:53,294 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 00:46:53,294 submission_checker1.py:3270 INFO] --- +[2025-02-03 00:46:53,294 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:45:39,396 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 00:46:53,294 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:45:39,396 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 00:46:53,294 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:45:39,396 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:45:39,396 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:45:39,396 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:45:39,397 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:45:39,397 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:45:39,397 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:45:39,397 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:45:40,491 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 00:46:53,294 submission_checker1.py:3286 INFO] --- +[2025-02-03 00:46:53,295 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 00:46:53,295 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 00:46:53,295 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 00:46:53,295 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 00:46:53,295 submission_checker1.py:3307 INFO] --- +[2025-02-03 00:46:53,295 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 00:46:54,585 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0676 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1838 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3505 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ecb9d0c1b..373892ba3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -581 +582 diff --git a/summary.csv b/summary.csv index 520c09c72..944b8edf3 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0676,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1838,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3505,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 748d3040c..e24f08fb0 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0676, - "Accuracy":"acc: 75.800", + "Result":10.1838, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 6616b8c74ae6dbe2c5c8ce4960b12a2001ffa793..6fc061fc8b8aac1de29d2d480eec0cb86ec27d7c 100644 GIT binary patch delta 2062 zcmV+p2=VvoaqDrg?IVA%7m@N-1^@sxDF6Tz0001ZY%gS5W{PbAM1%+_n(b!Fcq$+=m z1J0_#K4n?RtI~gzvY1NJ0ZoNFs?xjhI1FN175|aZG@wkA5C?8F4eJzF=QS1!-stKa zeWU6`MZ$6DEb{~Yy}GiSgh~37zDi)(C<-{2tv;K%LFkTxFyQh-uZ+{ED!moHmTbOH z6OS^MNc&HJmk>RVhBS4#yx5f%50Wu|aZ|~ww+WYxIH!MHkdee)VQ^Jusk)-;x>1$; zE_X%P2uEpNm@w*VH`m=HijtUVljmVJ5}}t3=WSbwXMrEM%4C`;4>T6c`wxmw$F7K? zp_jztU?Q+~$+iO6p1qtPo`5<}e^G&5DK zS}Lk;FKUQjsQAHnoaMDfnM)}hhHm6y#nDGfr&Co%!JMaVsI=$KNHRV7C5)GI?NJ#*Fro-v*W>w9D>!cjr_Ay&F)CT5 zZH$4Y!8XOz-dMx0B$k8OA{hQpu)V!l)HWp)b%gP^vJ1iz88)4`(%roU+?M5CE1CmK zO0j?SM5?LRh5x`pzD8E6NjK-HcmG!xIgV{2n<(8Rr3k)y3+JFBYd2`EjaqS@8+Wd1 zcfJm=YN{RSI@s%0A@Z$K7X;ABb+51PhlB22zjuTT;gz;Uh;G=9(e*)Q;$8OTo1ebX3ze9E2Y3e)et$ z!~0%;I5@kxzwYHv>$3Z6)|mvWIzkf`wSS2SEAB2qLf3O~G@4J>q|$~9TOD0BEN9JD zVyM{ev{$wgaj#$-*V>AO(%jkxnU+ur6D*bJfJ#K7BVB+g^CYs51&O! z636r5%utPvinnb`RtT$}XMqrfULhHJ)9D*Zj*yCeB#5XC094#Mgt~@tQf<3+#mfv# zG@v%`y{~cLeYbe~*#l$on$@)J@Ao~d?Uj&emc$_fW_X~08Gcc~*az~-YTtkTZI4_p zVe2n(JC>X6>scMx`|9aaIu&mfDWiG)Y|Af=K-;(<+8V7U%5_74yOr=(W+RBr z`n36)*$85@B%7_8JD@dCHg|tQ`$}$27tR_qKF)y*rw*ynm@I*6O3IEvFG1`lFRcD9 zI6!*ug4aZ*ZNzIf&9;SVAjh_cYM9lR+&a5LHgE@g2-#R>Z;RAiYJb&neb!;iA8IVW z;z=_mNKqox3?>s>>r~tWB`qfyw{}D{`|l+pS#Kf z)4i-Qo3Mtcx;(ypoBXwPRhNf?i>alFRcy7_rk1v9+qSbBT4E6pZQVxbvUfYY?%fKB zEq%z}q+5<3Wg{+(!n_NWTw{!%{s&M?0|b-dHyX2qDnkea&6SA4lglz7e{RAs5QOi2 zipclGP6DX1k|4Dery^A(FrN|h_pYh9RPx3XhDGv z9_Lw(FWvx7`MV8+a^+Mq^cL<~&*JFbdpR1ruFJY|u*Up3{#w4i538n9A5O&qHI?9$ zJ7B@Mroz`nJY36sSjH1?B#5Uz>UZ8wiyZqvBryOz#yu;a<@3vGAc2~K6w{OxDJfTk z(8-KWC+ow=oBj(>ore6Icm^DE-?IrcpaTW47m@N-ldd;c0cVp2I648@lR-Ep8_ku7 s!X*I!08jz|01*HH00000000000DytIlY%%t0S=SFI3othH2?qr01Qgx(*OVf delta 2053 zcmV+g2>SQyaq4lf?IVAopE0LZ1^@sxDF6Tz0001ZY%gALZGPO+61=Pr<7wW!8<;(mK?(V`jxSBV7WsQ9QKK# zTh`ZKtGm^9=lI7wj?j!`G)$9|jwY)eL=ry@!enyNd3?B#oDP4Y%u50o`nD8q+3e(n$GDl4Ns5R=}j7t=vZN=$4Vh6g#(`?UKXZx`C}3? zS{L>S%_CNqW`utxM34?h#@$hu-i^mmn2@^okDO#7p^AVw@ZxFIq_{e7u$c2kSLf&( zktZ_Zjzed85VG&}mEAN-v!CQu0Lw>l$e3vL*~|+gZxlu$6CZk|jKp>6E%&ux^L>{3 zgwj;lfBL(C==&rhna9M%p0Ie3j@gTs30}QTnP|j0VVr-AAntO5>oQYTB~?|mx;*fh z$HRs@O7WwVl0dn+?xt~^CRCX`5AzWZy?i)#EGe0VLFh@7X)ZmGgfs6yNC6ppJc@>X znvBDVz^-(4M3!6DSAdG;qjkkQn}k5(Jfd&?@92r7N%8l^mayTPPl7l9ygI)+LzmkQ&lhr{3LG7f8-UydE2`pT2le;&@TY{;5KeIFar$fHEr;>=(~9=x8BWp8B{gf&FiY~l%b_ZDzlmUpFW4oo4% z))RlJpZTb%@YP#72bEd7L2GW*^7Gudb6vah zb%14Ec7^L;uUo|sZ^5wbLy2ZM&tn*e_wA>Ec;-py8mYc6nQ-O_A?9`1hjZU@8r zUVk_^yScya6;G?8`y1Bj29{NXCNy?_@dzvLE>1#K3voP}PuHZvh6`J5Rn|;<%~nH` zvE@1|TMd4%U<)_ein&tX+6IvpS8@|f*--%%k3?Iz05ihE#B1o%MZ=K zt9uj!_o$E{l^j|R^amQyA80^-paK1X2J{CS&>v_3Uo3dKk^_7(z!w92F~AoCd@;Zm z1AH;S7Xy5CfUgel)d9Xbz*h(O>HuFI;Hv|Cb%3u9@HGIw2Ef+<_!ZuDSxZTC@JcvBfc~OZR37uYqZ)Z*DV3=R=#&b#HyJ62c_5y z*eb=%we4_VX2U0q1A=`avxaX(yMn#@T5`Y^sqrh+Aq&{2(LUkF!xzkt;M%X4tsu7R z)Anm-D~RoqY`1FefYw6U+zEf}E4j5@I2+K|xB$}ZCZtwlvI1%-DLVqagh`OT(B`|~ z0O`F8UK^RV5wG1e+ZL*Y9NQkMWmaEu>+A~I!X5A-WNVqdEmC`_{Z+^HS%)ousImNt zC(M{2MTO8X7_9fzAI%OM^zTpX!vz3JwWw(>*?;|)B*;a!YpPF3vp#>AP38%w8zw{; zkS{J^xByNUzT zy{ysOu*Q&8aeVtW`EyiR6^DY0silr(Y&vUGOG~ya%U%sFF%JkwwGg`O-43sNw_IWg zAM!Wpmg7g!hzqSWZzTp-C}pSr0kdZ+VF(0g_ua6Q%rYQqfbB_xVK+$?+BTEsuH4QbvUCvDSK>WSleX1|?Xxti}(oBV?F(&!rH89@nXZiLjE z8ojR;={-t(U|O)&XsA&S9?hGZid)K!gI8xPxFEqN5u`okwnp0!EXCOKEhy%*=+IGI z8>d)^rQ2YC%lJ3j04@l*!wLcuEHFF(Q#%pS0V()|v_GyL0fJ*_L4gh)=UI+V-T+Sd zs|}-a=2SBD7Oq;4;^f|WIT^dI%er#3#{4<{T)w=GtEN&9r{aK`N^r^@uwYzM;qxLM zu4O(f3OvvKutl4SxSnOlq*8$?TnTWU&F|Y z{u59ghWwj&1nhHPvkEkz0|lX|LO3QHXZPK(B>?~cPyzq| j5dZ)H0000000000fPuM_gE&3`4U@t+BL>Jd00000T$0~w diff --git a/summary_results.json b/summary_results.json index fced2cb5a..f181eeeda 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0676, + "Performance_Result": 10.1838, "Performance_Units": "Samples/s" }, { From 6fb35d97a6b2f1f90efe7d8b36e73f5f2d2a9075 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 07:48:20 +0000 Subject: [PATCH 41/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14571 -> 14570 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 17b20fde7..bc852a548 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 00:46:53,294 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 00:46:53,294 submission_checker1.py:3270 INFO] --- -[2025-02-03 00:46:53,294 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3270 INFO] --- +[2025-02-03 07:48:14,582 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 00:46:53,294 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 00:46:53,294 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 00:46:53,294 submission_checker1.py:3286 INFO] --- -[2025-02-03 00:46:53,295 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 00:46:53,295 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 00:46:53,295 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 00:46:53,295 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 00:46:53,295 submission_checker1.py:3307 INFO] --- -[2025-02-03 00:46:53,295 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 00:46:54,585 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 07:48:14,582 submission_checker1.py:3286 INFO] --- +[2025-02-03 07:48:14,582 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 07:48:14,582 submission_checker1.py:3307 INFO] --- +[2025-02-03 07:48:14,582 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 07:48:15,745 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -70,7 +70,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0862 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0215 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 373892ba3..33f752bd7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -582 +583 diff --git a/summary.csv b/summary.csv index 944b8edf3..e8b1cb8f7 100644 --- a/summary.csv +++ b/summary.csv @@ -48,7 +48,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0862,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0215,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e24f08fb0..d043a5833 100644 --- a/summary.json +++ b/summary.json @@ -1479,7 +1479,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0862, + "Result":23.0215, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6fc061fc8b8aac1de29d2d480eec0cb86ec27d7c..4ba5c19f2300792c8f540de1d8b24a9850bb6752 100644 GIT binary patch delta 4360 zcmYkAc{r3^+{foO8AD>SX5WUwP)(MyjO=3%H3pL{#YiRLv5zdzFcLFkX|ZKZ2w9TI z5;09RmMlX>l3ig;h3FkU?{&TJ{m1#|zOL{2{Jy_)zW2G$()GpbwSEHJCXS_^PpN?d zx?EhP)KY;ngmLz`=NAvyK<7cPIYk1qcC^rztC(fZBF)Kmk{5fax>kH?PBv)0Ig zK+S)}H+^gU-#lK&uTkrn(MYTqA+JDJ+=Uf=%uV+mPA61E`7O=MpnCFb_;JCCpqSjK<-!sJE90HM{F!4KC@J0To}*8sb2~sKq98YAZOxV0+yXg=)u&m1}rtOT5z* zU-~nm@?{#g7}q?i6lEgm)5=#JU}DOslyB=zr*$u4lkPq=n!^1gI_NXO-fj+Z|` z`Q;H$l{=|CNMuJ7^|+7$;pwT1SE6IS-sV%XULWY>KVzXtK?;#Gz2Rc#J~bF-c;8U6 zUvPI|Y2q~%iC<@Z^eN;MOtC*^(qvn;pAIwT!hYK_>PpC+u%z64&s>3jE9tzY?A6x7 zjm3#z9XXHX=#jLdckfo+@KPOqqRzZiEYs)RoNxP)2n9K2;yi~ODfa{} zfgFU$;gsZVVWl{~#ZKY-;QajA^MknQyOFEivnl?uA>z+Gjd zB~p3E8cM=Tn14J*4lR7B_-=ZuInJ@wNC0htuPY2gLxX%JXi9XnycHz5m2nJi1$Yg* z6w>+7YPx)AgcYDMwT`+WsIr9lhI??XT`S|yPk9V72P9*mRx-3p+dOp$3_zh8b-^b<96 zlurt98OtZEMhN!)?of5PD^@FvF zXv5$=p_Vn$`K2I0BnR+N3r;o6p7F7eP@*Zj0U0*P)0#BUvpdGY)#f0d}Yvxgb={;>wB=iVsK-@vULf7g2TJ|QRr|b$DL5*&9eX!qgxoAj1Shc}^d{9MELYbhwCcvwZ2bdhBgYp1t^7<+B z642m6JqZgG4$X;@xzK)6^YD*d)~XyrF?Ul3i|aOx57HkkAE?GyXPG^A$P&1Leh@~n z4&?x9t?9!P5s)!c1QhzOHW7;u8aH{_y(= z?FgfW^m8VfT=U}UpeE+^VX7;Qzz^<=uAw_QM(rqj5Q__C?8z3BW}h(n@^_04TV0BC z?na*88zG6lTP2DZfVSak?iPD|6L#CTAtxJ7l`vZW0p;yfwBAoMKkHH@V90eD55(-| zyi~h;q0i(I-I{z`5?^(`SuayDt;I?$OOfJ;vBH5f6|;f8QHLPkJv=8;$Cq5`pw{x| zGx+g3+9C?TSwXM&z@iiQx4!47tWMHrv$sqSyZtg9&pf(wt!rVGsP#y>VuOgba9t22 z!@4A<1nc-atRRU&Tl+YSpEo4Tei}u`Im^=gk6z&|4;)c?8}!-N8ivRfRX27>sK?^+ zy`krjy@g5jgg|ITiMQ|^YSbHw53!sP|H?7h6%QrDo=A8>>-cLqs)1V~3K)R7ams|F z>h)K(qwKF}R%Z29i%;iVd0cbpG$Tbb)yVOsT3*TR%MIh+yZ-AZrSQ8Mx1J3CR#1Q+ z@A&O4Ax}08grGMnjHWg7MncJnN{`q_a)LwzUA2Oa>y0s zq5WORPaT8Zj*`#C(8RT3M+NdfC4UKjXTW?Uynfg6d(mWngRxtgau}d)L)FMFQkxhA zFK$CSWGAgsl_O3H2l1ArKOWoQokPZsu$dZJ1pAG$-^K3LVGG%>R0uf&o0i3DibnUP zIiyaz3JNpG)4SdhQxrpSX!8b>LmIp=L}<{u@gL}ffEydF?hBKJm8g$I||>TngVU3Oj<5EIk+@z8vS9nMD%hRY(9xf^<-V@9Dlm#+tWihH=46ajDhb zQXSkMPB7?yWa*}mFts*MGRzNxQ2nYC+#mhOowsZ3y$28Wz=b|Eymv&A91?}?vwliU zgmr zs=fW9v|GA@CK}!*D@H0EH8ttG7&rP7*`XYPfdwrLMPh-eK|l7P+Q=#ND|@M6)%$D$ z`ovhBN#%=*vC+*n*wu=v8$Pd}*w^_DUw4i9P}1+cbZDAn<}~X40K2FMPZWX7>mk4* zJ`mLnrd}23%se&p8{yY!g+9IIhMR1@z0$EYRa)4ZQr?is~1gYWSr|-S%<17&8sVYDaI4bCvDDwevR}T3_;ks!{*+#ytmHBE!ZrMlvjaapZ!G=Z=dm85j|G-dAlf(>$kMK0|n75Cm$700&VlH$l0m0-`ABWQ<-qkjHb|ms2m6XRqGJ z_up{Eht~TP%%udO9GdpXmM$?DRY&ZEEj4w7dG4F3rj!i7G^uNdL_?YUwUJoJ7@V;N z=)^b)s)GUvTe-4e`@j7))%omb$PuK-uihbX=NEvdn*t0jIe)2pWl@pZotPt9@gQaeNb&_;PlD6HiXJB)NU?S6mXDhtmK}aG+c^ zpqWw$#%=k^VL;6rG!(>F?j(h5e>yS=V>rks!3nNQ_q+4upk8elWV<}XnK~(LPZ+*> zYQ(rwnqO$t%qpFjG{aNK`8Xy3vU~d%kWFVt7*v6d&9lN0;RZqQ3*~mGXRPETFM_p6 zP+OK5Wz>CTgo)}(GIV!*~QkyoHZTQVFBxuV?`x%FHM3chZPh` zv^rSc5`~=c8RLvK*g($nCgIzTpxhB|1~E@()0Uy+&&h3=6-F}SLoj1bjONFS%6g*# zwbICV|GF_qZ=ak^15BA)PK+{9%b}zI@f=FZvbRA;31ih;bV}d-1>gDoBu;Dnk2*Bkr}Az> zK_H=5D8bAsftVzOU;Uf6t9yBy-_6Ne5!t=GJ!K#!Ez9xMW2y|^ut>%v%R}{>e7p}! zcg|Janh01Lvvcn~GNjg)rgJc4OfS1n(Ry4D0YADEiiRHKuMNdQV7`LWR?6|H^Tczf z_r<)p&F#r)d4o8i`Jrk4h%h~g>^n8C&WfEHzuW_1s5~@Uq$^~-LC&iMGQT)hTjaKc zJ&_V~*uLqaIPZ^Nt39KrAuD9!0Y=DX10Kv1QE@H>uU@Lj*PzmS<9j*#Z+t&)9v~C7 zO!Z7-<#MXWRwC5$7_GbYrT;SO>~t3pr~tihz~v15`Mtwz4<1fNhKltQvG@%eL^4t^7&KfQMHO-B!LsYs%ZG0p``B9NhoU zhw-Zw{2|hWYX(e9?eVWXJRka*INZ$U`moojvB|s{7OR!&>7g0$(sj*Y#E!mQ+&F!! zI`-p6YI*=0av9Z~x_fD8Al#KdDrhrOr+`13zP8cY^o!9#qtBSn9hIpj9bnw9Im#Me zpE&ozh0*RIT0;+b!XBSrVonTrw2h^DzU{5w4BsTIl>MwP8fSm?A|R!9T4dj|Lw*-5 z{T}uFw;g^oRzjtHr?QMtxiUtR^w^fXWaoatd&sU+oMXD2=+u=YfS&L;O<>45g@6Y)a*AD|Vw+FRInL#o!%UT(D`{tpLa4en|GBl;uhgCaWW^_8?H1uUs<&i%|AzkvtE%{umVFq@~H*=yID032*RPIR9N0qfPwQx*m< zQgT{7h$rCmLHm09As@T@}7#?#+0^l;Wtlul~y|noF-cC0TRf zjo~qJ7`xJM4n2T&)>NjxS7&osSB|*>jvkHe7OqiFE{}PCtyM<+)W%J;4=6G@$_YiE z%lRs;&8@1Us_NvVNmG(&D;SJkA5L#wGy)G_?`uyv+@bK3eb|2&R4C+=nNMKbP)7^Pu$IBYQjkraDdu%QVG(h Iz4zvS0RI0EkpKVy delta 4361 zcmYM2c{r4R_s3_5>@$RnEHjLC5WYfUtYz@E##pmVgpw#U)_V-HRhY3PTThmfZ7}wI zi=mqATZ^)XBvd4x(et~m-~Gq==f1Ajea?Bm&u6*61g-^Aym?rZc-oVi$bLK;Oa?{d zVje4CB%YdQNyTIlV{cG0r!DipZ{IfSMMz7+kqN-h7+kDhd_asNC zI;dT<-=qGbP5Yr?3dtRB=sC${p=hCsGv-P;T1liOPv!iHTk@@s;>RB4sAweOXS}#t@#`>L5^g<-T!YMDWwKx=*{f^En5>+mgY~b0wvw{4#(TGw+g-+h*_-)k=YG@Wmu%NvwsAgdN9SJ#NBpA{ zyltxHR3dPGVpGt#DsCn5=RWvSy5Nyoh_a+)6~?HwDN2TyT@1!WnEXzN-vDb{Zu^@N zWgwoOgi^8!pEmF;w}#)%t*Nj(ffNy>%$%NOkxZ%bd^vh3xhQ5O#rTYWzJbzJo_Dk;DYtnT%#x;HlGS~)@JoTgRnir(9w z`rdfIm;JKNRh3Z>j=-v9fY7={Qs|pyymd}V^V3WC`Ym9awDK7wX=*dt#WiOl!Dq>r zhR2=w<`@30cZu-qS!kA87QS8wkJ~m<(``(Ml+I={s3? zHIhYurv+d{JZP_D4H;slV2N(MNPvX2z4<;EzQc&0p72iTwMrrBov3 z60DRZ4caXPXRYSy#Egj%hM%#fF*UaA()7zg6bB_xi?c$i4^3jgut-2DV`03zxW=?t z7DQi=yD+U7oQQs2^Dn;)w`iY|mZ)l)!5T6{otGwC3MAgeu&r`IlN?xAxU7=W>2eY* zsTKuxy&RxCbX5Ysp4R0je<;g0hBpuP z&EjJTW|?U&HTM@t+-!|w=m!aQAf-rwW~V`|_yA;KTSB0ieyy0V^!T5<uW9&u%L}_w4uaN%Ik0afE7i#w$RX@aku?f)^=+DTB!|>^MHo%GzCxFTMUVLP(DSDWCsjf)z4}xTk z-8V$W2C)7zRA3Guff2dr=ma}G%}CbaKM?V`bYs;SGx<;H$L&_i{g<5>SBsNc@eL46 z9{PvtKc-7U)PPKY89 z9JKMuVLZfdP!hJmGW8c)Nn-fcUg6gPKeWO2;QX-HmdtgBVsibwa7D-?-N)K*EWpx}Vo#n}O$yF`ru-|n zF3^m-;FO}+Y0ts|t^AJXqDd*2e2qS^T!6#8ia#3~47d|M_o#NlVNlhTI}r{no^NIL ze0)bly3_9OFo^*W>X$U^035H~j7yQp?OJeLo6CHm!$@(XS~@ZJw7ux{6zL=k^1YZ~ z&Bz7<5v$y};Q=G0HtE<qdfA6$?_Kc*sbtwuP zUa5PK0uIt~&Y9bDbJ}>*TSM^k}9%qL9ngleOT*6zEs^Y=yRgd0t z83eBBjBtgNzkP8_{zq_>cC54fTJEEo^2_Uv!7f;F4Y{Bxt$U3UX=A|-vYE|1?1(sU zGtUqx?vXx6WIYcG#J}&E_gfro#a6>};ih}raq+AZC=~a^4PPc_rlq@+uBdEW!jJ2N zl@=p!eNlmb9;es5D4cV%_XzPP<1(8RCcQHHdzHDbooV;;)Qiq7@S>5iXI$Qf{@bSo(CPYj zoFl~~BgC#!*GfaBLpo_ud+GdPjCu_q?1)^j0b~f2yI_q5J)JWx@dx z3Z|~=t^cpmwi;+u4OeR9lcV-hhaaI#S@ar)vTo3O+?{T?r$a@}Zg2%nRF{baF`koCy62cl{7ODfbx zcy5ovWyOf^yk(2Q`6*R;z)!{!n@(KrvB-%iS>{z+OS_hUL~&#Mo0jh+FVvnWFkKRi zEfU0BUv$zGC8kI@C6I^^PTEa}_*f_n)D`aqGWnq_=04*6i^B0OPth%|lBJI^ywuZj z+!Ux%-hzEdS0?ty#|DAl+HNmLhu4RQfX70M5Oc;WWuME=nV~G$jrH8O`l^s08wdB8eJI*BJKJ966POt{rMcm+-E0HkDFr@4vGxMT%X z9~$D<*KMImq&vbI#wV(C>K|H{WN5XI`-#>+l&VP})$?%D%m7h7gA>etqM6_m(=qrN zKK+8q9(gzs?bc7MVpwQ~6L1+rSgs{5c4JfnHZnqW12*b#G2W-f#CI1{O|QX%1nSqf zo?|8bTS98i2wLuUqIiR-RJ?<1njm9ps=$H_`K#8mbvnx|=RE{c?F;&^cFi-qw85gZ zLl`i8jL1WDqzqx#e`sgC@pe6$w&_`(s@GUmKwsDTn%2wP{|^}krvhpu+kL9I6ZrW~ zs4io7e!|x(K9hW)HfZqOUTGJ@rq691ZF=Vg|CsnU7?o7Y;^Cwoo%BFo7}g*Ec1lxO zN;oM~8p>H$#BOAe;7edQ>nXNlZKf7gPq8U;Q%Cu_U43oD%b-9nCt%D?-4Xz1ciP1GT+**6rR zVsop*eYk)iU_=%id(!=U_+kde<|NI^m34e`%Cc3XT@T7u7(=~6s4HeI>jY!`MNL}*_bQ`)Y@ zJ+&^ht9sqPeEKfn`lfOMNLs72tX;gV-t>F+B3fv1d;DV(Ib1zQar3Z7OUmuqcE_Yl z(9sdd%8dC$t*Jx=l;w3sLbnuoNEHPzJ+C3b>ja)j*vZ?W{4?7Kv-Xdc0|ub3oi4SrJ}@>9hOz$BxYkN0tk&N}5KvJ5eTlo?Ao2UgyLoIS)RF z(ePGigm=mC7U?Y3UuuCW^i6Gx$}ZKeytRHO^voflcosMGOJkNvi7nfMC7K-&6heb8 zU*H>Nm;04K)vXi-5wH5_NArj+J3c)70c5e)!X0ya(N=Xw@T(L3^U8b$%1Ug+E&9OcGUn1RxOX*{bWdK?b{UFrczNm;GT|Be%iRwa$s)h?kSZXZ zmpZ6|#qFW%(T|Dzqv`x|a`v%rg=qIJWY&5Hat=AI^6nkv;8{iPe2gM4L29|xV9fvb z%L1k0491Y-HMGGW$-Nr-U~e+3rWSabtgk73=5Hs#AdnRk2*eBe|A!FWOeSig!93(H MO-WYTx#Ktg1s!`2d;kCd diff --git a/summary_results.json b/summary_results.json index f181eeeda..36ea63ce1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1292,7 +1292,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0862, + "Performance_Result": 23.0215, "Performance_Units": "Samples/s" }, { From b18b3543435126495b8106dd58e9d4d708dedf37 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 07:50:32 +0000 Subject: [PATCH 42/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14570 -> 14568 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index bc852a548..7effe53e3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 07:48:14,582 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3270 INFO] --- -[2025-02-03 07:48:14,582 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3270 INFO] --- +[2025-02-03 07:50:27,251 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3286 INFO] --- -[2025-02-03 07:48:14,582 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 07:48:14,582 submission_checker1.py:3307 INFO] --- -[2025-02-03 07:48:14,582 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 07:48:15,745 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 07:50:27,251 submission_checker1.py:3286 INFO] --- +[2025-02-03 07:50:27,251 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 07:50:27,251 submission_checker1.py:3307 INFO] --- +[2025-02-03 07:50:27,251 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 07:50:28,286 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1838 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1476 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3505 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 33f752bd7..af368af27 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -583 +584 diff --git a/summary.csv b/summary.csv index e8b1cb8f7..6bf5f1bc0 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1838,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1476,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3505,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d043a5833..bd831d519 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1838, - "Accuracy":"acc: 75.400", + "Result":10.1476, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4ba5c19f2300792c8f540de1d8b24a9850bb6752..48482827c8d9a515c8661e8936f0918fb8087d71 100644 GIT binary patch delta 2037 zcmVmvo3YrN-Glkg)Ue_2!GI1ql{U!lsr%slv#Zwi>ALZGPO+61=P zr<7wW!8<;(mK?(V`jxSBV7WsQ9QKK#Th`ZKtGm^9=lI7wj?j!`G)$9|jwY)eL=ry@ z!enyNd3?B#oDQPQO9C%S6LQjdCA9P7^tWS58RCy7^rZ95*z`zIsQ*l2kILzkeo`nD8q+3e(n$GDl4Ns5R z=}j7t=vZN=$4Vh6g#(`?UKXZx`C}3?S{L>S%_CNqW`rh0kPb-3-BFj`jmJ@#kh=Jf zoMa)Pihwxq;%U^RxH@mJnDa(gf9L2MktZ_Zjzed85VG&}mEAN-v!CQu0Lw>l$e3vL z*~|+gZxlu$6CZk|jKp>6E%&ux^L>{3gwj;lfBL(C==&rhna9M%p0Ie3j@gTs30}QT znP|j0VVsO0?s9|cGE-J1RaLdRJn)#u!-hLb@uQTIK)Jc@rg5AmRGB;vfAbL!y?i)# zEGe0VLFh@7X)ZmGgfs6yNC6ppJc@>XnvBDVz^-(4M3!6DSAdG;qjkkQn}k5(Jfd&? z@92r7N%8l^mayTPPl7l9ygI)+LzmkQ&lhr{3LG7f8-Uyd=(~9=x8B zWp8B{gf&FiY~l%b_ZDzlmUpFW4oo4%))T3rUYGs@3;7yZp(fp&qu%{rU1Zyqjx0m! zrWrx-)mu6Tm07z%e`{{k^7Gudb6vahb%14Ec7^L;uUo|sZ^5wbLy2ZM&tn*bjn z-IiV6%~pbIE^uYt(rknt?tb=e2gCbbe>gb1xxelePphK)e;d~629{NXCNy?_@dzvL zE>1#K3voP}PuHZvh6`J5Rn|;<%~nH`vE@1|TMd4%U<)_ein&tX+6IvpS8@{!*--%% zk3?Iz05ihE#B1o%MaG$dlUorsE{C)99j?b2O7{HXh46U0sVmn^amQy zA7}tyEO@$-e*=6mz!w92F~AoCd@;Zm1AH;S7Xy5CfUgel)d9Xbz*h(O>HuFI;Hv|C zb%3u9@HGIw2Ef+<_!y2k%8sPC)AEGBjw@BaQR{iwESAzFnGa{0thqAYf3_`IA*_0y1wtJ8rDW(&r*9|) zLdN@%FeV}ZP;r|Oni|GQz3o;dKi9CK0JVAVeYFGcyT#kj9vF+)thQ}`zwcpfuZ2vz zBn}ZU!vh7(@QVV*I*?CR`|fXh?D;8Oe~H_%+-zUZn!w&y&w!9Af2+tSDe9*qzBB@D z<9=vsf3(^t*DV3=R=#&b#HyJ62c_5y*eb=%we4_VX2U0q1A=`avxaX(yMn#@T5`Y^ zsqrh+Aq&{2(LUkF!xzkt;M%X4tsu7R)Anm-D~RoqY`1FefYw6U+zIU~xwTz58_?Le z0MhIxq*i0H0%|BJI|99gNszwK=DXkk>AeeHe;b*$5wG1e+ZL*Y9NQkMWmaEu>+A~I z!X5A-WNVqdEmC`_{Z+^HS%)ousImNtC(M{2MTO8X7_9fzAI%OM^zTpX!vz3JwWw(> z*?;|)B*;a!YpPF3vp$$j<_V`8CPWyJE?Rd~ooijccJv!3n&J;1mVGQL``8UTZvl>~ zW1JQ2AEPQ{0F8*e*QC?>{J^xByNUzTy{ysOu*Q&8aeVtW`EyiR6^DY0silr(Y&vUG zOG~ya%U%sFF%JkwwGg`O-43sNw_IWgAM!Wpmg7g!hzqSWZzWeKWvBlEP)h>@li@cS zvx6!_2nDXGinb+_%Q7E-YQr!PgztR{qq{BHAx#ms18vEvfkItcdK1zn5w(7h)H;6q zDo)}g;8TyQnf-QlQMuf!Hu(kTrO_42GlCM(+z6?+6?$)0=>tl9U|O)&XsA#R9xdy; zl3U7+gI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf>rQ2eE%lJ3j0xk%-#|i=y zEHHckrgkKv6H@RIX@6Wh00hU-f&v{p&a)g}yaAl@lMRD%?~cPyzq|5dZ)H0000000000fPuG@ Tf;c__3zNY(BL>Db00000o8Z~w delta 2055 zcmV+i2>AEtaq4lf>mvoQ7m@N-lkg)Ue@j#2I1s+?uTW(#GY7w9*-j{6iVA_EhSw&r z#hy}*tpxA*k+tLy_SdhBod=dXB*9@%6y37E{#xCwwmZi^=23`dG-W{&pR^2BZy_3c zi66w1lh)(Ih0<;z#@*O=!z89BtyjugKTdx;W{e~8Xv|Jp&zw(>G>v)BG;*1me@tmC zG~*S5W{PbAM1%+_n(b!Fcq$+=m1J0_#K4n?RtJ0LRm`c(CO@%wE(!23E3}RXp z|B=x&piGkx2W~VC>l9b#H5Lorf9UEQeWU6`MZ$6DEb{~Yy}GiSgh~37zDi)(C<-{2 ztv;K%LFkTxFyQh-uZ+{ED!moHmTbOH6OS^MNc&HJmk>RVhBS4#yx5f%50Wu|aZ|~w zw+WYxIHz2Yk;GkLa8+ihx}xj4QI-2HcSYC;M`>P|FzRbJ*WDzFl9*|ef9GK~5}}t3 z=WSbwXMrEM%4C`;4>T6c`wxmw$F7K?p_jztU?Q%!E?e-d$nzvWVpHL_la|P0 zmS;!n^ep3v$ZqPR(JNXKe?#3HG&5DKS}Lk;FKUQjsQAHnoaMDfnM)}hhHm6y#nDGf zr&Co%!JMaVsI=$KNHRV7C5)GI?NJ#*Fro-v*W>w9D>!cjr_Ay&F)CT5ZH$4Y!8XOz-dMx0B$k8OA{hQpu)V!l)HWp) zb%gP^vJ1iz88)4`(%roU+?M5CE1CmKO0o4ss;Sq7|G+}NMpmjxH|MB#|5q0|j%^~F zDBUEb2)=p?=b$2Me>Z5YjaqS@8+Wd1cfJm=YN{RSI@s%0A@Z$K7X;ABb+51PhlB22 zzjuTT;gz;Uh;G=9(#O?&VMGf3o{))|mvWIzkf`wSS2S zEAB2qLf3O~G@4J>q|$~9TOD0BEN9JDVyM{ev{$wgaj#$-*V>AO(%jkxnU+ur6D*bJ zfJ#K7BVB+gqg?f46N*RtT$}XMqrfULhHJ)9D*Z zj*yCeB#5XC094#Mgt~@tQf<3+#mfv#G@v%`y{~cLeYbe~*#l$on$@)J@Ao~d?Uj&e zmc$_fW_X~08Gcc~*az~-YTx~Bk6bTd>o0LTmYePCSsmE>>giKD6>k+Oqj~*o%P);U z+qfUve;TbO%5_74yOr=(W+RBr`n36)*$85@B%7_8JD@dCHg`h%N^VUT&Kfj6 z&VdZ44ynZ;RAiYJb&neb!;iA8IVW;z=_mNKqox39zjpK+Cz|pPAC`SAD*D(BJ8uDw zWXhbC>>r~tWB`qfyw{}D{`|l+pS#Kf)4i-Qo3Mtcx;(ypoBXwPRhNf?i>alFRcy7_ zrk1v9+qSbBT4E6pZQVxbvUfYY?%fKBEq%z}q+5<3Wg{+(!n~DSV~n5v2T)4`1e4!4 z8nc2bLkI;;sOX+1lgct5e{RAs5QOi2ipclGPEuNAB|&N{PDQFzs-oU3djYHQk8HPr zx33c*fW)cC^~`=dyK*(#sW$lq=cUm#$}@rz(A)^AH#PcL&eJJMd|+Cz)@Z2F9z2>g zcNMpk8wYRBSa3mtPa;Tr%59CdAy|sB=UY(BXVIafxHe9)5KFhgf0prYwgFra@_-ct zCRkv&2c~u;q9ao95ov#1I{*a7(1HRTJkGNmpS=N`@;4g><-)0C*ju=1J&B`x=jCYZ zx-RR=!5Z`Dc(r(aA68AJ?oY)5HI?9$J7B@Mroxv+JY36sSi}>rC5Wee)Ni~U7diBS zNMZnbhArzc?cX$Ta`}000<>;ywTX diff --git a/summary_results.json b/summary_results.json index 36ea63ce1..82e6251b9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1838, + "Performance_Result": 10.1476, "Performance_Units": "Samples/s" }, { From 33e9e7ff94d81c9ad50417ef1fc3fbcbb467e041 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 19:10:01 +0000 Subject: [PATCH 43/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14568 -> 14567 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7effe53e3..553277cf3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 07:50:27,251 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3270 INFO] --- -[2025-02-03 07:50:27,251 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3270 INFO] --- +[2025-02-03 19:09:56,105 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3286 INFO] --- -[2025-02-03 07:50:27,251 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 07:50:27,251 submission_checker1.py:3307 INFO] --- -[2025-02-03 07:50:27,251 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 07:50:28,286 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 19:09:56,105 submission_checker1.py:3286 INFO] --- +[2025-02-03 19:09:56,105 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 19:09:56,105 submission_checker1.py:3307 INFO] --- +[2025-02-03 19:09:56,105 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 19:09:57,266 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.428593 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40839 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index af368af27..6b0823e75 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -584 +585 diff --git a/summary.csv b/summary.csv index 6bf5f1bc0..3316f392a 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.428593,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40839,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bd831d519..04cb45e2e 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.428593, + "Result":0.40839, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 48482827c8d9a515c8661e8936f0918fb8087d71..afb4bc0a4b3ad0bfc6c790987dbcffe515c9dfeb 100644 GIT binary patch delta 956 zcmV;t14I1iap!TctOp8Tu(_Rt1^@tJC6jRu8MDs^js$;_Ep5;2r_ev6pD6fe^Og=d z#2Q_L9AXXTkkB9w2@T?qhzD_~)Nx4c6(A0491Ik=QE-h~hD0ZrLufUkLW65iLSrhB zL*;Me@EA!QvJ71Zwq1uRva0WU{Zr_@qZ5VL8Zsb>SfjTfiCBY4Bs7RbLW4*o;;|r! zY8{D$0~>#mSmR)@z4=BWY#9=r7#buIT8$_xL?YIhDkM=oE{UtzD$Z7o&?#NKSjK)R zTfYV6G4iX2a+f!HKl4Pv+k`C>@X0|A!MqU~#2cYOyb&53@}|}Ch8^IUug1ZE zasc-vb{qg4hLi&|EfPDppzI?`V?sQ!#?&C5+Mj=bXUS#!6nKGC?n^wdm?!D@;WuzR zMm`73k>hn>)f{Q+vSKt9dGikpq5B=NC9A64gnJm$yQ3lzOXsZ5!jKDYlGn4!f7@D} zpBXFdNqN)+wyW*?>V(Ien>>yr_JU#!YZ#WC9b@2Xsc=lbJS$b{^5 z$+~|cds*g>wy2mWcrSi+2uE0^L6Qd9Yn4xN3TPENy(sZMmcg##;OB~c{NR2t+8nX3ZWLaM6qA*Lc zEQvOs7R>QjT71fj?k7pM`B0Q(dpxT5A5gPe4r?R?n^WuDlgTn5e{RAs5QOi2ipclG zP7tVANs!u#Q;{k)RV!|my@1vDN4DFLx338hK;qQndS<_!UAbKBRh#^R^U~-F|vlL*qDa$BKY2$o{(`3@BG zS#;yM1PcrwfTB#5Uz>UZ8wiyZqvBryOz#yu;aljoOaB#G*R6w{OxDVa3+oRT@6 z&en&KH~kl&It}?Z@eDZTzWoICAF~TI(*g-!u(_Rt1^@tJljk;90a%kiH$?$OlZ7`t z0o;?qH$e?!DgXd@Y%gL?RvpqEZJUu~~pXtZ^_<;7-9cY8eupU=X3zhzboZLJ5tj zKoFI`5yWF8b;vq&9oTjus>rIo@BL4qH;+yfVsFTRBw~%;f+S)MCXvt}5(y0=k%-5F zB&u~J5)OZCNMenH!S?1GiLhlzbYf_bL})dltPqJR5UumkC z@fi6WFh`EpfmL&)smqGdROHP+Fof=R#Fnh8b`x%5Nbk;yL@b@NJ_|!GxJh2mD*tV3 zb%189v?t|R6WFe{@2evoZ*KB9lGqE1HH;HhCuXuBo(ueEbr#2H_rI%h5uWRhS0NL! z*Cl`JitJ^XKiZ;VqTs#w)hQfdnFdK3WUo~|#WA2&==7q*`&b6Mii4jk_VI)J!DxSQ zjZdEi^^?#Q^7?J0dDdkw#RU~sud{UK?8ZswLCl4a`FR24rYQ<(b*`tp&XQ$$rHjHW z$+9Hcd|EKaV`=dzFS?&3+2%u0lI{7Z-hZ=L4s0X@uBnQ)lgKh4e`>=p5QOi23ZuI% z*&$64wgYX+sewXWT6z=GCK0uMkkmSU`zlW2B;ZqztC{_Fc2T+9t2X%s=cUmV$}@rz z(A)^Aw-tJCR_Oyud|+Cz)@Z0u4<0S+yOLYVje}QbEVv-SClRDQ<+eh*5G=*m^BpMW zv*^%K+!&`=h^5Jx9LMUH(Sk{Ey<?~cPyzq| j5dZ)H0000000000fPuG@e>gq?3zNP$BL>Db00000Pn5Ow diff --git a/summary_results.json b/summary_results.json index 82e6251b9..f3b1553d2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.428593, + "Performance_Result": 0.40839, "Performance_Units": "Tokens/s" }, { From c70481ffd57a8b918c4aab5dd12153451c105ece Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 19:24:51 +0000 Subject: [PATCH 44/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14567 -> 14568 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 553277cf3..dede5ab81 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 19:09:56,105 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3270 INFO] --- -[2025-02-03 19:09:56,105 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 19:24:45,640 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 19:24:45,640 submission_checker1.py:3270 INFO] --- +[2025-02-03 19:24:45,640 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 19:24:45,641 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 19:24:45,641 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3286 INFO] --- -[2025-02-03 19:09:56,105 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 19:09:56,105 submission_checker1.py:3307 INFO] --- -[2025-02-03 19:09:56,105 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 19:09:57,266 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 19:24:45,641 submission_checker1.py:3286 INFO] --- +[2025-02-03 19:24:45,641 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 19:24:45,641 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 19:24:45,641 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 19:24:45,641 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 19:24:45,641 submission_checker1.py:3307 INFO] --- +[2025-02-03 19:24:45,641 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 19:24:46,896 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 6b0823e75..2550c37fd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -585 +586 diff --git a/summary.xlsx b/summary.xlsx index afb4bc0a4b3ad0bfc6c790987dbcffe515c9dfeb..5b950ba8510a1f9885cdb3a19940a1ed57a699b0 100644 GIT binary patch delta 340 zcmV-a0jvJ!ap-Zd#4-hRAElQjlgTn3f6Gq8Fc3xed`0Aa;yg+PR+379x}p*iK~(H4 zd)ivW57~yae~*)#~{p#wOysP9T<2-6nctTtf102c+2a)g-*?YuVxW5;$NsY`-G1#zRTq&}2(e~S%e z-*gK&Pt!S;;3=n`;vN{AiHJrdXA{!=*meMLhM@%s$~&AT89sYGSm|yy49bO53V$$g z(|Qso_uh%g*mYgf6@xY8&+vNn`aY~03)P>B0jf*R2(v)Fwsnawi*UG>I=>1h-U#4N zhoIkh8y7kD0gplea*TUYK8xp_Aw$sS2QBeVHeKk};o43y)Le-lrD mWA5uuQ2zn53N%^-1#};!mnM^GI8*@(lfF132F5i20001pd78KY delta 339 zcmV-Z0j&P$ap!Tc#4-h&Q|sI%lgTn3f6Z>fFc5_AeTvBU#7+>XSV@rDic^s)HB~Ea zmc4-0_(!(ekhiZ15J2M8<9cSlon5(H>{Xlmg7ebo3gsC=321JF)Y}SuG|Ti6B|b1M zSZg#?s0WW0^+U-m<;KCAGZtKs;FAc_o^o5ET?m$9?D-B9^I3H0C~k~XEX2}nf3ao! zn{5FXgv_vlzyu2nAAqSHiRgqBd_>wG*A4)|F|?pS2aoeC#}{t^r~KW9LAi1&8F~wM zt!Hs`@4Xz2UDsvZWU$8kIsRI`z7MOWQXfvm0oA48lsjO-xVpsGMLb-~d|1U3ZzPDP zKI(VgPKzA-KqN5$J;psNpOfd8C1#20f)vw~6e*cB`J9qDozB*WkvIJppgIltH}MQO l=Dz&|^&hhfG+G1&n^WuDC6j76Q~?T;zc?cX#Wer`008gfq8k7J From c000c626bc04347ad40df9cee78903c03b912568 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 19:30:58 +0000 Subject: [PATCH 45/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14568 -> 14567 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index dede5ab81..c0e097944 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 19:24:45,640 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 19:24:45,640 submission_checker1.py:3270 INFO] --- -[2025-02-03 19:24:45,640 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 19:30:53,492 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 19:30:53,492 submission_checker1.py:3270 INFO] --- +[2025-02-03 19:30:53,492 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 19:30:53,492 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 19:30:53,492 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3286 INFO] --- -[2025-02-03 19:24:45,641 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 19:24:45,641 submission_checker1.py:3307 INFO] --- -[2025-02-03 19:24:45,641 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 19:24:46,896 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 19:30:53,493 submission_checker1.py:3286 INFO] --- +[2025-02-03 19:30:53,493 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 19:30:53,493 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 19:30:53,493 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 19:30:53,493 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 19:30:53,493 submission_checker1.py:3307 INFO] --- +[2025-02-03 19:30:53,493 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 19:30:54,554 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.14067 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 56.2054 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.4416 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74391 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2550c37fd..3aaf00170 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -586 +587 diff --git a/summary.csv b/summary.csv index 3316f392a..be72b34cc 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.14067,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,56.2054,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.4416,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74391,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 04cb45e2e..83cca4537 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":56.2054, + "Result":48.4416, "Accuracy":"GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 5b950ba8510a1f9885cdb3a19940a1ed57a699b0..223a45b4c3f7b0bb4cc6d91b99fa8f0121c7703a 100644 GIT binary patch delta 3587 zcmXX}dpy(oAKr{n7Bf++%*M!Ns3nyACrFOG_iYg^1^%584Haza^DhGC>q zXqJ{Em*kd;5DU4^!I6Gn=ls5ZeExi1&+~cS&-4C#XD`oOW?z&N#bdg04PjCj#VI^e zX!VuE^{6)%d#m%BICR1}qHW7rb3!oHp#yIkOeHZB>}1rbQCln}7triltR)lFKLXV! zK4FvIx+FI827B`y$4KF`+|X3cUzx6<(Ot5NuG6yvn_`uH@w&GN{~b8W8-zsX3EWr=yF61>d~e)&A_& z(6yBp(I2h99Z+hls<+J7iwbNE$Q+QmNhmK;ss*8PZZUi)f*g1xAJ4EU%=N4XgNsHh zW$N~}J*YKzwQ-w)X9aQ&9Dk?0i_d)aE33e6+LaN-+k!_v-3-ceq?Gu`E%G;&d=ft~ zy;%A7stpVzG-uPh=)wKH*Xlrmq^TdJ>_;w|kNe}XF0gQ&abJ_-IB(UK*? z5Hw_Pw*6NyNBAK+T13?o*QXX_5J{H#@{F4gYsd<5qJ#Y+TiZHm7x0}qDb7U&N(v|Y z$kIAugmkF04M`xKd#k=*Q3*pk(WdB+q~^gSCf4>? zutZ)7PNe^!Y=9~a8GYWP2)hxM)t!k~{CqT-a$3_RsRSBa*KEjzKMu??lMEh#lZaf^ zcZoGzc>iqH8=bd`IRGkL2@`mt4epQh1yD!U)XiBU^@6K30JW`R7nN%J52%K^hcKF# z)MF36Y{M&+^^&LDPraZ|R2T|rt55!k(#tB zH&@{^0mWaQ=={jdEOlpE~EfXcw||K?DR92$y1YRfq6x07Rl}6_8CIr8*v20r%gyM+mV?XNmJ1FE zy4K1o+QVSJnx>p0!W;_OuUH6%@~Jsyl+sDA20Gc8Z)KDB4VYEtfu)$T3)viVX!~TJ zT)g1~Otr^d{E73znZDw%?32W)mgesc@w$!{N`iHEPC9R{{kf2P|L*!z zBc_Ol+z^V&Os3nR$gSMbf zGwR+$Un=O%ynKUi+7eC0Z&>of;-)-#6?XN22t zQ=3N1=>pZJ1yOBQSm^QoN+E?U()OU|`*B}kTkfs9dQ&7l0FWJ!;*#i8SGzii86&yP zbmRx!_3wE+S-;gxyvH3p-XTY4)^N@12BHZP>agBuGE}VtMw)GiCWFg*kIs$#TDoDL z6FnRXnUmX<&h~Z-WqaGm4kM1WK3{j|??20Nb-EVKNR4=c7^2h?)vNS1647~Tin&qI zWVON?ps;R(YoxkBx2N1pOe^d^_=!=^mp%$;8vriUAiS)v4n8)I#*Dp9ePdAJA3JlL z+IO=>P0@k34FCrE#Fc1~>?ZG4nU5+Bh~%4&9 zL1F?1k0V3JvDqu&(}~8?nxM$7C=^6r`UoG;TPL-v*YnhEG10k;JmVhOiG-3%yUYf= z>pwWLidC~bFR4oD(MbqmRa%OFs4xdksYm(>*)TS-WD;^|i-~dp4X>Rsg8uQ3zp0mA zmB#^qDrV!yZIN%m{*!4#v{|C_-p=^Wm!9Q=>Us^lvQ>Ayyfqb^bnog4)+ zf~Ud=la}OHPDc?W>R=A>WGEYJzX--9UIg&STOBBf|G!EF1u08;LDYH3&%bPW^*yr9 z357y;{&rIH{-*4HySWccDQ)^mCCYJ-w?^4(HR>v{r@M<2yZaun@Nn2cX;B_-t1*5$%+B3RQCb$0w=;k8|H!j%~r$@ZOm z+IXC%7Pj`uXmvv(O)|dezH?(qil8vVxbxcNBg4ls1v-=Aq{j;R2tNyAPg=^<%POE- zL#Z=dyeV*AExnfyjWK>H^676yV~4#Z(gxuxw@a*gfdznJuyoOsr_&`pT8thnQ4+?5AsFku#yIQ~KY%6{A`0{sc7z*4C6-QbHQIh}N|1XdzhCh*Q2b2Y zHuL5vhR=wYhx-WSi_83K0@B{9K}T{YC3n#u=8Wf-*@nA)bNg9SGmvk*KcZgH8;h$W z<`2Y@)pGSIK!Fv5^??E_p7%3stLT~WlgqI!?F}%S1=6-)`MHR8(WoD0CK})Aa%s^} ztYhX=J}WTQ3?1><;59b4tExdNx)UL8L5v2Z*IF;+z>^M^{b0kWZ%TL&^wyYy3)pOp zZy-OVaGH_pJ>G;pyl1Qsi?NU`;RR@3`@Mw{vhzNkQE*;6-Fg{|`h>6XQS(>m&r43=?lU>W3_27-fh*TyryEiI4>Eui-)rBC+K!-aKdEXdXkLJn^l?j~TWl z4!F{2n|cOc%uoHgWRach%mO^EJ%Da@pV`her|SYSse>H@V!{rz{|b!tUTI1dmArKX zB`LL=6P-vWgu9TSX#uAxc+|0Fv&OHwybf=Sp0B75$PX>}e6CnR|B`}Dpo~hmf;^*W z2N-M#=&(KxB2h2C%tV-#)&RRC$=IhesTN{|AC>eRz}U~xr(Y59kqZT}CXow`{>j0z z1B$}w9P5k7D7mKjWy(VrWjil}k$pESf3|7LL~%GVfGzTXE45&Gz?Ith+eU3w_LG1s zuBDv;?UTE%h-t4;-<-XJxr=Gd?sA!5H!OG;?EPzm#GM(gD2|d>JgM*(;cNONQucQ% zoA$I6g!)Dfe4LL**cax;01vd1x}GIJIgq_iN(79O0SFJ#A?f&HHp+q!LlXb=Yq>(mpyfVNSM|Dg|Te93+Q~J zssl_FzN?ts6{HIHJw#bzE4oV@`-C_%YY-9s#2Koeb*f~({LsQmG9+U5lD76r!e$uz z$o!(+%4Y0mzK+g;#tGKO(YcE0pMCC^U%RwMZ1Job}Twm=kCAMYle)w3%f?9e*AV| z(>K4zm7;0)W8_j(3*zst@A(P0yMK9?_Wt|@aj=1i=J}pF)DWQL2N&ys;0sjp_;?hh z+m3q-PFpmzoQn59dQXCdEdU8ERInMQW4QQ_?j0ue`$9jXj)uQ?`A(mVyV!cy>*<|E z=i=?8)&AK-(@3*DArZ8J;zKWL!{QA3@p@Pq}+VWO2s~K~4rTePHzs8potY z0h|;NVs=9cHRG0L{~2VrS+%eH$~R~HntY9W_SdP;v1gFaYr39Y;k@2Hv*Y=)C9|r6 zBwhKHddE8QtpeNE(`jN=2~`thjGLqnAvyvSg_^&mS*|Cd83(C9CQ$|1D}q>+MpIj} z3^(hH{QRoz8-=uY5v{4Y2Ywzc(}!)IT9z1|dATjxt=J%`i-i39Yo~U~2>k!+3p?!$ Q-kaBYS{cea2?Dr#$T zjQURG7!{$=D0h+c`{?`o{PFtp`F=j%ujljqe0(M^OLd3H>r!XH{ zNE)uoee$lRwicJHAKptVo7q}PCS{pb*=9*zCf*BWlCC6BGj->WVdAjxK~pRY%Y<^} z##0v*%qn7@A6)Nux>wE~zYHKOaYM-=e)pWybYZDESvH^aNG}_^sr&D0hw6w3#Lf(J zfAn9_x^*VuT~(*Ny$hB>K32u^r<=3iS~Hv=QbY@GbyElt@r0_tRqG<^@M$#b+1><- z@MsTsKvBizzxX>Qx9JKPz78LNc2dcLmQ}r16ynMI^#|O&0Ixdf$HX;8bz@+U4!Ltr zhBH?J)^V-6%fUYKx0*Y>Fki`rVKHL%k$g<$R>2-CUz%eZ&Ukae zRa5J&0L?5_rajrp4VJ0Of^JSdmA1Y~*;=uLa$C*%Kc8S#3q*@ZsGAky(o5vESc$do z+3Is6U0+yhrh8m|nhs^|`+c!xdXBDJrdGN{Ct5g9OEHiw3ZqiB63tfdxS+KyAjeNj z3T8iyuF)J(+;jUbh?WHO?|u{XiAq9ZfxB{ACbqG46t`SY#3@{Retca3qLl3^Gl?JY zMEHeRPHg`SUb3?vf`NRX;DM->r~y#})@0FSD1YfF0Yvrqi`rcL7c?uga;@I$#bgQX zNzoWDQ775RzEv}aee*nL$aCY|!jYYR8|f(zde+fs%z@^0PX$1QVHf}>E|nTp2r;J| z0S+fv@)OY_0-D82HH%J|0#DU=_Fa%Pn`=3^UIM?M)+^qE``FymX0P~6o=BfBv_~_p zvi~w%Pe*zwc&yh*cyLkcFv?n{G8tOCq+#R|aflv-DQqCxvf}45)WVO*1c?@>RShnQ zPU510{$;*)7GSefyk6*989L2+rJgOhYFVhMYIJ88$kbs2jh?_6Z+I$07v)oe?*5RU6zSjR3r+?*6I(>Z5h4X|4RGNH5*QtTY9RhAZ()`bT3}9Xj}O9 z85%V+D@jTe9Scp8>Vm{ljXUxI-n^$=x=G$dVUg9QS9`?647$rKUC^*~_Fh#+NF=3$^q3xpY=Jzc`#=a^R)}wHe!WLIKv9fI zt?s`3m1aiDR+(H|(Kiun1$Q?e|FmmnMOBp9ryC)Lr`<F-m-8vol!Z4hT3{ODJzF9UNGlM+TNwT=wdGU{iax z?~-%Wdv>>{V8>XJnZtnRJ<3;oOq?uyN*@c8^@3|G@%1Ypc2gNwz5zdt(H;6Toa@VC zo6;%O6}pt_BGCZRe@0G6R#KDDL**LJ^sP)EyCw>N0e0por7V29Rc0EoULPxK?L?1D ze+_Y>`_PUrR4A+TmVB)dGA`!5JK*sZywDuI?kNf)Ff<(*=wmvexnri0$nI+;)N&$-2!Jn96G4FwVtDgVy(kAMEl9xHT@=jb?|x17KG_?BfJ~LS zf0BfA^I&6-{0)2^$7^3%lqR>aV$H_t%zDN_W)TLMI1D_@01Lw~=|3Tf_*9UYbd-D= zG#O;ZmPvQ#mkLj3#igG>Mjn6Z#q$o>a1=abt_phs8n1KrUeKcRy@w>RiDS}M59#p} zy}{sSDe(KCQVO%=2ZV*M1M#p?UTM(U|CBCu5`J=V#w*u1eieK1Cq>=xT@jVkXcITF zVN1V~Wq${e>^4P4Q_oLYJVpdufDSA`nFZBJEQh(RQI*M%CreyBoT_|;2{3GoBjU-1 zJz@&r%jJ8gEmzs2fsAWXfje?Ny7a5+hY}8+OT?%GXMaM)xsLK)LzV#se|K;4*teiPf9og~oQ5U3~oS_-bSSd9_7x&u)ttlfJ;C{U+tRB%}w- ztkOV@BKcs;#zFzq=1TxYr7edWR6zDmErwyk41zG;B`5Gt7Zc(=01_dnAxoY);5ysS z$G5~Ay4shy@LEEw&GWpU&7VZ|C3I%b>uK~!k`nh9uqRsy!6SWWG+VcM(NiHGoKJan zB$$Ennd{$vODXRa=FxNXR*&;`p92d3X+>5{2i^zsCgr(jMe$j$wGmd@>F+up2I+56 zw&oz!gq7u)!!;mHCNu_Q$(->tXlC=~Jmp4@xi*TO7Ec0Aa&Di4q6gYdpIbc>qg^@E zbWv+1yfLMuke@C5r={cS0l4NZ&+g(!*u{!yCntZBve;pM$dmzV5JUQEfXyW40ASkQ zKytveomknA^5~shtP$%dww1rnp%T;Jt@Y??uF%+GAc>4?RY4x^ zQ}SkZeC~6m*Lx?z3fT3ngaxOfP$I%a!tN>sZcOEkL6K44B+F?kYZFT13zECI#C<`o z%+xJ`)=Xr9C~vG*?%6SSK=;qIA(y1S-K;1~IxX;)wEdlv`fMfZHfgmYC3zB?V@e?U z&^WC)W3@L|Sr(~AW9Xbm3GUKY6GvaX-R9$Lmy_t9bu=PQGp zpn^E|$jfujt{0OPDZ~DP!4t0SdwVq-QuVfl4C=EwR7penSj@iPSBQvh5;a#Sa3ob~ z%t|c=f0};EcuQ2{P2n?OOgP)0+&9hbm+6XUP)CQf7NbXp&cB57)bAU8?Lue0#!YDk z@W%oBN{hSop$~S-+y1)lq&juZZ@$J(RU0KuaED=0ET}vGU?j~c?^Ua&N;N*MJ*FMB z`H;BV6qG^NHPttbR?e;*oDJ8^;XU7|E4sm}J#ly%i}%<6#%SBteo+0}O;(JQxSGQp z8r_`XgUQA2o~Ocu47~F`wQ53g8^u2C0A#@szIA82HxuSeGU8|y7ozk|rvM>C(u~o( zxaKW4#QphKz1L0_%30prsa_jQBfa0It*s*OZzY}(hkOSSPw;`9)Kh`me>7B^wJyn!7(V_+CBolBi1r8LWOX0mwt%^eJtd_QE1C`1kmH>2N zCqw|ckc7(*#gs1j8F7T{rWd0(yF@L_h_ijwFEf6k+U+X?8KYPrl=bDL#(ksFgXh#7 z#alQ5LUKD}L#exz3Etz)ndkPp+&0F(W2z%X(Zx`8WEZ6Pbq(}*lsd8!^3U4w{2j+t zHWhnhcO6%0)&)jAjW|e;dbQy0tM7*bOC>^fT{PfZ9vb>ABJ#eQ@8&kKy3prwbZ{hR zVsY_##u;}V|5oRD+x`<>zX~6YrBp_LSV~UwUxuH@wP^4Pi-GdlzNFF-^JIQyr1S6V zTh9YO{DX3pw|lsd8^Mzi1-oP-lMEhjgs6k)Q&re2^N^MGCk{qp^%^( z_k=||kKI*Mz(*QVrUUgPI^!M1fc-(e5z>2pd`#pHUOC+(7kg-eHy5$IhObZ*@5@Vq zmRbLcs55SObR}_La?lf&ev{aqkBCuQdj3e*&h1B6plJk1a7v^DK6x@*-8b>8!JF^n zx@o-yT$B}lcipO!(L&iDiOlhIK*pBl{Y_wN_Xk=hrmFz^@853EVMFNu@6YeZNvL>6 N Date: Mon, 3 Feb 2025 20:11:45 +0000 Subject: [PATCH 46/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14567 -> 14567 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c0e097944..f26870e31 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 19:30:53,492 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 19:30:53,492 submission_checker1.py:3270 INFO] --- -[2025-02-03 19:30:53,492 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3270 INFO] --- +[2025-02-03 20:11:40,515 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 19:30:53,492 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 19:30:53,492 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 19:30:53,493 submission_checker1.py:3286 INFO] --- -[2025-02-03 19:30:53,493 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 19:30:53,493 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 19:30:53,493 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 19:30:53,493 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 19:30:53,493 submission_checker1.py:3307 INFO] --- -[2025-02-03 19:30:53,493 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 19:30:54,554 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 20:11:40,515 submission_checker1.py:3286 INFO] --- +[2025-02-03 20:11:40,515 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 20:11:40,515 submission_checker1.py:3307 INFO] --- +[2025-02-03 20:11:40,515 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 20:11:41,660 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -65,7 +65,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1476 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3505 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1749 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3aaf00170..9745b4bc4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -587 +588 diff --git a/summary.csv b/summary.csv index be72b34cc..171e436bd 100644 --- a/summary.csv +++ b/summary.csv @@ -43,7 +43,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1476,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3505,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1749,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 83cca4537..e870d9e94 100644 --- a/summary.json +++ b/summary.json @@ -1329,7 +1329,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.3505, + "Result":23.1749, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 223a45b4c3f7b0bb4cc6d91b99fa8f0121c7703a..124e3456141537790fac1711d773513d04e70384 100644 GIT binary patch delta 2113 zcmV-H2)_5{ap!TcW)21X)xfy8vuh4U4S%X-T^~Rb2b)AGiKNxTLb0@ZNC?>suzFYq z*d)*K0XBV74=Vs{`kWpEP7fT(*l9yH;ZZ%L8LpBb@R*7Po}UBawrV_PQ{%D5MyYF+ zY&l3IK1(D$9u|tF$3sF$Vt~iPGC(4Eh!2qH(|A|`Akk;>7_fNYNJcsV>6M*YkbhoD z2-yU&$5br#JjUifwO%WkUl5kU?1|F0a+p1)?ZA>KE|FAwSV%rCr$a)BWq{hlGJqv{ zdJkafGkaJ8z|tr77!Z5lNXAYZNUtn}pDY7pALzetDwchoz~!^7*GlFW#HFx(ly+LL zCUtO0tP%MujSpBza?l|m|;Rofg>5|1f*AJ)g!Ht zLD{Nrwn^;y0oH3J^9y1yl?=A5pY(CYrR%7zYqp(Bp6ifmVaX%w93IC zaZQx;cvwh&0P2tsG8o|TunaI5;PF_W$HNK$gFcVPg2w|#GIrYJlRJ-C2==EW1lL<> zA$h&kZk_ETT-dV}rK5L<(QthCW2qQ{*J}e~nT^$laN&`F} zmH|owJfHgvr8L0v@fGOa|9|j-Qt*7(X_L?NJc4FD(hQMgr2)w;_k3=Ve3sx-$=rgF z)B;JZTnnc|@UcV-QBv?>A-N6fkPspnAo#EhAjuxa15El1A65arv*oMMwV?RGk&N~t zr2!VgrIY;1F4)q_{`-5H0zOO2qM@! zwqkqdDIh-Yreq~E3<7ae-7zMKTwa}Zs5_Q8AWEt`EF^zbYKMdn!~k`NWdKC>EFJ*S z=kBly2%=Bku^{ijk$;Tz0=FgENZ__42_c&BEFN32yW>UkpWc(KWO6}NiYM{tmT~<; z5$lk4ED=kTly+Dsk`^>GrO(=76_84wwqrrtfg>61Mc$QUA>6tK zm^{#b-&Sn$gkt%uTauMbE(lBUTex(ZIB$nI3?54i5+w~D7Jo{l!9zlbWq`rMGJqv} z5)WYMQ+QYf#M0;RSa5jYNJe^ro08C~M_M6*Ane$Rg`IE&pR+PuHF(O%%94tZKn zEEh(SforeK`+tFJuN50So1p%vFNahz%b=^UWsgU-8gc5{$!=IY?FU|oprfV5!$OI) zct{9&4X}7v26$!9>;YbV8V{?0y!t#I3my*~$!ITdp$9}B(hTVg`F&irVz=k1bUyR? z&z2fK$2Q%hv-qg#xJMF49M71zJ6Z}pER;yWhlBu6mVc3WvW&!&4G8dzeTEOK0C@T; zbYoC_;7CS$k$M3O!3Rwe0%B<)No-uP3G^tj|HLn-WO5;(#v4!bQH@p{Ic}K5$^j~I zN3^tnSg0i}AQD1Q11unx0Z`e1092m_#3~@DJ`X4c4+xHAtQRm0pjnSJLn=e0pSWV_ zCzQ%(DSrl)OfHB@Vg1DAP3I<++Ce4JN3^tlSg0kf9}+?;1FRpG0V>IW+L5Hvr~S|h zK&8+7x&Pqy>6XYr-*IcZr83ZQyzyio)o7O=9B@Ntb8(U9%U6TtV1IX#&(HHtetEg54QIta zSvM{3kJgGaw)%C`FFvZ(8?5;dFZ!EJN8@EN?7`*h*?clsF2~dJ+TuU!CNGm0oxgOn zDt|USJHEcmYs+s}`CdK$dR4qGtI5`DqpxRgM|<^RFMc&2&*tOhx0~M2ifY%FgZ;^D z^y_+1*VFNTuJd1i8UB*I_+^;<_OF`$t+rPyrhXSKX1DHT^#?Yr{oSjb3d!{NnD<%h}cXPjS8WX(qN@&L($%yT}Kle7^cyadOXQv*q1iR;T34 zZ2tH9gzo@lV3L)vtuG*5CvDR+odFv$ucT`I{}2OB$WX5L?tAOK-?^Q+gijw zvJGk99w%+nR_cl4dS<_!UAdlZb(j5u56YSb(So21jIdIfb%WlQ^ZWs2A#x*mXALxH z2La96n_4&~tcO=`9r&mqWC>&f6Rts<7#+hnhz)2SisUd!Tv@MqOr>9A$Hg~a11<@< z#~LDkmppPj0CP7H(E%yNgmgc?8v#;a=s<&s0n>uwlQ)9b;cCOEoH@0MI|o;-M{#m* zgPM$e-xqx~T2p?CKNl}=I(Iww8hk4?J4m*7&?ghwFHVi*(|Z0{OU0`jxlC zBKtm&Sqi}RagWMJ_4K@)qqZdFT~5lJRLhbWFiP1i{TfDI^q+wGFy!CFBVeEV@)NXw zvkEjB0|ormz__@RC^uUHMw5g$JOSO4!8bt-WGVmvcx*3oba`xLbCXaw6q6%38UofV rlkqGSlTip8GC*!-u~YbEmw!cv$$QQB4xv&Xa@SQ5o0l4=hN$*1LXNC>eEPdcH^l$su7hh<84x^9h4GbL`lzwh2*}gLqbStfak+9 zKxu&IbDyD<26#Tc0^R!`K7UXOo)0^1@|m7T(5y$AA(E^#Ai3q9&n=SA5_~F|TM&|3 zAgPsW;dBT-mS`bL3O+0(w_zO;LL>tOAC>_m*~55%NuS}vD!_NPd=cTUyzF-&XANd>Kzivo8z&g^}-K3{*11piOOA=2`u&u7A{&J``@U>CkyB zu}PHFc~~fsIu8jUn*llx%K)3~VLZU5&*xzkkWHV^V?pSFBN^>Q-j{@CJ<<$81bfF; zZ0|e;#OK|VtYn5kAa1HV#w3x;tFsPu#}WrbNp**X(~+?FIEL=&FHV=H!dylDQ@dyeEFnCx7uw+l- z0W5tA537J!`Wzk$4i6m3NH1_x5?b|0D?|{49b2)m6OQ0>R>p0$b?m0CW0Yx?-bR}U zI&2+FWDzB89Tt+8{yHRt2nN_XECUF#2l1p5L7%R}Dj0 z!bmc3?Ui{yaDVN!VuNQB)F1WbkV9pT}dteGN&1q9XS0ma|}!I6yh0)_!J>yc(iWr*|>S1kR6 zQu!>!pnsCd1yL!ipSZl~+@w-Fs3iJ`mevmowWRe!LP%wR^}{kiB^gjVl2rP%A6fyZ z^m#w`KYXAPydT<&{4~l!@>eePNC-gpJ6NqI1Q)buzralvhR(}9YeI`&0CJ-FSXfLqKWhM|{8t||m zS_m-R5;^EQZf&IZs=?-F7kZ&YOoya?@sdhdEUt{FBi4ptoSGE zrse(7T5-l!zi#@)N40u`H6P+df3xXmyex)2xO_dEPX^27czRx2{Ab~e7yX2)B9Ob?fP=CKbehw zT@UJdI{wdf{_8KpUy>KU43ppfRnx!K_G-n{@1n)**1fF$z=pNIdo?)Ej|TJe@pMtU z%+HF6MvYdzHXol~-2HhuyITJ#uGc=zmL<#C|{O9KRxUpE@FWFlb@1qfbB_xVK+$?+BTEsuH z4QbvUCvDSK>WSleX1|?Xxti}(oBV?F(&!rH89@nXZiLjE8ojR;={-t(U|O)&XsA&S z9?hGZid)K!gI8xPxFEqN5u`okwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3_&3`CE(p29 z3Icx3Ow4O+kuTN{W<}tNa;(mW0l3zlM<){U@M04EZ Date: Mon, 3 Feb 2025 20:15:59 +0000 Subject: [PATCH 47/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14567 -> 14568 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f26870e31..a01c8bbd5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 20:11:40,515 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3270 INFO] --- -[2025-02-03 20:11:40,515 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 20:15:53,910 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3270 INFO] --- +[2025-02-03 20:15:53,911 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3286 INFO] --- -[2025-02-03 20:11:40,515 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 20:11:40,515 submission_checker1.py:3307 INFO] --- -[2025-02-03 20:11:40,515 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 20:11:41,660 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 20:15:53,911 submission_checker1.py:3286 INFO] --- +[2025-02-03 20:15:53,911 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 20:15:53,911 submission_checker1.py:3307 INFO] --- +[2025-02-03 20:15:53,911 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 20:15:55,038 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1476 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2341 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1749 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9745b4bc4..82d45208e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -588 +589 diff --git a/summary.csv b/summary.csv index 171e436bd..a61b31ff5 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1476,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2341,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1749,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e870d9e94..04351a580 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1476, - "Accuracy":"acc: 74.800", + "Result":10.2341, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 124e3456141537790fac1711d773513d04e70384..f3561e2c2e02e7876d2c5a696e50f0dc21f5bf42 100644 GIT binary patch delta 364 zcmV-y0h9jcap-Zd>mvlApE0Mi?<3g*0Zg;WC;SQpVwe5CC6mfBAAf4YFc5_AeF~$y zE!n9{5w-(u$*F-tQ(AHp(k2nLevs5Ue)}p;;w0cxkE@yec6L!U->WwH1?Q#FHOe!B z642ZTskb%ySS`{=l=#53V6D+mqaHk(H+L1clp6I>-g`M3yROT+*mvl2YrN;P?<3g*0SvRrC;SQpSFYQoB$LWAAAe56Fc5_AJVoSt zVmkqZtR$5H^+Y8kia^{fd)r#XKe7#J-ySDz(^l$<<9cSlon5(}ZFQIZf)C1?2GN3` z42-Z+nRSESm-GAqWg&7Sd1nnYXa@n!+M8N9Cai~7ZyorkAY=(-0u!!5n;0F#IEW2s z9*X2JNnBa4c}%5WV}HlRH(vuT3Ax7_B9}aJJOFby5zzrD#e{S}z8e8jVCX=Di2>7s z;*&Rm*WqfzsGK>qiaQ5atw(WkZ-bhQecu;-HCj`Cia!@GZ{w=DGQ+6^pthER2@gD4 z-`4oNNQdiqh>LXMl>+&=OZt_!!y@}WkXZ`A_HmEONA>i)oFk*QB;{RB%A8cok}yiy zE&UosUi6=U`Y`0*#3Nvz`|=aCf3piToC5@zYrN-^t2ep@SFYQoB$H}5Q~?T;zc?cX J#Wer`001Gws67Ax diff --git a/summary_results.json b/summary_results.json index d6680bee9..015a8a386 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1476, + "Performance_Result": 10.2341, "Performance_Units": "Samples/s" }, { From d83b4bd4ba04d7d5837dd4c577aff3be4bb9b16a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 22:02:23 +0000 Subject: [PATCH 48/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14568 -> 14566 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a01c8bbd5..2ea22bd37 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 20:15:53,910 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3270 INFO] --- -[2025-02-03 20:15:53,911 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3270 INFO] --- +[2025-02-03 22:02:18,245 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3286 INFO] --- -[2025-02-03 20:15:53,911 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 20:15:53,911 submission_checker1.py:3307 INFO] --- -[2025-02-03 20:15:53,911 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 20:15:55,038 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 22:02:18,245 submission_checker1.py:3286 INFO] --- +[2025-02-03 22:02:18,245 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 22:02:18,245 submission_checker1.py:3307 INFO] --- +[2025-02-03 22:02:18,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 22:02:19,339 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -65,7 +65,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2341 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1749 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 82d45208e..970664774 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -589 +590 diff --git a/summary.csv b/summary.csv index a61b31ff5..ab2ee10ca 100644 --- a/summary.csv +++ b/summary.csv @@ -43,7 +43,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2341,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1749,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 04351a580..8d597db77 100644 --- a/summary.json +++ b/summary.json @@ -1329,7 +1329,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1749, + "Result":23.009, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f3561e2c2e02e7876d2c5a696e50f0dc21f5bf42..ecd6c017a9f6a7eedbcbca4c3ba897d3ca4d3868 100644 GIT binary patch delta 2131 zcmV-Z2(0($aprNbW)2Af1pf}V6952Rvu_SR4S%z`3>3LOeIFhl`hG@c*dPp-|B*EV?75h6shr?~Pc+957 zV~vec*CN?+kVtfvNNPMR6ibbVgpkAljfZ7`MDh$DAkk;>umV7$PvS8k@xYOcbOO#R zJAbv{ypj;I30{w>*z0+W&3`JrRx-aJEQQz;rETR9draGbB{5tgY4)&?d{|D0gb>RB zvxj8>OY-m@z|tr7umXUk&+9SZ^}vygoi=b@SqMK^2Dm=Zf8SK>`aFTlXGyP>%rA&b zA^RxplwM8h;F3rq@;4eEu#n`SLqf=9fPd`6GQcHKb%0Bs>%$5Fmp<3Wfa?QCGSUe+ zuh6PTS|Nk7Ro`rr*Yg9U*GlFW#9%5JY+1)eR~vB#EpbHTuM9t6A^Dc=kPtA)GLk`- zkqnXyFlc>h4=W%U^cCn@P_3hSaJl|J(tz>>74BALAsDIim z$2w@0gF&L2D5>$Vko*ADAt7WiK;vNsgp?spoTSHy+BP8d3Q&-X^8mK`HS?l+=7!Nbb8j zB!rX(Xg(|hlm=)%_ZdoQfac>X(0{%E;RB_h`LNR_AL)4n&3dF6BFRbvl3T9%+#>lb zzo(M91tF;gl3KYIPKV!Pi4~%x-@`(38`dEqL^8ndVHrS@J&OmJ^a(z!0(@o5SD|ac z@PQ*4?L|5REQCuZ`Hfw$rIr2nZN)mzm+^Eo`?BEQ7x^y2KqWH_+SHa^o`2P^>Pk)N zL*XWy4x7gknM6sOhlLVp^N&(cz{ZuwZkePl|F69g0=%kGTMv0 zE6GB*bqz3ip#Q$D*yIVt@>#beE16snmg1Li=`?ZP4sjSfmKY>T8h<=2lt_bzgb>RB zgNJ1ROZFfhz|yDiunLH!&*8D)@W7Fb^a3{}p;eExLIgqBu@wtD;RrrwW!zR<$8Op> zMwwRWZM2D?!`87x7E#jHVIg_xuR}tJV1TW|GJqg^4o@l(^yxaR0wU=1bu9Qga3rI> zz(pPMu%1{hj3fisUVoYQ1J_}5b_#e@vsc=${yJRy!tdARsnhSc{~<89ypTGUf@Cxh&-ej(i!sWxNOC4 z&r|7q=JlT~HGGb3x=Cm8QPXjcB#t!aa6nt1Hk%A8i0e_w>Bk^Pzi69LO_i-9_FJOtvGVr zFo~4|RN{_kX#ufNOIkoAgrEjkKr92GvH=08J`IRfKu~=iPz)Xr9LZQOU>HEN9%+VD zhDbkg#nMkGm4DAt3@Vvi5S7CEiOZYLO)9m6N}`WwY5lNJOIkl9gj5DtKP&@Ok^!|N zNu^Kwp%s8ipZ9bB!v`wC`=PzaPopd(f8$b*gb>sa6{s|)Nf?rbI*0(gb3mmNbD#2r&&XfmjAGWrhu4>VGqVXa&I3X9C4w0>P1t_5!3?WdY&k#dj9pQcwJVLt=C3h&)$yq>cw9CYCfLL$IEXwy`L4; zt}h4sliBFk^`Ne&4@UWX^|#{Wp3P>< zyT7bX$(Pyu@AV1Y|36Sm0|b*`HyX2KB4H2(l;I2gB$LWADt}p~B?#Mrw&c`6p(!oB z32BpvT0cl?oxFV&J8=@osmImKemlFUp6_&*{eln5ng*2xK^Yifr81iay{{Je3}qp5 zBY9^HG-wY2&D*FiBinuX#+R-(bhZ zH{SrJggjskk$+1bIX(b$HxbbhDaC|zKfW6PQefyngNXr`MTyVe2wsPq4TEyw)GF>B z+_av=$-N6|GWLC6^wnTZ`Aht{e0dvI&6PQvN&sqWDVXrUqxEf#FN<`zj)%BRCtfR% zkNc$GcsnjK_JPb&05--wDIe9-^J;-wO6Yw~XilmXr5KD*R?fZ-Bd_|;Kz$tYZ{i6s z=Dz*}?H{uXG#LX40R;aJw-W#WU6U<0R{=$ng*Q9_+>^pLK@DRn004MwFLQKxY-Mwk zQ8*NnBsdxY)ssIsCL5ID3;iSk002+|000pH0000000000004l2wv&K3J^=}nzc?cX J#5Di_0090r;Hdxr delta 2138 zcmV-g2&MPtap-ZdW)2Db)xfy86952Svu_SR4S!Rto2q49A3zfan?xyzq}9Vhv9x+f z2-ys+8~kyLwFNIos6LqdpU zfZD?{fF*f)4`AstdsqR$(kJ#95PRTA#!eeZuPlV0ECXa8=)Z3&mVKVU<+H5UO6C{D zrLcXJc3Q6{b#O_n5&0{P4_HWY&>k3V?g$SBN^!g zq*rLwBdw4@*{W~0N$mLn)@vp63t}*p47RM}s;iASgO*4l@^^+Gu#kMqc1Q>qWEsgI z%SZ-E1{kzHw}%yw4EhRmEx0{!BqN=G^a`zdq!lm-t@?KBJf81sy;d^65C&}|7=Kjl zmSY{X%E2IUO_cO_SV(>V>W~mJ7~t`+3@{kr@mQb7!wLX{K99$O#{)+)cG~2VJC9fh z_NOES*IQ{JdA-$co$VuB*s~R-s`V^Pw^Z~wwi{37QH`j48E=zP?VywhBT9NcEF||` z9TGxH13Vv=0ZIcrpZg4@G{E!m6@Td7|L}oQ@O;>5lh5=#f@VF^43T7|0m&`*d~T6^ zmf%y#+=7tQ0!gi03#UWyu|x||Qt)9Rxee=(5F!~M_^=Eh$sWc7O!^ETRsp`V<*U%O zp!mR%jP@d>0T#lgll;mq*wV`W`?g}A=gW9HntfUDFN}N_W1x~525oA~GJntNcXg$v z^r3K*O^427iA|!U&ci~9)Okn<*$mKmSO(Z+590wgeLfGXfNc7N9t%Pb9LZ=e^1dWA z>yc&%BG@~&VteN)AU^M=WF<2U0&!E_F(!#zUY&KQJC-;gN~$|7B!5+EhlCKs0Ck6D z07UjI9stqj?yw37qEFtjAb;<`k&N^LwyUOV5lfVmc33Ep(hdnBl>yQY%K(+^Q9MAU&)Q)XkV>DnV?o=2BN^>Q z-j!q_+`0ysJkWpNR&4TwV)?9Fl9fy@2utx>xOAF0Z-+Px9!m@oC4UVb7D}YSLqdpU zfWgBufF*kp4`At2cvuC*(&zA4aCqQIMtXsplF+J0S|NfU?AVHhop1!7vodb0tz$QB z9ivRE^fuZ=&|&LXB8w<#>#&f#^w%LFL@>bCVHrS>J%}fj2>Ns#Rsj+8`8pPS9XOKF zUf`k*d0I~_7eYhg34lpsTQDk4Lo{aq8O1Zdg3+2VRMw zqou{eLW#6^NC=O&ffK_$^gw6uO$s3ol*5<)5itRI#ED#?J_ zk)+b6{m=?PrO*4h|KS6b;Qi2ECE(fX|?nOf3l0vI$gd z4BC-#dl#Fn{%#K(qp2>NA02FoEDmMtgx> zE;E4u(}0Kl&_aOemdHWhacjG!GSG3n@nj#>XqO)xa6@NvagpcCSA*qXe|M74&+|@x zdAX`gPMUKC0CltoaZx`kPHh<7F}I!R71Od@@)r$J6uL;y>#q zFOwIYzkhVJDmFYjzP`+B%WqftUOoSMRlF{%$<}M5uV-&Zd-Y;3el;J@=HunJo8Hfg zYS)*8{mE?f>v~Yv)A4_<^Iv}%{*t`-WtjZ-ubTd?wpS~reitofx9(;22R5wz-K)WQ zel(b$kEe^;WqwvnG-|Z!wfXq`;_lDO+12__aVfp_X|`O>CU<|k$Oof*zWQ5na?fV7 z<=tOar{v3Q{`dNX?*AW9O9KRxUN;)EV%+*K{tHl@hWwj& z1{`zWeuCx?vk5dA0}1@qz__>*003T-D>qjGMw5d#JOSO4z&Ak+WGVmvcx*3oba`xL zbCUr!6q6!28UofVlkqGSlLa;q8)BFJz9j(w08jz|01*HH00000000000DytFlYTfp Q0SlA8I3oteH2?qr065C!YXATM diff --git a/summary_results.json b/summary_results.json index 015a8a386..80ffcb11f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1472,7 +1472,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1749, + "Performance_Result": 23.009, "Performance_Units": "Samples/s" }, { From fc353280a389cc12e1c18f3a22247014d4da1659 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 22:05:11 +0000 Subject: [PATCH 49/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14566 -> 14566 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2ea22bd37..0e6a8e363 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 22:02:18,245 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3270 INFO] --- -[2025-02-03 22:02:18,245 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3270 INFO] --- +[2025-02-03 22:05:05,657 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3286 INFO] --- -[2025-02-03 22:02:18,245 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 22:02:18,245 submission_checker1.py:3307 INFO] --- -[2025-02-03 22:02:18,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 22:02:19,339 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 22:05:05,657 submission_checker1.py:3286 INFO] --- +[2025-02-03 22:05:05,657 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 22:05:05,657 submission_checker1.py:3307 INFO] --- +[2025-02-03 22:05:05,657 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 22:05:06,739 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2341 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1671 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 970664774..cd5a71808 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -590 +591 diff --git a/summary.csv b/summary.csv index ab2ee10ca..eb16024ff 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2341,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1671,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8d597db77..761189289 100644 --- a/summary.json +++ b/summary.json @@ -1239,7 +1239,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2341, + "Result":10.1671, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ecd6c017a9f6a7eedbcbca4c3ba897d3ca4d3868..22f6ba5276fe405315e187e4391454a477550719 100644 GIT binary patch delta 311 zcmV-70m%O5aprNb#4-e5@t7Wy$TBN`*&(3_+kv*^)IgywExie8lZaYBNNOFweHABh z67VU<)y#f7yQp05Rh#^Q^U~-F|vlL*qDa$BKY2$o{(`3@BGS#;Xy&hf{GtbtyRI4p=a*F7b5{57#mu*73v}3F4`b`o!C5kz*f-BnF_z zxM$^a_WaVUP+gE>o{}OZv!*B*D4A2T_&$uh>AwKgX~^BgGvJu}_7l|qvk5d>1O#94 Jm>!d7IDhUomi_<$ delta 311 zcmV-70m%O5aprNb#4-ew;S2qf$TBN`S*9fj+kv*^)IgyrExie8lZaYBNNSzDeHA-# z63D5?)y#f7yQrS;beH{t56YSbl?6c=7-6L{n+CnF7WoWiA#x*mXALxH4*|{FyIMFV ztcO=`9r&mqWC>&f6Rttq7#+hnh%IOyisUd!TwAYsOr_so$Hh0_0H%aIU=5LfOCC8s z0CP7H(Ge-dgmgc?8vs&Z=s<&s0hdLI&)x`Lhno$9a^ch}?j78;p2W$$3u-dZH+IBbhwU(xJ)NrE0B--q~CZuE;9Ck%u)b0 z#yu$?)zkB8fm%xFeNJdjsuiUej8In2z78X=`p-ap9P)4C2{7iq{siqGvk5d>1O$}f J3;mO4IDc18j~xI2 diff --git a/summary_results.json b/summary_results.json index 80ffcb11f..25da4e599 100644 --- a/summary_results.json +++ b/summary_results.json @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2341, + "Performance_Result": 10.1671, "Performance_Units": "Samples/s" }, { From 4cf1e2416a5af30f246cfc04a596a3cdd4dd3983 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 22:07:28 +0000 Subject: [PATCH 50/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14566 -> 14566 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0e6a8e363..a8c4d40f3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 22:05:05,657 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3270 INFO] --- -[2025-02-03 22:05:05,657 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3270 INFO] --- +[2025-02-03 22:07:23,011 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3286 INFO] --- -[2025-02-03 22:05:05,657 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 22:05:05,657 submission_checker1.py:3307 INFO] --- -[2025-02-03 22:05:05,657 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 22:05:06,739 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 22:07:23,011 submission_checker1.py:3286 INFO] --- +[2025-02-03 22:07:23,011 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 22:07:23,011 submission_checker1.py:3307 INFO] --- +[2025-02-03 22:07:23,011 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 22:07:24,111 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432134 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433254 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index cd5a71808..7cbee132c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -591 +592 diff --git a/summary.csv b/summary.csv index eb16024ff..9c67ccf37 100644 --- a/summary.csv +++ b/summary.csv @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432134,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433254,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 761189289..ddc0c4048 100644 --- a/summary.json +++ b/summary.json @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432134, + "Result":0.433254, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 22f6ba5276fe405315e187e4391454a477550719..bfa12d8833d0eeebcee3593aef618f2e720dc269 100644 GIT binary patch delta 4450 zcmYjVdpy(q``*k(j*ECq4jY?u)RGPk4Z|FYB+c0za;SvJB(_3MJ;h|J2w_Q1BXXKU zOimG#66ryUoGQmJtN499&p*F^?$>MoT%YT{uj{(s_iHm@GhwU?5`rfN2P$3!OKd|Z za*Bo;7q9k?ZSs2+Wi0RW3gZ#^ckpO`w2Ed2LFYL;C#uRBd`V8hVa(vQC+a@@G}n4;KuhP2JXRJ7^bNZr>{KBD*rl#5nbM(<5gw*6~noX+(ya3PtZkbZ~jW4e?MpX|octmqjynixDfk3^sev=vq27&7TjA^RxlMBwMw z*cm0e_vN1&%u3^ekUKw2%lve~=s33G=tRW`r*Y)kqN_}6%eLdx`r1pnj*n$eT}Vo7{+!nw^oLT36WME2;kYMPnP@XnsUba7i?bIan-6MiYXRP zn1iR$Rc{QcXv6|TP!$@&0c1);pba+m_9g7YWUM=VqlqM!?XsLqWH!C&NW@%9BxAEI zJy#F>ec5*TM!RQyYJ=bug0Xz*=pE;Za+W#RQG;H?gy6Bgggwa#aC;iq2%>|enHt5) z!?X8J&mF!=fO3Yn5URp4|ls=CBxYwOQBJVriX0R=j)b+A$0HSm|iD zC8}R)F8U$(^xi(uZru;3&1I$%q3GcNnV_vsp7}A_kf$rn?!xcxaaFWnn#KY2UuQh3 zPOJ%+F`;W)UJ|8#cTu7xy4n^ef{UuIP0c%`>d&i{IHeLST`*zd4yi;~C{QV*RQUP4 zvm7mCg-OY`?Ig+iUSLAdPz;JAWByYd`>oaMQ5!`s+ihO*ap06>(Ak!?);+}?%G_u4 zBCfIu!k>8(Zzdc-J;}j>)DiIZIq75~G+{UZKC$x?d^#Fk*j$6jsdl-i-;tQ9Y03(9 zE)z5sAMgo_zKwF~QHYnN3{<0vXtz)0{Blo16%8q7c-=pV1W-|Uv+fI26bIX;hJaVk zolPJ@{f1|S0o0qPc2Pmu|AL-#d^x|63eC$Cn5L(mRBzPT#*3Fm&`wkg1t)D4_I$Zi zv}KO%wOIum#sRG+`BU*5ECh0>c$+0cD6s!C#HIA^T4E0Bwdb}QmOqi zMv1hVCts6grzhM;hS*A51ueQF6<^=8OvHcAFxh3eY;FcPcDVyf#l-xS^qxB#M}%^? z#lHjRMikmJ?rf+SyU4cN(|J>NZI}lpjXuNbIenWh9|G!}wvP{GuiOi%9b>x5Q?lJp z@Ra|4p8We#I1XUC1u*RcZ2?SeH{6kv+<+aHNH^1dXNddeKcbfm14mTL(@n{^^a|o^ zv`GA0m;H6`ejXp14l^}KE&H;SHuhvQ@V+I_g9U6-XDt(Sf*4H0LQDJW6wR#&0+2Du$!ms=|lQZjb6;OL0bMOt@s*vONOz=C8v zcZr13o=}o96NY?JqJxDWj;mT2E?tcwK*{*rO5FMQF^Cn`z>aK!(O<~8Kn}|)F<#l5 z)#y_@%O75kh>jJViJ}nwF)u_L%Uoq+oC^I<&F6EEaVSdWf*FKI#yWn?kLG4LOdmnZ z#nRClTh+}zWKxwRlkUJ5i4Z3nhoXIp#cvn}zE|xB**MF8CqGDS&K1|7B!r&t5j3yY z_b<1~w&eZZriTR@->yeytX(H+&^;yhpcr6J$vzN+=VIE%v|k`WskV6|zHRe2RS|}l zyP20L0HAl5NICzhkJoIMc|EgwAVst9L}%`~F8}qgOtYkN-(+nmYmcNvpOS@ixT9SD z0!z#x8BsJCu)q?)J+Xh^urmcoL@uy~VehqfTla3R=6W~N)mRwsxTvLCn&$1m%BPF- zXz_^aHLK4O&xo??Tp93X#r;)VL9>Qk4(QuBUG;cGcVUDmRL;>$DQfO8tE!12 z2nUI?qKL3S%-jEgU1L`yv_UoePl}N55(+(|LFTUJN>k=$^{=3hyPV2aDZdr;>qce| zwa*^1Y|4EU;3Jf?$gsD50`#l7mJV-6AlF!8ZPrR(A))*mO3t9kwK*eLgwiou)iBIQ zakt<45Ui;1`~K9}T=n)XqmR4H%Cr4T10oR`;j!Cx3V~9EXquy>Xs6>m3WYz6nV*BQ=&+fy3KemYU9C6h^!K1Rt}7#P~PeO=y)Af1RCKz$bYBgjKhA@9<0sZXv?@0Auym_ zGoRtyhOC(%FCQkiRIa)9{5F%T$?$flb-Noms3jDSKXS(L3(zC@nYmfFH}!P~whgGh zvGS7IATho^MM+d%a@QA`lHZMdP@{P?xO)gQk@wZPDYE*W1L>{v{pasr9Gq}`J3AU; zbl>Vo!HPfolud2C<8+!v$rda9Q&c2DSW|rP1`(zK)bEGyb3y%G=ksHBI)rRYS4qXE zOt`DUkJbr?@w6-$y3T#I!D7WF&K`4B)XY&7O{on(kR#FBC8dqGZcR$$8I%EQ)sOlQ zuvXDAvXTeqq;C+Ro4xc(vk$kXfIy6;MZk!{jMFRoo4jqcfBhIwjm*{iG8eev|Eg7Q z#UtUS%>}udk^@afrCuXhkGM(grwkMg?qRSgGy_Rd40i^F@;lr%Lqd3G8m9@3uFX-5 z$^@$CR-<(`jY5HErTZ6c_vJ<Q-S|9z?i2rnG+1_&?wV<8!Y&ui?^LMEF;b=_(LxN=25(fw3!j?>Tyf$fFlOP%v~ z|LUh)x38x4K+k9@aSGjDuTy^E%go2B@H%H4$4F1`HiqZ~Cm6v{2jWz%z6{DTxd`lQ zt=O@<(93{%YW|8oPfHA4koR7}?ZmoO^+>0FeGy{qjBP;bbm~oUgQnSv(+9~qhY0ue zftX~#fSAz1uw7d(ni>JNo+8P>ZSy*OUrp`C1R7#vM%i8d!naty+OpT$_ud%jtuKFh zccLcp4z`>-9w==J?7tM9f}-*cvlXsaQUd%H$SuhqWV)e{>!k8+IFbm(56&(Se@0Fc zEMsn_o#2s-tK8+RVHM5TA3_z)&X;~(y2USS+3(n+6JOCzH&z-6=9FAiZz3(%D3dX_ z?$Rl;yRu9J=l+qlX%b>G%eY#IydDIovNFF))*=6^N?W25JeqL0=rHd?*vHg~++yz5 zn}w5Z!J7)>5t#$ig-^dISu2DY`A&Zi)ZAn_s6dsa1dLoPr+ctc_6Eqd!y5>bGJ(mt z(!sw?mx0EYEass)g39{V!pTfZBDyN9Q*E%2vn}14PwPpf%3m>hV|Bzej8{*R(iVEg zQ5@n5_&lN4Pzl1O;w&l=hVzFAH~^P)DK(#OlrLw51eT#^+d=9DW3E_EaETZzjtF~(ra*$NXf}Bp=GuVRKW2dg z%<0zu%wsb0EL{lOubE1FP|r|W4ne20Zt)=|@y^&`_YA9-(qDlzX~vHflD0tALcs3mU&Q(p<*+P&zmHM|iNu!H2Tn~Q>5)i`#UXi^}NLC@qZ?)O-{Zw-1_>pz#vCv$8VyBebQBX zV06CjRO{w5=PRrkbN06@28w?(I$1)D%7E1#?*;|?w-?iP%CgyZo)O#Jq35GN7dQFX z$^)$)Ut`vPFPvL$EK_r<{JSN$S#fD3t$B0ghEeuC3ge%U>nF9Jdw5A~sJ|GhTrl;p z@7ie?p#1UQv|X8IkXFW9gZXu1p5+ht9yJdp8W|(XqpXdnO4#hykawI(u5K`6%jM9} zv!Ul7X1=VZHJ#K8IKObRbA7D%WBBpxPk)wFl<%x;&0p#4G0yI%M^RiJ)AU|!{j(t5 zxI=NUg@6S!J&(%2@Kqtf1;f~Vq|9!tGNmuL;~?DPe}6;@n-91=l}uQ|xm5?}58kJL zPtdouU4u^69|S&oRt9pANgmE?If5=iTHVt eo=pG6m#(3_*rSht{P#bK+EHWhf5QtnM)^OIDnHHu delta 4450 zcmYjVX*`ts`<{gfXJWF|B+HB;##nRGkr+ZVBs*i@vmE>oLi9{1B#g18g|R1OY^CTR z98FVXlC5MHN!doW@_%&R{NLO!ujjh&>$>jm{dwj>=R%nm1$m%6dx8xCg1_K^-|VQ| zlfQg|V&i7h1!PTD`#X>p##p+BaAvj#PTclW;~ARACD{uLPUdSW{wmm{**VYC^vn|@ z;(Lb2GSB0Gn9mrA`0hw5$!h$Jyne61?VjL`DmH%^ce$0Z{^dj1Nom)wv4d&Fy}iqh z{u1rJV%Gc=fZKG%8yy@noIG!8h-exn{?el2FaCSb1(bqvn7Rx?xIz=zKk_y=r6SCNIsN4t0J0WI8xnE@D*J{-;~w-(r{e9o@Kd+W}dpQY2ef?^dXa zeagxn5)dL8qm`q(|36+@hCPAW8Y2+y00k!ATg%9TcwwKcm9KT9RG8H$0`uZdO5Ez~ zM-zFmM$R4KXDc^d7DJ$-c~Dn;Sn8SF8Bb%hEKAN2N;B0Us<0qWZ|jHEWH?VrL7o=C zT$zM>$X|+NYGq&>;CZ3sL|TN3moIbPp!K z;ageHCzisd78Gfjuw>x)ZG&p3%~HG*_kCZ3-&LJd+t+gQm;4>TVg4|)8j5gB5-oju z&fioi**iPa5yg+W1@%O+A-4>dphw~YP>R$Rr@~`hpz5bBvj!wzszsLu(U)pf z8hhc8A~k%w8fyocUyn8_g>Xg?+Tubi#Qv69PNv@YvB zX7%8Hsg)!aEoZhX$d5^dZh#C^y@_GnQ6;g6Lsc^0k!@VtdHllWu@{-QHn1*^euJ`a zh+e*B)QG+)U?sC^T=h;!vGHds(f4y{W@zRJwQLVf9|w{STBr+e^>6Fl$c_ z@7_j*Jl*oivMg{D`_9_Jy2Xu^x>RSI%QpaYG=jF}npal5v2EHacuxQHJ#5-PDXle; zL=7Wz`fEr7C4<1n@ zV|WHRuQnR- zL)jPFgzt48aGnJx1Z$?Frfy<35$N@CRkMkKX1_u4icZ{!?)W})XsBs-p@@q(P<+Eyn=Cox*VMV&KNEVAvx`Cd zlx=|y^Q3NdY7ntkhLN;+Yk#z9b=MT*8TVfrCSC&ZGMfJ;Kysab0hr~Xd$zX1o z9~?C)Kb+{(daW#_L^D#ECOCbr!~px9^qb)(2Gga{zUhH(r=Qt8z+6|;2EZO)a)9$5=jtt4ZdQIng z$LH!ar-03Sek;u~5=Mh7yGBbEv6?2bt183bCRGxX7B)b@MWu702o{y5UpU=~mr2lN zoiojv1Fc$m7+8*C*k&{uAUdY|#jfZ~BIWxW4?MA1`m6tLK*pcsnbzhn7FRSa4Y3Ow zOj=H`5I-gn8Z5+yB)(qCEWW?-^h_yC2is@QsM24$CX1Gk*C`af!zG<)ZGJT%wUun* zQa5_(=R(Z0)_-`{TJ);x=_9-@cFL(|9`@JEM_BkD^8_+5@g&?EDe-#FR}6+h_rc{5 zWGiv`6z@+*3WqxdND6Vh&k(Irr|5O>5dTPOE@obY$SEDXtaHBGwGWZ4XV6^b9glIo zQ$JW@CL4AYV4rG~I_pd$#hf9Cn${*I(%6vNBp<5jYPHM+r-I9bwU_mMQFrIIXWqJh za>nW;00J_-uG_^~*VV2~V?J6reD2Ejx$oKcaJqiGnOxL4sNN-(Qu?UVpl%?7#HWaS z8$m%RbRn(g8zLxB#@pkIANQ888PFnz{o#vZ`>iuw9sHTDrXs__YHckWjx5o$jrP`& z5%e3uPlShR$%<9l_&Av?1?kN22#P{3@CaO3`Ors#Y)Jl$gC4Qm{JjS+^?YHaSNi}s z(9=PT{yNmhuQHgAuWr0NUG5qES)J0B>S)ixB|L8Mxl%FQ8<58hs7|n($}{AA@+hK= zp2j6MB8E>B;EnXpJCGXP=p-?SW8AlN=;Z_H-d`IrHJFTL4CdoRe6JUNFdDcjc;~~H zwIh>jM@n4G-<+s}U)W$*l)eS~6^$YBO(Ib-6vPBBV-Gxe>!idG@Q|dA+ps?YvUZvAb6RmVqJ-=&4dN^K*W}clvHk#P>sZQ#q(tz=Xh`ybn655MwR4VdFk#fP zzD4zk|BMpi>cFQ*K{V^W1;%Z@ZtGlQ`;u#OUaKaXVs;dkvR%!-OwdWPt6E2;nY0bYaKa+hc*wU-)n{oG6>9jWFQ7a*i0gI*V#ceJ=NK{D2cein1& zl9b$0j@+d|f1Pprm>Tb^8008P?ZK#WkG5Y>%e8HI@DKlGM~y5j=^eJ)tL6kxd%*NB@^34huguGpDzV!NxdpL zYPqHHNxp1UUvb4KdmxrV`qrY2OUy?2=@Q`k^#|I-pE@_K^!EAQB)p8V#e63(y%5dw zCZ-gPWtqU~PLH-eVz&rw74)aPm`Ywc;P_PhLejWbidb>!c&9;_#2^M^sL}r215Ko# zNj-3TonN;a7VE>dFZ#P<84Sddkv5!=XnK4r&4pQ(^Ycegw{~CNO1S1u{lxg;EtMBU zxf96Ifn1k@_qAV4%ofg0%HI{d+asKBNWPnNUD$t(hN?%quQ8EKRzWvBa6253vV)GS zZ|R7~f)?y|orxpwzuYK8+U3CR)n032>?bpp$;Bf)Q{_*+&I>HdjG$iD`i?lN2_I?T88k;6REuWp{ZlhM!{5E?@C2un?~KZw$_rW04E-b+dPFDp2+riF4k4SA z@Y~i2`?lVY0c~Be%oIp2>zk~bSO#OV4%3n=0``|bkN$9-*dM1S`E-4KKH53;1dnPP z=i4m=HKs5+2kQ*QEjfc8nCOMtvj)lT)S6BWs>~HZbNr$81?_4Wc782_u&Et_D|y~- zB)Q5gps3u!!G~P)e}F~lWYkIhpO)hXE4_2|?LF9183BJz4K!tqF{4I>T7~t`nF6W% zvaEpT{!#Z8CUW_e)CjzFTN6|znj5VnuJc=!2?L>Iol-Ohy}H;O*d5$wy%0OcEmZz9 zHa+MW*>}bAU1$a_{ZxT^=;cPySc*bsybwRMfFQf#jZQf(ftorD|1vhg z#y7BBn0UO<9UM;=2QZ)yH+zcATYB!LxLL4f42Wr3QgbCoYfgoIR7=T$+f1eV`9te0 zOdW~1L&lhH+ijx|{wl+jBU9J=Cz}&2&z$_}tVD44S#ht7-!aezt*8k+E_4L%HLRR-ecG z;y#RWvCFC%GIs0xl)mw$+jTcUx8O|=!2X}s^Fp!rk5murWR1)ZaT$*)ruUpbxBb%C zm5W|st8Fd0{rS@=sMhZMPZh`4!DH+%LV=W>7W;&?T|YHFY4gKP?A_|yegl5*|6W_! z>K-f1YpCKDPQ@&}3w(Od?Ax!%Od;c~SniXlnSwVL*)46=7g%{CyWhF|V|#iQrf?`v zmaEp$1~04!ipOo`wp7+j4Qb#_K&Kii@q3&=NsICbrikyuwZ}69#3B{t=luRz$&PxY z0LivEvE!k_%ZfekUtc}l^1%6~wJPYVpxrTBDg?#J!P2F3q3vH11{x8S6PE6e1*$^4n XYv9*J!~gq_BCmA@`rq(;PXYf2T%bO! diff --git a/summary_results.json b/summary_results.json index 25da4e599..2704bd4cc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432134, + "Performance_Result": 0.433254, "Performance_Units": "Samples/s" }, { From 30b329b2d320d64f69b95da078beafef4efef807 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 22:15:27 +0000 Subject: [PATCH 51/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14566 -> 14567 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a8c4d40f3..12c17a4ca 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 22:07:23,011 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3270 INFO] --- -[2025-02-03 22:07:23,011 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3270 INFO] --- +[2025-02-03 22:15:22,004 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3286 INFO] --- -[2025-02-03 22:07:23,011 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 22:07:23,011 submission_checker1.py:3307 INFO] --- -[2025-02-03 22:07:23,011 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 22:07:24,111 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 22:15:22,004 submission_checker1.py:3286 INFO] --- +[2025-02-03 22:15:22,004 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 22:15:22,004 submission_checker1.py:3307 INFO] --- +[2025-02-03 22:15:22,004 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 22:15:23,202 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -64,7 +64,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1671 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434095 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7cbee132c..651c401af 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -592 +593 diff --git a/summary.csv b/summary.csv index 9c67ccf37..f170bdf92 100644 --- a/summary.csv +++ b/summary.csv @@ -42,7 +42,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1671,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434095,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ddc0c4048..28efa28c3 100644 --- a/summary.json +++ b/summary.json @@ -1299,7 +1299,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434095, + "Result":0.437053, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bfa12d8833d0eeebcee3593aef618f2e720dc269..e4c11cd78a992f860fae24119a9659cea677a0cf 100644 GIT binary patch delta 4661 zcmYkAdpMJS{QvKnGuaF)avD>aV@r}#Xt4v)oaR`LA)<0VZL!)2*_P=@hY253m@|o3 ztP&p;MF$ljTj_;Os#Pq((tMay-J83Dba%iAcc_ObhPn%X|^n~g)dF1JeG?X`G! zzj(7YVeHsoQxw@qx8-4~d26#btNw0ex|JVX!>35jOUYH5GJ`a*h41I?HIH{0{Bosk zex<+ym9t0SAhFLzIScy?iy>ZQkld19E!te{%{42Dw(4!+G|^SrH!jU$?Oz5HHWbca z&{0ifEu-sZBXcBW##94pYW4Z9Z^=9~i*ML~+*s?IN)%rU>g z`syW|Ba7(Y914>@l&3ORRzDXp3yr4nFEWEF-Wb=kDpNJ?1WM@+Z%v`L^%* zs3CzQ->W6LSRh5sYI4RHA;^3r81wPu4kiaMp3>_N5knU26hpE(Kul6`lLRYHFX2J| zKgU&i{-!|nt`UGNjcAGZhEw~ikNV{KReTv5I*w{B`Z&Nm@?y{0;A}p$28(Xj5&)O3 z+QJJiA33`nL|RsytPyFyyI4&ZgC&9TLqy3*R?PEptViipG=tpuX^NErrOp{tpWQ^& zztsS4@v~srzef4_e;h0Ll$=~XAD`DcnP7v)w^>*qsjW=ec|5vZFErnC_G`iWE)jn# zv1TqX|4&+LAZV;0nBqA%+vwwuyT$Hu|GWLOH{Um7(DPAK+c5HIpv1$CZ&QQ|zsSh( zx+h@fF?)jl-v0~y`c+9r^I?&Qp9he*qS%8K@^1dBXSWARZYOz2!b!Y8UDj)7PwTLx zy~CZPJ|yxV#5Drh&jBN)CcZl1?B07hj4@ax(4(WQSr|<5eWBSj=wNsTLh*GzZ>R^1 zv1QuvgqOO$@5v#E#%6*Y-B3?-a(LGMm+tmNVP&D!@k`B216D1L+Zlc=Ot%(>)8{#H zk?cu18+`##jRbcEif84fb)gYQaKTne%^VS}Y2AE=NMgX0;ZjV1lnU}kE)>OV9fK)=kwZokMYIrItYDxIkf8@dor}f0LaB-C zj+#W!!t$2zE(J%#pg}6JpmP2DgqO*oB0lOHY`LkHD_Z&R_pYd+xZ^ZMPQODqPLCdZ zh2agALgat~q0fOF%q(a}S_D*=(&AIB+9eHK8W*W*;frLk^)oCn-k*s$e*<3>xxy`5 zWZto;hV_A1IGKrhj^Pu-E6=9=jP1clSF+f(Sy+3VQ|n`97C7ux0|g~&-gDz8h4 zUFT|f0F%o}M>xR1I}vkSLlyCvM_L_!u%C78JuzzX9xCNtmCt*|`DJF@{Q-ZH-tP^Nd3W3e=74{8tn`Gl_>s-1!V3?-V?ETD~7K6O^ zh<--hmZu)R>&>48Zmdtb4$hMc0TtdjZmK-K=DtvL(`fO{kdrqbzZOhJ>s<=~uFb`H zp&}|DcTLAA)^_R+Oy1}jiI70LE+Yl9G1(;X9^v|ylJ8jT)`il%W?g}OZnR6|2_xqx z6OX>OpdEcVAF6-1Hs2Q$$N%21UB!m?Pmm9CnFlZGbDFt|Q)U0=)ya6$h@4{w!U-ip ztK4BjKnXnr7y`&8^lg%7Rt_HFihTLb%KkxKrBqll|MdqvVRpWIML%g~ZvHNj+W1w| zRI=Hh1>ej!9Vh&~5o5lA$c@1HtRF_+<1W*%lAmSgX(aJcB-qX@O+>ze60+8Be@k%Yg@F1TIV;iYfbXG}$13nKSF@iL{l4OC^FE1ou{nlx)K zURsn8Qcb8>$#=5GRFe2xA=NZ38(Kk$Tvxp#q*iTFx1acbP`uE1zv|H?SDW|5g$Ya{ zzql-*;OKDPsoLRy&9FtYo#C7LIW+?t2n2F(`$mAWCc`RH9J^e8h%3UJbS0B(YtPDk z#@Z)?3fQ@rbowL%TBF-CH=R9OY;dP} zKB&JpC!N+DO2aE+$%G&lfwpn)m=dGdNe*l8rTaG@dRAHUZ>Q0IbY_pYqaLLGFJIMgbNGBRVoJ-+^m8PABlmnN0 zmXszVTvA4{PcKUKl~O~k24syMpz9p0?-D(^KYhu~MTQTfJ2rkmD!9S%wuSD`120E# z25Z&aRLHcqL3B+Rjz_z|HA%rrmIW*0be>~TxLxpT2cGW!StSC)A3y&9^iU!%c&7Gz zv_x+*N7+Vz?^J`hRuAz&9!{Fl`Y^5>5ls+x0&gUM9BuX!Gn84yh`8l zzBh8MPZKAr;$+Vp(JG&=a?y_yjt^;YtQs36y*E>FJyOjB*t0A!|8*$!HZ)i*ez-mP z!j^Qdr-ur6Xh5`H^u^qRM;>TCboR{^zgwI;KH*>1_v?nZJJf0PAfGuP{nX&(?2-!M zfD5HmbfRr`B@IIIr(EMmAk9}Nzwa=P6BhL7&aha0(E}D($%Y!lVqQ7vW0LIP+}%NU zwBYt^?4NQ{h)U2;x#vLT*qXIXh4aS%F^WfVOHliV z^V-T5QP@*G+NoH{1C6(ru4a?tii$J8)vsIHuUbA08**K8oT0G(Eb8hm3-w-5huN8d zsp@Qync!Y3aJ0CuQyREweXE=D66}DEAGYCUZs{Fz3I%mEdsX)>Va$qmk8JHMju*MY zH3T1d&{v!Vb_%?LA$||IL&94jsbS(+D}8VnvwIb|Ze26g)fj;_SJ`z_m^j4?(#6iuR>IWC6bC153|$VpvXy)Li7OAdaF9vAEGLF8xEc*xG=vu<2)arG^*#*ChaG4Q7Adg zXh8V{!q;37J(FQjIrBJLZ!oDXr}W;tLb1PXJqhySsA~cJ_v>dCx#{knBUh{yPuEOU zCFwJ8S89_;dl;q=DZ~wssgQnSi%!1pJQ1Lj?^N6u)G;A!kJ%$13;&y6wxUVh9{O{Z zVqFs-u5g1d+G?J@Zt)b!AXM<=&UM4YFt~HH3uDoE;Z-+d$B7?RvXmZmK3H@8wVDUW zcDcy{{$9IQWywQxt(Ph*A7q&U@H> z%{pF!v5YFk$5cXt!o)0;))ACtdzaJEX_sY^_+P9Og|QBXd=bW48d5%fzu2RSwJ?HT>sxEHker3=V93lFilc-vKo=y#&SuZCeAdrO#5(P&J z2cYP?bBV(++u{eED9sG(|#`iS>GMzR41dEQDEl4^PGg}v!c~arE#>$6$ ze5i7)QvJ+AS*?77pT)avmr7UK_2kL1wY7;tTi`-l9)jRPTOQ6ZtlBG_zY1OPEbb2K z_`2nanBg*|=kRsJEyUxz=N>aZEel?r^#A*g*g5gG{=k&Jb%gm3Mwfi|zFj}^*clKR z9pXF!(as7%7U^p*1cOB^gNi?QqJW|{=?^;;HMYvg@o%@Aw`Ebu6p}XEz#A{pP8 zD1q}7Vf!S5I)Upe*nJ!{W<5)foo-j2$_{i9`ckGi=Ay(wBnX~17g3Hy3+7vtf)8idCOz&Q9cU?;q}k8yQOjlPXGHd9eQXlI0EilcPPS!1&Y?-c>Kd%Kin^4Q{k#`#j6d z^gOBGsM)bwo?|s$Z{Gsd=f7Q2+w+Y#*?;W)j#3i6WP}{}eD%PH$gZUH%SxiVNGDbS z$5}^=17~#Kz-HrVQOUeJU@`MV-(WH86tLNNv}w+>3$6lVQe(JP7B6LPJaXo@_4M<9 zRhG7|p8K(a;J3o&X-0cy^z3{7=f*{4JcL-fjdY#?CH zx6;)Ze{~MoVz*5o(CWoqFEl_1KbP&=eUmY@%;_5-%tf>8jF7-@7aHYqaX91CQs$kH-X5gF8HE$Az;#7v<(wNX{abJT2fUYIrT_o{ delta 4665 zcmYjVc{G&m|DG{s$T|_PvCc4Kk4%!nWEqU5NMbD6_asDSDrO4VdShl76(LOW+GQ(i zjBF)FTBu$$k+D>^H&f~Rc)#D@Ilt$R`#I0K&$-U^xv%T`eD3os#LUOkhe$w=j*Zr~ zGR{gI77$=D$|an^Q;bcCAC$XBxqnJ2)%j$f&)+^*Fn_#mX>XUjuF#M6^|$#`SKjx{ zMbuvkekod$HG%b3`gI|mbf{&1 z&#An@$!hG>RFMB!4NW-v@}6l94cyZI$e8};r8k;;>-tPw4$CkIdnpu_c;^KL6Q$An zp&!@z-Mq3ZN926TF9z@Y3YEE_eX*8o(6sqm@#E1hME+wkMt=W9rs7gUR@rOI z4ff? z(L~mm-taoMqD0P(kD^RF46o`GOVriADzDKj@O8@XA7EBkx%KA?AWg#akXiDt}(;g^wp^EQQ?)}r|{9B;hD$n*Y-XtmJB z{W}@Ps<&NNiwCpRN4HGeY=ZbJB1g=+jg`dQ^QG;VX2}+dla$s^k+_aG+tmuzckj+N zbBw4jwC_vxETmn`aGH+5*D&Wu_Z464JEotZCvl(@eMspUs0gk44ZSx*!v^%?WwM5m zkg3`8_VP#UgPSuM&!xwIe%6ca#sw&t(>ly6jkSKoO`mjqqUmOk2j4e=&3x>yx@3OR zy0CvU23$s>DCSUZZ~`n=a^N$ASh_8fW=F(NbncduIVd@6h47En3rQ}%C z!LoKRB3rES!4+M4yKqlw1po??&C9md%^4J>>Z(x&muO`cSGnW*WSNHp}2WLRChYq#~`^QLZyzY zF8h_IH5Lq|y03hc5k=OaAsv^_54>Nq=QUw!`Q5^=wdbXpx_HkwC)H!5Oh}&&JNasN zP&BmaWR!DmU_I9||39F-N*xhI-rV=OQJ|-)!rVp#S;y_D@D#{GdHZ>whznT@$)Pr=ma%C^;4)x!_+Rz=E1d#bbRBt*s)*zYdsq%hNw3~zIg`q2f7X|ZiA*J7^K5+9 z_y2j$;oFs7pSH|Lg0qPHZ;^*@xYgFwTY+6PnN2JR9!nV}>`707JJG?W5M3nQ!ZcAH zUa)s=@!(Ygls$0@sRo5R9Y`g%eziN@!mBC{9sfy^t-9qfw!mow4s*4jymqNeLW^)O zo5OU24{543Bc&|mAZ#QPESE6_IlzC3bO9x_;*O*Ps*gbZ60)FD`#l87=}o;XdWUD5 zE5)sHA9UunX|P5+OK)80G&3WU#qs>aUtY|~*ypKnE?k;+gMNnucxe=LFFlw#3Stqq z3sqe}iyI!+)u6G(^-Kb|{GoUpJ<6YzTf>jYs=238*2bC|vpv0a-A#vOull;eKJ7@k zvxyz+C?0En=ev@Wr7UC_#RSU^QMnA9EQ(SQt-uQK4a3Tf%9`XS$EVhlm%xArrp{f80*$u{Kt0fM6F^+N(knl z7;?Q+&%t_SGoPe%^V&sYYxzWJveybVD6B#2xxazfYTQlnM5dhB2*@PyPs=i zOdt30q&tTE9Gq4|3ubBVNB?p1OvBL);c6CiW7}7vD(EIkv{FyU`e;OX!^K&ibEY9r zz0x(4VB>*_5qHic!lK2UDVcAmaQEU7nQsc=06fhRsTbpX%D^-c!-1!g;_f=cuS6f?3D#mcpUbV{9x)0|D<{lujo?Qzk;; z?|**Q!hd&b^0|< z;S3*1Q9~?3T|JGYwo#?FmV@_~yXtgWE-U-57t1&0IqD0KkRT4yw&Ba3NTsLQHmUdz zG~6!3Z&sFoDZlk$shG=OCH)ppB@v-)PQ}l##Yx3p+Knw$GY{EbCnk5+u?KSt_c&le zqj2QA$Lr8m*PSEd1#8(+%~LEdihP0hQLf6Lf2IG7j5PGwG2*?);fnuJZN)#O-^%yVjq3uKYx7aj_&FTWlcpNp|D%B=ploi%mudswaw z_e?#|EZwwm=oB%SCQcV3MhBykaRX_m&0&f3Y^M5iz^Vj_{>(S?D&_E0v1&L{#O}h0?>5%4g|=*EErb-4S82AkH}7#k^8$ZjGme}nI-fux24fzIKCbqZz3f^R ze4JOpF{ZGUtpsU=J7&5;thZK{=9m$Jwug<29kyYx!vNQiCX?gL7l{)mnMI=mE5xsu zguPOG4zhQXUnSkj>?{)3q@+Z<4+&be83xzb7T9oq?l5D*%&xZ~X&aY_noJ+bJ*a%J zkK_m_pX*`K!*W{khf*DiCj)zUSJeu4%|hqWT{@vIUP|x7P?d~V1Cc()uZ3DkW}PS{RHfl z&Ti>`osAs7PNsT2#xE&hrJ1gEJ*@8jG7l~OJ_=Lo%W!dvcjcR_pTwyy^6H%_x@Ul% zgoSu3F~QiOhe=e&;=lStK(}$H2B?+5fnJ<4FAGGqy|u7x%M+(1IQFlN2G%G2(OgTn zgPjW<**ej1cT#U@k|t1UR+eo~pmzsct10V06%AV_7YuqA3~btq?t}&i6)xvH+1&%`)!N8`_acxR z^;K<%xBB#E@?h;O%!XAQ_JuxhOvKOt$Kb!jh2fUxp_uo+-^K-df}0%u+DG!v7dz~f&(o3w(2EP`%8B{JE z?Q`#Rc9SJ6xT@=|I;Czwh)@TNoAf8H*f)(gvucgz#60KH2YnGqJ(g?xr%@=soM3da zt{MW3@Eha5RCY7Re%2XlF4^j#-G~zy)oJ3<+;L08ki_AsRI(@Qw)_w~@6~}R$b>TakgoAi`9t_>;z5d90%_GSPb3xS7 zMU;YWj@@4<(LErggSYEWOXV6>18p^k`Zv&4(U)Z<4=hSwAws_oGwUp0Uz-I2F_jeu zBMRrASW{^CbI|$mWjZsyNdMzv*jDh9ZvC}0DOc@7iSv0K{Xk1(;>wY(l2FA+Z5Bl!pMx`Uayvh7@ zw}Se;qkx|G?0_))_=SO6=-BZ1SN^NSQmSV7YJJYno;gDxQc;Ro$NFaRi<34R< zIxb=0T06j%Bl?aRq;_?hj!p^dEh9zt^KSm}oN~#jkv;@Hsin*=^Lk3z{DH5wnrgu7 zo^+Y=zlXOsL8rLF2qwA^*G7$Rpgi1JpkHezPTho_1I*L(2icvT8oea%w`Tr3(W_xd zI`h-RC_6XoBcyJ>{wybau0v_=07=(?kZTCUga!j*!UV&1ZM|$^3fOwqUk2__+!y#{ zb~h%_D0_Xo{0zb7^3|-PTz|6$Z#g?yZ724NP;IAM)+j{U3>6ll`#ktK;bLaJgBaqilLowI)#;rZiwku zA1h<;NOzaehf=BX=S`p49`cOgw)sox2t8mcjdKJ7?lGIF1YryDdMXiS9t;t12JXsL z)DpgFiJU1CXofyL2dE)t9I;9orP##`ooWu7A zpppo}m16ZtMA!p6G$g{7ZeP4*r2~llWdXQ=xnBFn>N2g^#)Gi)l%>1}^#G;q9DXA2 z8Xtm7cEdI!9Y@ri8w|p1pMFCb8c8^2TrXwqVO#EJ>_g$eXBekY-Nxg8y`KNLp4EO# zKh%BcSpVi!#oO2;1@C^XsH)sp+vc6`A2KU=&P`tfgyvjaqs;d@yA zVUVgnTrj3%#Gf&QRiTVT^c{d(|MN?vtaHD|eaVy+bFapTlCfOoYKoy9zD9`dsX_7W zw<{Fv;Kjy$DSUMBLD9MR{9o53hws4hpcV)@e-?HL)~Ua+4H>rN+N_PrhmA=H+-W}C z9PLPytov~E>q(12sX;+??_?u`qt|2L3S*}GhWzPmjXvwQ?7JyXZ)TnEf;sMa)F-c+ zZyMVnG#`a7RddPQyO?z4FsQs|>?%weqL(FA+_|?yf+3z9JNr_5RngJYY3}m}*dh5J zUHbfp`ujgIJKh}~;#Kp*%T5?jbMrn&(#$x&rW-0Q` zL$p99Y%?vD$%%Dz>`hPm&*x5ro%jpGy7BIyXX1*j&t~yezA^W2w)9NS559C0<)Mfn z0`k8fIhu#Pz}AeE!=~VOj3 Date: Mon, 3 Feb 2025 22:19:01 +0000 Subject: [PATCH 52/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14567 -> 14567 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 12c17a4ca..e8ab43750 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 22:15:22,004 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3270 INFO] --- -[2025-02-03 22:15:22,004 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 22:18:56,426 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 22:18:56,426 submission_checker1.py:3270 INFO] --- +[2025-02-03 22:18:56,426 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 22:18:56,426 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 22:18:56,427 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3286 INFO] --- -[2025-02-03 22:15:22,004 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 22:15:22,004 submission_checker1.py:3307 INFO] --- -[2025-02-03 22:15:22,004 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 22:15:23,202 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 22:18:56,427 submission_checker1.py:3286 INFO] --- +[2025-02-03 22:18:56,427 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 22:18:56,427 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 22:18:56,427 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 22:18:56,427 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 22:18:56,427 submission_checker1.py:3307 INFO] --- +[2025-02-03 22:18:56,427 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 22:18:57,579 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -44,7 +44,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412747 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0219 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0425 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.20426 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1671 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0455 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 651c401af..3c7cb2996 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -593 +594 diff --git a/summary.csv b/summary.csv index f170bdf92..79aa14cf0 100644 --- a/summary.csv +++ b/summary.csv @@ -22,7 +22,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412747,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0219,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0425,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.20426,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1671,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0455,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 28efa28c3..acd0ae109 100644 --- a/summary.json +++ b/summary.json @@ -699,7 +699,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0219, + "Result":21.0425, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1671, - "Accuracy":"acc: 75.800", + "Result":10.0455, + "Accuracy":"acc: 76.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e4c11cd78a992f860fae24119a9659cea677a0cf..4d5c49ceb965ded513f4f6d9b278ee8dfc5db1ff 100644 GIT binary patch delta 2627 zcmV-J3cU5_ap!TcW)20@!FU?Evuh4#4+XT^aZ}Q>)*U_qf8>HV;&{fy-O*C;VWC6{ zJ|qNqvW&!&Wh9<#K!9iLGkjPDz|&Wu8-wBlM>5)r)C*V$K4_8<5K9Y5V&jTUpht=Q zCw@UClM4Yg-guagYNT=GxM3122dKmy(b589p;THxB!r*_SU@ZTpt1o0s6GvdRX|XE z9#9M(5FE)^e=lGdK(iichE#?~KXJv0Xlebh zP%5n-5<)5itRI#ED#?Iak)+b6{m=?PrO*4h|KS6b;Qi2ECE(fX|?nOf3l0vI$gd3|f(KdWb3uj*R!{y zy?U`1znYI{^YQZAP48z#wd>2l{$w`#bv>x->G(g_`LDkWe@R~aGE9E^S55y`+p85* zzl#>LTlcd10~^-$J0gaGCwOOIvO?7dTl;FzqtGJa(1=;Q(Uionk|>J z$=%;B^1&#dul`n?+_TwidH0vqDfu#+|Ghq;`~L?}O9KRxUpE@FWFm(F1^5dGGgOoC zBO!lFQ{y-gzVEM4WiK-a+p_#hC}4^Tfue@jCa}exQjVk(t@!>*gw-DoQ?7LwS)05UK zWvw5lza2Bik$5y_C#`4Br$?H`yk{D@Oih2LG!~k1l145U|E3d-O;hUn?3q#?g&M)w z)*?5ETgXeYn4h$4r_(}N9Q>2fZXpqCTBpY>I6da4H;GTfV~w94Yq_8h4m=vWX^>Rq zk8!|RRoJI23wc$VQWjH5I-sd=M^$<^9*03ptKvU0ng*0<65_y(reU4p>b%Bc!5e>F zouhA5ov27S4xMFwz`s{lc9Sqkf6`Y8EE`1u=d#sjGdBp`Q4j`Pe(05P8dasY!q<|` z_i5r$#u91&>F*Mv=h2X+E|(X((&9lf<}Yq4dG$8ovJvN$3o??pD-5p6EEOvlW4$W( zUG9po5suQlFk#f!ZmzpY6eTg!CeMGvY$QT28_wId63+rZaFxk4Qyyq6nD-wPpN?G- zMME!%$H7EmSGYQ)%Pp%bK*jRWs^XnZQXmN);kW*G^hA?5|NG)7#Ozp4k~janI=?zY zm-He@J&O7dPsqd_TwS){S&`>Se#EB2ZznC0#VpT`*6CTs6OrB2N26D?B!+*wH)v+4 zM72~@-CooX!BFvo@i@zCjWU-~It<;&C5nSbN~cp*M!}q?Zm6{9&PXyn`6Y~(bL~+a zac%JsQJST$C*?#LI0hg$0l5Xpr86t#4j^v>a>*HmT$j#pEx_|CycXWZ2s;iY{>ZX@ zC8cAUiYiav0(|-^lGS@2&WL|%s=C5FkC@8HWmMbZ%wR+jyspRdr&e&@3Qn2jVWL;E zO4}F%O@nQUslBm=Ur8(nvqdocpJ01?v8ZiIDC!8~Z)F#RHD%az;!1b-7I0gZcdcj+ zEGfm-6RDDs@2som}_&>V7!r-t~J&NEcpdTc(M1+i}{a-9hN$>U^Ok zhNapa3!4UR>!Qrcg}i_Fc-hmBkgcmC7<7c*1mp-Aj@l93Y$>=Ff{tp~x`WWe-Ot|b zV0ho_4+m#A_t(ArX)J%czh<3b5)~sfVNv^+h_K@B5+pFr#nEU!U6V>1E^KwMs$0&Q zt){LLyVG9TYKnV>*re80B9z9~HpsMuQkY-On z@Ff6W0`MgOUjyK40DKLAuL1Bi0KNvm*8un$0AB;(YXE#rfUgPgH37aRz}E!$ngCxD z;A;YWO@OZn@Rdt?DF^sk0ACB>YXN*MfUgDcwIsgOpU|Qs$?vo*7O&%4*>6;O9zKhd zB#!69nXc*`m27|8maGs~Jlzef3AP!y5hj6Z>!hfKo21n@jdz|D`eNAg5#DC#2gS%qFu~&<$cL z4M-QQJ1Wn$s$V<$jT24zhY!m>78QN$hMl(nM`dEpO7@RY88U!IM&4`EX@7oTn$KP3 zf$3h>7)@AXs#qT1zD@qxSjF;Ca51$sh)S&X+SJlkZQFKMLrWq8qK$2YE_=7b>)x%9 z*wTmmO}gdyQ8wa2FU(uXHOBbqe^5&U1e4x38nc2bLkI+))pZGz$ub{*%WA_g5JmU> z3ZuO(+aYZbwgYX+s)0gXTDlX`BoVb9NNSz@edT8o$g0cHy>sTypkjTLP4o-a39U+$ zB`JzPG0lbAm*~CT#48jzPZg()R#2i7IJB&$uly;AQ4Q1bS4>(WLdo00IPCdmPFg7z0jY!UBr1`N;58w<#0}_;X zI7>2o_Ij|=-E8QU3#SzRWZl8;Tr1vHk^1TfIpsse&cOiWatAPg#cuTJ1HNFr{{VkgQ`5umvNfM>7vea zlC8+{;j0^Y)qe)camc@k6JW@F{Ryf+vkEjB0|nE;cpABrC^uUHN0Wp%A{+P%2QyR# l001>9000yK0000000000004l2-jl&MqXeJTbqSMbIDa$(`~m;~ delta 2627 zcmV-J3cU5_ap!TcW)21Mg@A&%vuh4#4+PZ8wQ#f79X0G6BaI`+4U|z|>~~ z(F%a6&jgCW1cDET4VHub-AO(_&pY|$<)Sv675`-2w7frBE6&*Jw@ttJs8(;V=0m*bZ#Er`m&LFL zm#=4k^T}Yj98b?{i~p>fyi8tn{?gH^*zoN5`ZBLAzg^{f_5ABq@w%)gTd$42p1mFI z)r-CO)qFggkC)$WdOs_wU0)9NC$rJ7>p@*l$N#y`fBj|nOY-8EVe;F*YWla@Uagq= zU9_0px|h`-*s%6@uLkG&(O`Z)o-S&a`B^bK(WsHuYxD8>#oeEmv#a%=;(G1VY`L6G z?*4X>4@UWX^|#{Wp3P>`} zA%9s@<2Vq0-(R81zRW!Ml5Yx_qC%jk;o1bY*r$|ZE5SQHvX&ge{`!@%b6~kc5*+r4 zqFdJ2U#q*-cIWuVJdV(eWHd~Zla3~<9Yhj84Z>t{(s_Kikem*p%u50oQYTB~?|mx;*fh z$HRs@O7WwVl0dn+?xt~^CRCX`4}bF!550UicPuHHg+b^^lW8tJkc2buKS%)?dpwGU zewvKKiNLOObwrk1)>nXv<)d}QJDY?+;yj{n{qN|Bq)GAj#g?$)nooi^|GYZCIzyM_ zBF%h)`VUXYP+hgYY{9c8FOvL-PPyMsIy{SMksY1Wvz(E2`pT2le;&@TY=6j_M13C{(#WGk+2YJ#L>|1J&x)s3aQ+HTniXN9)v`+2 z7z1sCZHlSAv4&qstN^okF#Mljdwa30ZBoSR2xV_&7lbuL*lgknclQ=>Tb6gFYz|Bz z#nuz4pZTb%@YP#72bEd7L4Rv*)bjJ(xN}{* z^L2n_U3P`*V6R)n5O0;HAb?J;dwq339CYvcy(6S?uQ;Y*sG4Qlj$ye7U0j_nv{*M~ z%QaO)R~?O)S+S7!?;bCE+7YrenFoW0(3=1sA>Ec;-py8mYc6nQ-O_A?9`1hjZU@8r zUVk_^yScya6;G?8`+pnO=?0cngeEk0e(?w^?=DV4RSR)EnorlH!iEc5ZB^Dxd(Bou zldv_3Uo3dKl79nyF~AoCd@;Zm1AH;S z7Xy4Tz!w92b%3u9@YMmnI>1*4`04;(9pI}2e06}Y4)8Sqz6QY80QedJUjyK40DKLA zuL1Bi0KNvmS1jq39N=pLd`*C_3Gg)mz9zue6!?;0LduS$xYP24zm6+azftRX*esUP zB$*Fqnyk4p-hZ|&Ss|=?o&`c2`K4s&Pp5Av1wzLAkuWAA08nw85SkjsNxkh>B|q1& zp#Zgc?|roc@4Lm@&mI_y*Q~Z}f4}cxZLft)yCe<~Fv9}{%#yXHsR{QR6d+hls zU4Mz&vD|E5&zivASI>ZuDSxZTC@JcvBfc~OZR37uYk#!bDAz3k?pD5cL&U0>{RgGk z4A?5g&9&`tU}nQ7jRS&xA+v^WM7x5$`&x3q7OC+o)FBJlr_ny)$HN!QkKo#`n5`hT z>(lmYW-Ey8l5DqX?ts=p+1v^3E4j5@I2+K|xB$}ZCZtwlvI1%-DLVqagh`OT(B`|~ z0O`F8UVj^zwh^!0G}{)cg&f-+s%2JRa_j60*}@(0A!KWry)9CEsr^;Q^;w54f2gti ziYLsNAVr1HFc_@&)gR3c8}#o_?85~BO0}qIF4=$mmn6tVwri?SNV7heP38%w8zw{; zkS>r~lWB`qbyw{}D`uxDOn7fJt z)4ialKWo$ZYQ%g&>EX!UEEin%WN3{^T?A;Epd$(L- z2_Nz|>6YV1(TEGJG;bwWC}pSr0Z>Z=1e4x38nc2bLkI)_k7k#X$ub{*&1%Ci5QOi2 z3ZuI%TPCFl+kv*^)IgywExie8lZaYBNNOFweHA-#63D5?)y#f7yQo_3RhRsN^U~-V z65fiSLp*vd|+Cz&S@8xXl`#$T-(Hip?_;da8Hm;gVeK-{d)Kr2~ z?tlg3nhIYQ@o*jUVI5Drksuz2sNZ-yE;99jNMZmw#XTt><9000yK0000000000004l2-IKvLqXYntW|xy_IDh;)1TO#p diff --git a/summary_results.json b/summary_results.json index d2850e641..9e7254261 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 76.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1671, + "Performance_Result": 10.0455, "Performance_Units": "Samples/s" }, { @@ -1724,7 +1724,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0219, + "Performance_Result": 21.0425, "Performance_Units": "Samples/s" }, { From 6915cf3244563ee81770abcf90d73575e908e65c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Feb 2025 22:19:55 +0000 Subject: [PATCH 53/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14567 -> 14567 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e8ab43750..7c16577ce 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 22:18:56,426 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 22:18:56,426 submission_checker1.py:3270 INFO] --- -[2025-02-03 22:18:56,426 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-03 22:19:50,677 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-03 22:19:50,677 submission_checker1.py:3270 INFO] --- +[2025-02-03 22:19:50,678 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 22:18:56,426 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-03 22:19:50,678 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 22:18:56,427 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-03 22:19:50,678 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 22:18:56,427 submission_checker1.py:3286 INFO] --- -[2025-02-03 22:18:56,427 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 22:18:56,427 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 22:18:56,427 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 22:18:56,427 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 22:18:56,427 submission_checker1.py:3307 INFO] --- -[2025-02-03 22:18:56,427 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 22:18:57,579 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-03 22:19:50,678 submission_checker1.py:3286 INFO] --- +[2025-02-03 22:19:50,678 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-03 22:19:50,678 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-03 22:19:50,678 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-03 22:19:50,678 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-03 22:19:50,678 submission_checker1.py:3307 INFO] --- +[2025-02-03 22:19:50,678 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-03 22:19:51,750 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 3c7cb2996..42a0fafb8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -594 +595 diff --git a/summary.xlsx b/summary.xlsx index 4d5c49ceb965ded513f4f6d9b278ee8dfc5db1ff..58c1772456a440b8d0eb0331e84630c47406b200 100644 GIT binary patch delta 309 zcmV-50m}a8ap!Tc#WDmdxbth1$ucc}r4_aVZON*ELQ`6@6VfCRwH`=no&0^}XA(*xiMtaTJ8_sNOL*fdSjEIMn* zpW&~~>wC9qEL3|c2B<1HBg_Kz+ExX=F2dnz>ij00crSoIor6B}HZ0Qj0gple(#IW@ z&&Bi0c8#h$&6jbS$LV65=OlX;B+G2qj=br=0Oc^`-^3Bn=f3>})gQA8G+G1%E4cG( HlV~`9FRGh} delta 309 zcmV-50m}a8ap!Tc#WDn+)pZGz$ucc}Z4kBtZON*ELS0(A6VfCRwH`=no&0^}XA;P& z%hA1a=FXsEeUwe~3)Ts(N|Yrjia;^Vh1!?sz23wt6gf{7r;S!nq7yi@uI>tE2-6l` ztu|o202c+2a)g-@9lSRLW5*64sY`-G1#zdXq&}2(j}2wtbPqUB(|atzQ%*g9#T_s< zGZBqQ&Ss?fu}u%)3_}AFly^8wGJN)Wu+rUZ=#>kn6#iu3rgai$_tA;j*tTubE_!Ro zpW)B#%Ui!{EL3+Y2B<1HBg_Kz+ExX=EW+U$>ijmGcqf2Ao`QbkZCqsN10IC{WQaQ{ zAB(5wdV{Jw&6jbS$LXTZbCRtV$nxQ<8+p}#2Fh{BzljrI$bJ0@sz0*|G+G1%pVf5< HlV~`9aXXRY From 1d6176195476453c8f96b35c12c0ec0b9ae1ae98 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 05:42:04 +0000 Subject: [PATCH 54/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14567 -> 14570 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7c16577ce..e89838535 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-03 22:19:50,677 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-03 22:19:50,677 submission_checker1.py:3270 INFO] --- -[2025-02-03 22:19:50,678 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3270 INFO] --- +[2025-02-04 05:41:58,875 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3286 INFO] --- -[2025-02-03 22:19:50,678 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-03 22:19:50,678 submission_checker1.py:3307 INFO] --- -[2025-02-03 22:19:50,678 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-03 22:19:51,750 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 05:41:58,875 submission_checker1.py:3286 INFO] --- +[2025-02-04 05:41:58,875 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 05:41:58,875 submission_checker1.py:3307 INFO] --- +[2025-02-04 05:41:58,876 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 05:41:59,915 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40839 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.414195 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 42a0fafb8..7d3028fa2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -595 +596 diff --git a/summary.csv b/summary.csv index 79aa14cf0..c2ddd23d1 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40839,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.414195,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index acd0ae109..74832582a 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.40839, + "Result":0.414195, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 58c1772456a440b8d0eb0331e84630c47406b200..7ff847ed40f113384f78cbbddf909b43aa31fb63 100644 GIT binary patch delta 969 zcmV;)12+8Uaq4lftOp7^0WQ&n1^@tKC6jOt8MDs^js$;FT8gdj0g?V4{Y1gPo40fb zBG%{{1QBa6h=c}#NN5mK?aen5Vat%{#Lys#&}u|kAri61R3VA#aY9VCfvl3-klYRSUP8Y7KU7Klf0f){@d2- z0L@rwPs+0Yw%w$147x>TWERNCce^=!qJl7wuLMCLd zOV)oC*~>D2v_-{4!F%zmQ#ish4U#m-UaNeHV?e9W=|ze6u?%(<2R~Qr;|KSH(f;5X zpFRufC!s6k_1j4Etjk`C3o5K$XX(t@jg!uUmoC9)Hbl!Y~ko?|q8M z_ry*hRcj?dYAa4fs#L0?-Yk0otMQL)w}H2>6Ci-ZsmJxqemlEzHQT8+`32{t(KX65 zf)dc&2&p$U`dH4>2bB20v|z2#P@_F~G;8iEZYeho-kh=Ef&`yLkoJ_@8f`Z8* z1>>3uUl#FjE%RX!PrQ~Op7v3{@pfF~&<7%k0q7y_N%@>Sza}i_s3}NMrld&8bV*7& z&1w1g)sMXDKLgcq$iImvz#;ebCusfvvkElR0tq_-F42Vs003i?<~COWS(83DMFB{Y zgf~0^-;=>NK@DXp004MwFLQKxY-MwkP&gEmDW3))D rCIJ8dPyzq|5dZ)H0000000000fPuJ^e>gq?4U@h&BL>Jd000005P-JN delta 941 zcmV;e15*6zap!TctOp8Tu(_Rt1^@tJC6jRu8MDs^js$;_Ep5;2r_ev6pD6fe^Og=d z#2Q_L9AXXTkkB9w2@T?qhzD_~)Nx4c6(A0491Ik=QE-h~hD0ZrLufUkLW65iLSrhB zL*;Me@EA!QvJ71Zwq1uRva0WU{Zr_@qZ5VL8Zsb>SfjTfiCBY4Bs7RbLW4*o;;|r! zY8{D$0~>#mSmR)@z4=BWY#9=r7#buIT8$_xL?YIhDkM=oE{UtzD$Z7o&?#NKSjK)R zTfYV6G4iX2a+f!HKl4Pv+k`C>@X0|A!MqU~#2cYOyb&53@}|}Ch8^IUug1ZE zasc-vb{qg4hLi&|EfPDppzI?`V?sQ!#?&C5+Mj=bXUS#!6nKGC?n^wdm?!D@;WuzR zMm`73k>hn>)f{Q+vSKt9dGikpq5B=NC9A64gnJm$yQ3lzOXsZ5!jKDYlGn4!f7@D} zpBXFdNqN)+wyW*?>V(Ien>>yr_JU#!YZ#WC9b@2Xsc=lbJS$b{^5 z$+~|cds*g>wy2mWcrSi+2uE0^L6Qd9Yn4xN3TPENy(sZMmcg##;OB~c{NR2t+8nX3ZWLaM6qA*Lc zEQvOs7R>QjT71fj?k7pM`B0Q(dpxT5A5gPe4s0X^E4cG(B$LZB9)HVf!!Qs<_x%c^ zy)D}zr4_aVZON*ELQ`6@6VfCRwH`=no&0^}XAMtaTJ8_sNOL*fdSjEIMn*pW&~~>wC9qEL3|c2B<1HBg_Kz z+ExX=F2dnz>ij00crSoIor6B}HZ0Qj0gple(#IW@&&Bi0b|j6eJk6JJn#bv4o985Z zB+G2qj=br=0Oc^`-^3Bn=f3>})gQABG}8hJU$D8Iga!ZrVw34MR{>a)K{rJKN0Wy) zJOSR5#5X|!WRnIsIsw*`K{zHGE4cG(Bmn>bPyzq|5dZ)H0000000000fPuG@f;c__ P3X{P&BL>Aa00000FWR&{ diff --git a/summary_results.json b/summary_results.json index 9e7254261..9418989bd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.40839, + "Performance_Result": 0.414195, "Performance_Units": "Tokens/s" }, { From 4604c3e69fab0ba6916e99122f07f311c927b9ff Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 09:55:44 +0000 Subject: [PATCH 55/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 14570 -> 14572 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e89838535..561fd8b5b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 05:41:58,875 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3270 INFO] --- -[2025-02-04 05:41:58,875 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3270 INFO] --- +[2025-02-04 09:55:39,371 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3286 INFO] --- -[2025-02-04 05:41:58,875 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 05:41:58,875 submission_checker1.py:3307 INFO] --- -[2025-02-04 05:41:58,876 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 05:41:59,915 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 09:55:39,371 submission_checker1.py:3286 INFO] --- +[2025-02-04 09:55:39,371 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 09:55:39,371 submission_checker1.py:3307 INFO] --- +[2025-02-04 09:55:39,371 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 09:55:40,441 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412693 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 16.4667 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0425 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.20426 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.08903 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -70,7 +70,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0215 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0183 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7d3028fa2..c41b47a30 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -596 +597 diff --git a/summary.csv b/summary.csv index c2ddd23d1..f86168940 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-re MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412693,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,16.4667,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0425,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.20426,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.08903,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -48,7 +48,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0215,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0183,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 74832582a..d7f2378e9 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.4667, + "Result":18.9092, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.20426, + "Result":8.08903, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1479,7 +1479,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0215, + "Result":23.0183, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7ff847ed40f113384f78cbbddf909b43aa31fb63..b50e3cd74b834b13674e3c588b23022501f6c460 100644 GIT binary patch delta 5177 zcmYjVc{G&$`<^jOc4IANH?l9OkY#LBW-PBI%h)x@5+xx-&m_ri%rGjiB}>RUW6fSh zVG@zGys~8(vMVCr>3#qB{hmMWbDr~@=Q^MJzOU=P?(y?VRGhnp=EyfM_m4+8lR zgSgK#NfZ-{x#3*RPZsK?e>XdCU`kGqiKE=&03kAvEMji*kT!8)x*VXL+y&N;2fk-b z9Ikg?&Q-l}`N1-_`aapkt<6{CWLHv@I_AQ4hyA*r{SyHQVWlMC!$5XYc?9BArH+M4 zwsxXZf3Jsb<+N@1Ii9j}Iyz@~wjF9`cHwy{w-VoP(_T}a{GP=|)t9cXr@rl8x2Ko& z1KYK22WGnk+YSLM{YO)~7S4KEGcC_D@Z zihiY=8@(?rp69NV)Gt)s%;}Srv!xc_S>PHm}b^66!Rh#m`~Ipp%%&nevNP20+3H z*2sKz{v-953BS0QMbwUrhQ0s3sq)Skw)hE(YASBg3S8I(^S@9gtte8+pPs1fdUGBP z_w#bAJ*T#`8cWSqZYCnB8b>)u=AmjdvU4jA$*ct6bvJk=ms`Uy$h@V++6aqTxH+V9~YfhZ9c500kCHN(2(iBO& z>!j&(h23%Y`upOSTVK)e`iX$YF(%+=WA68?1MC22#ITa~2kRKLltWMIJa_w4EH(X(SKN1Vm~es*lSNeOV7T7YcrV zzPX*eT6qHS z+}CmHeCZkK@kN}{0Wf&EVQUD!(+&XXQ123nVx9yX%na?9<2%>UZUj zM@uTf9RL-9lZtp?L$*FxZYV5`v97?%OUk|Q76|P^jy@g`W2f9~#Pu~(PKI#J?LeH9PeGF9(X>;+XnF?6d>a|q_=2;UW0qA+!q?`f=!co^~CxuDmcL&G*95fo2eV9T*tf6rL&q&7*v%QWC?2`|QNT!9K z(lhY^tl$kcZB=DOLpr^^@>A&4m?Vq*jy0bjOM0@p4TqFu>^TEYtS~OUR1Z{W$tA_I zu{0`sg3Q#3eFYMk!ag98lwx#|kuz0GNcynB%;7Y!#;v!CWDdzMJX2i2j(Ss*PC3M zy_It1hPd41S(3l8xZT7Vr!{#y#o!z}?e~(8CeaEbk_77^>N*}qIpx7a=Bz=Th5tBp zD`P+cD>$Ds)7Z}LDkGX_4EXjg$qf}hx~VtBbs@}N`_X4P!Cywi*Fg>AuS6;oqb!`J zof|8?q9tuw6r7Q0Y11K8$&cf?SpUbCMIWuw$C!9fS9*EggrHbiy@ka1$yJCC^cumsE|nqQ+@3Q5C*W6;;k}$SNf>AkBaX-7&GX!- z3KCBd9Vl9woVbzM6%#)*^{2n2Z)MHG()C!I!$IGjyDO`H;AfU-H^<)g^a^-H!4$U({zp z&;~7?VtS%1J3!k-GDRriu;-8=Tt!z&J|hi?izErG(Ho1m$Wvk@pN(){`kloS&__y6 zWeVu~)5aeJii5jD#lfKftkD*%{n496&L30kTsG%l8K!0MOn3Z>wgKaqAvhE(niB}Z zVW23lv>Y&BG#{)S2S0#IC?EZ8?AzNTCow{K+Cz*NW0TAd{@GqNz8w+U53CH?Q(7gI z=D3tBM2gAu_d_$keKXMWZOafw6`S?pu^4ERz7hmOF#&-|i2cWQ21?OeX{5M*$vUqP z0l)VYSgs@C-34&g*Tbd6oID^%WzNSnZ_0*i33z{dA{hqN6d=!~^tGnTw4&x!DvZYr)C zND$)TrdWH4T4FzOPnZ-Uq}F3Z@yQgMJxNI~HG{ot!X9{YRzP@X0#U)FzBC+N7vj(T zTt>PO_Yu+-S#(mJ^-5aR_!65>&2GifSCT10HuCBg`Gw#m)hcbcxP@0b$_b(5RlHzq z#Ci?4(4`KNtc@xPE{P`bl!16?+k#8}_^5mt&*4~EJEKOi<+zg$Pv@n@?aLHNQC0e` zmRVzs9`}E|1MKZ7&H1#K-rXh_Cr6986;g~>&mmz6MH5{{_RsX-%(j%|B7!$O6V9LX z{>&WzJq(zwxY}S0HwIA_{v6=(k0ApaYDF0ede>#Ux*K4yFa4>-ROr;3V)+|-lNSoM z#OEwjM4i$kRZ#FYF`>lbF=L}1Z`@dydZ$c;KP(7X9*)F-zYh8Ao~nst95fB?qFWj9^||GN(cHylrjItuNylVi+wXT3deD1rV0EFbWU9q>#J z)6*ZIa%)YDS)rdZkmPspus6aIJ2jl0Xt5PY&_R@#J?QJsvMw}4><5_dFSpb= zSog98f9YDrZbwjDWs*22Z;>iGHe)BBf1ZkBg@YAB1{n6zGP7W*8A zM}?43{<;jPl*h4nDk$Z#Hx9XVL0saa&t?siL4oD$If+dJvnx>WVrHOIkQSE@j4XS$Q?t{%=FR){aY%U11NQO}uy0A}}_?7JV+nmTko=%?*jOsKZ zJY;k<3ryjmfwGVzV3p`99z~GLfr~h#3gBT#7#8IA{9g%sSGP}`tzJBVs?MxXjel*c zKHiX;RHZS?=`9bWQ@Nns2l}kD`R|j^L7$)xCTf>4&&& z_CM-XV{JPKqhH8fS8TJVw8Nl*@Kd(v&t&nszSk1g78#$yF{oS+gS~(-=U_MndWsP% zEawY+T&GNok+A9xi9p>d1YVZbIRkb z(#(KGt)#k1L;20i8K0ssD6zYLRKYbE#ZUz!em|cv1nPh1kbm!(cEZ$f`&zaI;>&+- zh$LhJP{>e`7RPz>wcocWfPuXy{^=|(}iuKU~JkE^s8z-_*T6_Qz@ zwv^wnx>eC$r+BfL_{Ks*R3lABJ%OOX!vWM6^)n{IbzDDVBAmS^#8b089D!l+8eZJS zY((TUoUi8>VX)GqSu$Q?eLQg?rv&%2HJiYIWTjf%dH2MAerTnl>kAnPwjTO&O>#e{ zjIq7IAn!#FIt4d9t*i8j0Oet0 z_$QRXMAE9EF0L4`oVg|lGt2Apd_BWxdNPxy%)eA*HXJ7la_G#kB76p#DjQO#>Ozy% z*QlGE4)mH(rS!`gb)guP+W{R9lRD)=ui=#F$?YPVGDkOam*ZFuqC`2`x&QcGQ>n4o zC_hb9y0^35;oDJX{a7Mbu7wrw$;Vj1(R zstZT4MsU?Is2T2^whDHOKFw2(Ydz2v;}Bygh5Xj73yF(5msL5w7NL+c(E7W! zIBtM!p}DN|*G17_Ly_CMp0nS5<$u;#nXx0_%;A(|D$yIB8(@UQUq3L} z%Grrw%=u6Z!`5*^E9Pd1$7R zO1$Z{BkID#4;Bjb4^0cw0AIhO8U_|)aj=DA7>k1~flVg8XsG09klmno;MXh00+#vp zXwZ6oLXM;Nmdi>yEar3pQ|73xe5w9dkvc*#%QYvB2Z@k}aP?H;;JB8W%;`kZSr|us zF&ok--Vet{RmaJn4**@wV?!!}ULg48p!}>1(XfjwsPnU`0bJpuJj%>ZZYGHAi1GwB z+CsHcHN{p7Powu=gEd1mPM(Zt|L?*?cK=X!eLgEIX)?(Eu(x?RV%kZuVSMFYiF=f7 zxh7^?(k)ptJM}FMJNb|{R=e+2_2+?lwQ`5+!oA%? zy4_}Tg%IFeozkA$Cb~9~l6}xQ@^A-I<7f-vO&UGJ%CI)F`yk2Q6{GeoMUA=rlq)zSt} q5Dm4?p80dsvtSU&iU|bb1^wTHd(cdb(L#WEh_AJzp?w$seEA=F7Op-3 delta 5175 zcmYkAX*iT^*vDs#Eo2#cWy>hWmZ>Dh+Qf_{JCikAmdFx9m?_E{W+r6G$dWupmKkfZ z#KeS%5DK9z!(%Ta%R8R;!~5PJt}pj}{EqXy{{QQLpU1Tlyc}HZ&3j_FBjQw(AOPS@ z2;#lS0xltx@mOc5$4uq%@nB2G3+fgfaIO`r=_j+wD?E`hdZmYZ> zvqgTlXR=eYW$(Y-fABqQTB@6oeSK-5Zp22X|2C%Ta5J1@pKka0oTqD5!e`=`A!B@I zFC-HFchg8mrwuJqrtETA{F90({b=;u+^o}bQ~+$Z+5XDTMD~VHWDV5mtysDtV;M1a zr;|<%QD@M987GDo;T4ifey+6-oOg8#5n)mnp1+r=# zqd9>(a%<<~Yf%T$GOtF>y@lm5-`39h7=P(J(2=1~ElIwum$I&eCqr%Nq7wGdfMa99_@Og^^ct zG)>gchv!Bd`OJ0}C#AP`eK1ufmOuXO&?h9QtTt>UxF%&5-5!sk+QeK}e|2iOyfyV@ z%mZ=)$W(qqWi2By;!ew1f#4gM^8ydI?|=oWTVw=)y;6{~Q~H4@L_R|$DRFEq1JO0! zf1!vvuFd>3{DGL7yV4pGky>hw(F9k0CPLhI|0@4jYhXQk2|oWDAZwhGU40E!9zkw# z7vH$ckhtiO-#A7}75kL;mWN@q!RIfPB6u`9-16WL`R(Op3(1}yfa9#w8JeNC|Fgug z9}?x0B7$qWsr&v%=|`(avH9;-B@UaX_?43-J~JXBgAdBp^38Gyy|ZlujC+Ii{t}c( ze34e>&1m^b{#n)=T?2P^!sMokR;&)pZr}TsnA;m!P<6$je+CSlGdg7%@UFv^ssk zOa>I-F8>{X{<2wd+H&`yV~(zV2#da`61HW0$% zGa1p9N52oo&EPr!5#=0kf4-T>-22ui3g8(yV11YF1*2Na^cJq7ey#CW#6!ycdCSqJ z09A#zRneL)%?^b6BL7%zXAvpK0^Xa#)?C;Xh_b%OWs;g^xW(u!=2_kPPNo(#Ri0+V zaZyh^3HcF;uzG?++e8j;>OJXys!zF;r{7SMz_tD37Z zxrP?UEj2*rxA$j5FE}~hkgrTwU@Nts<6`-51Drg~Gz0j2TY?bSj=31Bg-%aykJM zi9ai{uu=AN)2$Y}Zb08drm_?lho2LQ_0{9BPo>||%w1)o$9;oLq_b*8N zWpp3+cSm3nkUY=kmqHi{4Ae@7icz7jz>nPDrWW0O-lJ(@Bj6nzC4S;7DMG7i*+8{$Uv5U2JuN^3!lSR$CccXAZ1SZi-iW(ExhutVJ;V z0(Ewo!pvt*M@5ydho`kH-q(W}w`yrY;#=@M>&CEl#oHP7t8#Ja*_ug!^?zbo z+=26%o>=>})ka4b)QhYx+au8*4pgT{qxSEbS|=bk+&QZDe=V6j{sZ}0_a20|G&FHh z`~SSTIbm+;ssa{gJ3z#aP!H0U$;XU97HBt%Y!#bHiR8a)XXK}Y1n|va4 z2#E;gnDEzxZ#CnU2^18m)7R2RfdVGbg;eDnWeQA7abwXY9+OsZVzuZScNP7J4_6nzkuORy(eoM3f&YX|D$t$37m$ir1&YP^@hf28Iug%xaRE|( zvQ)pNvLt0GJ%S?~epsPgl%1dzHpwjOrUkWx?^fB1anN1_)e7TV84~k5vNr9fEnlRN`MVSKr1p?KoJ^tK6%5C!E@G5e@Z-- zUNLApv~e{aBP+goy_qsdGRl4U;gfO04|QW|>!9a_eC2Qz4my{sBkj*hmV>G zL6G8iiNTWkWJ%w1BM0|Mp^mX~D0>ngFgwpxUHm5lW4lq1uev!hX68UOW_ZFBbKC$q zYhgE`ewA%q^dKQhr^3);UeBZ-2gG_CLgNWaMzi-Y*RR+NQ=&~5R60ke0@R$LmSX;GL`2Ni4I=gyerQyVwdi{<90f9&F&G#yTw>clfvT#QvY! zrqaW~B$2}m^`9n3c|n@{GEe7BVUjZo%Q|i%u6P<`j{$u>wlrfOjkJ9%xqcfeY4tF zKx9*iuiY|lOr+hROt>BTY-*hLJAqwgOWF22Mn1q5C~^NdZ?YM5ZU$^BR`&$2lZShnu^tU%fMzbxP;F16-xY5Ell=SZWDLJsnDH z;SSe>eKe*f#AkE>s0m(JeVDW-(-^zeTs?*6o1ceDuExuXFR{TTeMV#}GxXOZLaCDG z8gVr8Xos@xgB#ngCOpi*S5@C=evvd2!XcZrl{`x)uNiQqSCQ=R|U@d^rERngwj^`tS{~iX+W) zh4L_vqd%%dT#shlDZyu5Ge!pfOy~a>nsbXY`xR-&XE-r`dK$)C+08Es!MpHrQ-|m> zGV`!@x|{e69(l){+V!UYPI$O3Yy#R={COV|PAw`rPxbfXLTJte{? z+M^yHw@$LRJ4G5BVy$QGTYK})4o2l)9eL*bqU5Ec@9f@XjDnY~QJqu|UR*s&Ek;#!yhz>@^xNxV?Tk6VBr&bq`2dL$INa6JDP9S3~V7Zi03DdD+?q0 z?~qhL*WU~flR#$f&k$D2ZZFZs+K_l33_DU=3FL$C-Ua09lqCoQF2xw9155wYS0;D8 zL~@8n6t#ZelzRAS;75?7vLxAe15GfjxfXS~JPuJrMp(@;QGRepJcQi`t^|VMy>|gg zIw?#NI0l>!Ok$Es1cO}0%h?t(<1@_GqfDtT10Mc|)=PDG2Adm3p{wBacPrkSZps-n zxKj>)n~$58@G}(2S;KST0h+xU!nX!gigD*t1fcg?Btb5GXJ}u2K?|9W*WWgej2i}L8XcIl&Zdgc;B!0}^P*)h?HBM43D*4rE z5HdX?BU5roZ_7=*khwMuSQvvYNx44!_kd4Io~=LOg*pQ{AnP+K+s}`(JC%X?7Sg{U zLi1*5XZ9&`qLecz6T#L)n){L_B!0OANfG>HHPgU&U@TuR0Kak*-d9JfvybYK_$?VHO4L0_@Nnx4~ESE1RTPTl>jp zIzo4Am#QO`aHwaLNVGf-qt16iCJ(62#OEF3Uzsns31^rQ2Ru3!nCD~l`c(FR0jAFy z*=?t^250M9`2LLK`9xp(Zm!k`2+pK}%zLjJ%!WYSOL#mVXv~~kABo-ix0(mrBS{5G z?|qT51L|)#X^DK3(7@wdX6hlQPkhUQ$RGqIsUC`U_c1L#hrnWid<#v9U-FHViE z3D)rXc7tmWDBpy~U=`=5G%-^nSiY0_aY|1`ouS#Q0YfHuHBESn2|8S?es;4kDm1NJrWkQni;7Zv|8sL z=Pmd}YUg`~@on)?;rbQK#%qc|gR|E7$&|94c13|u3MGOmiwIMdDxe5cm8un-c6){S z98(p0Vz)=f+_5TbD%;qeYq>HnWZKet?N)xnGe3E|9F1FbE_|)OvZSQrr#b&fV>aSL z*Z0p>>uxU*Tsjceh}zUHf)}K~Uk~fOaxogs);?_i@-!xH_PElNFWhKO9UnYDzX>zquq)w76^wBV8CKSM*S-V)Pe&U{a7o*;FDsYB-SnFCfl&>ScJ%=ddqy?LL^pmlM^ zMd>~m--BukXr&a`016^WWdWfytOKrOsO(^nORsd`Z`08aQ0C0Rpv!!jr3k()xZKEq z(=U0-ENy zfu1vmdyzNj%3QJwBb`N+rH z_>$bz*t7Msq9dfbO^dd$5P80>#p%lv@+^^*m-WZ!4prRopngeIYGnE3Xmq0JLI11b zjasqaGn+g9I^s&%b4!r{8^ka|HZA3otEPkS=>iE?*I#u%8!Do!3NY`WF4-fV7<%33 zF>*ukyQmt`w%l~@t&Pn|vo-JQb@bkjLr?X+Fzwg$rCr@IMpYXR2IaC3GmZUgZ+2ui zSs`rQmLfuS{8PIJIz4*$@5l^X_f+3N_we;qn6BcvQJMc~xX1db7|X`DNKT g0ARra00;p7@4-H7B*bgz08bJ+HDuX)E+4=7AA^CO_5c6? diff --git a/summary_results.json b/summary_results.json index 9418989bd..21fe7a6e5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.20426, + "Performance_Result": 8.08903, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.4667, + "Performance_Result": 18.9092, "Performance_Units": "Samples/s" }, { @@ -1292,7 +1292,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0215, + "Performance_Result": 23.0183, "Performance_Units": "Samples/s" }, { From 397f78baa4548d3d889fa26ee9183201fa3dd5b3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 09:56:50 +0000 Subject: [PATCH 56/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14572 -> 14573 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 561fd8b5b..543fda5e5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 09:55:39,371 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3270 INFO] --- -[2025-02-04 09:55:39,371 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 09:56:44,633 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 09:56:44,633 submission_checker1.py:3270 INFO] --- +[2025-02-04 09:56:44,634 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 09:56:44,634 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 09:56:44,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3286 INFO] --- -[2025-02-04 09:55:39,371 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 09:55:39,371 submission_checker1.py:3307 INFO] --- -[2025-02-04 09:55:39,371 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 09:55:40,441 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 09:56:44,634 submission_checker1.py:3286 INFO] --- +[2025-02-04 09:56:44,634 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 09:56:44,634 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 09:56:44,634 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 09:56:44,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 09:56:44,634 submission_checker1.py:3307 INFO] --- +[2025-02-04 09:56:44,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 09:56:46,094 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,15 +62,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0455 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2398 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425199 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0183 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431711 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0278 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c41b47a30..c23a6231a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -597 +598 diff --git a/summary.csv b/summary.csv index f86168940..5ad13f50d 100644 --- a/summary.csv +++ b/summary.csv @@ -40,15 +40,15 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0455,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2398,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425199,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0183,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431711,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0278,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d7f2378e9..18609270b 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0455, - "Accuracy":"acc: 76.400", + "Result":10.2398, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1449,7 +1449,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425199, + "Result":0.431711, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1479,7 +1479,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0183, + "Result":23.0278, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b50e3cd74b834b13674e3c588b23022501f6c460..9266084b6c39333cc3377d07b8eed33c8b8c7020 100644 GIT binary patch delta 6660 zcmZXZXH*kWw}ul!6C%VQ2q-Ot-UJbl-bp}2klqXk(m|BoCMrnBkU&6?CLp~xrAd{h zRA~xGl`7IAf|MJ-d+(onXRX<5&H3@nS^Is@-e=BidH?jTbYh^o5-t(lNN{J61=AAp z7zBXi+V5wLRy&&U2i(8T{%$T;|Gt$a@v&yz*5>A$Ah9e4r`+9xkczW~)6EROmU?rB zx4G>r?rFcqKdi(ae>gmMIvj96J6uEzrPRG?^QkiXU@l%^E1e#@pryuocx-S~r$x_h>bxMgP0d53uZ&BYC8McsF zj*|4Cp92B4&F0MIHgP(+A->t0bqgf>Tj?jh`@Dn|#+%HMj_+JgpNu!BS?$*^{3*{d zEq&m7K#+h50nt&-0*L`P_MU25rYF8+m1Kl?luqo7vkPW-xk~czd+#|fp5{g~zT{Uv zhF=w_3(03WcB?YFUE8h8rQZxl?}C~5FX@q(r_Eh&ryNVYI9vZgOOtwc(Pob4((Xin z7Gg2=><)rreG-Yl1Zp5Ch$<+}=I3+xD7ifw_@#>P08eyOZn~XjY;Le7VsY*Fb^9*s zxx}bcy(}owHdh7RwR30{`{BFgAI4<4N*Y!}%jR$nJi04=g?S*#f8bZ@$YNW|L&q+k zxf#b^l^6hXZhq}gL+18uQ`T%oE`y_nHvbe$&mzii!>)iBz%iHDSY#e>Zui6KFL2^@ zez>#+c>LO#B)xDp+=?3P8Hn99h+l9YeQnKho^iP;W6{9u=w!=BQE6bcJ5x->=11(x z`r`4JL$kRDz*3O4r2WNua6hDF#I^kQz0Y?>{szh0hKc;vkF~#A<2<+RSj+e}wf;y(>1x*GIbg!IGSL3#vEjK#L!kEHG+><&ZH_2)O!aHcQEW^&wW7S=yAv5>izrgb33B&AwJh3Nr_Q;ahga+)@ZF{ zSr0~sso5xv)P_D>)h z>W#g`SG10y-a%f97KlbEN>T*8knuE{2+(sO(>TE#-@mBAHXZDubhO)OEI0TEMYTkKXxgK-a1MU+PyHm&kr?_UJz1J%R2T*u0 z;@1r*+6a_-z`RyXf$xu}fa@87-Zc-+JnwsvtYEe_E}_4Ct`wDA%7^WG*}&BFtuD=6 zz@w+!WOs8@g4x!<%dHRbFezly4@GN%WWyvnV6{GR$M*Li-p7JOgXnxLz!<|L^DYXp zKrqgcGEfQsbe})?6vJqh_HzON2U~*NvMAh{-7ehnox8L3duG~I^XKTwioWepSwRY& z*J^KU7!5Laxv54&y;r0URDpyv+fxUQE8F(zi_VYV%O0QuRebFcVNtb3QwNad5}H<$ zzqsgGlYmR*CZ&99C&jWu74^&w_B0hIKi2gk4v{N$9T2>sww|qIcr$>9?&wOji9E!a zEiEzAb5~?i*qDL>NSAw2;N|3NhsJss$#4O4Q`$R~5C2Ao?ybz|vM~F+y~*dZ#N0Ho zKp`j&X*OPF2^Q_^j+4^H>(x?lr#N7(@!-dj&zCK^-gha19xG8bmV$6r;vgr>$F~<2 z#p&C}ZQ$xY7Cfqxo`7tl!TUC=KFeoOb?KAI?yvIn?K$qms_W&<;=822b?MqiZ-ct< zP?(;Ss^~-9)Rc}WlTni9#JwCnVeaZo3N|6^72>uO;}v4%mBl_|_Q1~#(hXwW`xk;2 zgeTzPPSzi5X0KbmukT=>Lgko`5*c89g1mg(aCJpA^$xTW3!tcX4_s8;wW11!C=tvs z1B0h=msJ(pFWYx^2$)QlRD@+}?JrrB+KZ`W=?&-(A2)i5_}U!$qSWX@lj>`qp|shG zs1jivtlZR9&^FW;PV7hlbt1*PCQ_eTDX7w19MohbIy{Qk!bPGa6ix7E^ajZ$!dl>f z`pjy=tWgTE_FQ;Cv%6^2a^-ky9T$DsQozH7DKQS877!VZ8wBMuV~?M z9=zdb*DoBbnia!X1+7H!BZpYMl7_hLnU^yb8VLF<{D4P8YlEj%-?4h0pGkex*Z9H= zZTXgSE&*eO5Jdcom}Zs$RVm%DOK&yQRRsr7>K5AYetybu*Q~o;!l0a1IvcUN3=QYL zH9`AKo#8T_Ms)N!he$@hgur@?7J}Pn-%TJ_@q_RjVPC8DcoDs;9VVZfTzVZlawz{5 z<3sYb1tQC`Is6i}_ciA*3f!;kk?4n5n=UEUe~~}j;rIYy3hp~N(y|QGI)YhB$?GEe z9JXIJ8>*{6lFyG=UN<%qy`)6pwr+fUQW0%Z{SkMsSbRWB-ZihvZ(#!$D;*JC+engH_;V`o-n=WDQa^;pa;4G5yT|Y4{f#Mw=wiv_nN1QG0yM);Mhm_A^ zH>2;=UXQL7`c63&K4u4ZNxE(10#A&q9z`N}!f&)8?b)z|+q4S@MVh4)9qUh`w}C&P z0(WvQJT4S$P4Ld{boDKB%XyoM(kNXVTwLCRnib-2QfJ5wl;d1wM+VW;q{E{B<0T_<>Sxm{)yj9_!jhiIy!?6DAXHW9&FP}~{;Yfgn6 z5dgcauU=Ev)7AzT5kCW(VNq-3w+co6+Kf@k(wRCb5cl94pzn~HYa8t{WN%Woke}g- zmx^wd4ONxu*FB63hYf%ZBOO6{ziGMKMg`6*>7w!t&PGNnZ-tZ7h={jf=`bKJOX|1r z$*DBjuL`^#_+v5Y)L;yeA4zA&-NDy;&&qo{tFq@`I+|X_0uD*Z?63}|r3t7U0YD2kR1iVHQTW<@foQ-{_wmSCaBK3rlMX=JPR3StgaD>&C9 zpwC8fMF5$g4mNkCO~rTl_@U^zq>xeo0Vh@E`R?nry`IWo;Tvj;)ZyQoo;8B`24P zuT{HR>>9>*rF~J$_l{C}L5oO~4mR)At%(G}vAV=W4-lqg|M1 zQRx%={^~*3p`>ZE4ppl|=o0PiSsk7|A(O!Vh` zPHHOjO%|1MRD_2+NG-cvO8~gD--UOd8fu zr**;(cS?Z-0t*@2*7pO=!Pv2aeb(W?|B$-m|EaOl+$YK{=ViLJuXt&2wa_Z_6d7SJM8d(|uyK z^ujBpysMeW_V1F+e-%_BJ3Ze!1Kr2!s$*5lL`Q~a3`azKkJn8eKe@5vcFXd4;zD@cLqbMcAucqbSkRCL| zu1C>%)1B&jRkt|c>Y|WO04EVF+V>y?q?a|-Ne3b-SM*>fA!2L_?qnV;tP=Vt6mvp0 z3Mr5xWe9I8VsRgqqW;_Pt58eYmv+8H?d4xG${3zLwf~rI`k(3EY=BafC&@+;4=sNQ zo}{sqlA@KX7%tHX;(1&{}qDa0CFK9$5MkOJ1VSN*2f2GOHYY$95P3=L%aaX zvs0l9?JK61C|l1uD}leX%)i~WigYt0)`>yNJ$_3$ip7_bC6+vliaCOP9u|;J&x;-`$W0UnIIqvc=Tm6~r>28t%4 zlWGIa3;8@a;p#FFcQRY%A6q|buv3^jT{)}g*$`P~4nS)zCEQbK^r9MNV2`QjFGZ@Z z)!C#{hX^%p&oqUPwTKi`CypNpd~{Lx9}M(;-(P*@;h~glo3Ud9B`U?xs2&t4lu|rf zd*UzpPJyhB;e0(Z{lW|p$)m}{+5JGo)W6yJY%Ca`EGizQyUWd)!8+;1QpMoX1FzCV zyF^A<0!CzDo|NiWJGN}RQZ!BBfdv_8v_9r1>P!ZX7%tnOSvtzxuzirFLyvBL13}_Y zBcDaz;fn|l z%e8-tMDWo)cn}FCp$QA;u&SzLN|*JcQ=atuAHW3cLhTWF%K$U5hL zV7S2Z3|!AJ0D87{OR-3ExL}jq?y`p_tL8UV^|W=K#>y|-DOD*-;-S{Tj&qFG!DUnP z)08BgP}w}Z3AIgxSg@k7OIMMbK$G2WEmr{FvjidwBYj<7H+etrhVAZ@0@_4^9T;~u z#sbW#md96j&36K_`EqmHA~<1zDA~Ll%U7OE$INCHj2?a3RIzgXbM2cCVQ?|`t%i6M zZ-Ue984fWjE*4jGdt^AsfS4Fi9oI$Br`?GQeZg1)3OE+_d2l!69}m@njhKJVe*s)o zi$ltkPf-RXO#zEnjH`inu)9m-ql-dqxK6@V{;J1V-I+^bV%#jcc~qL}w60jHE-8?3 zIt;rpM&3P8j<15d=fP@vAGN`qphj{Z$&=DNjus(?sTYM1?eCIhDZD&kU_IyKX;j2v z3=h@R+l-8WyPV=+Mr@eVO6XlQ|Ca=0vhvSJQ;=}z+|Vh&$W=)BD>4D05?_rwiPuwt ztKZcy&K^w@rXtaVP?tHlsHGD1v+W-YC&VpjIJ;j==qFD5f8J zwe5=&#Rwg?XoxbVo$}eg{almvLkjR)@X&Ei1ZYew@p5*Xz{WJTY*PFF z&|Jdrjc(f$SLMvlJp*2{xmnEPBqOyXHI`2xZ?JXHoUsqmC`>G{Q(OOyhP2>_f1ISIdE4GLU|bk0@0Fh6;SEI>3WSd zf&iwfypp(QN;u#Zr~KeBeTTr0lmnn9UOeo<(Og{w!o*T%#Hc6F$Jr_WPUKsGGP8b5 zl{`1W?~o?MD9!TeFJp{BPkGdv zoRqF545CWJnqzc@IL@4Hf8=bkJIF7DqkOqzb*>Qf2L1Xx)cf*n`3W<^@@JKb!HU6Q zz7~|x%Qt78Oo$+z3URBQ(%yZfiv2zE=Nqxtp6H~rc_6wJ-RrrLJ0fBU*OG zQn=Vt!Ziw0_@_PK^-7)gD?K5O7oN-Z-6I4^etv+TTL3#lu{`=J`7sTy0sMYBPTdpb z^p0gdklB{r>q(wMLc(dH<6m9f9%*Ki%vP#D2PVjpL&ev2$%QXE6d9Qum!*n5ydPeF zLu)&^baYQd&Wozx)82JSVT*i^xTkO1WxBu2nRiwoiO;7ItY-$5EiL9>WcV)ohVM31 z(|5oDHkpifqq8kkGsG1eku->&ZWT$hxM(usyiL7T+q4)hyISafZg%BXf(Ig9a2AP#$kKs2q$wz!)5 z_R_w}yxRNV*0rJ(^EB947iYv!Mso~-tT95fRjy;t+CcyPfR!LB z+Vq24S?6Q_U*K<>r2LZxy3aK4MiHWqt+=pr6WscrK{!?d{OA7$;NM_BUrQ1sL=0O& zwNnRbWoetF_+tsbzhU4!lDO<|$r3y1UA84Fti+h5r3>%bU-)!4J|U*TL(i19B-80B zEW(zDEof~0s%`=2n89>$cHz+@HX;j^6egg-j>dm`PEY4Js;W-@OK*WU^i{N z_UErdrP80tI2UbV7Z1h5fNU&Vm|@!#&0e4D_fwCT))ID`65a~L^q-p+98-Tc*6+zP zdoc^;2A;YPNW`mtd&l*ay^O!#R|DIW{A(6gB~D-!G6O>ikAysdb%WkYl~nzQRoPd& zjM;JGmj1?>ZYCf64z3EM?J;r@Lq&y;r!FdZffM>)CvS1(N;!byLGm!Q)qg8^LvrZ5!Gt`$j$y6&)~x|v_-NvgYJus1}FV# ztHqR!+|5-AkoMXZ_7sw3u+1Ro-4FmAUNBSgn`8cfZvRAS>g3Jyj-I`%r=d;Ap`wxx zcKoOj+c7i=D+d(0?h%EUH||9 delta 6614 zcmZX3c{J4T`~Hk!tTBiX!!T$VlCmT_WybOrS;o#FOSTFTdQBzSml;OoZONLfW6xfO zNV4xqME2}ivizp^=lkF9kJo+9^PKa#&U4?-b=}u}&a-xR^=`QvJ3|V1W1xZL#jXSv zA?3150`pbVC-pWPxB^DXpY<|Q_ryYIGhc6hNWgI9a&@T6zi}Od+f9eh_sSj+A9_;+ z4fUROk6)QC0=m!b0_&%}zf#AK*4wXVT)cDr@e;o55yjTN$zAbWTTG}t?#gY;gR1?W zaW8d#bPVvNH!Y?pSe>urnvrapO0;!Pm!oRQw1HXBC62;N*RGxC*tV>k*+pi``bK}= z?tWkK;$#*dTK#5yJ-(xT-D0J%XS=fL&|o)v+tO>f=Xh#&O!(tL>h-0C2~Sk!?aCI* z(;yO=11{ggTUO8JmKw7Q+#%hi$OJSg@`p|pr9E8h4ytOz@s!%d>g9x1zh0@H*~MQ? zI||sACjI14;7N0>^E|#k-1y37yJlv;^tDB?bM-Ds4u%4}IfIQwH*e2DqN)_dgfHRY z&sEv=JsL=i3}S2&=cLRx zAVuw4039w)8>O~LUYos0UFyi16E*T(qZsFkaHNctGxC{O zH--xlJz|u)rOBM@% z-$h>AEx%>Z$i9x5GDA0>%57I3Yn;@c{<-@bsClrza!7DD5#&ryNIf^2DbO~ad-Waq ztK8-=pnf`#<}Lm|K&w;LClIVX%io?U#Wa7sFeNlUdF--q~zyv(A#m; zLkHuJ-M;O22eXx9t*-bcxCKV%!G50 z8E{&p{wek8=6v?aq|bKjYUOD$-l#q4j=iDxF%TatH2v9Ns*x+M#zQau_(59E=#3p} z$T3JvKSm=?=yPYMG#96D=`>dNiJk=Hd$rnkCjitt*fdH0-W3sRD8~sn#=8{2gL!l_ zA5<8(91ng=(z9BqX~|t}-dfsN_B5J^1}YAA{jL73Q5C9?BA*3~V;ktLhRQn@$F!_I zpbR#4Q}oREOo$2Ww+ZrGNy$l(xol#TH(L^$nh{fRW4XR+Qw_Ndk%Hugn6{)?uP)uA zC}q)?W^j$9b4Ffk4~kRMLB?E3LBvQZw-~`~`8{_7BqQUO z-GJTf%ZD4(Uo!p2i?a7x0IE7cH2AS8#pH0QCg;r)lVUsyaUtLJY(Sgt$m8s2<(QQ~ za83{)m>4^Ne?Eagv?jSC6VQOzV$vl7WDAcq}U|Quy_P2!*!l{_|Pwg%>V( zj-}}^FY2g;kwUDdPu$-O#6Qq0kr-K{-W__ozq^~tQdS^oT@lGIK3y-wrOk_l;*k7~ zoR>SZw9W^Qj*#gA0#^uXGK6Yla7}b| z_^OJpWMIR(mqK=~e{e9#^3lpt-iOJnTiMg#t_ughRuf@c`tq&PU7)fTTK3fmmiM?> zgLcm@R?xK;{oagbNVFt^?VDMZST}{8{u>135xMtR3^?S;+YFOb$ zMsA};kf!`6_b%xO;J<&c(4cmB@fcyDC52rx8D@Vhd__C8S1w#V3frdtuUnxff*tnj^9$8pvzP-cGb# z5lgLO)GRH~OyvRH>t=8?HmE__esx9E)f;F>%1YBO0uyUD_E6`7q^%5Kl)&wJq711D zTioVh;*LVfV9}LE#YplZyC%8|L_tERcxY5y7*!r|9qXdTQ=dyGx`Ikfq6>0blHK-m zVh%xM80IC7h8o@^5=)_`jXloTs(N+*h-l3YN& zAb>T}1!kNHrCNjs9U6+5f;V^0c4EL(ZbO73`rpD&_F=o*=vTtMc%D1VDi>v>wN_S| zONIibMiC6C9TP4OhQDa6qKNQInrwJ}Led*GP>%5#QHG7h5lKc0T`T?_NN@`Om`p}L zQPq7oQ%Y>Kt}7}wMX|J2?k0TfCx%~qvK%uDYZKR3+nQ^DpM>B{SS&eGVV z0tpu-gH<5OqVxBoG2q-mHy#R|WnQ zw!|e2VcH;Nql*>OAHWpso&E^-d~TfEe=}*6jstU}i{pbohKb3QAA6x2qdgu@xVgq1 zjbjiQvV2{XS9709hF|FV=WV9?U5pZHv&A{@*1-(Yj)I{N_j$DHlDuHq9&)d_(^6O! z^{vy`t%}G4l{>W8J0DxB{$&Ly=c#Iw9dOU+JnzMbycLV=Vk?iqK|5)2EIiRL)1E3N z@>0D8)17cZSli%+o`-?_%Rgei$%(Mle>@QHt_|}T=q9vUPo$`yI^$D{FS(???Y@^3 zFW^yZSEn*38aIoTninOSSYT}-bF^yyjfx2Bu-iEbOJtGp#d_d$9#UE%o|6MAMTXbm znbZxSKlTh*_t#zT8yIDC%NUA!e^LS~&iKF36%Z$xz0yVI6kmzLFwL;ZMd6^|k=)wK zVN#q$1iwSu>^~0bttcCL6@Af}bQ4xUJx0scnNfpBH_%4w0rx_rDNN{;XT0efJH^4 z2>Qnvw2@#QW##k0%!zw4cCKj&LMvmv1~ME6t z+9hvZN1MrsFyAH*LR5z4-YW{2JrhH-#GMDjCC#;K;Y$(cEMLxv30o9O zlS4~2?2J=KYaJi$eYCKssL$$t>)NiTFfo$LE)}P%Ka27O+>4|%9N zY7H!O}jw*mvK z&!kt`lwLGM6{$#tiI6S`C;129E|5#vnGr9+rR?3Hmvca&hGxF;uoDBpFtgCPb?7lU z)?2Y}5y#{cm(kHJfm9vN30Z*Lci&xueid@>8oRAytpoq5qa8$GRY@qsEgW2R_bKXi z4)1l24FuFQda3NnWjpvW5arky{g`Ic|+k`K#(#0Tn)X z1sQE=3TqB7?3B|H@{N}+W$kDhbSRz&vYC5V5mSDW%qQ`MW77v$1$~OzofT!^cP30$ z>*14ui5Pz}##5CB756+mkqU}??n=$4`wEdub~N7+HHYYC!iY6;4tvM@ASRd*Hmpgr22D7@pq|_+8HCh=u=v z;0~#@RLggQ)u4_4OLZaT*#=dvIx)H zdxy4a6Q9zqG!`&j|wQX4bb!iA1VD`1quO*0J~*d<-gHwt8c^w6DyVsMg>E zKF0jiXp*$*ntk%-EXQoW_m9KdoGXceXikIVAXNR&$OStdHJ6OdF}H77)gq;>Y+>Ztl{){4d*9ga@~M*fkop3u z=piDMgPG=^P#P1-t6HjrJmVQB9+*L9oAdh_+NHeG8GuRMt+%eE zCVr|aAXa{jy2)m_QXYU#x}IDWfWz1yt|Y=l`5afu*+e=s+DQ8JVfE}q1g67KA=YO0 zzkXNuMh8B^-7O^6)mm-&^SHHoH2O@2(M@2;O-IVgwa6e$ZZIPBZ#2FORuS_UoU9ceM?;<><>N_xW6E;k z0?}j%7;AMNv#xfe2Z5O?Pe5Jv0^Q1F)|Ce3t8-s~ax>9H!y-5@&&?!vCTN61hW>?5 zl;DmKhj*c}ToeORbRlu?Tejs^pq`E$rni@0VXK~b3zZ5)e zEnPFV98%yAYF4C#+eX;ODy7ADbmJ$Uc8^vbxRl$*7c5k_10#0!HTM1|1O0EuHdgl! z<;!GR?B+vuk5x+49%HojWmM3hXQm=CrL|MqMjSkG=>9?2Q8~21`ztQAP|U&_Gy5MXO1haMU%k{`>c$Y%}Co~vhoQwxi0qH@f3 zVOKV1v^{EJKYUgjoVW%_Bhs!V!UZitmphVzuH2R^ar;uB_o}1OP2ioOQT`R)S9ggw zWY3-EbH^FoT0UHTX$pTP@P2GWR4LM?)1cx-W|ddLpthCZ5vkt?!Fk8J;3HaD?Pq~f zlyRGLTqgX^yDS*Hsz3wak!UubaJSXUe5?S?8f^Z`l=l`^C9BX+&-|eh{JEB4(X592 zD`>I@*KBKCzEk?_a$^Vn{dv3e9AC5aHyCC1bk|3%EtZynBsnZFV9AU=e;vF~G3<6( ziD}9N2DN>(m~vOZJg@e{D6wbHW7edD%j0xuohy5xKQUi6^;`omFsmj1^P-Pb@UG%a z1?NICt30tznyvH`nn#+?xk+lz?pJg7UsP)K#3Z1ySj=QKdyiCv`^@s}_C8Vcba-vbC%EZk4oveMHUw;43MU)Q;|r zA6c^M;7*zv(#x@|LPpdpx!-=HTWw}GWj_Rae{=KcrgHb4IQdjl6}G8P@#v?dOA-ir*hGnN zz@ix|uge`}I);Q$-gxU@SZ4a+>v}F+0dO7jpVfb`$LnW9eo*4hjz+bQa2Mv=pF4rD z9tcTC+XqxG)VMgEE z*G8kJr2@1->9%`^%a4>7zwL(TVb0lm*yKkAOx~TypVC^KT@HOM9+5X8#F_^6Spl*? ze|pvk>;;VvCM1>=iSqe}f1{sc;*X!;obp1|@ji+79IJoY*EZGrZOt)gdWD10O=)km z>Bs}!&~iK7iHk?UJdk^Riw6j>%u}A`HK9&T?wkB_h1m3CN;WP?1@=h~6m{yqGf{VARNr_`n z&qpA~kJ)z^Rr-V%HG!=ETePNPGGymL4s;vh|6hQ7zXJ+ALns4pD1e%wt)rlQ0g%Ig zn?{5(<_!gKvqbAC{r)SE1NZkkbfHGT-`y00hbYE!m}mG-=E4MmKyaZ+d?-t>Kgkc}1I$R~P6~gPsKoW1!AqUNoGZB|)m#16;o4ywx6#3K zj&5~cp0L%3^jeUf>lgjs2`2pVgc~=$-5Kxz9yz}1wVQvuWIw`ZgmZq2gHJabolM}` zw{H4fs=gAbU;KMwXu!gIGSEWDK$I>!&O2N-q_%D4TS?;B!1E)c`uTkm#aR~Zi=S<} z6gu{ofJ%2MCnoxS_S;VoBa(P0xsH2@Tz@WTgELypca1I z{ru=!!##z`Ld64yg}i=+PnVx?^?hY3R8ly^eHqkC-fyjPEa|_QqZV}K&Mzq+loI>5 ze@TXyC4o7xG?QU@s~?Bq8b((y_q*7@NnId_DgHGBgG$i9ZdcVqAj?4Z diff --git a/summary_results.json b/summary_results.json index 21fe7a6e5..7c4e66bb2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.400", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0455, + "Performance_Result": 10.2398, "Performance_Units": "Samples/s" }, { @@ -1292,7 +1292,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0183, + "Performance_Result": 23.0278, "Performance_Units": "Samples/s" }, { @@ -1328,7 +1328,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.425199, + "Performance_Result": 0.431711, "Performance_Units": "Samples/s" }, { From ccac3538db7e7ac242eaa24080c222009d890f73 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 09:58:56 +0000 Subject: [PATCH 57/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14573 -> 14572 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 543fda5e5..796c5506d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 09:56:44,633 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 09:56:44,633 submission_checker1.py:3270 INFO] --- -[2025-02-04 09:56:44,634 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 09:58:51,244 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 09:58:51,244 submission_checker1.py:3270 INFO] --- +[2025-02-04 09:58:51,244 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 09:58:51,245 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 09:58:51,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3286 INFO] --- -[2025-02-04 09:56:44,634 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 09:56:44,634 submission_checker1.py:3307 INFO] --- -[2025-02-04 09:56:44,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 09:56:46,094 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 09:58:51,245 submission_checker1.py:3286 INFO] --- +[2025-02-04 09:58:51,245 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 09:58:51,245 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 09:58:51,245 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 09:58:51,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 09:58:51,245 submission_checker1.py:3307 INFO] --- +[2025-02-04 09:58:51,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 09:58:52,515 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2398 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1507 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c23a6231a..d77dccec3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -598 +599 diff --git a/summary.csv b/summary.csv index 5ad13f50d..748377822 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2398,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1507,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 18609270b..41c4a9343 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2398, - "Accuracy":"acc: 75.400", + "Result":10.1507, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 9266084b6c39333cc3377d07b8eed33c8b8c7020..f50c98b6dff57162978120dfd66828fcf6ecbee1 100644 GIT binary patch delta 2000 zcmV;>2QT>TaqMxh?jr@EpE0LZlk+1Xe_2!GI1ql{U!lsr%slv#Zwi>ALZGPO+61=P zr<7wW!8<;(mK?(V`jxSBV7WsQ9QKK#Th`ZKtGm^9=lI7wj?j!`G)$9|jwY)eL=ry@ z!enyNd3?B#oDQPQO9C%S6LQjdCA9P7^tWS58RCy7^rZ95*z`zIsQ*l2kILzkeo`nD8q+3e(n$GDl4Ns5R z=}j7t=vZN=$4Vh6g#(`?UKXZx`C}3?S{L>S%_CNqW`rh0kPb-3-BFj`jmJ@#kh=Jf zoMa)Pihwxq;%U^RxH@mJnDa(gf9L2MktZ_Zjzed85VG&}mEAN-v!CQu0Lw>l$e3vL z*~|+gZxlu$6CZk|jKp>6E%&ux^L>{3gwj;lfBL(C==&rhna9M%p0Ie3j@gTs30}QT znP|j0VVsO0?s9|cGE-J1RaLdRJn)#u!-hLb@uQTIK)Jc@rg5AmRGB;vfAbL!y?i)# zEGe0VLFh@7X)ZmGgfs6yNC6ppJc@>XnvBDVz^-(4M3!6DSAdG;qjkkQn}k5(Jfd&? z@92r7N%8l^mayTPPl7l9ygI)+LzmkQ&lhr{3LG7f8-Uyd=(~9=x8B zWp8B{gf&FiY~l%b_ZDzlmUpFW4oo4%))T3rUYGs@3;7yZp(fp&qu%{rU1Zyqjx0m! zrWrx-)mu6Tm07z%e`{{k^7Gudb6vahb%14Ec7^L;uUo|sZ^5wbLy2ZM&tn*bjn z-IiV6%~pbIE^uYt(rknt?tb=e2gCbbe>gb1xxelePphK)e;d~629{NXCNy?_@dzvL zE>1#K3voP}PuHZvh6`J5Rn|;<%~nH`vE@1|TMd4%U<)_ein&tX+6IvpS8@|f*--%% zk3?Iz05ihE#B1o%MaG$dlUorsE{C)99j?b2O7{HXh46U0sVmn^amQy zA7}tyEO@$-e*=6mz!w92F~AoCd@;Zm1AH;S7Xy5CfUgel)d9Xbz*h(O>HuFI;Hv|C zb%3u9@HGIw2Ef+<_!y2k%8sPC)AEGBjw@BaQR{iwESAzFnGa{0thqAYf3_`IA*_0y1wtJ8rDW(&r*9|) zLdN@%FeV}ZP;r|Oni|GQz3o;dKi9CK0JVAVeYFGcyT#kj9vF+)thQ}`zwcpfuZ2vz zBn}ZU!vh7(@QVV*I*?CR`|fXh?D;8Oe~H_%+-zUZn!w&y&w!9Af2+tSDe9*qzBB@D z<9=vsf3(^t*DV3=R=#&b#HyJ62c_5y*eb=%we4_VX2U0q1A=`avxaX(yMn#@T5`Y^ zsqrh+Aq&{2(LUkF!xzkt;M%X4tsu7R)Anm-D~RoqY`1FefYw6U+zIU~xwTz58_?Le z0MhIxq*i0H0%|BJI|99gNszwK=DXkk>AeeHe;b*$5wG1e+ZL*Y9NQkMWmaEu>+A~I z!X5A-WNVqdEmC`_{Z+^HS%)ousImNtC(M{2MTO8X7_9fzAI%OM^zTpX!vz3JwWw(> z*?;|)B*;a!YpPF3vp$$j<_V`8CPWyJE?Rd~ooijccJv!3n&J;1mVGQL``8UTZvl>~ zUz`=}AEPQ{0F8*e*QC?>{J^xByNUzTy{ysOu*Q&8aeVtW`EyiR6^DY0silr(Y&vUG zOG~ya%U%sFF%JkwwGg`O-43sNw_IWgAM!Wpmg7g!hzqSWZzWeKWvBlEvuG+|2n1`L z)qa!BGAw^+DrzM`YAa4fs#L0?-Yk0otMQL)w}H2>6Ci-ZsmJxqemlEzHQT8+`32{t z(KX65f)dc&2&p$U`dH4>DN1}`TCmn=sL>ugnl*P7x0D+PZ_Ze7L4r>rNPEg{jkY0J zim~TgP|Ro1p`*AqPO%V6x51Y2Z?*wk5b}T(1SWr2V7LdSb|j)BQt%OJe_T5N1jo>V z0v$ZgvmBqj0i5zT8wTaVsbtt&xM@9!qkHG&XzaQ!>n4LW=FjnJ@%lcjno8ZDiUVpY z!6|pZf^kiSFN=7%mie%VCtgbsPy49fcsnj~=mU|&0Q3;|q_~ delta 2018 zcmV<82OaqAaqV%i?jr@T7m@N-lk+1Xe@j#2I1s+?uTW(#GY7w9*-j{6iVA_EhSw&r z#hy}*tpxA*k+tLy_SdhBod=dXB*9@%6y37E{#xCwwmZi^=23`dG-W{&pR^2BZy_3c zi66w1lh)(Ih0<;z#@*O=!z89BtyjugKTdx;W{e~8Xv|Jp&zw(>G>v)BG;*1me@tmC zG~*S5W{PbAM1%+_n(b!Fcq$+=m1J0_#K4n?RtJ0LRm`c(CO@%wE(!23E3}RXp z|B=x&piGkx2W~VC>l9b#H5Lorf9UEQeWU6`MZ$6DEb{~Yy}GiSgh~37zDi)(C<-{2 ztv;K%LFkTxFyQh-uZ+{ED!moHmTbOH6OS^MNc&HJmk>RVhBS4#yx5f%50Wu|aZ|~w zw+WYxIHz2Yk;GkLa8+ihx}xj4QI-2HcSYC;M`>P|FzRbJ*WDzFl9*|ef9GK~5}}t3 z=WSbwXMrEM%4C`;4>T6c`wxmw$F7K?p_jztU?Q%!E?e-d$nzvWVpHL_la|P0 zmS;!n^ep3v$ZqPR(JNXKe?#3HG&5DKS}Lk;FKUQjsQAHnoaMDfnM)}hhHm6y#nDGf zr&Co%!JMaVsI=$KNHRV7C5)GI?NJ#*Fro-v*W>w9D>!cjr_Ay&F)CT5ZH$4Y!8XOz-dMx0B$k8OA{hQpu)V!l)HWp) zb%gP^vJ1iz88)4`(%roU+?M5CE1CmKO0o4ss;Sq7|G+}NMpmjxH|MB#|5q0|j%^~F zDBUEb2)=p?=b$2Me>Z5YjaqS@8+Wd1cfJm=YN{RSI@s%0A@Z$K7X;ABb+51PhlB22 zzjuTT;gz;Uh;G=9(#O?&VMGf3o{))|mvWIzkf`wSS2S zEAB2qLf3O~G@4J>q|$~9TOD0BEN9JDVyM{ev{$wgaj#$-*V>AO(%jkxnU+ur6D*bJ zfJ#K7BVB+gqg?f46N*RtT$}XMqrfULhHJ)9D*Z zj*yCeB#5XC094#Mgt~@tQf<3+#mfv#G@v%`y{~cLeYbe~*#l$on$@)J@Ao~d?Uj&e zmc$_fW_X~08Gcc~*az~-YTx~Bk6bTd>o0LTmYePCSsmE>>giKD6>k+Oqj~*o%P);U z+qfUve;TbO%5_74yOr=(W+RBr`n36)*$85@B%7_8JD@dCHg`h%N^VUT&Kfj6 z&VdZ44ynZ;RAiYJb&neb!;iA8IVW;z=_mNKqox39zjpK+Cz|pPAC`SAD*D(BJ8uDw zV9K18>>r~tWB`qfyw{}D{`|l+pS#Kf)4i-Qo3Mtcx;(ypoBXwPRhNf?i>alFRcy7_ zrk1v9+qSbBT4E6pZQVxbvUfYY?%fKBEq%z}q+5<3Wg{+(!n~DSV~n5v2eW4?VF&~w zjWa-#%rY#0AgHyHAhi{zB2_9?tKKYo0ju$kY`1~8uSp<)#Hq*i%zit&ay8$nHu(kT zrO`FYGlCM(+z6>RHF{qy(tDKnz_ehk(NLp3crrc@90kaP@oC5{07m@N-ldv~e0cev5I648^ zlS4Qr8zPM}KqdhI08jz|01*HH00000000000DytJlY}@v0T7eJI3otjH2?qr05`1E A)Bpeg diff --git a/summary_results.json b/summary_results.json index 7c4e66bb2..78f1d4698 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2398, + "Performance_Result": 10.1507, "Performance_Units": "Samples/s" }, { From 6a03f6708bbe8e12492a3a57f972b12804796f68 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 17:39:56 +0000 Subject: [PATCH 58/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14572 -> 14571 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 796c5506d..750189a9c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 09:58:51,244 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 09:58:51,244 submission_checker1.py:3270 INFO] --- -[2025-02-04 09:58:51,244 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3270 INFO] --- +[2025-02-04 17:39:51,435 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3286 INFO] --- -[2025-02-04 09:58:51,245 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 09:58:51,245 submission_checker1.py:3307 INFO] --- -[2025-02-04 09:58:51,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 09:58:52,515 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 17:39:51,435 submission_checker1.py:3286 INFO] --- +[2025-02-04 17:39:51,435 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 17:39:51,435 submission_checker1.py:3307 INFO] --- +[2025-02-04 17:39:51,435 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 17:39:52,548 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.4416 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.351603 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358788 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74391 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -74,7 +74,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28764 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27277 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d77dccec3..e9f960cf4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -599 +600 diff --git a/summary.csv b/summary.csv index 748377822..15e61ed53 100644 --- a/summary.csv +++ b/summary.csv @@ -15,7 +15,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.4416,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.351603,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358788,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74391,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -52,7 +52,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28764,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27277,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 41c4a9343..0cde5279e 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.351603, + "Result":0.358788, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", @@ -1599,7 +1599,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.28764, + "Result":6.27277, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f50c98b6dff57162978120dfd66828fcf6ecbee1..cd89bf3f8cb2b3360d2e9dbcf5efc3107e582de8 100644 GIT binary patch delta 4449 zcmYLNXH-*Lw@pY0RT8QohMG_VYQ%sD0YWheB7&6ACBj7zk=_&&DMAn_i9|0TJ)njb zno?90kfwkjQoKkJq$^dWT>QcvZ;W?-?DOM{GuPT{&OO%}dpT$+sLDr#*Ub+l(ZKK* z(dEGxGgu-vlv;Ne6yM;7$zaU;UyBp#=KoIY{;ApZ#{9Vvlrh)iY;&n`gzbTP;CL+S zefz3Xp0L!tQG2Y`tzb;HHKb>{a%1D$v|ldNV(C1irzEw|Ap|XLy`(GY{?63zU2pH( z*1aMBNW-quo@}L!z2GWT$*bqm9$JgLO+`U#HAB&U8TGUdp1*XJ>wCGEk=A#N+3tDQ(d|x6 zPy5cRx~}JX%trTY;g{cstX}-|HgI+?6CBolTV(T6zkh3NHJX!Swcog%(Wn(TXkQnw z`xHO^D?pdA3C%fDOlGC2d@;)UO%-4yv38REQKw2QV`NPK{A4_ZrNJI1Uk*Z?*{Zr| z*(~w{KVHuuh*Er?j`yE=#u=RQkH%mt|FE&$58~El3-><2GkB=83<1;M#|AHyFKwBGxW6viqf$xy|Eu8nT&-}$>8W)LHg1a+N$?^6X$qY zjuXt9?N#zzO@KsL_?=A_DT{2gLF)zGj3%z1(Z5KC(uXV>Cq(ZtPRs*trO$ zHH0Wl)dJ90&u6_X68T?Tn$PDr)$r@I?`I=2b~luN0wubHK1%ViCccDA*dI@}M@W68Gn8$I8-wMVn$!iF9!_!kslSOy0NE+hdnf!b94O| z55=u1z6O$43>$cwfGL?Js~lq0L8--x3$!K$R1f7`Iz z!2!s-g8GA5VvTFbZcBkci3dP;jo`b6%o(zUjB*#!8JKQ~HrDO}G(0p3E>ClvmIE~8 z^0+wwI<*c{Pimbm2@HCWFuABLpx9Uj%fI?ecD&SevfN9{q;2f|60!c=fC@*Xlww}M zO0P$7-eEX&QsEf@2LWoM3&ZZpATU2gd*UUK8C!Pqbit&uQez9j$plu@yVeBcazZ3J z)53j2>-{y$wVQ9;pC{LWr(_zSkr(g?Q+mz_U1KTL#=M*uho3@!Rb{6NHbc}M_H9m=LHq@4Z%PV*5$10Xy zim2lY(&$9ZB(M7!&M*PR6d)Pa14uDj0ZPaC!PKcuE_p}W1Oazvd8(S)d9<{gz)?w? zZ!RLE>Q?QXShf;WpC$D(qgr7!?BnWlb~zg5_7#1(B7frx!sx{8B#PJlk8NiV@u3iY za}+2PqGsMbCy~K}FYVJH^HR^}IhqM7JW}r`+(|geS{`zA@jqlJ)2=mi2m7>Iw^sjR z31RVfVSMmIQ^7h$ySaPxZG@mjZ3mmkGM0jqY;`JdQ+qT zSyY67Al0~q*X@3OuOXL&GREvjm58!7W2vFL z`;?puE2}h}BO2?@lfW*uZS#jB0>A>4vVgY?K`k2wdm4iiWdq{5G3q{s{}&@EO3vh& z+=2>nKcXzu<<@C}BT5WdknM3sW_H_tEjQ=b`p{wPWQy+kpiH?}0x;#}@CQf{yK+O{ zpLeC|u1=QfI^;93OG24|1}!Twfd?x_5iwYLm?MZU={q*CuGyW~Pyb*eMo#b}7KsewDLg}d_D7Ihc2K2c-p{&;k; zY_8tlN5QJS$+P*TubpySbEgC;_SJ9m#eGI4ZGDU2nV2Wp{`#(p_oQfh-Fw_JYq}|l z;V>XqMJ5o_6`yXEhicPM$i}+N?K>ZKi zylQ?Vmdmw7P%&I2T?Ja7Ap+x~!4<@CVxXf6OrY54Wgn_p9=MI7p+Lh5;4O{jV9sc? zAimCDW!=;)jT`IqO%KQAZG$)8jKX|1o!nu}H3(BBb&RvbV731HL5uXQNf^h(#f+Q36^#XT>A+#uLBcB4o${MMU>l)SK* zaLZHq;xC?1fC~u1y`@hyQ_z6gz0N+B1I&+9Jb#05Ka~V|?q$u-qa7m$V^dvs%oMcX zkGPnfYLnw8m2wLS!f%RVL)*!H7o~6ZHUwF^<%w&_9rY^e)CYrB(qnWMmDjaL4yx!a zFNTAFyyTfc2)BwRc`L0N3US!+vzDdbDF1C++OVH}Hd-<6(&l{fz1%>EOP@Y8;K@u# zp)OMXs&mH1^i$LI4`x!d;@MW5gEJNp4}*}fC=d+Yb50Ho%j2!nI& z?tPH^Mz|MjuiJ{QS=FD4tds|6mplB9I;SaW;}I|9q>@>~Tto1t4!5FPkWqfHz=gqR z5^!?BXG^*&Iv+{7oOr0=KW#3P3LGcwmY4P-`JdT+I(@?1$*0c8o)-KF4zEvDtiU6} z;9Rf@QykutTG5I&R{Sd+msj2@2uQ#>&2jZ85nI+OB1@|5FF)IOd#jrmx8>~I!lZ7G z*9{Yy(uFUws@a2n^Jc24P4JSk#m7HEf17P56;s%)I8JC7p1(^tFpLE0L|Fd|M8`Z8 z(FAzyoP&c(L~6a3t8QC#-~wyn;sZ5hm3}--o_*AD>0fO>H{@O6v0moNBk~!x+b1?h z#F^$HM!1ofK=%ES4jVXf=|`1tKQVU`RJ$_+cK}@3YB&S=2Y#h%#M9K-RPLK8?)hup zN0bh`vW>l?I?smYPP~~rVbPSE=R@YpSsuPXDBvQgZ>GcE$S7=93AYgB7D0hKH{~1v zlbZ_$kWje;BoIgBz(@_QqlNf;vn6*8tm8MLst+>Lzs@Yr9rDiaoDzA+1X+>qo7paU z7LbUBi1;fIydes7h0eDF$f96N6D0vUF~l?=i>saR5tvovU(kg+s)W^ubM5=ReN%xh zT5>hMYb1sl$29sBEzYd4#?1CB7s}575s!deF+hP3^wMnr4SSc1Bo-r<2BdM3(ENIp z%l)w%L|mszq+toUa_$7?&y!8(uASH_39|La^O;K+!eR@BF0GxRbAyD(VoFVMN#4}a zRx}kEl>)uAAbx`cY#Er_l3wNNge#LAN2xbZCa?mUGNCKkgteYrcN)0L?Gw|somk=- zmg8V}AfZ@vyXhX3gb(iLPB3WWpPn+4^vC)wtiCNzUaDo@(&d(|+sqVUo?gw(#SHh( z{AJ3J?&q1UJ-|X8eT(eU@HtoWp5p#8Vpm z2vn#GY79Rx0v$Q75pdU~!AO!m@KS&28mT|>V2f7RHycR_mncMBU_+)9{ejEMr!Fim z!O+F#Dc#4jk(JxW8IM#;^E2z-+9R+TX>b;$KFEpl$#a9G#8C6ysJdV|#wEocGQuUr zzuIRFkd1wmXm5;(!KU%%aY@lnZayY(&4pT9I)5>O#0&l~_nj0Nk&jd^YN1>sV8owm z5pYQ=2FV=`W#1z|?3MmPXgayVGN(-p^iNbwpvdrnR3w>39}p3$E3W09UvFwH_xvJn zh(Rwb)P$3Giw8gJ*RKEp(5b!ML zndwnT8WKXIL!ex=*OdP>^ZGzDH(*@NT#V#4(Sf;DQXsdW-2AKv^FzunU?#9||MoZM z^S;E@v3Fu&uv(N~SyJbzFY^l`F?op2N|`Gm7Ks zz3CqByfwpb~KN)PHkH7Q`aJk<}?}3~ve| z*B?jqSw4A_#M?6HSRflWeUPd7A}orq@eC7{rx)`nnHE&bmP`_wYTv6CWmZp6R$p%_$a?BQ%h&@dOBlnC*LzU zBGZxBKcaUWc_f*s0uw?GfVqRTP79bU({f)1Lw(W*E|dgZ%u^GPh!X%Ga1-9}4$N~7 zl)Ggr1T1k;;*04hUtx9|cdWgQi?LJIs&P;Z76t$64ZY2HhnA=GoNql2Gyiqdd?scZ z!7?aAWCFs>k%-vj5vQNEbxHEohkXCY=@JGVx|3&re>olEGehP)u-X**Bv4Mgk5o@K zKJh+bj5%NA{uZhhoU3tOwmjdAFYt=ZxWqeHiB_}IwU%h0wxGf+|4HOn)OG&1sVl~| z55S3n(-tm-7_}9Olx+9Q1*yQ?k_b|4X-;%D)7DC|?ZQEMK1_*cLX>G9Vm()mv(Q;rLL zyy_zPyP7}o>f;S|;PgGfeklJ=+vh6$1fPK*5HJ6$xXh>Tn97!zeNM)lv~*p7cm_?^064*Tt7`)Mi@~p_ s4;*2b>Z!;clvM=?09@b!07L=*|4?r?Fyi&FKoLf(o-(BC)WMhk0hbW%=>Px# delta 4450 zcmYM2c{r5&`^RVOF)_wEYHW>d3`K<)Vr(-ei7bO?nv`8BX{<9@&e$6>j7}X}h(Xzx zEM*xnlwGLE8locRIGmR6IM?sGe$OAD=a1*QUiasD-|y$X@8`1~x)ECKD=sLPa6Gw% z5hQNNCsM*F5qBhqw>RR&pA>a22R-~Y@p3tJ|K;v)-`#pfv@;Q0^2-3Jz6~$)%qz{dDEdypdc!=Ic;?ZMJKM+Zc+Nek z2$=cXC9~yQt^s35;xC2z9y=xbHu9-0pp?2Kjw>^b4>*^zum8udHAXr3K6pyDcL5Xr z`AR911e?ulVKK0hWPe2PYVP}~wVMW}NAf3syC^LWbpCV8@po`jgw5bI*|p{gapj@h zMbF>)SG{`;x19{Dtd8knmyF07N81HPq`qH{`EL+xY`im*NVk8USB0STp@8FC z6zWeP%3(J|wUcoe`X|AYp$9>cgU`DkxqSI~QCAW@I_w00R%TLhnic$-e6n@6VX%bH z)J&4){@by3TuW--mFKNf89qyx#-6j1ws1!OPpOCz_DF1Rf6}>^UMi(4a;5R}pKqbp zqDkHtBDuk59!VSzH(R%H^z|h@tv@_hmr^3IxmW15Fs7caiIwg>xlDTU4>AefLiqKA#6;W;0eW_eOwbBNrDl-*t+M^oo~84SXA)ZONLpFRXb2daKM3Bo!7 z9_+}h*7DK&)A=AcCxA|K+t7!2&L7z(ZmSc=?U#O=CS_NuQP}sJq*~uMJj81R_davk zT3i6~t4H>T{Zk?N7&=u>P@PeSj+*QmcLkwFSo_Bb1JmQiX9!+Kk z{aI{JgqYbhiF`wjD9~D}YT0fpPlg$m_JJDO;qJQu~C~6+@7Lwq5=~dm#36JNEUE+e6Li{LG9L&)?rlt%* zqp(7oplS+6XluvcJjgn!bVxu|2JGN)FHPEqqx>pi)Eb?&! zcEkHEH?;3~?cTvyh{j|!*2iINpe(^maF@KYPz|U9GpS2v~Ebniu(UPS-G=lx5$eC&M zha_qOZ4gkHbd53yD8cOgrRoG&+i|z91Prfjr(l4^b%(>~SLV-YMR6x&JuMwH-M?`4`v-Kp`)NPDAd0 zF^GA#!%uBnu*tt++PVG5HP%wBRh7fG12u|JFA=khZR;2N3)n5PO{{TIcI64m$gc4_ zLKydsH}z=&EIiX@+iI3zqWmWjj=fItV(Xh9tm+9q`#Y+xK+evdwMVS=9@s z^Ywn4BN~0vxFm6D&DjkpY04M4=Dh!-D%qv>742fV?try%Kygj@{AXI~sT2)XRw~BE zT`lx5Efp_BzR9~yC4k-Xz1RrlXJ+lIzo-(;{mirszGgajAw}gMU1-|J<2rLMbKdBj zPCj3MBDr4cqrg);TACNM>5?C|p$~;baG#2&FL2iOlb?tSb4s)@~<%CI15` z4dUq?e5iEe?Xj6nHfKrMDAW}xY;B9hEG!tA4ofG9H6RVyOwC|(h3Y^}VM(>`DCR}Xv36myV3}4R-Au+Ss_gjb) zE2Wg?64BvgU2Sv6Zp4fo%q_5?8#ubuX)p>@gtXs!PM+=u-brFm;P+WDBDq?)y&=agHpPF za+_g{xqiRhE;Ru>0NABo1K|D^QSKPk_+2fURO0Y!WVHHB0#8QOba&Y<2j-HVeTSOS z*%so8#$N;fxKH{vIrAHjJW?Rj&eI!1XN*Jr%oR!`_okO9K3Ulc;6o}oKm{H*XXB7X zN68RBQHF-g7bz|&x-Fsug$lNrtWF$jr&*AkY0vN*5u))!XADDF>$e~G#K#9oyZ18W z8!Z1GDZAg-O5oMg%B&cS2oyw%!3)X5rO-k#jS>|U(0-VbPVPf8&#Qf!gdOuL#kE9G z+(?nOL1Y&+$aEdmTtXj8p3RD`^tT>Ntq|wfO1Z=7ObIb}|2-NQEl=312ATIEl~^DA z^Or3V9o^rf1zQy0N`9QFQ}fvD1MAQCxTx>-QC&W#sO7IaOpYycbsl{DS(BYzsk>Sm z;J)RU&VEQ9w!zFsfIyyhOu9nj)of zRlMpl0H~r~qu{=eE6!*X|-}OhEp}@su!%O$-OOb2&?io zvA=dOV5l6jY;0m21hc%9iTP~IgL&~_T>vj04EjM;F{knlHV);B{DWI8FsVwvZY$0B9Am|i5Bst z40r1TG$P6*I@Y!$@IcB~>9+iE#D65V@j*5=+c-#NqZ_Up+bGGw6;3#BXRB@QjghW7 z_f)AS1P}33$&L^#i<(H(;?ls6p~~E9PytqPmEVuN|CgdGV0cH%R47Z4=bbB-r{@1O zEk2S5@BFAI^RLNSYIkr0>GhMItM-r{@wExKI3#%xf=%b4SX(qOUA|ze8brh7ed34j zBY7mu4i+T@=aKr9=h|v5#RjzxsXUtF9vG#U_bHDCs;!H4IVG7ozH@!{9v&=AX0mT4 z7f*Mizt^0BLC{G4*NRLal5;g0fXkoi76mZPs(LAt{5nvRARZ4KTMfDyL-@h-2{(c- zXNc~t?S2`YgZ|s`omTt9@lLf&o273CW&{x&pw4OiGks>Iil-982}Cb#*EC+*(Ku?Yz1xyr@I73oItOe59rIaH-;TJ%6TMnv8R z%26J`2w{cf>&+UdGBnV>XqZl!=`cOhq9Y>F+jLY+KEXSt^^U^OZuY^ZMB)t~pz`nq zBMFaX0r2(zBeNjPY|76K8yuE=jFLH5c_ zR;5p(g>uM4lUMbsIj+92PZ?^WN%T+RVw{hEzEe>V(Chec1q#SCO%XQVVVP9&#jW1H zi)=RGy}RQaxe@;8&<7@5#Y#%C=c=~r-8R2pV^Q>Mq;8Vkma<~5{NxSU8gc(#NR1WN zpPK05!~+)Vo4>mPEufnw3zzM<^MFokW8p|Y9&p5Q$qOsn^)!&e;a}Vh?ooHpWDd2c zxpNlcs-<6M;7WB?zJRHnSCcIs7k74}b^JHUG%yiL*sKN-IErk3{JylJE3jtg3`+Ls zfd@5jUW7z)F~_aI2{6x^YN?F5+kOe|QH*fxp@{!p5JVwF` zE={a#;WtSIYDGmIiHhI|jA8M~CAr(vDKmMcqkFH{&w2*@bM$=}V{oD9iKTARu?*jH zA7DCyO0ogiPHFQ~ zY)T+9Yl^%e=Um$izOcyq+bpwz*k&A2uen9B`yi>KqbzGzENF!m37`nF^#O=SV9Mt) z3SLlPVg=YU2eiPwvH^AjT6jgeSl?3bELt?CO z>(oSu|4$m&2}<}`4Z46;AJ2ein|-0W0uV8?L%cd&Ch(1#p?WsGmi8kZX9_`IuyigM zeTWd`3P%e)_3^iOz_c$QULDRzTeS2GI{E-|*cvC5#AEVhf{6+Qz8VmrA9SQ++?Q`e zl)@U)6iDe5h&vdUS}$Ga13m!`?H48j#&j~{Uo5^2$6YH}BrSHDm5=Dwxv1Qd6bQ_n zh-D1uE0N!Xbe#Yve2Yox+hw2kLj9socU?XH^rxY% zfUSMA%7=C{`6>JXh)%7$9dV)diN{k(&CI>(Bn@vI*x}!@k0cY_TdVw;Y`_k9w>bKI z-eq;>?Q!X1&5KUOOR-o*|1%}mwkGbkeLl-S5!&}MS))T`m#g9T3s5J13d;X~XCX*K zPau|&WQYRJF?tLwfW8c2BV*t+18t;! Date: Tue, 4 Feb 2025 19:29:33 +0000 Subject: [PATCH 59/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14571 -> 14570 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 750189a9c..4740918e5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 17:39:51,435 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3270 INFO] --- -[2025-02-04 17:39:51,435 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3270 INFO] --- +[2025-02-04 19:29:27,562 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3286 INFO] --- -[2025-02-04 17:39:51,435 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 17:39:51,435 submission_checker1.py:3307 INFO] --- -[2025-02-04 17:39:51,435 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 17:39:52,548 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 19:29:27,562 submission_checker1.py:3286 INFO] --- +[2025-02-04 19:29:27,562 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 19:29:27,562 submission_checker1.py:3307 INFO] --- +[2025-02-04 19:29:27,562 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 19:29:28,875 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.14067 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.4416 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0741 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358788 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74391 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e9f960cf4..75a2e0d5b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -600 +601 diff --git a/summary.csv b/summary.csv index 15e61ed53..7867199b8 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.14067,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.4416,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0741,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358788,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74391,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0cde5279e..49f6d74a7 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":48.4416, + "Result":49.0741, "Accuracy":"GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index cd89bf3f8cb2b3360d2e9dbcf5efc3107e582de8..9d85d08c042b5bbecaab125d563bce71aa924694 100644 GIT binary patch delta 5137 zcmXX~c{Ei28y*cq8vEABHq+RbYC;iXNwzGB7_u8%yFy0BjBnP+GGi%a$u@Szz7r#c zESaoPgfz%Hh?3vb`Q1O>bM86!KIi#7_kG^yJ$J=#*{{-@n>FE!LnZMxw}zPMq_Ysw zL2Dt#nu_hmoYQJtQ*HEFkz1~j?b@utSkHB9<-J>F_-o-5 zoCgf;i0(}-@`y7kk^K)3Vz)OJa~hkm2o+qNgQbV_>orxEVv(74ciNrxdoBrwevTpb zHh-p3%8Rr+yL-!L?pp??e^8i;TwPi51%G=^{OTVRK-PMV-OoLlm2sxOOSqyb&PybA z%_Jb2ed zsOZ;c`>T_Mx$~ndn?G_Ie$d(=JHe3^k^OUPyCJ*z=h`LQ{4^X}5z7%{i*ArIha*R| zif?0WcQ3Zw_DNnriMI|ldo>?oZv0gDgEzONaY>@PdQ2}jzaBcwQ_V__)G%1VJ-SE< z7pkThz+mepaOZ>&sieW+h0})byL4}tsD;#mIOWI=X3-{M+UY?@K3}_XV-ngZQ>Mx; zMG=1-N(BXqQ1slGy#yJgYbV^js10`e>T~*amv_TDHznP81Wa4{o8hp}q7}vlVze(N z22zLV2CPc=jSV`s*khs}58vd+@fo!l!M^T%lI8RBe-!8A=g-4@b_Qt}_oDw2Ki?xr zVi@H%LpJ8*pU{(-2%~Z*uP4ms&w)FV4FMU{A&h$H_srp|p%O2MITl?Jh$JEyT z)|yN=C%Gi(l%)1K1h1$E;w21~2wA75Q|5)Owm{Ywa%1WD>hLkNdE=sOGEK_sx#x zxldlCCytWy0KM@9PPA3OXs!plr#wlid)#iV@N5oT(&;_G6AqI<&1V8JbhlbGUk(E| zpYk*Q65ON^KKFQO~OpQbh=T$LeCR+w60g`smuZoZONFJDIcY1nus^sc2`ui z+XQKAj}orDm)jp>qE;CrI$%^l>6Z}VGHSO6*-{N9`^L`mAjr0CeZ*1rjcGFeij&UZa zf9UASlHI46OXR0^q1%l7&L@N|hdBwVj$E#E#Yz3Y??^*th0gbuY;HbdOSH2LlTZY- z1amIr0k%xzpA<+}p6nD*Qe>4yRqs=f`w#Ca$8u`Y(m9DSg37Aewt(I;Vf=Yzje-Y$ zX%Vx_D#xKsrM%g$Nu1H8BwY*6 z*@*n+_gxXvs}Vtn6hr6jviEl{Z-#U_H^kSn&Oz)p&2AFU3#mX&SSSK!P<5q{>nW|E z0L*%*yA#jH*)dq2>`|AZ8AqsJ)x$YW1s~u*qk~jEdD&aZ1z&7Xg-#|&y8FDqXiC8#GMpJ zkO@r-PwWOfiT1=_K|nF@f^tJZd?ixr;P#yx-{t8)S1(_+FiKKU_Fp01%O+QkK&B0g zCPmW0M%CS8-5X7v<2Bo@0iPVkL}0Zg#aEK+a#Ume1CMkp3Il%{h;j*GlpiFZlYhg^ z(!%lAQj@}~;^4{V92}#_H@N5<0DY`bu*0oOzlI4kD~c9R_RTDRBLC4PTTP;i#fVz0 zf$sN%aSMxQU#mha2SvJ9nL<(1RdI)O{H+4|j@UrH5`4n} zBj=YSa#rEyLruLEJSzk>&%^-H zXG)9?=N-fWh|!(^3j{QrM&c)Pa*&3kn~vsyudAdlXY4U9AI|pD`#kxfRre^BP?u;Tzs?4kwi>=vwtF%uvO+69X^m zrfW?YYU;vsOa1Z2946ogcofJOoshx8kLBkr!NU*WQp$&~nOaiiq(-Y=_VVyy`SsNo zH-JvnOWPc;jy~p>d1CZX?{1^gFZQ{Q%U+H8m1`xFv{&$t;h#iacgXCYO0X-AD#5~v zI#`gjWg#=DIu4@h|Joq$T)+NLDnYB@cmwN_(fI1`@JA4xm4nsDTe?QUPQVcpg6Mde zKols?N;E0UoH-xui&&C)A40NO$lwRmOkP&U11%|AmPSU5u7jo~6Ow4v(frgIJ8}XN zZp21U@6IHh{rURQd?c<3Y2=`FvDYvufCDrI{pIt~Xu+vcn_y+zhlGcNLA^*NAee9* z`V{#o>V_@|+mw)MF@UUDKntD?&bG^9=!4#V|DbkCdk!3!86Yuzjk;u9aTPh{4_oTK z5w(yTqr`909~Dtr6JpE(1y_ckKvL+`1e>**p)IJ;tQ)3H?U*(ffv0BSPWnck>Gv_& znT@Xs#E^w9g@6IV&?u>dzO^j;e*MoxpM$N4zkO}-&q7;w^KZmyh^#yV9Hc!=sO|!N z6aQ;VfV3jlV=c1*=ONK#9sSl4&oHQCdhuAUxgt?XR7KP~imNes<{BS{9U%G*8e6x5#@p9p6 zm|0GU!N{oF8&s1fJRA*b&{P7V38X)Om&hAYLI4D|AfCmV-Qi*2gl?_}v|&9N_s)}Tjas7P&j>J`vk?~+}M#*9&wl~ELX8Rbk&UTc>=j$HHDn8TN=@-YhJBW-!W z;(*=aRs_@>Op628xN8}diK9Tk6UAyQ10|s%iAaCwom3d4GoFj>5~P6jXCDPyum2YESzVD9#;4cBxWa_+H8HQs2D?{dmmu##{1L@dI z+0K0}+~i+s85c@yhn5l779Oyh^@PcsGlj|TR9;icm*Ce|1?EYlvZzvlSb%vJt&0UK-2+6!WK};z-WV7Bz&loZos9K zh>fS7_!ZzhkJ&)^kU&F7D#M}w;J*gJs)P{>`Y4)PB}Ns7tKY;)oG{^Tf`>c)Yj{Wg z8|)2?Mr0}A0sRpc+<;{IN3R1D@y|cZso_Oz$&{389jj`S0-dM2x{HfA-K9WkD11rn zS|JG$3peYo>UT)>8-wf(5zO5pd#6ad6ptRZ%l6j$5tjNId~8sWu}pOCkYb2VJf{V< zRWNW&R{AGvl3cIl$INT_q&(HjNta_vmiQa+k*UU6-FY{NE3=Z3&zXG&!ipxaJ^$Bd z3#YYsP#n7?IBVKXUnr(=^xo^$(>4<{p?n35aKWkNeX*(9sI5ZVJjbD|nkhZ`%`3?t zLr}(R&CJ6lgM66o{#*BO5K~|Lw|T|kX6EqC&)%=>l961)0RA=#jA{xboGRa+hV3i0 zlTvHhphav1+e`OM9iqN?#0cf&ed*GFcxp@v}4&{FpeWjkA-;l+r9CA zyTLqiJS>AnxYJs8f5xdYKV_ch&Vc4n+9m_N>-o0*%$8F)F3iF9WKB_7lU9k#R0{>kAW2UAE&goS2`LIs}c(_kzZqS{9<#>?+T(D(TYB!seyaXjzz#4ih z!!)YKBL zVVmrAUPM{Je(xDVsi&VF>X}PVes%nIU^K34mQ0bd8ku!_6R#R%%s~d94MKq`(L7&S zl(3vUC3w_nbH13othz8bE-# z0O72?CIAIUfoNNR8CK>RScdnngcFMh0gkg=L1R41tS|0Gi!sFO>rN>E#oyiL*|jRG zIhe(t()<*=or7uz2c#&ctmZYW4=tuB}^>uS+p}#^M2*-e`Sq{k}YBoQJ*<=N@WgW~vDvHLh*ps^c98{?| z61;FK%d3C!&cnU>ZTFz-@9l7kJZtnjbuGX8mlH!?4ji>1vzw3DtBqkeUN$}B7YfqE z_Xr7`)77rCf|2WoLehJ8rz|{E>wqav8oZ@)pCcAZPpC`6Ka?MRW_ZkiTKOLmTh=jy zPq?WUj6SfoyDHgL>6!@U0^m2|`kQpuN7Ws3J2YVs-{kOP0j?hkTBt>Xg@}z?$)Nd* z6CF?U1`1V%-)dyI-J!Fvl9L`ApIJ-cqwJU9ipVmIS0C}Ti)}r}#M~%+)4G3xjB_*7@rELw`h{ipNz z%>AxHXxd@1TsFd6H5U|ZUGRQ$vHMIpbnM6PXM1839!onB^}dN@ZqzIg?J-TkA;5n>`l=N#XtVrZ UzPlG~Sh$Ft7f*wF)d0-ve`5L3{{R30 delta 5123 zcmXX~X*iUR*Pbzq-N=?D`;2u^k}XErnjwiSW9;H@3E6ke6tX4;Q$J#8EY;YBY(>ac z))2C0iJ=%t%EI`ab##NK=sT1+AP)J@9fcO&5@hxX{3MpN*~6|sqq`x5&q0tDf>(3rUa3V z@5zKELg7JxYK|$mZ=qs)dwIbt4`N8UmC;w6`qC;;nctM6#_RA|&+Bu4|EG5B@u+v0 zc27xPj>PtHKqb8R!#jRQg|(w5lK)oCXoT1Hv3JFZdCSjZ-kgX)Y^e5Lm;Z^I*#4Pzkv+%|!$?`4p#5l!sXQkHcf$F>Wb|M4skJFzHC=oiHldbI^*YtHBeyHtv zr%q^e$YFc`Yt*D=&rJ*CP|7lnRs2LUd#^dTJGmKA^}^(&@gTiX!EeN}*7xWQa(dra zE%OOR)1gIVR&jd@@g>J>{BqP4G{;`(N{v$OK}2llsMBumtMi!i@ow+D{sxXVXWMSq z&HpJcuqwS9yo30iGrhcGIX~5rxK4FYjY?w|HD>?fg4oX(jxrZq=Je^G$MSPqedgj$ zOlX!)54%i#sE^4>f69Le0`e&x+Z+{?$?x#GBrfGo^;q~zN`yR>LL71l$=5~|@g3f) zGQU~qQuI`6Mko6ZjyWTt=rG9}ex=wrT0rk~Z<&el`RxV!8S!&lV_^mo3(rq(Nzkp1 zLnWNT^(5#BI!OKIm>GP6I+kkBb*^GoJTZZkj`AkqzoV z(m{1>?%O7Rn8o~oWT{s&@|$6rV}k0i`eaRHlC-dIJFYNeEb9tI_7fh`Wj@N^%^AvRKmy?%P7Y;{Uo6$a4 zz7paO!>@Mz+fgk;UgdjtzT6r*iqLd;Airml>>^a-G4snkBx!osq#$>Gbhn?n?dq8s@4Da999BjzD3w-t`|z z3n?qNSsa4`X^nT2#SWV@q7QpaBX5YfxCF4FsL*p7@X+&9=_wa_ERMPT^_6`Z zuo6ny>XX0G0_m$Iya5pp{5AQ@6{R!#<5w7ptY--|2b(S{HNHH-A&s+6X`t z9p)mcdg`NIs^|Hnx(n$jC5m&<(p(Y#49J5@Y(dW@N*e+O>KIMeIl>|uX=xMPr(&eB zA@>u1-3U!T+Lqh{^7OEN_yEG2QKu)w#TT<&an^XC!=;#JFVTucDL(Axn(Uw(1I~Km z2QRBw1r*dAaldPtYP43!W&PJN_8tvY^TXPgZL&{i*|#?N58;RA^V+EJvQ3<8<`CYR zP5kT?JlkfHz~bniuui&ea>=!4yZ{Gn24!ZNFpT*6sDKd*UmU8Cbs~4Isv70M%Rk+?U z`GsOscXL>(ilfA!peU^_gTf;!hi6GBExBgMG8b3W^tn9_wjLLzoMq^+t?Try4@}~?GEKuiT~2H?lzkt zViGLVLLbi_$^~TwMR63sNF?MlzQ$do+Tb^vRn_l?ufLtoW_w)I*}vpuKYwrXaHi1* zMqy2#w49JI5;B`i4LUl(zPP=ynW!QtN5#wmdt?p_o^vn*3*eH>ZUQKo98S(R(OBJc zWI!w}NA1^;e>oDuMXuM2tV+R7LQ8{eA84Sg;atFi9LK8yO9z%)c`uZxqvuVN)Tko@ zWv+3+l=tI9U=q1vThp6iqcT|~Tbc@<0`~AoqLe}F5(MB`R27Ggl$@_(K^6}v4O8WN z;|K9WW?U$#KU%HtR2!8VG+W+dTz=gT=7 zkfg9-LcqBm88WmS7;vr+a29ho7uIT$(F`4zbR~2L@P5`R*#HN`L;T>{fISAXIC-D&PUP4MLop-OV_AvVZ zzS}Vg6Yt1l2jK(WPhQaAY?qJeqYm6zlBrXDuGU^KWJYWf$b9Q}qv9^iU- zQy5gt@z!;{@}Xguu3t?|te?D)#P8=xOSxT6e-*v#VaaTh{e+ENm#bv^F>)0LCee`# z6&W9gb@BY2#yKqL^t+O6&?s*Q^!9L3*b3mW+i(^9UD%`No3kE9Ph?q7_gx!9ZkI*r z_VERk`q7n3A6G6JHsulBT^L`ikKaZW#K*bF?xb;k6cF2~WNSr1iRT5VL>oN?7SWUo`NnAy_CLt#V`m5wb;iF0(7t^n|vM8#kwN^a8V!BEs8i`OX(bG+E!%eg+ z<6z+_5T{k{NHnl@cx9J=Ga?^GQ~ZTB-$w`mSPn^xs_@)F(-SNNjSLv1-E8G+d(HxL_*a^(MFYFe3^-#V90?eWhv8EjHJzOPfUoHrpez`Vt?v_@1n%i}b`Tw<}+x#Xu~vAQS9? zcuzM?+rGD1vLRQ!sq=X^2Fq?VisMUF#2S{e2NduF)JQoSTTVPi%(M1KabfTuZ30@{ z=ujIDt#_tX2?ZU~s)SC|gyB%cxpui$>$)i3Mm{#te;ZA}Nn9%!KReA}i_v&6YgM`( zC184^Mv3(#I{miSC;o-5itQc|pTfY2crb(p*Y?lVZD+2IUz9F=V0yv`Zo-TCpB z`lA7j%f95YyJ|kdq?>rnym>F-x>Q(o0=|xewZ|As6NVn-EldE?PXvZda$P5u(-!$7 zgk#sE3Xp<;FoWV+TKq1ygW7>q_S?t`*YVq^b%}B3{wlgVon~Yu5X4)zzV)i`T)q9( z3HD^^=(#K=O$X#ICz-KWOukN_Rfu+M@V$ zT7R13XE*tf$ULK!%N3{Iujd23r*eY$M6U=3T)*J2b$8*HhwL9LO=T7Yls>FFr6a=) zn&+*Hb6>w4g=T0Su`UpbSvafIV2uGzFRFrKN+^nhR{3{I4U#%exIg6MJL7PI_q9IX zxPn&?_fs5jx-FqVyWO`zaN6y@JrN7AAcAS9`xOEr1BNisP7Vt{_@BYE z#D}q_c2zsvMoTq3F-Cmn^@}PWEKrLBGmJL!@joRo7#(;fk`f=A)(fD-yWxzK3smqH zhi^qOz*jrrI{P|N`CQGSnn}khy}DF2#a$%Rb#}9dM)%(Ed==dkc#AJ$aK@-UOnP!% zP@pS*@SFNYm|zk?ijx&KOfMncq0&n4B7kkp6i~&z9-x283%EmM=80jZKhq|p;WItm zO;Fyh9xJfeR)R6Qt89bNd#!8hQ%tmlq=Jt6+~fY`1&M)Y72v+MlV4CGaV7Vd8ek^W zpMXROlw)$^hufu%wk66$lrmbeYh-(JL5HGFT?$Mk4Z#mMbwH zWla1W-z_XJ`EhEu^5I1JxM+9bnKPvxJGyhfwgpqQ)8tXrxGO`}sr20V?ZDo@O=>%i z(z5@mxLxB?U?%x`eEG+Aa+8jdoVzter*n6*WFkN#=lkCc3DFel!P@=KHtn1*gix%F zFkY$VV3q=EJjK=^C9s?f`~5)=r7}J%i(vb1wO`~k*pD(;j9!2iX_Z2=0Ko<@=%b`> zHhXVt6GW@oJx@i{P$O2oNlSlbGy>fhU8_)q1nc#H3F!}BDTbI88#VypXpQT#x?V2^zkeUhPJ3^nGGzT zQ1AOyKEJ%;Q1qjc7ZFKMTfe2~tB7d`r7w4>3ga%bu_JHK%)@&&Cn#-3Y7>L9=Wod# zx3w1TLQ{XWaFeQhrV4X|$I^7#iRAHp-%<3dAFuiI-^`_4Wc_;2miJFJQ~bSG+ho6m zCxDY6ro)b@O5`k~77Gw5x?fo8KABy93lXT4u<$({Wk`;#o*EHA; t)VAn<20CINtwCzoRRlo)e(lxBsc8fMpFeMP3m`{Eo4O>p=L&%K{2w4Akk Date: Tue, 4 Feb 2025 19:54:40 +0000 Subject: [PATCH 60/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14570 -> 14574 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4740918e5..057dcc136 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 19:29:27,562 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3270 INFO] --- -[2025-02-04 19:29:27,562 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 19:54:34,730 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 19:54:34,730 submission_checker1.py:3270 INFO] --- +[2025-02-04 19:54:34,730 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 19:54:34,730 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 19:54:34,730 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3286 INFO] --- -[2025-02-04 19:29:27,562 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 19:29:27,562 submission_checker1.py:3307 INFO] --- -[2025-02-04 19:29:27,562 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 19:29:28,875 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 19:54:34,730 submission_checker1.py:3286 INFO] --- +[2025-02-04 19:54:34,730 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 19:54:34,730 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 19:54:34,730 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 19:54:34,731 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 19:54:34,731 submission_checker1.py:3307 INFO] --- +[2025-02-04 19:54:34,731 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 19:54:35,894 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412693 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.380515 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0741 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358788 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74391 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71954 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 75a2e0d5b..85c3d27e5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -601 +602 diff --git a/summary.csv b/summary.csv index 7867199b8..7682911a2 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412693,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.380515,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0741,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358788,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74391,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71954,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 49f6d74a7..7002c16ec 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412693, + "Result":0.380515, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74391, + "Result":1.71954, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9d85d08c042b5bbecaab125d563bce71aa924694..f9ec47d052cc256277433444dbac3fcf39b93a17 100644 GIT binary patch delta 5482 zcmYLtc{J4T`~FOmA;cIFl^7;j#?quA%Y-arCmLg$5Xn|iNtR~pdl*aJS+nFV`<^9v zhsKyJjbsbaWS4!-@_p6kcg}DAc+Q#gdcCgay081X@B2B=yw@+U3KuRGW9FoPbCL&_ z5-H!KJer2&Me%+#M%R|x9(PJV10j+h?rht9Pudx;AN4eWR}MC&rF7mWjmh}t>iFGV zKK#AFtlGMoE%&i{j%ar!*H0#!%O!7hBcS|n@?asehG>Ig?f?AU8`Hf#Ke48LF!B4S z`nNOY$loh*hHh3`BJg?R2 zfdIGleUI|P8}DZK`O^I!lAerhQ@swI&GsI8UP&>j9-b(E-z>B=kax4@#PHyP!v<-l z*6bPPZHw*s-j@Bu_+Gix4%ZkpTABFn{DkcKD5g7|I4Blyg}}A- zWTslKcDFI2W59m1>)+GXrwg6mD(}sZ=2zT2x>^_F4JXGFdhql`}R z<)1;^Ypqm9(49+JzcpO35+`p*VyF*dPe|)TMQLe`C*hg}chyBpyE}Db(;}W@?zr!G zoIGXfSeFJ1QF^Mddic3LLYG%KFe>gJYpglPQf-=~_>D{by1dK6Mggs%dhHg0ByG&= zGyO#^v9ALCt0N$W605Sy36Z|;&7wSB#-xipK^yLHo{DA!511~D6dl*Nucwu>t6J#t{W{3h++KidAW=()TyxEXxctR%9BN{?AEiJSCbWZ3c_Wu7X| zoHDW8-Cum9sz#sbcqwzuZaU`Y+|*vbQ!_z^#9x^GL+6Wa&w9YSKDRQ`SMzS;&wGB# z#Ani9jba=HtM87Nx%fm6FpOShP7ZALtgSn_U^1Vn9h97TGuEe4U)ewD;fKMN9~EP} zUWBav%sm{vw;pR-eN^IT+|hMKF1+sW8ujh-`yUJL5usfF|Pm{#ybZYL*l(GKf04l|!NdR}QS7J>PRK-F=*$hjqC@5Cx?yY;U0!Z$ z=AfmARZatU;k0h(h*Hq8?YUj)Um$`Q#i`_$O@(F3w5T`Xo-8R(_(ay!GNT(6+ z11+M0SKUMv6^^{ z!@J{uiEfhybv+;wl+*vp80)`(wI=az8bV1!rJELRM3!LXmI+qa3 zms>JWS=t&S^MX09*!F3N%d47<|YnWmcFc{7#)fs#6?ujt|?DsCdAI)1f_ai3m-wq=1QfP~Q&DwKsvyln6p=aiRJPJ`j;q z8*hmsH&ZxQ<^#_?p-UGf>Dz9Sc*+3O*}XT4<>hsJ^^UCN`;&79VZF$o?>;~%W;zDM zOQC;0(MPvR-$}6i@q>>;q&(}G9UkPnd8!i!W~O9p*QK^PvdLAHz-;Ziw|IXuM;lTM zrDpZr^=Cgy`;@k^nA$>pi#o+fTLemspmQwp#a$M4SsJJT31K=yX|-7ID1Q*skNuGq zSft(-R93^x_|R>~AKcW044tjgB$tiddz9@RQcr0(&f~7QEIo|YOK(@s;15gH9HF$M zN89xsqke(fDV#6^v^wfe!qAW|iuXpc)>rda22vtbS;qt=$%?7J&?Iif)C#^Sn#Gx? zz4Z!pG9BwD+$W``;h`?JA1`|fl-e3NL>$XEMd=klw9s9+6Oo}X6AD-rf)b%_E9Z-Z zVfCd-MF1JyG?1Yrd1#~t3L+1$WwqZZ(#Qvu-3#=FNrO*M}_=YQQVl3sQFOcmX(5pyiau28g zx0@k)12nf`rCkE*n_G{@f%7l3DC^ZfC`c~dbBif`MY@>xwqT@B_vThO-m*(UPv};% zDc3{XD<4McJd7scI`ss{X2n?%L4XpF5Q=52VwVf7l6{S~Vsk63jPkq}agKIZj!_VH zM~%U$;Dc{G8E+=F6($}s=~4OcD#3E+WnD}CJsaA-X1>qOx=4mja-G;%%aQO!3zY!T zxbWy^(rI6HwEi=m*Vi@GCFnR!D;;nFt!SdjfQr(JjXe*WDDt|$JG#0-JMgXxko zNV&pB$EaBzg24;M49Td3ay8uc#$zj4hFPAF`{^>S23(#mBh{OSCQ*zm1A7C&f7Y|G zsrhOEw|$`S^w`k~*zzed?;WTN8T!_dt;GO(ux-G*v+VrPz$lkjhGZz}*~JaUOg>7} zI&&eZP9X}XW914@gnbaShgWj8>On&v>{*E7!IK*eTHDtG$mZt(!HYu8F$;-zd?%f} zVYW)rsUcZV7cZ};;nEhgb0M-B&X%}>I3Ek^{Cib%55g3zk!#ZHtjN5QM`1X&Va^9( z1n3YFqKOHtf)$g<9(x|IehXtOB;%cGPTT(vHwAD-WXI*29AhwguZD^V&avcMW8k}p zTwTr1*NZLX(pv%N9oDlOYf=p5*yz5zbR{=x8ObD-Yg;$ZP&C^}I3c!*aezRyqgxLg z1Y0Sd;NyfnrZ65{xk0;TeFhWp0FV)PJ2h-2>B8{q6K{{t=vprBhdva%ws5$Z>ZX3t z&m6+gqnsJc@M;Wp!O+s+M;(s6<^lmdwO$5x*}O6st|~Jpq2M5&PVAXYp50T6WFYJ+?h}X* za1}Ql6yl`bnbW)Fz?Yzr{VT7){2`kzn!zEU*7ru&8WCc@F&iD1^#E$q+sJjVi1{#A z@q)0sb>jN;o9kTt6QYa8 z7!DP|UvAhmX|kh!erPWA&vaPCgp-H|f#wZpaRfA0xgah%Eup zNG-;q146p{Jp7oHmCg+snxapM(w>f6jZ!NihcYa4+*=od!~|75?MdSr)#qKIrd5oU z_BM;fO%L^5u`f;f;sPHEB;D0U6ht$X@FgE#Z3;{UYcUoaUJ?A%kfA{9RzpE-DRv=b zgY2(u?4TFeVk&t6VCyrIU@%;w`|`hNKI=`Rj+UPIhzADFnY`wykMZvK6bb6f0VNNi z>$Zuwd1cfo1G@-PXu=1OU4%1P^Ape*Z6f$U?15E12i(FE$E_kVF%`sDK+!`MQc|D` zwF*Hp;Poc=@)W~MTK-p@Eei+wqKwa(s2Exem@ToQqtWd53WXO!F zba>;NN-S>+)xh5)DO+3|9gJ}VBL7P3xPxF$%Hjs3W{)X^2i@IlP=S6EQleD$hu4IA zOkdZ0+scwjwJ2HNKc`SXFmov#+dE$`^>(ZGawWug%l+T{zBbLuQ+_zMNlus_0s0-8 zz6Z*{lCpu2l>(LH!I`Yjofd0?2&~*W2he1s$Y?2i$gm-BqSF+st2wY2MAZR3J52VvkC}@^H~SO1Nph7 zLC8Jic?g{Q91VBzdvldJN00OgMEdv5wCHzP)S>-ChWmwejt@5?)y!0GmQj&K5Cg}z zf6dq6X&ey>w1(;kXOKZv9Jl=(uRj6Y%9z;{Sn>Og`)GVtsT>t5Oago)Jp2B>luuw* z>(A=^LH+5|?F9*e$v*n!ua~}4aAIpu&I(r~FmtYJb~=+6+t95S%pL=_Tlj0cK`MCx zupQ%sQu+UiuIlE5N7CRr zAoTatP{Id8OpLJ@SPzpX z@FnX3f+o7vavX<&iOZs#7U;GdZO`GKRtJo-IxnlruMX=3(Ig4i^$Jsi5mjWU2Xsgk z1@WkQ=Sw!q?rD<}NylbWgL?rDMq&bFPyBIoFx%SmQo{IC_Q+m;2`d_Bi25m&)igYb z$(S!``&Ow1He-BqK7h)7kbFGwr95rB)tKA{FT{Yf;3YO0f15S4-q zz54;LTIcwf8Z&(@P~3{f4F31j+11(h0MDW-)0eED zT?^o)U_Gh9b${rQVgjUSnr|PSlz*CY)KWUcJkhac6_1qu#}<}4HRv_xL*G?ucUU1f zCP=uh0B~o3XbDgOAX-9IoG-{A`vU-%ijANKxBZV^a_FB~!gJh#ibpMmitS`b|EL>L z_UZq6sS8)vue@{)F{LJDj9#WC16Y`k#cN2HJJJ*YL0KS;KT@p>&n+j&n~?v9a?%S zmD$^TR9z6>Tj_b9VHNP@Q74O6`)X`dQ(nRMHcc7Hodno-8X9hrn&?{v24_|F0u}<9 z+zPwL8U5ElH{|RQJ2`rq>L?jxRGc&>==_DP>>ZXAL)qhBd~aVC9>7g?uC^laT&7Vf zxdklkno%`m9Gk{TQU#d+Rd-%@9&1A zNl41~gFYg&(2Sq*-m6Gglg%xX-QuNrLCzt$Hz$Q;D8trYb;1J0$>>zVhJO4QB>CC2 zMj?nGkthxvZ(f7NfgbkyQw7*CP#jhYO4~Ec5ikV!eJ0wc&<#ae73|A}t=Z(q;8$l= zWIgLGpvun`Fs8tP`7;-j!FS{|R?~dVTDoUw zp0sWJe&=8O{fg=9&9nD0LUAuTDp+5XyR6gf8_t>Zy2l)qq&Is2w@C$gH!CNHS2vxB zHqV;6w&!;Ti>Dr28PyTb2CDh1j1-@CUoGDC#R$v2AKQ3_w`giPw`g)F@77>QW7hMd zhnYuv^pzK1V{V0)P8{Cse?PG0*70RIw`0-x>${S}*6`Z*{#TTGBQv|(}DY_Ux&>ZcBMzB2Hmso(0_V(Z`a2J$al>YzvsKS?SZiwKX87m-|}giRBkeR zcm%G-iIQL}-tdDB@uW}a^CY@)n>{nx1YOpJk_5PhBT3nd~GQNyXs@j z9AZ^v2X|ijWwndSx+Ols;v3p3rzi1pPjP{ zsXnP72>E|^qSA^sV1uLxMOE-rQoEuC*fj~NqzoQS(ohmV`&U;m2xNK;1mXex{~tN1 VIVo5P3+74sPe}~=>B^rc{|AoAMJ50M delta 5409 zcmXX~c{G&m8=f&pl6@P?Fw@wVYQ8Lqu_Rj}ON=$k*h&i_BV(p7Yh;Sm%pw9*mW;xYuL4M=_?u*|o#BQ6wL??SXKv^Umn0`$M7oBf+ZF&cUIncq-<+cJum$ zX5Ur$l{&fZ%{q+De2;d)`!z<8wxa8Rkv-9)rA+~GRyCR$co4t6xs=!3ibbg68XPS> zU0lXQ|!7RV@kLBJr!o>aq)4s1mlaGFZBR;Yg@)? zDRg(Q+2z((Lx%+ya#Eu&8m>OjBu5I>k_}<7pQdovq;Tn!!LY?MMjyKMZkMTr*RylV zlN`-sO(k?PLyvvGcIU?>b&#jbRBo0=9XXZ@3Y4Pg`Eh$mvPidgaF0@42kiFMXLReE z?}v4rq}+J~%-X1}aM%~IDicEq+N`Oe^kJqUtMWY)!>+9paj_4Fo%s2TJB(ppcRtJU z1q3}v@C^v!VSYM;G)j0*y~NM=0G#}oe48N`CwdgIy0PbU;`lZHi4dMMX7L%-R}pcc zwQU~Y&Aa=O|2RrCjh83jo)5lk<=Qvim+8^%Kwm$gO;onDLwFyw>Mi(PP zR$|BJB}-q%Me#tqBU*m<9&<9!x)-qGGI^@6}D=$_{N?opw7i6EkM} zv9!GD_sUa^zh6`YHU`=F-VVal*ZDQPEMko^`pSg9i0Sul_S43 z4H=z;ih=>_+oO_!Qu^G>=z31$zpN) z8J?~6J2=xx?9#%=r6n~#rT%aBG_QTq5BpByggwsABn^C%hDjm3zkR*Gtaj z!KGY20KDKZg)@AnU?UH!rE4pXxmv{oOlHGc_Dmk~O;&~AL_VjylAzfB5eU#jyqMNa z5xFMu<6(J1AJ>T3MV(8!kVLx-0mbF+TEYnR9(6K2sAI>Wv0uw6U#aj8ASYN^20z5( z7iIafE)Q-cF_f_IVt!fXJQ-q?T2K@Z?jjn(v*EB!GRNEt%m#TKUtIhsDJUb<8=_5m zWE`a16hDp2ks|Trnz-hgOQh;I(BY+EUmjAFhmVZTAl)yt=w4XX6LV8=i8 z%e_>9@QAjoiJ8h#vX-f2t6fi3Ri|yJj?O3ncjbfp{umRr#stv?qX6Qc>kFqPT?rHc zq8NhTNi1D?TclVvtt}R5D#z9V9JG9G_uwt00rB}6Xt#-V49Pa3HO3#f!UX@(TuXa8 z(!aD6?7Cl3oWXf-xi3EA5j=mfwf3q(pArR71qZcxuwzD%u9R!abE7u7y_|q5+*GfU zu%4?ad+5%W#MY&>Po^@J2b~<_Ow^!=(bZ-9&vBPXPwXSM8HMj25w;xXAr$$1nJQ6( zzrVE|@ z(K0z(HlMnz-UkY109_RW<8eCaiylVcGZYy|VDeRjo+<^<{9He zs}RQjqjIt6en3Xl+=}W+XiGV7?rjb6nPTU;3NICX^3=|%S%cY#7Kyv~%Im`95IE_B zI*EYFy96M?L1gAO#e?DMyL{A0L79{t+wUoyvE{{j7MycYg{>cO-BB`YQK5)5BiHSU z4|gtahQD)dN~~v{2itF&I~ggB+fBR%Cm-7i zQY<<(#gX?U6qf+>;H3f*41WMcvA1{?@U2cc5@b$5{3uIxYY&U0iUTVqmhsL;22;Ob z>riYhi1%4mxHNe>I1X9z#vEO1m#Xsd$uOOh9_o4aW?0jX?Ke;-0ZhTju|603z)xtPMRV8SZ zcsG|+I|80IDxDO`CUxs3* zg)l0(`$_23KQQx*Nc{Eml*pO{c!F#@utOyTD+m&_k(c@i>F^}!Yv0ydRF4)mc;kz31cKdQmS|jh^xRIhT0XIpD z7}EkP1f{^#5YcZ&jE&?S1Q27r02T;nB(0dA#K}=SB-3&{4|-K2Q=DJGZ!uY6*)ktT zr0C!OTJFFdCiZHHvIMTYu5z18E5ObLn$}QLQe*1kL!0OHt20@&|F&mh@ zs^Sbqf;%}xQ!i6{!bnRGo?jk>H{melx{pV(o1l}jfc#i~-ZDJ=04}X^_=+hdMP7Qe z=0zV5AC})heQ5*aQoFp(@$&dnVTBh)ANBq=DpRxHV_Xh*HRgA|m2Aph(SH<=l6gJA zb4S%&ZWIWR6?L#EWd|fND7ubf>i@Mt!Fj5|FA71s=wuV?vhn!ZpU4Mb-PMD&=v#Wm zVJ@H%Q-audg+L5@ft6TF&NXIaJCp)bcmu(?KoZw|N;WU6^MSUs9k7`Zr{}1p#e~EW zt-4yDxMD|6fg_FC=$Sp)#piy#daw}Pf;4v2*6cG%3Fcs*g8ugXWW4B7twXS~>qo*P zx!C)V${;SnN$3;g=a?IM?AVs1bc+FG-6C4>Tv)Ds4#R-`?e~vr;yUwOA=$x@>Fbnb zld7x8u^`xT&yASH{5WNPQ)*09c^xj?gagV|9gbp`MyDs)uGbB1L51erF&!*ArwKfD zi~gA#1!mvJ%cIp*q8a+HUr5lDNOl1;{9! zeyD9e;5sChs%y|*=JgorocU%f|C-WBoM-PX3CAR;stLRuBK)*){IWrx7k<1O(Itbq z#e1Jl3qt|tMtpWetHW|-d}2;Fr_LM{h_9huk9U1qTm2KtpY+!6tH)$z@ z(1hY6zzgJ!7$E=xTa*a2IpG)>GNG6632mCxH)u*bo$}Cpz-4RDvpPbgzB2s^`yHRM zU5nuCZUa)?LlX|a)71}A?7lLV2S7*c7Pk_h?qFIHw9Z|(xK<6V zfbwiZq0@8!4tm}i!$fdrzFgTBo!=JCcNrr3i-Rxm>;N^fNFF7?{evI~NQ=3tYt@fY zC;sY`t?`15b$K8Yn=SWlUwiT|i7bsWyP*}t^~L)q%zKGl;=(u(7H}1UdtFJ4oLHG^$ zK^GwDKz<4>9B-J@!$gWh#`WUKOe8StEBg@Nw-^~HTA&SLQ#-yb`tsN|tilVckK!cB z^;<8s;I0w0s^QUB;R6t1Lec=8z;vkFF4Li=kOdIxfDEw%R5LI->}?QV%A`AJIW21A zi5Gqqw7_FNP&p*f6rRp-q#pd+Bv_L)VnH88bF0Q(fZ-Z9A*W2aTi}t-|5?eI|Hg?X zMl-S;aG(AF%hiM&CJ^HKa)%7ABlD$n zYKJF9Ejl$^HRzJ2ny@<Izzc_sBz zILc(bmATqfc3)<2{H=RnGe7*d1*PFuW~>+H?pJrqO08o6f4c%kF#{3AEBB{i`^ufg zxb%89Xek@P?$TW|$Cz2qIHA0P*=~bC@i7^wA)@Q2J6MmP6My14<0P`2CESZ@|Jvv6 z26M@Yuq>ePI~%$E8JFt9v;`jj0j;5oO$K_`>uu-REtkm0j&`DTr4=pOWjCkNz!^ff zCDoH;5OD-`o>RdE15BHH;|G{Fm&H92=BcLBFeo-OZFw8DfyreaL4Cgof2|3PmHs)A zxs+9m|JC-AP+xwIgB<*z_qoO5$Vz$lG|3QXH8SlE-M?hh3Xha#e|||BBwb^?35Z1o zGeeUXyPnAQkSQsGb2=UU&*RSoc%<*U{80aal|+#NT$p7|dJmhG0uDmX7qEfe$})?o z^Nd5~71VVbNGkkD=CuY(|ET7=PRS;NY!AvM?SQ2Wze~b(l7El!txEW*`7yPw#mAc` z8oY^ef>fUwLb+FfKI-Ys-oo0%?U2~+ITBggYGls+b>f9k6AlvBxlj~)HJaxuP#Md~ zQ-((!NI*_;^VH#(nck#uvi+by1jg|$vBF{Er18c@pZC}5gV$OrJX1}rd;f8+=pG{( z3R{hgxkEAZs;dUEZup9B1YY_Px1}r}gRWK10{gxlto%#s?<_Ny$BjW*DqAMNLF2A3 z_L$|idgz$`q1Ei^Qm-_9d-pO(l9^+`w<$`cgEk!psxln@IP#UrvyRCVbGn+zGt?_- z4d{v$WAbb}(1x6XFjFN+uxS_sg%uyDSiV*P`Ok( zP>~WjVq#6FMfkGijw~NeA8$N1+lcrszeSTjMVW2-BnM?xGh};HUTSw8*X1+pm@XXY zH_D1r)L{2ROGp`bE_9{($~$K2DW{qUFb63bK!7<&*?3O?ijV@ab^vp%>~$_#-oJ89 zEFlCq&Ycb&<56LKem7QvA=%gwwQv{dms04t*oe2VTt1RuevM zUf-(O&ikE_XxC#@9AhtX#dH5h&4i!u&9l^Tulfn^g~e6+XJVFjvq?-{zK8D?6NXLupUgNe$sPzYXoD$DE z-M^vjH+3aB{Kdd=2Qs(y_(bhh7><`s-{iTX3~>kn!MR-RJ|`Ic^H4};@6MEkS9${| z&E+C*x%`)?#qv|?Qt*$J$6pwpGwkjBfyB0-n88Qf6b<8#tetMk_BDE@g82aWjRb0o z-p^5W=lm`$7}zg0k|o&fV^JHWbg%@maVwR5L35((Nx?vg>hRl(S?>OHAS)^5p~>0x zG(Pfv8LpHh%Xs+-Pt)w^Wsz_v^G$;;>j->|LuJRk<=^^LfzGUX zyv=piWP6wuePz}3)r>LK$MbW8RjXJz68Ep}!?X9gOQ0EtZ{%|kJ{R)YV{M8)Y%cYj zt%Q#K`15p6V#0HIC#umenZ%8n1ED>qiCd(wFqz7R67vOyY69(CM+G59dB^8)w6X1t zE~w|z;w)Lf?F07kWWR1L-vuz}|F@cFHJpGZ#CQ#Dmf17Jqcg(9Sq(*?H<43Q3phc% eqAB}-ZSQLe0h@`@nn>VjVvpt-_C7TL^Z6e#u}l;I diff --git a/summary_results.json b/summary_results.json index f26b2d5c6..5d0c57e3f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.412693, + "Performance_Result": 0.380515, "Performance_Units": "Samples/s" }, { @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74391, + "Performance_Result": 1.71954, "Performance_Units": "Samples/s" }, { From 5885bad488ce145319ebce555f53e0fe232f6c6e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 19:58:25 +0000 Subject: [PATCH 61/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14574 -> 14574 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 057dcc136..24657619f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 19:54:34,730 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 19:54:34,730 submission_checker1.py:3270 INFO] --- -[2025-02-04 19:54:34,730 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3270 INFO] --- +[2025-02-04 19:58:19,996 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 19:54:34,730 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 19:54:34,730 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 19:54:34,730 submission_checker1.py:3286 INFO] --- -[2025-02-04 19:54:34,730 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 19:54:34,730 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 19:54:34,730 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 19:54:34,731 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 19:54:34,731 submission_checker1.py:3307 INFO] --- -[2025-02-04 19:54:34,731 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 19:54:35,894 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 19:58:19,996 submission_checker1.py:3286 INFO] --- +[2025-02-04 19:58:19,996 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 19:58:19,996 submission_checker1.py:3307 INFO] --- +[2025-02-04 19:58:19,996 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 19:58:21,120 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433254 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.429127 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1507 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.639 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 85c3d27e5..4b1e299fb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -602 +603 diff --git a/summary.csv b/summary.csv index 7682911a2..221d65c33 100644 --- a/summary.csv +++ b/summary.csv @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433254,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.429127,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1507,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.639,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7002c16ec..92873cc75 100644 --- a/summary.json +++ b/summary.json @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433254, + "Result":0.429127, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,7 +1239,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1507, + "Result":9.639, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f9ec47d052cc256277433444dbac3fcf39b93a17..0166167c634c726bd50d72d2bfb7054ea74a1260 100644 GIT binary patch delta 4487 zcmXY#dpy(s7sod;cjit-!zP!xElDoJ5;H_@ExBY(=u08FWwuZ*6}B zV)d#WBdF6pk~E*lr-<9?YZS0ESD>Lpsi~fDNvBV_a_!7ok^G(F{P1)gDQZ*D^|Heq z?GLP5j72Qq1B}(9TiFQ}zmn2K9z?mw=Pyd-$4sw;V3s4o-0dP*{wFhpHAB#==4ZXV z!-^{frz#Q)4t!=*>UZ;N z^8zJ*|K9YlYNIg1Hp3?>SgQ6dnsCU3`pU#V(0=!ezr|Uu9DIu-#?CMO?4Y37dG18j z8DvQ^zjVvbSqd>#Qfg3$7o3Dt3@6I)!aG3J?=pAbDj@4!zTopl_P28F+cCjg<4{naxZT?)d~l~{Yg&4abT8mgXgUYklY zHY62%eUSU=T$l-})2Bo2&fiZgAz>{7TUR1q5x{dp4&w8%3u!TmTFvZpOmWCQ=j=|rM#(3oLvbGBo5Sktfb#vzFt(v&M?r`<1Gb$ZM__9Dk zTu|~Q)A4Zw*}eAI3pD)Y47??>PnLqGfeDt7_!a@tLqFraFp0GG2-RRqF0)|=m?GEL z)_%8z=~mHdjNt>E;s%@1TAde$_EdLmagF|ZuPf}CU{2aUDrKaqXMQv zXDm?^Zwi+gDn8$qlW+`UNK%3pP{Jbk$i!UG4DU1OswBFg)0xEDMm{{!Ly)-6yX)%W z96kJ~ElZ=;QI?4MWAYLIf^KOAr`8 z3Dli|j5aiLna-oVatSLavRC12s9>e0tugA%jyRrID*s}hs+XCfPgJ-)Y)Mx2SD>A> z7u!gI_RGM;%LIC5vcy#~9nAP(kI{wk5+Hq&yNC)U_?gMkWyzA%awiY}vg12M@fycN zyU#m6^{(I^C>6}!747sCQW-omZZE4KgRt2X;+4M({s!#=-6fiAC2pFt1r(f3(K1o1 zg}qMwn?|)`vuf*Br_SsRta|mP2J2c+~V|jSIWGt1tu$VCl#qNt(BAyF2cx*yvy@c#uh(xnYXNkNa?! zr5l!R%KB^7?dq2{Irj?HV{^h4jxc3Yto!KjV+|Sv8QA{Gi9wMfL<%Xy-8}{Mx7m^V z_gE=(5Z99Tb=ez1a!S01Ms21}rDHls`wCPE-&?2e^L5eIFF zPO03vE;Nlbj{p}i?wdu$FHxc8D)k$yiAI?gqkH)p{)hqocc@RzsHE|-ReE02uRdjZ zyY~P$Jf|p_jO8g42q5DjOyUI)n1N=;Ihi0yTgi+&-SUS475o47lHVg*R{L^jG_w07 z42-8;TYKt^>+=qhdlNnlQi5r@t*oWyr(oYy^Eq!iC!ppP6L_pE#K#25=~^xuO|i=4 z2C_~^i(suR7NJSRT&b_?Sx(877>H*f%&ly6TO5*k-T$Ng$fDWZ`!Rmv=i@GB$(ARZ zlg!Xu1~4G)wNse(inc<8SBD zxg36sMYY}`7`uUbPf#r6MeJpxet~Nxs1A#(&IM|W$P!~W+w_r|i&5YG)zlui_4*H$ z&*YE~T5i63srw`(-sldG0#)ZGz8zN)7fZo0sE+0La%n^YHT+T}L0Ap;bVp+8;Ma&L zZ0ngr|Ay}OE;zMF$8b|i;-=9dyWc#l5*qKF0N+lRWZnDNqe&8lfj&leUc#t55qM>W zJHVR`b_aOVi6gM7{(Kgbqam+nMVow)1>io`owL^k8kI@7?M!kgE8U)z|7d@Hu{qW2 z@z6sjL}%IM<3bnOpy%MUIwAfR`w@BhTrHCkPnj&}G3?WxVNWg!CbQ`JpC)cT(_zni z!J>0M!;0Mv5ng$X5$ZM5@~K^igo6TCO*&uv_l)EfYSmMGH#M{Nj_}U6!S7wkQ!>;vi0E- zGfJwegk}j*_wM?3rtHy>#_ABBySQ01A@_9?rmQj)%i}A^48=oy5C*?M#pv)j36KkE zJqaAjokU!#lrNUgV5g9w8Udz@+7hdV#@yZhtnl~GdM@4XEBUo;y+)z`I;g6$!?Z)* zmJ`M3amLCqz|@sl6H13_7q_h?w05mMZ$ATAc#4JwH90+DUXp9*I!Uys0pO$PFJ zr0xDIa5nGFZz-3U14TF3&?iG5FQJM z_!uHN;k3y}idkk0AX@@Dp^efE$kxxRSGqPBy^X`PD}@<7Ba|*`>;5U&ap~BLW{MMR z2hWhxR<$ix8felQ``*J|%@~lEZ$>q&dCD}<{AM}vyUlXWAm&cE5#*Je9<6Ku@e{3<0I#z=0w2N2XSXMc7?U z!g%Z(F9Y}if%1FA6fbxToKwIhbG(TU3n6jUNe%*F)H4m?x1Ejp`J`i=N%c1l_Iyus z2lf|FjIOA_3!Ok$?&2UVtfLs>xSS;I2nBGOz+cG(AY>oI;rvYiE6aAaENk;`FVx;% z6ya~&O^uxjQB#Lss~In*lVbU9R%bgRdfo+F8D(CoFH=drV%AypCwE8)m!O%QyU>IQ znj8Y!LdRqqXba%1?Xkmg>KyU7wGPaGWv=eH$GWw(|E}X zFa*z|`FJigw2|ig?Q;#_VUN~G{I!d+xghM%U~#ew{7CSUurp2f1MZ9s!>3Yk`zeM`flUtLmTgg)Vz z*_OjOqncpQmNSOWBx)=5SG7WsXH8OdLh;z&AAkq8d^~tTs0$SDrzQvL%U@^m@nrW* zmQAe#(RMhJ5o6e?GB~gNL#9M>F@AD0n{AtE>5Qs{kyTsF4fVAV*}4V6SOZ2p0*5mI z`1*l>Y2bb!V4B}ixvYNgFHAmcp6x_g)|dbAaOF@9H%@E0rMK7Abi*-ak;94|I3=mr zydO7Q>n4NHLV?nT3~4y-;C{9cAX^3)0!%-KpQ2ntfBsh{UwmHAblKEb@?Sn-#MAv< za=>)7OoHEFng_ku`Leh=X|nRhJ(T6jXw#6spR5&81mQ0$Nvrt{*oOx)3JfZqa#Gm1 zsA*xpx?dOmkAw0VtmdtESX979TDWn*Min8(TWs(MYcd6QSteMjdTsN0 zu1JlW+L+*7@=?-M8jrq<%RZ+hz*!2G1UO6my5y#Y0AupC7xv7SP3;@=8^?(VlR5eB zln*H$@4eJQZa%>+9^3fiS1^6uZz3aD^2l-7Ak)KvhL>j6eN=uh?X39F@B?Ax_nPTm z@F#vI6z{cTfP)$a&OJR8GxOI#gZy|h588(Yg?i*(s>QL@{=pNEicbH%0+DWfmqsnvRsbf9}2mW`JejF0~ zF*>Lu;iqMR7jZh`>NYS*TrRH}Cf=9A3q0vmj`@KcGI`C^K+n)($`CPtuTly>j}qh2 z5*%HPljN|jF%#M)Cs|FXr`<}Ly?>wNsHlGJaKI_=>w(n8u5&T-7s;Wto}TS9QQ@`E z1-)C0&!14=NFGi+R=xXrKok99=ui)MIenG!=K zn%qC>V@o5ftdjiMKdy^!f1~#DSTpIKt2N}IooiLEU9?&CKHBv+Gz*QHo?kZH1wU-( zH10(jx(#YSUH`eq%w`$zN8qXgKI+^kD*B6B&_qNz8*IGJ^GP)U| zTqWd$pc+q4R#s32)>}SmRNVT%-AFOg$9YGP*6Ryil>!d-kJrK0ZEg zf=uZ8abki)gD1R!k3$Tp^^w!TK(tRpGHgKKuGB_cw2Z3!mA4&!3^J*avjVGB3Ki&2%Zqm3X3& zvk50en;oQBhpx4q79zoD(iFLwC#gLh+g*q&w^pLX2U5jVYCYzYgY|Q`*ee$9tjt?0 zdUKzf#imkz?urePD@1f;IRE>RLf1TT0c=gbd%}=oR+0W&QIh`ogev$7olj36{E>cI SPwD^K2IxtEU(@gC`TP&h(KodK delta 4487 zcmXX~dpwivAKr`+VkV-<#u7G%mXO1e(oBN&$)hrCt()#}HyIBT3qN3QeG8K*3wgNBEl@$AXu*)o z(%PSW;p}>Tn}AR7w`t$xe+-%jK$K|vAFEnhzI?H$^u%LdS&pSZ@*+05$y0vb(9Sfk zV?7vL98Q)=zr~q~89^eM89t~qj6yU@8nJ?FOGoihEY3x%p%JAY$x4VC;I-%18-tv3 zm&TAd;tt2~_joLc<;sKK4%e%0K3kS_m8MIqNFF~&E?v5Lw49r{ujw*VkIS`J-IC_<~HgQxZVuy9TB`4>)OQxYPus0>}~-K?+{*j64gRW zATi4x8Qzfqxj##D#86sj!b@{eGS_<0rJ0tl>twM?0Ci^TiFQ?0y+nf-f7SNnEFq4K z`tsrxgl1<(a6d}@^NA&viN28T`1!Mh5TdGp$Bj$`g|F}H!h^Zlc|YpuOfLcT>T;N? z$J-6@U)<5gY%7%+OMlCm*XXc{Ha=TdXn1Lt!Ul|YFp&AR+h=OpvpsZ)O zkfaC@_l97o6ByCp3aYH*=Dm9BCK=P*jH1re8c{08E{7I|#5T|xdByy-zoUn-7P;+Y z-8{)Sy3q)&9Xr}?>6P>i)J_wI5wM1s3+Y2c<{0raStcLs?-NuIwFNxV$`oz-H>{{O zy;@?5<#6CSyFs&FwPSUU|D?(^oO;pq^|3&y3Ri+>0uRF$V^IPz!FJ>KBvN5EG_XDd zgP@(#k2(O$6`Ptp7LDg?@3%)NkojOX8gZnCC5yB5b7h4W-))CWmgNxozH(~eb*=H(V8$^%U5kdGlB|j?zm$smM~a`9jP#kG-H0bT zc57P5p3AZoxq`nR#-Y!_SW3QA*YE;P!qo{RCfSM~ONRB`6)wecWc42KBF~ZK)CvwcP>J`RX zcjb;a=tat>#wWMz4>!bG-V}RiY-Fg^gEw+AgLgxDseXmF?JA)At%7c<&+?6RXz$$H zb<&EQ(lj&SyL!YX)t7k;v}6*c?`>87t(EO3;WoYdH|J8wM>-B}&fX$o+dyXH^Os`Z z4GdnFt&a&CGClBCe3PKx+Xo))AKP65J6=Z>zW`ODsL#CwS_#md9|ZBA-+itSPCpP=wUQ6) z76offhUSUP*hvYlN8=P-@VYgFF#Dxz=d%<76bo{tzH|Db-$U zKI208q<09+RR>LvEr4DO4!%x~M?0`&V%73|9PuToVGhoW%eAvXahlGk74#}Us<=Ef z4lgh)927@_4xu1MT2ZyIGD^T!!2MrxxN7Az@4EdS|KVl>=4kR7`1*+QyhN}sNeyk- zaTc5Eb;r9%RwP~V;3%Fy(K{I6E5DP#~RxM`- ziEPI*cRZz;v_LQ~X^SgI1lO#wPC6gZN(cf}#Gj|fEoB}aez@m3?^kojg>C8;>67!n z7wCS5M{bbqAsh?Zfx%+&1SrGqKt|zdApV?6!5Od+=^}8Aa1bj@fJZIr5Ky)2mI*>t z_K&qP+?ma>L<5eXG_@iow8M*SFBRGQESTU{D6Xm{Qfuce8xtMYg*&h=wz+SC0+d}V zx(68xt`+S8#d;fd6|pxwCDKn7ek(4qCtndT$8v?V=R zrQ9nIv=7VqJ7=s;KQUf;XCj%z_|j%XbjG2mFenL!fxxgm<|@}lE^ej^Ry!3`89K&t zA?8>utD#c(y>;$(J)g{4{2l1!&lmy-<2DO8SQkw0@jT0vSb(Bspor-yu@ zwzZt4_BMxw^#H?7h4;;tic+sil>H4xH0ASkh~=;EuS-qEm~iGj?~`sApr}A{8(<)= zG`Cm^q3}bSAm|RRtdlYTA)8~vM$!qx<>vSPO_p$8Ki$#Vo0bqn@W~gqr(Y%d6SI{% z?+xe#$Qrw5;OBHP`v`78R=onbCHPQ`-T=Aik{|&RjH>P35tUQobxO$Bs&>B*3_WB% zI~zLxOfy;)yxQzvm2Gv@B;vS_$FRVf8wz#rH(53Z8J?KYS9a>F) zd|iCNw6W21CYdk4)1iEITSl{C;Oo&`9DA-o<@pBtSPjH_!~Zd(ug$1t-wnLLq%iCT z2|9tw-2&y|3W1V!qIA>1`TV!N7wV!({BLhCq}{de^6?J7O^zx!@@i9mxRPc8b<^%4 zvgfpHxk3C^J<*CsE#;Nht8FnQhu}t<5XK=oSd&GL$59@#XFd5qrwcjf4QKy$I8kw7 z*!bS0DSTVjzUJlX6`ha7l+FNW{SPR+g`uClC=qy4FWe#((5-YC&`qjn8PE+MwhD$v zNs6LDs4dhX2wYT#g+DUUQfn{Ni{69GomkIFeo@dlwEc+Ve|S~s)tN*CJH4}&H-=j*!E(v1oHYwLqLRT!!q;3K)2m;b1Q zMHMi=JY@`8PVa9oNsr13v#ff!I6=cJtXw-NSDnr+GB)b+p)9mvnTPGJ0=Ap~YrDZt z$~-V6oIN^Kz|4_>p{qSR9*{Y>3i%Jo!v&i|>OAduFc_*DY{FF+J?Vkmq%Wq!G`g^~ zQ}hgBny|*DC_hm&LV8VN#@RMv0&k`Z{t9>(kM@8$uP=V^)gO1fQQ>_PDLjJLc+6=J zmaXgl7h!ig>wP#0e-i{)4f|Iz4PYgc3SR@XVsWXRF|PqyVIy400&@)SygfW-w4yG( zWz4U^dP7WQVL16rw!Tom>U!ku#n%t@pNJ_B{>~)oDIIQ2q}>x0VqJ9v=7Ixy3(N(u ze#IFp@9m3>;1y%IrIx$6R>IU{_ZeZJ4Q z`qO|;LRxMaU{H5FW#L9B$TnaS)UdlglVP*uDKUX|0ghLpc%Zew2`=q{Yzd`}kMR+l zhsoo_^3_Ob8}p~?LcgOjzRL?cVB3mJ1Bbsj?QkmVgPm$Vc&?J}^v)={y{>kqOrMNWbYgL19`7R`T%KvqN*rNukc%XFx{~rDz|Rr@bb#UkK!@7= zaEfK&GXO9f7uXq7{~tXSmeKRjy&Bhxc3UfkxO+NQoYGL}yRI<$)L`&zSkIb#y5}NsTf+k zu9DAg*?n3P&#nplhvO9aKD3K3xPAFnQgd<1M4OSS^3QbG1Pcqdp=X5Gg24qfY`{Sv zn{#oOg_Hjp=#07t@+Y~-!pIP#ib_gFYegSAyixd*FWX9u_kH;JV{!xdsjg)viYQ{6 zr1zkNuiYrAj)E6BwYQo=f*Sfjcx-{obQz^+OTS1;9|5GQJ5z)5=TccJoN;q1xunom zrLaWG0o#aBiCY)Hvp+xA!H+-NMD87D%k#d8^T|IwCXr79O{RfMG8mjkJAHzNCG1~~ zyOWUYiJd-o?jt)%+RNZlb0Y!2L_vpp_9aEXDeR(bD^-F1+C2^!VNRD0j{5p_wDLRK zI?ZU3QidXMy?F>$1S0I=n`*FClp?GGl(S`3BxMEgd*|+v-D8DdYI;=3ISXiv!B5QS zsgVO49H6?lwY4_vQtMiRnS1l!c9<5ZpW921-5ImMon-56e);l1D^FtAd#tYIq%*pA zXzqvg%b$;l|5Q&Kx6J&bC7XJuqniIt)x}koN281l+dpNuJhvs_#msg|@!6Wm;pKH7 zcbA*Z-9P3w2g|0eI-Ra}KNw{Yp*K>t-+#G`ycw<~r~Yzm?IzKoxm9Mt=0fqg!PusP z+q+ltcei?$?tDl&7hf^{`)vQqfepWo_un6MEPQxT{+k*9>}AAp9d;UYb#*z^uK5i+ zIms-Q;hvkuR{!?9CC{y5ci*6Y;f0h{rU5hk5oyYen$kbOOQcy+njXO6ryLeA5hKp+ zqQmiZp4cWP7o`%c(MJjy59 z78~ej{_{=cfmynct!l3(b?3IRdEu&*lZ&yq z Date: Tue, 4 Feb 2025 19:59:40 +0000 Subject: [PATCH 62/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14574 -> 14575 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 24657619f..eefb28fba 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 19:58:19,996 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3270 INFO] --- -[2025-02-04 19:58:19,996 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3270 INFO] --- +[2025-02-04 19:59:35,814 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3286 INFO] --- -[2025-02-04 19:58:19,996 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 19:58:19,996 submission_checker1.py:3307 INFO] --- -[2025-02-04 19:58:19,996 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 19:58:21,120 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 19:59:35,814 submission_checker1.py:3286 INFO] --- +[2025-02-04 19:59:35,814 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 19:59:35,814 submission_checker1.py:3307 INFO] --- +[2025-02-04 19:59:35,814 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 19:59:36,859 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 4b1e299fb..103a99dd8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -603 +604 diff --git a/summary.xlsx b/summary.xlsx index 0166167c634c726bd50d72d2bfb7054ea74a1260..e75f448dafd638befbbd8ae5b7a6a65fb13233ee 100644 GIT binary patch delta 340 zcmV-a0jvJ*aqn@k%Q6MReK9pAlh86Af6Gq8Fc3xed`0Aa;yh?Ytt6EIbwwp4f#~{p#wOytnW%@2-6l`tu|o202c+2a)g-*?YuVxW5;$NsY`-G1#zRTq&}2(e~S%e z-*gK&Pt!S;;3=n`;vN{AiHJrdXA{!=*meMLhM@%s$~&AT89sYGSm|yy49bO53V$$g z(|Qso_uh%g*mYgf6@xY8&+zB^_B!CR>o%f)w+we&kjE87Rjg|0bRQ m$K2PSp#B515j0u^1;TwXH71jAI8*@=lf^hA2F*190001srkNA~ delta 347 zcmV-h0i^!#aqe-j%Q6KGZis>fFc5_AeTvBU#7-y`wUQvU6{jLqDpgT$ zmc4-0_(!(ekhiZBAb`ZF$Mwv9JG*i@->WA11?Q#F70NS$642ZTskasSSS`{SN_=2i zu*PVpPzN5(>${R$%8i3JXDqlN!6y-|ASyAUkJ*z+AI=CkO~QQR1(Scs+De`3q{ zH`@X(2zkH?0uwAS+yhfP644PU_=vPWt{DJ=V`xBu4j$)Oj?dlzPWhV+gL2_iGISPh zT2JEW-g`M3+qTWx$zYB7bG%-@z7MOWQun9gfa+3k${ny^TwUVJA|9?`J}l#jHxk5C z7xf!&$3+f(Ad(n>9^#&q&&l)4DQbc0f)vw~6e+o1<&ShaqXqfuM_%=xf$BKq-^3H( tko)=*)PGP*0|b+yI2yAsG&}?a4Q_~nC6jSDQ~?o_#yBGe%ryW2008xXr3L^1 From 64f9faf8c0f0ad86f39b83afbcfacfe1585b4435 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 20:01:40 +0000 Subject: [PATCH 63/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14575 -> 14573 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index eefb28fba..3cc1d387f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 19:59:35,814 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3270 INFO] --- -[2025-02-04 19:59:35,814 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 20:01:35,832 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 20:01:35,832 submission_checker1.py:3270 INFO] --- +[2025-02-04 20:01:35,832 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 20:01:35,832 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 20:01:35,832 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3286 INFO] --- -[2025-02-04 19:59:35,814 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 19:59:35,814 submission_checker1.py:3307 INFO] --- -[2025-02-04 19:59:35,814 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 19:59:36,859 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 20:01:35,833 submission_checker1.py:3286 INFO] --- +[2025-02-04 20:01:35,833 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 20:01:35,833 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 20:01:35,833 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 20:01:35,833 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 20:01:35,833 submission_checker1.py:3307 INFO] --- +[2025-02-04 20:01:35,833 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 20:01:36,893 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0741 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358788 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71954 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7134 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 103a99dd8..695668ddc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -604 +605 diff --git a/summary.csv b/summary.csv index 221d65c33..45c54e523 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0741,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358788,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71954,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7134,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 92873cc75..02f7af862 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71954, + "Result":1.7134, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e75f448dafd638befbbd8ae5b7a6a65fb13233ee..24e36dd80fb8801a79cc84c2abd82b1605417308 100644 GIT binary patch delta 2582 zcmYM0X*8R88^#l<)RG|>N)Zi-HQJ81#V8R=+XzKbQmr+L*csIR#8%5p2||le)z%s+ z)=@ewV{58*RVl?#%~)#Z*v-?v?>X=D;s5bEzx%qb`##S~;MYKwhYaeXS)jXLm4=QA~z|1EW1N~qig3S%9fxpuQqV;@m6cb zZv~pjmB%;JFTfOUQp2WQR7WE;A0CK&alncp+J+caDBsf=<}!k|v#bcjbp!eGEtVy& z0VQg97FLhSfu3cT7_=7mFSzj1ObbUSgoRi@LF$t4n_{0)gVlUhXzrt&C3yP?4Tlewg~H zpxgQIth)m?3t``vGgsjHGt*&+StDeh;wWkoo|NdSvO-4=oj$X|LJW{J_n`LC%R*|9 z`n?fZ*n*G@dyHyiZ?wIRM&F(W;(95njo~pudE*ru53dz;pML5v&dnWzD($P=3~^nG z$a}h&gaG)CE)EuusTU0~FYE&+{vDs+P3whS)d>CA%8Yl{8$wj)oAJagx_9dIi~Sw- z$lH#MiGQAdr`#h%S~;i;Rx(G(;=^VHN060s%!T>Uc~EoV9!Q>>J|IUJF5P7D&`SFM z`N?_nd%(tGjZ5g>ReAosv~u#)?$0uQ%nphs7Ui9(9J&@#v+V)J$(vefnJC~h6zcvtI)@k2Q@j0^QHe~Mdx~RN- zR92w+fdXWX#9~EmfA8MJ#Zn`~>diTc8lOVilz3PcZB}-z!uBbOTO>f1}CbF7&&SD@$F!>N zY~Yq4Gbd~y*- z8W*-u+CF^?nipkSo6$-!5jDKvqJ+9t(>!~5qY$4}=@w*QlNt?#1-p`reXZu{p*Rs! z1|t;YD+UEsVb4H8Rn8S%LUI2`5RX*YNHG}yFJ87A`gSw?q3!@MW$Uw^;JRCRH8b1n zET8GxcvEo8T$Ier`7qp1M8<_MQ0)hBS{w`kPK)E6o5*J4{MEn%_O3aE($*VljJR0la7VK|V}k&~v^$^?rkPPZ%UrnG%4=2u?7P1+ly zJahs2^@ASUrET5pK99*`j#WE%tWj$ohxgL?9H%`K*46Q8&}!ACDCfs0BHyZ;REkm$ zQI|u5^pAOf4WgA+8*0Hjzk`;aVj-gj01Wy}jznfhmv}xQU3A~#AdaMXipT3^O>Vju zCh&8Zv#8q1vJTn7As*KvV|xN7;W3;{z`-8ZLZM~&AChknETd7D2ozF=O+M!=NvK0n z@7dfV>IZfXStw`sTOm|)ZzHq++DqyAweOm0LpPijx!>-#bz-~XT*-qUupj43SS796YxU6<}l z1#125C*`WdJ$$yjOP!)@m|ElwwBx2$ep*+L>sWlxhq_%i*6p-XZbK(9dd;!XG5Ba` zuwra=X=hKPTK%=dT;$fl;*GWDO2u2%_g_8wQ}*loUmJGbM*+I|=?|v-BJ|ZN9Grx< zmFfno7Y!UNx{qJ<0f%>XOcy)PwU7EASIkd&MK-uD+#PfcU39bsoZCN6{dxL#?$!9+ z-LU2|hc6zT#!s(URrhvtPu&F^`<{;isf!I3b#p9t zV4d7IUVl&uK0by*>dfQ;B(L0fWH>f!Km!HzdbOWNCagpYJZ(|uP>nfn+ z1hYo#w3~$;)4Cdc>q0fjlp`6S%5f^`hIRSAbs!-`MSD(&eE#{_wGa{E59j4<*E(+T z&y@eV*D+4bpvWa-;E%hTJm!AcKBz@IK50JFX+%n8OjHXRnwWcRT+?4KIp(n`MIS3Z zf_6{*Q73dk7WV(&xU36Wc2Lu--?VgjW)!lH6(q8zwN#k28>lS~ VeVLV{jfWz$y0jJGJr^P1_b+x6u9E-& delta 2584 zcmYM0c{tR28^&i$#u#fcqs<^Qmh9UQV=Sr0SWju1lw_xuW68e#WGPCP88e+y28X09 zGfa!hN#O`FCu3|;LMD3}DpKB`&U;<&_mAh#=eqCv`8?0{U5H$YWCzJg{`2t2KFPf5iP$bL0lJsK%7AY=}y--sAogv06uyyciME<(ZGaO47_j-HQyizW7MB za3ztd#XKV?sdf!|My?xj&3{mJysv7NE!I-Wn>9KMthN;9-oj5E#m^N6vVF5CDaSbV zqDi&UqbG%JMFF=2H;&`FIDuxer#j?MU}=wKrP_}Us;N~N+79~cDwQ`OmGK6A#MA6q zHRm^5C6IOlK2oYFHf)KOla)l1Q>@iA= z$M*})KzXu5pTMV#`q&bi=?{NosSER`c>!t8th=p9$aUhejt z$29t~ggFsJw+Hw9X?BA>4T8qUGKvRAq5P;)$6OkTY^|&4RDCH^z6Vl$DFEUEG?dAe z&YVxALT&j~D?8o)9dNm1(wBn3_`+>KhsXTV#$MTnf(fHybiSFRq80uxlbj5P@L8vYbx zLF4e@y*g6wCAuMl66;IFm%Uh;5o@|L_i1b5k=p($mACrtg;+hp_GNF3{Vk!3J@BzW zK5CchceZqdqlfZ%pK$#bb*o0>>mYQ|V!b-tS)h>evNTj+{$^@*1M;hr(GXKwV<3_* zIDFI2Y47&6?p8S}>8}7Q>^A9!T>pXqYz7u4?a3NxMUvnt;*Zh&ps+&MmV_*dbsutP zCMD%b%x>OS#u=UK-iQqR4(Yq5k*6hUqYOzN3OF+inx>Klk|zBtGOz-KfvROtR;k-0 zr*!$(E;MBcw@W==FUlN`)zQO-KOCv0r(`0N9+Z3FcrT)ztXY0fINJH=9J=mrRtRLt zIR@nw)2;SVV*+q3Ol?KrT7b2Q=T3&1s(4>m{2`~JCEj!C7QU%TxZM=ovTUr54C(i# z>GM~n8aIFv@-#ZL0!{K$o#i0h6v-~O%8~O4!PskY;ff;<+$8xgYLRvNb zshQ0esHC60{j!p3+0bd(Zb){h4hHhN zYLSgZl=H?oJ~Wr4gqgu+3$%usbqOJCI|HZr@C_ntHUsRh{5 zCx1|q zs1OY)@gqCzSaq%YRnv*y9qIL3bCDYDS1Vp^`}~4r9I?bmu!&f@@0$^%(66#ORzMEi z6|MAOe)&$t_J>|N!!lPifs}Cu!qU@oOD7uCSr0{nI_pEbG2;RdOytXZyh+aZ7MNee z%oA;=)Oref3%p+i=)@+KkcB2oo1xY3&xgJ#jz+7PDM#7ujU@O@ECy?TW4k*ciP&xN z4Y_Uf0BlL9ZCcR02}n>2|3ulIjEP?W^-EC^05O;*FYhGf=sS7YrSwS756x6N99;@( z_IBvi5i^=a6u~fu0rQy13l4#D-1q6J?yz@<>8diJ?!##i4;H*_zv>;LoEBUTk>_8d zWGG#FGT-#z{(}{)N4_(fMO{JKaE7yfHreM#+lNC!4aSHa?)4EaH18K%a0a%G&zvt(06Jr`9pxkA#xK072$vduV6+awfnj3w!y1G!C9(1I&~`nXEe z55@`zLz$tuOK36YhYqsqd4SU!J;sH+$*WgWJk<5V_eLBHIv-gm>^_n4;VckGPYWP;jlPMYg2?5SCkH&`!rIWiewH2?g=+;FlnRlkET8kw+} z(=a$?UO-{K!$b47#xjFIY>fLxR3cYslus82*-j~^S&kf)lv81P?JKVgP zg%Z>mNPMRce;iybt*e^cJS}|o(|DtDb>f=O+(SUH5`Mqg(`fZcu+ftkgL^h?pM|9} zGQFZdpKl}+y!!P;jlSi-saFq9Pl~QQu`vibF=x}hI#M+pb)b0k$HG39+|O&X;q9H) z#XZb8;N&fuLF3xHIfch!n?R$}K0sdlMT(J`2AyY&0lHmV4doAgO+GZ0GC>GFREuRm zVh-W3>9+H_Nh6+Za>Ez7KtBC;h)zs@_*^Ch)kRZ0RQkuXeVES5 z+nNap#RG1rs#=PB8O(xPvqTO=@X=>#@n@SMO5z1knb(mEPtCXab;PPxJEb2KkNmT? zf>;y_T#*TQ7cFJs#y`Lu*RCp=!0ar}$v%t?Qs~@yejlM1 zldA0cbbE`6oZYi!ynfU?43hdjOuuYS5Svt||5QiQ- Date: Tue, 4 Feb 2025 20:05:33 +0000 Subject: [PATCH 64/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14573 -> 14573 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 3cc1d387f..1d3b58a14 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 20:01:35,832 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 20:01:35,832 submission_checker1.py:3270 INFO] --- -[2025-02-04 20:01:35,832 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3270 INFO] --- +[2025-02-04 20:05:27,882 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 20:01:35,832 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 20:01:35,832 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 20:01:35,833 submission_checker1.py:3286 INFO] --- -[2025-02-04 20:01:35,833 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 20:01:35,833 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 20:01:35,833 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 20:01:35,833 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 20:01:35,833 submission_checker1.py:3307 INFO] --- -[2025-02-04 20:01:35,833 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 20:01:36,893 module.py:5501 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 20:05:27,882 submission_checker1.py:3286 INFO] --- +[2025-02-04 20:05:27,882 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 20:05:27,882 submission_checker1.py:3307 INFO] --- +[2025-02-04 20:05:27,883 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 20:05:29,169 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.429127 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433075 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.639 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89007 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 695668ddc..924a92614 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -605 +606 diff --git a/summary.csv b/summary.csv index 45c54e523..211d37f26 100644 --- a/summary.csv +++ b/summary.csv @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.429127,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433075,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.639,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89007,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 02f7af862..07f6aacf4 100644 --- a/summary.json +++ b/summary.json @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.429127, + "Result":0.433075, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,7 +1239,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.639, + "Result":9.89007, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 24e36dd80fb8801a79cc84c2abd82b1605417308..200753c1ecca88c802d7fc1a5d2d0a80f302da12 100644 GIT binary patch delta 4430 zcmX9?dpwi<8{UlNu$bl0Du-##LydZCjYeV{C4@O2lea@`|H1a8G1ZkQG0F57pk{ck%UYpcI-o>8eDQxp!E+XIg3B^HQ3!vZKu}N zQ8H*__Qj)lNqaegohv)&JuL&*+d7i$kCwCgmQwqgE{7laE6f~kqW3~9xc@d$^+;ZL z!J}%^23HYu!M*jg#LsmW_h$`hFB!pT{T_S;|LG^46c)l%)GnR&g)X2KlXa7c`27-* z0`o|}D~)|y=i1>vspI){JF}zpse@-U_}lY|KKX(}`Wrt41&?AKd42{cJwd15OhPgb za$Gq?c(Vsx(OsjoJPSc8sG0(ut9(=P?xm4wv(T(5?( zQXxSpXr!;AY5T)PJ~3Adl$tBy?Fyotwou^{?1n)!q@_t@ghR7%h99$MT5|gvApc`+f3DypHe z%Ifls1D_oJ&AK${iY?7IEZ6^QcSZkGc~wFPa->Krv;Qo~_+0jWj@LtFP#qdF#(}X? zk?4=&cI`PZYgWJ8Uy3;x3a;(tT#^0bs=M^u90&feQx;qw@w_LdWBV|90%6S?YN()f zGSC+gvzI-TJY)my5S_t043o9%A#gh^0V4$QZ(I7E)!^F`9TDyh@y4ah_9$6eA!wLV z--#cjG`lA0ItPxKf9~xUD0S)b;>&K^!E-~9T{YC2P@{M?S(X(aog)ty>h0CmL|EZT z&;?i}lL-C397qVZNh;}*)KZ2!FdS~ADG~5ZN`6Dlx-J8cE3W4rU;66K^E0;0v+E>x zi(g(U`&K_c%ZgPJ>y_5T0z8&VvEj`-HKBZ1Bk{dDv!?{gmZ+^?5nA#Z zmvfwzNHlCX&2f`6$#F3_4vrM$R&j=$ZcW`Z=(H9LB_SqvI3?N_b~A9Kq7&9wa?mK? zCLAPAiwnjp@FNn8P?yfM1rrrJS(+BZ3rTwaRVF}f(@5X(-zx*i@57%GNUXo*rTn4nryB$Rt zVZZ)k+&}egA=zw~!J^LySg5yl{I%UVHNTJY`z29?E$_K}%Tu@hSq_u~ar{e)tSytjr`=>C#wZcHnEl;qiQ99_WU zFlwRg0Pt1}CM1=08T%>2-ST|GHeYY^8fT~dkf`OrM^jKgjgiZrO-_uwy7kuRK=R3& zeaSWYM1B4eM zP7d&F?|zBT5?WsV!lTGX?YhrsZ5>BGIi-CCR)3l6jmkMFatvhG3yWVX_nv3BSNyHP zN~`Ks)5$*;D?^V;}IJ)8p=r}a=5CH>0S6v8TFs<_>TMRUlEo6vxM^MU<)ngJHI(%n^>9k{f6IwOet8s7Y+9q;_t zdB3VXA2YxYrcP`p>XL%IGEeT=#j**szJBnqCcq}t+N#~332nhPYJWmW*DliuJc-ch z9XD#i!nUjx1)i7pSRoqH-~!oRcZiVZ;ZlYPS*4vYPVsBOG8DdU+3T3jyYFC zfNOVHpyOcBb_)yy#sQ;lT6{`$AwXm58JXOZZ&7=ML=)d-LYaDCJS?`S!0=#HGcwqG z7FS!u=}I2Xj^+g(e|N7;UTC|+9nIk?$hiluvEWuaeb*{sgm#QZ>0n^q_%VaVbM%)I z-rG3}dxZV_s3qj{ANIaom@vqS9y^y)?yb7&z${Sb%Wm*CX_R4Gh_*ZugC!H4kc_&Cp*nfte^%Bnr89$O#g^4_3k zcRoz9?)^ahiz33I;asSrbSeAVoRfr^3@XUx6 zMKZxMw?7X>mD5sCAIkL0aODXn!$u?fsU5``K}T66Mp`EEF%F|-=aLx5`Wxhu=+CsT znpR*L9rj_~?}H0gm_HuuKCI8J z7HPlOHJhI^rOnR1O3~vk%FCe2!He>ppmJJ)M*csObCnW=GKZo4`E!lzvXx-@VDn{JL^n=D>S8Z@=hiOm4E;<*LtC+A-=XDMek!w(7d;HBi?BecMKs zr|~>c<8(D0){no8K3lTA0ry+!dLde~_M$6q39tBWvMwUW9y?X<&W&=j$PF zmV639DoKp@n_gzYA0kJVD`EL0#*ZbiSqxdh5OiLgrVg?BE$}fjYNvWH%!HFae>-L6 zLEh^x0y6%b^C({@*{|*8Rh(dv*a#7U?vGmcMdphMn=#J(nI%c%yzvHC>?=*< z_}PM*KKg+R+itZW*{?)Dnepgtz(_NOcxiCK#>u4+FY#%OE!UM=Ku>EN;;>H==<%-R zj9yt}cDeZ6y_#H&k>5zrPn@O}+{JoPlIi*6y;_W`Kf!XsXTgbaOiL~sR+UC$(U_LA z&etelVSWYSW2^{BtXD>_qWiI}2lysn;AbbluDyKsCosII-8Qk7IXW4$`7ZL%1~7 zosqb^G42Z$snM#`d=3(76x3Hm6O6WF1xOB1V6rt!}aPdz~Elz%~TmniL4+=L7&2VhJKDGM{sy1#()< z0{}~x1COZG0DwSx&LoxQqCc5J$j0i0RviaDZdVlAT{sq7sq`w{?wRMiHkAC$HpES9 z(#@^gj-XJ$6d9o+8#&)zLnvY;;IjPX!+4cF>W3Ix`Td2inPIyoX-Xdv_)&HDG);OT zpZ)WJe@~F%2vmn>tw&<{*<{Sa!g9bqAI<=?t_kD`%sNZKb_Y+bslITEly&#yEM)=c zR5sUc+{4i-aw$GLNzl0$*!gX9FIdbzuA3n?U4<_He;I(Ka@!xR5UWu zR3R5i0V_@rQwmsd40gkrGkr{ol;l5?;MNgv6FY51A^~hR%RWc+Pg>sLhS`X7TEn#F zbA(ck`nWe`SCB!>aMy-{#v{r3 z#2;Em)2mqP@pk42t5e5L=l9+>kXqRm4K1eyS>EQJ$oHeNY~FlFnH`k3PfmDPB-3J* zY<7bvy?^_@8x-h1|Ag(nGp0ArOEta^kZARl0P_1GQRa0{0$dEfw$W|HoaVM$ehy&- zJ}<&{hcxi{nL&l)>5z+4!N$gm(Q84Kd;Xn2xwu9j7Mhr-KK;V`sXJRdzR}|m^2YU5 zgt*oAY1GXF8l%S<)zjx2FbU?h(dPFYpq=B9Q9hpQ0YhW{v&Sb*)XGz~cV$$VEQuCB zxx94idgY^B(HT^=BAS{V_|)U`f|7cM@wTmcEpqK!@oM!$(aN|l1G{^pUj6+_*m~aV z`o}W~uWEeOvB4WZD(?PQE^sp}zOHI??%ACG%C>{NjiIt8`gOD0d6^}tBLC~I_GUro z6;X8b^^n!c@qdFhN@{(6nRyxqWIcc7J$BR4;>9_0?xSVZ@rbm>*)sL#A!DRQ*0N9CLIk4|k>GqQA5rW{Qi`?` zX?%o%n)aLJ6Sie8VcG&NflAijCRo;8HbCO!{wUWC3Ds!0pLXHXxo^RhZ`Ke0=k=;# zds(s5YRM>b#_4j0TBWa-_h_e<`V*=J?`!fiM9W##mED)VYPDIwNiafYh(kx)zti+; zt8mK{IyohK@RF>RaL0X>)Qan4E}O_lZ%~^$xuCYekJT_UHOT+}nHc?H4*ow^7Y{N1 E2b$?1%K!iX delta 4430 zcmX9?c|6nqAK%R6SPT*6+8l+U?|vnTVZ=5Sp;>M-p-@R0VI$?J95WLta%+w`EA=gM zj&dhbawNChHAj9QegAp=_j$daujljmIv$_-YxCDQUZVWlCYPHkFN^+x&_4Q|51IA1 zW(Bm_goaJT2`dwpyX#@r=12xwB8lm)B7J^1SD}^jQoLZbq#!u;=>DXJfa~RQ+}5tQ2!q=c135|zxq=zD9i@dv$N z_uW)#jUko1!txFe-2d_|hJ4UC>9w)nHJgojKZ|qF98!}V*4j7qT;E>F3wz=;#!#gR zA~H>z<8%sBdVim&5I7#C988fFQsIJux0f8_v0_^&n`LZm31CwgT70 zy^M0wzp^KYc5B}EShn2>jHkw*sp-6TEhm=qzk#`jc?)xi@MQwJPLm5NduZZPonXHV z7oo20o2uZeW3Mm8K@;s6V3&Ur#{Gj;Z`j1L|LDA{j(f@P?7H@2?PJXtac(5V$RMm} zF)jDCJ=GZ9=FQc-`}Y&P0JRCWd@1xb8T|dDEiJ2E7*zIzPgb4&nz1t6FF}HhLRnIg zBE^H@MQCe+zOzK?Qdo~{U%YI`UcH$b1$QP2c2y5OSg`e9B%)K5$n!tKR2|iP zXvw7L-s~(WxQAG)gXLOeh~dD~%~?BZjT5pwbmP$x_#aHk=%Yk^R5I z9Q{6pxkm5H?Xtk~+w6%(0k#bstNtM9ENkQvjIs70{i9 ziZn2D7|mn6cAyp&+o%iIRjx5p=2*36igp*&D(20ZdYOs(6y%+LJWaz7$v9^t`7;sT zBdfHp9M&b9wNE|M)=XIGF@_x@1=0_55m%=N<{KZHmM&#hoOksrKekHQ_uTH_Mq%4< z*9;*|t!Vt7M4P*)df(a4Hu6YW)iYb7LWq0db$C1I9>sV$ZpoYnLlR9evf;DruJb)h zC!&;P=TFsEbroBr(;}KX2+v3VAvC?|q8xdo6{NjqR@k+-|5v}CW}0hEUum24xz8~@ zLAgV}cE+PFMO)=|TPOYE6T_Kvb>0Qy1O+{1@rT=S*g-nodR{V)1Xb#@f6C6Qxh`md z{(MhnQvB#_(nXswS_|R%L~3$i6H**r%d1X4_rz7|CzpqGrG+i!Qk9J!cM~qk)J@%z z_d8O%p=J#2%Lzs*a^w?FcPFd-uTGmRtF#*-Saev3IGaw{*g|T(%?{mt#}dh< zEF^?lW3g9HpxhUV-baL)R2Sl*&Z@WSH!1_wfI@HTcCQ zdCE*A8JDqaN^G&PHJ-YXe-U3` zcDu0kwUpC21FEv#(U4qjvu+ObVwuG7&#OqK*O!#jf?phQ_1wI)$5|8 zIP(y25$l0jc+6B1yh8oe&)GP`OpAf70&PFlUXgd`;YM`)=knR)yoT-W!^y2)y?a!0 ziUVml!E)GD8cBddaRh-`7+&;wV^r930_*MvgdD(P&qGhd76o79!>3)(>y|ocH0spc zS!?tv?*OeUcDRoo$jEJ8n`(Rt_FC9o zQVA$~&J^3~MxSdy*D1-u(NmRSBq*h;V3X!O-6!`h>HaF8xmeXq60ucD#;}&0bLdfh ze`V=yM-ASisWaS`bmI5dFJ}@%k0zZ$`tUf3DJ(G|L=>1M)((n#zns+ec>Mwz?~ zu>lJw=NCg@OQuNmbs#laObERbmOULDvm|}76^pXa&G2&t(HnX@v7<30?KMA;TUnINX>_J@F?qG>n zZ6n-Xbx5yc(Rp7>J`wU|$p5BSJnOwQWp*e&T0Y-*rf?mS)=2vull!azTW)s)C&-0a z-yjLJs$~2DrD1HNr9kMg?0E3GJ@J&es)$o9*u~=8QCHOujx9Wl`}(-WZCm@IvuZNT zp@TK?n0$im+`}#xb%=G5zz0X)A*=5W)B{RCIj}B*?grQ)Y(Z2LRJGA351A2_;|}=c?Q;&G0Ppx^eJu zKp`Z5!Z=w_H7PKJbeMG(Fi$%8EMT5=3e8B-xVK=H!$;5WochOY^{>EYk~MRP4~s~l zjrbzcH{J<3e|DpXuByUi#^Ip3TV4`-VjC~U<~9MeRZ($*co>ID62Pfs@IbMcSb#Q( zFwq2XG(cOfq_xXn5p*IYD)CHL$mu4BP7mKdwv*Ki9@Jg^^^?NK8lPw$jJ^m}ai_oD z`*N(n6dU77A80_+RR^2)a8`p2i$QMFy&*7f!>cMD1$&wH((lAr;Adj1-;PVMF;*kX z7dkGhu854;yuc7nKDO@-x{7}!&SO6iBjEzN5Qy*Sx;1y%5ynIPx;(43zpk6{FkS;p zH_gwq_x`9%D@imNe0HS)9W9jz11>nNLI9ZZWBRgiNl32L5y6s;pxL8Oe$KuoqTj*={8VJ@e8ygg zI!w4F7OD|08K6{*K!DR^VO;=aGC-siTa5#-Ofs82OveBALcpF*W$DN@b)ousq62Q& zzuNK8rl91y@2MiflRML_5KK#uhn2#|80O_oJX@YA;cABe5Pj!+EgNVHANmiVEr57E z^C?rn8}o+m>!u>cS~y+oq+h_5PGxVP%iwt8fy6HXA10-_UOYou%6lzAyjEh>=OybN z{VfH{Vz}zIf;6L@1MF}HI2@5hj_~3GE!R7x&;~%-s{PY24%|hbTec9X zS~BL&pX=1*Z#{|+w`{@@lO>Zbv;ang`kOu1{K6!(C7 zS~I}sM!)+4DVu)SXMvNrl~N=xfrUff%d^1n!LtUSY^$&CQgE}(S043WcZG&3SXLy} zdEh0mN8Jx8IF{FrpIXVpBo|WxOzh%lVbnk}!OYKQIxz?(ZpmZ@0r>KOgc^`sAfd*f zxq)$B=Vt&OA+yHAWavM5DHMcLt-wQrF4{Mu?{XBGS8TyZIjt>dxis`5@YO`9+}IH+ z>K`2QC>6-I7pN8mfdSQ`1~CToJgZRvo@Hi(NBwAoGHn+h0n1kKxg!|K@z9hGdM6&* zk#fz^PyUt9tc)k4ulx@C#(+Mn6-oH4k5aj^4cJE<#sLNuoHDdnr5cHG29_Ov*qeB$ z>t9`76q&Bi#7^8vqKph{u7r;aUv3iUR4X)^d4))MkDk)@9UMQ0SN4BJtT-rbwLz=X z6c4Pb+Qq2_;Bk$EQ)4OZwj<^~R{pjbFQM7Tuq+_J!gE7b&u_huU!lgLF{mfEVB(dO;cX zxIoiiphf#R@K9Qp3POn`&1h_v$}mol*tFV;B&hA@yY#v|E#Z>NweM6n3C%r)jq=?+ zeB`HQ4!}~*5+DIfIjaH;F2w8wP8{nn1#6fBT#if9ag#BwRPecLcNb@Lt*OWAa7w2Q zOe4(+p7L%Zw*6km8I9`OgzN-wpKXpcvP5xL<6@d3ATpQUgR`0L4flp-^-OMm*;$Mr zEQWjQEcYH3;kv%)Q%3L~GlOb;D6+Ap)p>Oh@};AE2KhCl1$-!Sidc{FI;Qtwhje2U z6DJAW=Di?E;5N6tH~`AWNX8IBr^D2&VQK(AZyipXryAVt0OymK7XL&4Sh-dCR9cOr z21LH);K9v8bBAHQk-Hr~#*m$e->0SGbbN!EeD)C*bzso@OKziERpi^UAtW1 z8fwQrPFg{@tRByZZ*C_K+;0C=$)j+b;tM9Ln`r%KTxwhb|MYY}8(5tCy@4*%dh0xK zYi(=x{8DYPnq%36H&6bRpZ|ET`u9g#s6j?T>No!oBh6>duA=LQD!a>OO`NS-c3yVS zwmpAa&Nk^Z^k3O|HvR3=t!nbj&2DngtP7Fm*6{h;zXu*oTHNOGsI`U8KfIdEa?EYY zKD12kze#iHDD9`k%~o4gP6X(_+#EWEk(pTe_$@B^mTrdD&p#FVs;(DT-hMh1xWl&t zskuH>qb6P*p4g`!zS(wmOl8D2ft|J;6m;&T zUZENfI+@ok)%L$giLNh#MK`53NIJb1%-?lYZV$SxpNbkd^xaf_&OEs1N(bqxNu(7L zcdO0s&N_k=S3Y^m`n^~8)0`K_&X&V1pEt0UuGB!B;OG`%K1meq$QOw1Nr`SD_1`g7 zMy0Z#NDpu02vLQ}3-LO`ptC&gU6Ypq8nXH#$^DgjwPG&u%{6|NLAoT5KXSAZmv7PM zZHqQ-uNmJ^*P9R}9Vyjbx*;z1>4+k6snJnrtmxiG Date: Tue, 4 Feb 2025 23:39:16 +0000 Subject: [PATCH 65/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14573 -> 14574 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 1d3b58a14..f48ca73ce 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 20:05:27,882 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3270 INFO] --- -[2025-02-04 20:05:27,882 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 23:39:11,564 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 23:39:11,564 submission_checker1.py:3270 INFO] --- +[2025-02-04 23:39:11,564 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 23:39:11,564 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 23:39:11,564 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3286 INFO] --- -[2025-02-04 20:05:27,882 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 20:05:27,882 submission_checker1.py:3307 INFO] --- -[2025-02-04 20:05:27,883 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 20:05:29,169 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 23:39:11,564 submission_checker1.py:3286 INFO] --- +[2025-02-04 23:39:11,565 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 23:39:11,565 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 23:39:11,565 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 23:39:11,565 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 23:39:11,565 submission_checker1.py:3307 INFO] --- +[2025-02-04 23:39:11,565 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 23:39:12,679 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.380515 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.375178 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,9 +62,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89007 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.119 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437053 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433963 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 924a92614..437dcabe1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -606 +607 diff --git a/summary.csv b/summary.csv index 211d37f26..7ae2cfd48 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.380515,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.375178,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,9 +40,9 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89007,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.119,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437053,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433963,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 07f6aacf4..c8b575050 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.380515, + "Result":0.375178, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.89007, - "Accuracy":"acc: 75.800", + "Result":10.119, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1299,7 +1299,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437053, + "Result":0.433963, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 200753c1ecca88c802d7fc1a5d2d0a80f302da12..f4fde1b1230c375cab4e3959958071badacbac8c 100644 GIT binary patch delta 3931 zcmZ9Pc{CK<|Ho(S8Zih_Ci^yHdB!#mnk*B8%D(SAF-WPbGeVXk3|$@o1W+U*YE!EIp^L#Ugv$^@AoM9kyn ze9Q4#m$$LfR{Yg3w8KzHdQ|~lw}MqI&2C5BY8#v+zzc>i)`VHoyABo|doB%M{gnKC zmqwedb~)L+Tj28b56*AIU$&0cR7BcH9ll(=-P4t2KHv0yIVn^j`KFN9VP#TUIvcZK z6fm8w7ZBCAo%}ibw-qv<#wUFNIyfS5NJ@Twk@F$8G|K%tnZ>UH-lOAdUII-It+-)l zKp1Oq4By8q{aJ>SrBUe zBEs3Y?az0lC5R$XK2JtnsCbCWPL2vOtxuLo|HmF`&GbD4pU(=5o8{20iiw`D_?w?Y z6Gqz(Ne9AuWRSRmffXlufPT)Xi~350FqT&pliO4c4zFY}49h+LSYo*sL_-(|u!m+k zfC-=WjJ_>3Wz|ROQ=WG_U_wcnDUqzqf&^d%)mTFNFi<9}lnn~7Juie~w29#(VPdVtOUUr;2b$I+Qz*$&grTiec_+@gAdi=MZs*U}_m1$xm8R@!hh zoRqs)a{VpNbGP?vHTE$1oRnybgz6*wrx z7>QCeGM;l1JOmd1Tb@p8c^q+_E@6CBRV+se5D*_#)wma4^)aLh{MJC1$+v;5x&5$g zN@YSPqtXMahc(-V-D()#`Mdr~$ab-IzU`!e^;&M~fso!^Bur$?j_RErt-`f*Aa&*( zvlJ+(?`&}`PM>c^7!QV0!66|KjT(s8Q*<>MH=pi}Y(OJ>H%eT6HPCk>@<$S5QbvF5 zTRKNeEbUh&***tqkXL6bb<$sUCRBZod~%SV?wg66#j1cMe*R6hLmgb`{0S zv(Om%w7Xa*>zCEA-tXd|N9o?DwuX3XEeZVOvg3Ls3$FVi1qcpXW-Y@B9DU~pt+P^t zmff^WNuY!1r-1XARkz0DNBCv4&UZ6iMe(3K_}E^(H5`L$Gel2o2C zufQ6$XFW<3pj6l>m1lZ{t>p)OqXMb=N<=_E$cm)-h_mB*XT12OUXSZ3Z(Rd4EaPW7 z3nrx(=D;znE^!)p`K^C$yO{)b=5^exWO3~I{R0RKyFk6I<4pyQ9stt9X^pW{{~%jV~Yt3Z2dG0SjGM`{odr7t$phrh*+4oUva7yX*JI0QNRX$Uy-QwfJu&?W8>lXSR!b8n-iubV z(x&t<&^_)%w#AqM)WWzv{3Y>UftqpAvyo!Vz^d9L5v7W8y5Q7tEQQAu{vK>Nm(U+- zDjX-=MT@umTU~TLHk})c1&NYrgaqjnNi@#9fV7KD>U&=5Z)u(^@8Fy_akJ>oU|<^{ zYY#1-4@VuvLDR!EOG49~4SVLF6ki%Ou(f~SYh1!_$`OXrUx|t4@H+hyvg0W=WL<_- z?G(M7-&#;W8ggUdEQZbW$-Y)da*nnY`s`BygjujyP|DB?dnfJ55iK&(99}1It5?jh zN)9bNcL3(Q!dQsX`bwsAevQgf7m3PX$WVvTX29DRUIl`sPLG_gK#=5yRi)|~fW#WN?J$D<#_T#I5@yMS6YS)RrygBngdoh}n$%3$f&(#r%uR737_+;#y5>o=f7?O)H8&C3bn8o6(uA6xsTrB=R*4+qnfc4_H4{ET z`-Bn6L-;@E&B)is=ZrP_LIps2LUAU8Iet%A5UX!8hX8-vAR4;K$}>(@pXfAh~WH{CkY}Jex6HRixF);L=p}$204x@_N;-8^4TV-4)z4IC5$q za5aDvn%mhZJ-RDfKQz4>z;nKavom1O$!#XXXh*?18Q6l9iV%x$8pl3U`hV-~Co ziP5-KM!5mcVo3GkppX&@ow6y*O`XoISFm>3Z~RQNi_r+yY`1Vyi-pBQyqqqUY{3k> zauc{vJ>ns+%rs>LX0Q!kOuNTtR?zTiETwW!I8J{HK|aBz|%;kXuyBT z`bq`uDgCHsUq}VPQ{F*YDn7%wpqE{mrS|RC<15({$xU)j!(+Y`jk=da{~8h-z_$5_ zXv$vsGFi_tKy$-Ctw+HG;t9*xzji4n(T@G5mTX*CE6Um8ol1VMbQx5=WBY3tKVmcJZs)hPtDm(>UeFD1FZ|>u zE}h%~8%P{ovg+hao*LG9V^J&e6#GHh)4do~XKFfSKMZa)+=|>n^m-@23C1ccQ{AHR zkve)sCT7~LFN=JOYZtzt;nxpEpA+t{8C|`H*#aA76;z1-MDZdR1eAgA>AMT$p+8v# z{AdGuc%y9zU;L#>J5o*D)16GJtYT-~)nTFqE?MzX1~W9FM^Y&xA&@Vz*t?}4KB>LP z_}uMgI!WnsM6owjbnO(G`%hoElljvr&!Spy@-pr1NKAZ5bJbG^T^M7{O}UdC$B>Y! zVvL@|GSi%&8&8y?+xP>D-lKhPf1C72<({lC$&OLZl0ti)Gg{CgpH+$2)XO*1rw(@W zi24X0lO-|9o0}sx2u3ceMy!T6W;lN5`@8#QjXQzrrdn(0u~z$rc*z;5Kn*~)?f%hn zIAU>nFGL5;WA9~C7#BEs?_1%N#v)}o^tI^If^Ynwe;9mM^7=kUH(~c5PYk_CDlZed z{2*$YZk~xZ@f-V;kK{Yc6hxjGCqNor8Zx($_pzv8i(vB?#72ikLQn?3#E3BDs|UJ z8D|Kb!~@e}Q92(0CZ8k6%a_PH3O^O=Fz}z)vDn^_RKR|5szAY2YqniRmBDL^>eyV` z(+Jl8HUI~qpbuA4X+xPw1i=3v7jZ-&{<;JY7Wh$&G)8xj1#sY;^`;5EjUzH|D*idN zkXb6YI_nJ#mBtb2H|qc&xk+Ow2gX>jc0rH@Rj zF*enjmV0WTD2d$*J%j(;nK2BAGTjx{K)C^ODWihJiqG+PClk12hvG_rbwL|pdsy0y z#*WDW$mM5aZwoQlmJKHiG3r*02rIi>8!dg`<};d3WRijT@w#8gd@!>OHg2-&yN|=b zGHKevActJ?xD3tPzv5vN?nU>l^K@|3Q_-m63eETO(2b|)%9p2%1(s~%5*__@YyOVr z{o)CC^LC|>7SLBbbHlDs0vsuK91OcU$pU8$s`Dorz8RZdgoQR6INsTpL?nOV)&J^`zALp`W#7pr5%5at zj!k*4CZoTn(M)O|xKO&u3PTD8%3Kim$t2J9Ey$0lGjUDbm_QeEfnsi>8zzB?#9nlp zsRh0)TDqP-D_Qi4A}psQ&z^w(;+}t4g<+dz(E@R#K0@fjL$`w~$#$5qY$l<<<>Jt7#h!jZ%C!(Tcuoa;^#7b%BT zqh(u32EGe?p$UMuXQvKO4&9nXP=T+0J-V0GwX~)pi{3(4pR6FZ^Umo~=aR6T;-sYG zZO&nA4OoGf_P<{+UPjplcncq|EcCzUxk{OfW=MR4lJTBMLnV@Jn2}Z? z-VXO1uX}X|%|JH=K5-w_TQDmJ;Y{JZ8jwg9x^4{^hy@1(M`%pPm}}C7en=)5_k;O* z^kp3PG)n|LqFBO*-b()Utc?e$zOGW6j~j@aeTgddxb-Qsgsn&Kk`)9;U}v`SIL2~l zUv)e#XVL722^PL-#R}{_Q0RJZix;^@CH-GG%1{ zX>cg>QsCE`3fv^^7r2z9#=8`;<)#rfBAMeR1m{GB7C*ppX!gi)8WK|O;f!E%J}(B= z5ws9I6&hsy8`=Z(lPzOkU)~Xy8ci=y;8f-o(IbeMHJ5AUfZd&38!EWC+Cg$utdzet z07R3ZaZco>K&UTPA4$-+R)bx>lMoY$?EoaixL|ZE(%G>q_#jK5(9rVs)yF||`nOep zi>d!u-?KMhn7mL%eo(rv*0prg-(E7^frHSt9ye9&rw%pXx8!UU{mr(=?uiN{(vmJ# z9R{+**`WY;vd7z*00fOPGG+{vUv6}{RAoTp4+R`9fK>|=Zt{7xXc_0V})b15)&^-`T#-$i*21miw zhnaX@mQh?Yt6mGv{Q;tz#<-NuqTJ8IUYVmWj4$xMjU*nXAO)PeCPAt`1^6wCoj=mX2js_aII8m(t3_L8(o}d zZeCZr_73PCRkBtX%SWJx2WsK2>ccE4&iTL|sd3b!6cpjBSbg>t+t5?C~ia!j)zXlvrl?pEij-uu`j-wEtAZdu4|AP6JQ0d*L;^oO{P9b=5 zaNZrD2b5AOUh$p>T9vDc3P|N=X28{!?}h0BogI32LQTX71J58a?W2zty<-sMAvEy@6D%rNCo_7*uPV z5ZC!ymu-38e;y-c)+CuyZ--5-bxhgAI9P5Tx{Mm1F^8B)j#hw8j0-@khSkJIcxA-j zl2&(cdEIi`|F91eg#~f+}8g;8_{A zUxY@pmjHq&5>FrA`!7JxfXuD*cR6XCJirD>TN2}*80ox2rpVb3oYTddd0cdxQ!j}g zO7g=P&Rmp|65>J|=F#eEGk9WXJLCWoyd0Rd5sW46h0n{t&N*Of%8PxPXrRJ+imSd$ z?A|2{*jeQLV_0Vnu7}oCyfnMEVPz)ozG@MK5=$6kXBHi%{5H9T^4nw2#!_@;jg@v2~>Z763!e z2DLG(gl?^`z8>}Ya)Z7NBvP_x-eMx?(F#&pdIxpvSCyG| z0`%h9khO0Yyg7ij7bL#eG!Dao{vJ8J& zJnKLbQD%2)XWG2UVxZan3bD_#Vt;C7eR6upzR@zlp`ToN9CW-4Xvo9sB!C?w%*~x_o zQ!Zg}BLz>${uBxo=6krj_`~z8u-5)W!A9B*U*BxE5Mip}a8{;V=TMyk#Ihc_D%MULUa6q2pDZ;_Fb7H?5o3@-lmUgM>rp*v|R0+qHj|dY92UHL#O2zIjXRXoe-2naBC910zHnAB!sC;mCH%UaH(A@9 z{f2{4wb(4S+>du?WDGA+x1v?M-}l}>G~}62hVJF zYuFV_L?PJ_EEWaIhS+lz_#Z#yl(XhOVC)xxzjBR)>SEdSyS|vdnoT5|reQxEC1*K< z-^62-x92@Z^J-Om%F}jlTcDR$cvG&+%_r%&1#?tv?WcZGFs#g8FbDMM`F5wd?@xYY z7V)`!8+=LdS^kR-|5d5OYRBDB0wr?z!33I8k6Af^2&a=fI$xIuVUs2kUNPD?-c5J! z_#2V+LTJ-3VqMM*E^2OG{XuOzjocosYWVZ*OPpq1-DZJr7W6^78asOTB6}lBIS?kE z9;LNnfcKDA$41I;-P8y2gs8t^FIECG$`<`8O6SB1uDuX+XHLm+=tfH3Sk;4Da zNM%;OTD<}0HqqUH#UJf6X!MGN>@8|iCxWxyT#o$iYd?C;Lpty2z5zk_RD4Bi@yu;mZbtmUP zI_e5EydH=zgg^epWSnsYk^8yAaPlh1JHk7@xRDg?U{DIE43a&~b_@w2m3rUrSZ8qY zap8_s)SW=iGX@@VJ+>hP<|EmRz@1~4ir?73`UPS>zGn3X`X%e_?3sg|^E4p|rn6SS zqqY6nMuk4rs$L82^=Ncwcc%7FlmYTc1&)5+nJS|Efm{x>Al|0dtlElQ(E zo-a|k08y0TGM$8|W3aRRN2bbB$)!)xT*rbx@s$+`d$=Va6EbzNLLr?#B6=_rAPD|K zu$%3NhkHpscKI@lj1<@mJ9tFjeqxVSR<5sW-n?Q`l6!*Y9b2U%m`6et0>XYWNN!0R zYnwCss%4mr=e0#M+jyIg8HM!2b?~G`6-gV`Frp)~(o}ri{jXDWDO?tQmy9M&fmf1s zZE6AXZEYQ1E5mw_hpg4!axQ44*piUTpu-}pP%57_`yTzZK|yy-bPn+BIo*F7fCDV( z0}c&B-Du(df5jg|WZV7>Y*J~lpg&pzIPlMvfKY#$ihw4%mT|z*HGl(X&I^EI#Snqp zf0`Ds)l|MYFB;Tc43TkL5!eK68NYmlbJXrl%BvmOiEK`iG;cOt51xP#h9T+fZW*O@ za6DD<*|LVzk;_Cn>T}VZc4#Opc6eT?UOCaM7FNdVO3FDZikMWCe0un z1~OV^Ee0@Hadb(ynD|mSpEZbc2s zUM)TREZEP+ySHWXqn#FDSY6~HJ&8Vh1?j$3jG7*E8nX1qq=riIWLdRUcr|JKL-)KLp=OI0F)68^-fSD5ot|Y4!1-k;c-Y>GCeD zxi9MEuj3~Vtggg_Clu!=xgb@%%*(5vS7@Z$>`G333i6Qkgc(kkJ* z-cT%anSzg~g_ujFyM!|XZFadIw)YlW7v`O|1b z-;vTZP?{mQg^A`8Om+rjrh0KG_A*L)N^qP7(0W55N0UBz7e;pT0TiD0M{CtRmf%rg zRw}#h>nGTd^}>7Fbpu)6b^7-&+vWz=l7jn(h4YxT`;UA1%N=d)=KF*N{zYh%evf<4 z+3}EjpV#NRFj<4cfJHOe*QA&HV>MW`QrllEKG-n+<%|!Lc25>p0x@jC{G6w#3w#cJ z4Is~qXXpF^RbEu!0sZ$Y#>*(#0Po-vl!W>IJlJ#qfTB16a2fD_3-f$l1b;5Vh5xD~ o0rbE#DXRdd@wb$r|Ev0A Date: Tue, 4 Feb 2025 23:39:59 +0000 Subject: [PATCH 66/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14574 -> 14575 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f48ca73ce..cd7925b1d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 23:39:11,564 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 23:39:11,564 submission_checker1.py:3270 INFO] --- -[2025-02-04 23:39:11,564 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3270 INFO] --- +[2025-02-04 23:39:54,997 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 23:39:11,564 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 23:39:11,564 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 23:39:11,564 submission_checker1.py:3286 INFO] --- -[2025-02-04 23:39:11,565 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 23:39:11,565 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 23:39:11,565 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 23:39:11,565 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 23:39:11,565 submission_checker1.py:3307 INFO] --- -[2025-02-04 23:39:11,565 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 23:39:12,679 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 23:39:54,997 submission_checker1.py:3286 INFO] --- +[2025-02-04 23:39:54,997 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 23:39:54,997 submission_checker1.py:3307 INFO] --- +[2025-02-04 23:39:54,997 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 23:39:56,052 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 437dcabe1..ce5acf709 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -607 +608 diff --git a/summary.xlsx b/summary.xlsx index f4fde1b1230c375cab4e3959958071badacbac8c..61ab6efe7745d1977ae5d225ba9061ed9325679a 100644 GIT binary patch delta 326 zcmV-M0lEI}aqn@k%rXT+iX|r{lhHCOe<%VgNhLsCQ3;765If7BrWWx-wjs^m<0fs| zN?EZS-#cgSj9f1FvWyoY* ztRa7fKi4mB!>X}Rhf^^?b;%iF7O2;@F7ag%4%brW*Wtul0sOHK`pnyLk+BbW6atVj z?n(J5o}M==ROe~Fh|@eyZ#Q{C7>Wn7SbQBuUiF`W@;Kz*#1ml5ef? z>Wby~-Z^t;ZS)J)39TxWB`JzPG0lZ)D)g~l#t$fRo+?gTt)M~&aA;BAl*|yO zExcK6z2o@_Mk+U2W)@lc6{o fvoJI~1OVh000003|*X@ From e9018489eca518fefc8121deaff9d65cbfba5abe Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Feb 2025 23:42:56 +0000 Subject: [PATCH 67/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14575 -> 14574 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index cd7925b1d..440002a8a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 23:39:54,997 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3270 INFO] --- -[2025-02-04 23:39:54,997 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-04 23:42:51,663 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3270 INFO] --- +[2025-02-04 23:42:51,664 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3286 INFO] --- -[2025-02-04 23:39:54,997 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 23:39:54,997 submission_checker1.py:3307 INFO] --- -[2025-02-04 23:39:54,997 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 23:39:56,052 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-04 23:42:51,664 submission_checker1.py:3286 INFO] --- +[2025-02-04 23:42:51,664 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-04 23:42:51,664 submission_checker1.py:3307 INFO] --- +[2025-02-04 23:42:51,664 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-04 23:42:52,762 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433075 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431775 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index ce5acf709..cb3e25bbc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -608 +609 diff --git a/summary.csv b/summary.csv index 7ae2cfd48..656deeb19 100644 --- a/summary.csv +++ b/summary.csv @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433075,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431775,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index c8b575050..7239a0870 100644 --- a/summary.json +++ b/summary.json @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433075, + "Result":0.431775, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 61ab6efe7745d1977ae5d225ba9061ed9325679a..743b1badb69a307451d8a8fc4a92af550df53f19 100644 GIT binary patch delta 2434 zcmXArdpy(oAIJAKMhGdg<2ptqBuNeoOb?E#%oWdR;MyVF+Xy`PC< zR9xYUD^{NqwczHVn5(Y|mdF$-uc{d54wK$qbfhOqt)^@Zf2H>757d71G2fO{M*#(; zcfhGIJw>huQkd=sajAYH+d4ZKg9oUC0c%QCF=Loc7v*SDAKU)kR}#~~D*Q7WKOLj_ zM{neUZoY3HsYY*u9R@ukLu##636y_`WVpymbqvfR5Vj{;XNhnt=0DTWKtZk=4$`*{ zw}54-nKC$oX@P&TEQ;X-Bzlx`Tw&h)bb5PeUi4q3T&x%JlYZB$n>rCROb83(u?u%A zJ%z@F+)DSO+E(Yw(q@>^ctE3ndRZwvdJJP{22AGN_i!UPNPHBn=yGRopgN~clL$xi zz2#Vc&S&z=u3P9hsN&_@v>86KQoKu%X2=vgEW<(%if3FRL!2mu@7pCIEX#^6>M+#a zD2i_tP33@;K`?Zjg_b8Dgog_F%VqZSQ=)cMD?gE1#V#qV(8^cMbiYARA3%)kKZ zrOpBjGRvS|0(W&Bq+z|x66UUM%AmVg#iZga1|;FR6$1Z)+6U@m{|doAG1|ChVK4X@ zCjv7mTAGxb-vKFJeMou=70secS7-Use5vSpY#bio6;)cjPvV1M1A7iv_C2%r0FwKj zU8{Hbd+SxtRqZt?VxL_SI!`BOzUaIREHn5~ac8^rD%1_MtPPS4S_alGfidX5(s~d| z(fD8Yb#5-Bd=U6ThTKS42_E!X1OCQ-2@rd%5!!T2i!CHK?=s4VC0?F(6I%EdhA? z@5y0wGHVpe8%d%Ki8z$2%4%1%FLAh%X9t#uh(_sonw%z_GPYSIt--4s2Xx|-WXmC= zA6C!Lco>qwXyfQT5UH;v%~KtwTY2s>i?-0^xh2b&3eT%na>-h6rw`;ee0pTPQ^`=s`bWBGMsn zSnI8vEroV7q+_)QYeoKI!*ldk5`FX`qpMc)4lY1-Do0x~gE1u|#q-YRBM_p;TStg+ z2z`MI)esD+;sEq|qq*XUDi}0jjDu9(gJK~Ow)>;TISS?G1mm6A6Ff(qgSI6f((MnG z+79XD$pm%Db8H=gK)iFHUa^rCp9rTEF+#aecL5s|EL#sUSF8mEK|ZX5;-4i~sXVA~ zGqgj;^)Rb97I~33kyLe{CEKaqv!@FoliVerZ0nG`+wB+v!8K|fBf?u53++(6-~iYy zXZ=@{q6*k8i1!d#J4eT5Xh@jX`4A7Ov`u@#>)#D?3hHtwXTkOGZ1700xRR;&X!rcB z4ts;=pbg>)o&$8%(icO9=}Z5%_XtG%(wCZo}lCaTYPD>B)<3(7qT3Z zBzXKv`5aF#c@bQ0wLIhTx~tkv#Fb*%W_h$dxWypmI07+ovUQvYAIn?#0BznKObQUQ z5ra}h{hP{|oooiG7(M`BjcW|)2fQf@yDe$1{qp80nQ_;`eN#e z!47TdDG*l{J-RxJ%iJrmoE4u2PEfQ!_r?{s{{|=)kZKeEONv7bUtdGfu~^L3S|XS##x;WID$7yl2B~;4(Rb58 z#~9E?&)N`41B~bC{usZE_5Bz0gem8?^g4$cUj7?;mGt(ZG?BQBx8Yf~Tj_);0#KR2z()M1SJCPVD);bg* z(s5Y?os~bBB-gvs4t?K!?_(?Jeds*n+Y*6SS4=-SE>B7 zmWxRxkEhyC-|-XDBtZfz5hp77b*J0#{h~ zWOOXSb35Skgx~y`>0_!DzwhnMsXVsEU;T9c)rH;`f{XL4E)F zMZXPk6SnYkd0R|^@!i7PPj2!35^&DO!N?6hnVb-@IX(GLu<%I(;fJxOK>)L<<~&EAG$8%$T95g@_b=b)Y-<7a+58w>5_g0-D(x> zqQ-sat{q+f!X6OM{oY!8xuW0gp&H6#_@;J!uCjP#4?Y~7`3bBZZ3FoTTW#BHKp5cONvo97#EI{N`Y@{S_EH}KC zB(K22-?*AczsbY$`aJK}KIrIwxq00<>e3xjP1`SP2fs%OS54UW9OF$D#NF>CVHc@l zf6vK#A6yC7Un8ozHq$lKemr$;zOVS}w?vE&BsMD!XX)?Sd*rR;JLI8T#)pC|uf5Ld z*>j0j+5WoVrXd}#VhjiK=mk^lZpOv6*2fD<$86iQ@V vi@Bqv$Q(PR3j{NzOwE81rk$zIe-?+EDgrN=>86&zK4zz>7VNK+5b*sUC_IE` delta 2427 zcmX9=e>~IqAK#E4XE&o#be1u{=STCi*$wmKHox+_`H>h7MX4xcHpC)1EK}~v5=EPg zq4bStezfG=kr_^Y-q)}E8cS5%r@PM|uRouU=dbtk@maaBdVw1a-IsOc`q@@q9CTM& zO4pDVr=loN2-yZ~Ol^*5;&ieo{LRCHGj@f< z{9g!{`6~wZ$bX_D4Ki2f>Pmvc@hb%IY`x8JPb>kBy@WoAtj<@`Y{Cg(gwsz zL5*lU1tMG}3;odo*57e~?+uu|QJq%xC05D2PSLF^Bz>kBs(`2ki=n-s+Q!m_`M9(HKY!<>Q)FwB%5Ek*{2H1k)s$>L; z*~y1XyD0r25XCxLg!jS+Ag0AE>gw%0$=!1PRI@AgXWwZ9lj3D&#Dp!whJdtkLmGc0 z*X4*OkZE0fw7u5ITRFle5g=DMe<>Ma1CZ0cOB28GG}f;}UW>~VMi>fH;Sn3dQ}K4TS&ViE%XNQG(jSI$Sx%PTJ~QEnZE&?C{&g6wn~!b>ZUjXgUpsIa8D08_tr9e{zwzWGOk{QJMiGKIfRA* z+AQ-d?kNG+yIK=@&Vg_yqjIQ#UM!s2B9XK87q+;CAvs9|`64Q0?J7yOFfw=@#K3jX z{b}dpwha@F591P$4_XP)wCuG3VtUdcRBY59CAj|gjVP=}UmNPvuyflDGcN%oOSLJg zG+JW&ztQfft{{3}M6P>9Wgj1J9J^5nYfuZ@;6k{9=4r4M1*62K#qFfQ6>)$EF{a`c zq{ULv&+=5Q*S5{%lsN-U-0J;E9dn43cU)&gP>~y(2ffszvd@Wl32057`*I;H^KjS- z7eae|c^V9(EJE2d&7HqA6jz`?(57po;+ryXw_;d@g6+B}Z%D(mY;W^Uqz*%|!si43 z!G2$@GYnUHV?-{gqOuX7X8_>&sA(>9sSq!L)il_UA_M^ab{aJlM*%>9Uky@mYQVC6 zHe>g5Gnt`xpf~!*Vou~m2U6Zdch#8j+zgD~5s~XqQQ7fL?OGv>tQNM$g@g${PJvOB z?*O%=ogd(e&~H67UGq{2k`RoXVAnK)>$&?2CNDk7A3L)o@(8B(=rbw{j6-Vq!g?oF zZ(#lqDNA*Z`0HV`T0WYlIhOOtSb1{+!;#XO-$+7z!hFl6-9w|{%m_l44Wkq$8fD*Gs?#x=agXWsl+J^R!>AS zskcPb=|g_#7MRz}&Av20ZECJ|k2HgH++JtpI#U&A?c^FU)A2QFOtP0?HnaMG@I**x;yIrh4L=o%liLzK zTP61Ew~)iW0%Jmyz2#EuTK>_|2vmYwo!g}=`DJc7UM(u8x&#t-Usb^s5Y-4nuUOBL z%pGZ5Rsgz5)4f;MtKN*DJ@W~senjq}ljENQGUWSCW%kHxw`FhJqoi-6`Lbq`JNAuI zVT*cfJqFR9=r+q*ebzYcqc(5zUL^X-azNY-wobGWK0KuD zUY}mp-xskz)*0{e&WRf&6&s}m0bM!KJ;S9b@!QLW6YROub$P>4mLsY0o01R9F?FGw zuDbuvjHJEza(VD`p!f35MXueQo9U;N7E>|1{Ykq^k*569L8j?y&FP;{dh7*^u@Cyb zyrIeq-fGy9J|f&*8Jfh}slV;-8FXGo+ZpMqTNbCr<8UvZecrfuxoOuoq>fT%T*?!S zPnd{1^vmmZ1AhC9Hxie!RXSv_jsA776?gyho*d6jiw2Xy=jrG%`q4+Jy`FqCnDbVe z^DJc=t1o3l)UiM1t&pn)v+ce2P<7q08PPW}qZ>c!8d(3sgl+Uneu&9d!_D*#%zgjt z2BfpH{=*h8kL>r>>`!p#-+jAc*$4fq<>*X_hX4{j`ieay%Q-Yz0rDv@^Sys}D<|xO z2#QQ1CRA&lYbtq1AF!f|ygqsyX-_G8AFgok;-Uc#6k)FXr?Oz=nuT4n2I#l&7>7(T zG@6*xAF34ad?d%ACBb#-sx(i?uxZ)ybnkw$RQ%*H2SLrT8#X=9e_ZcH_dRBxBv1=i ztsZVRS&veN4*SdvPjv=^ztk)3ydAqxRIikLK&tTA3UOQi-aTs_Bfq)Ww(Emf&-2zP z4ta77q%;keP)T`)*k*R3bAQ-d%2$O&JD#$In)H7^Obk=&V6Z#mhBa1dMw_v#t;(3N oHUyt#d}o6Jk29QYbpNyVyp1Zjjge!61FJB)Y_w(jEJ48c59R82d;kCd diff --git a/summary_results.json b/summary_results.json index f35fd156f..18bb771cb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433075, + "Performance_Result": 0.431775, "Performance_Units": "Samples/s" }, { From caa55b679c68fa6ec4974e62ad5836e5b4dafd9f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 00:07:09 +0000 Subject: [PATCH 68/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14574 -> 14561 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 440002a8a..325deb219 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-04 23:42:51,663 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3270 INFO] --- -[2025-02-04 23:42:51,664 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3270 INFO] --- +[2025-02-05 00:07:04,540 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3286 INFO] --- -[2025-02-04 23:42:51,664 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-04 23:42:51,664 submission_checker1.py:3307 INFO] --- -[2025-02-04 23:42:51,664 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-04 23:42:52,762 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 00:07:04,540 submission_checker1.py:3286 INFO] --- +[2025-02-05 00:07:04,540 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 00:07:04,540 submission_checker1.py:3307 INFO] --- +[2025-02-05 00:07:04,540 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 00:07:05,582 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,9 +62,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.119 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2605 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433963 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -74,7 +74,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27277 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22992 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cb3e25bbc..502380c6c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -609 +610 diff --git a/summary.csv b/summary.csv index 656deeb19..cf0b160f5 100644 --- a/summary.csv +++ b/summary.csv @@ -40,9 +40,9 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.119,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2605,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433963,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" @@ -52,7 +52,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27277,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22992,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7239a0870..31af17a4b 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.119, - "Accuracy":"acc: 75.600", + "Result":10.2605, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1299,7 +1299,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433963, + "Result":0.43494, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1599,7 +1599,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.27277, + "Result":6.22992, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 743b1badb69a307451d8a8fc4a92af550df53f19..ba7a4e168da133f4113370942fbfccc9a07caa74 100644 GIT binary patch delta 6827 zcmZ8mcQjnxyPh$O61@dc$0(5zA%*D1U=Tzfy+m|Eq6agAC=q2aL``(jdoPL37b7D` zl<3h*l<3^a_q*%fyYBhpJ?rds_Ossh?B`wQYJii= zn-z4z&y#Hhdo#*f(M;`-^5zijlIZDq+q>-nd*Dn-dcf9@Uu`RjrQ9M$EjQ3RXQOWZ z(Dq^ax%WQ5x7l1liSx`K!-VFY8F%3ubzZFja_tts8iLX!(P|cPyJOZ5zt3jk+rRzV z;D?oiON>?9Mw+EWy9_t7OMZ=gPivs*^QEb?UxZF zc4S>_;z{}yO1Sgd@db5XYNc&bDiX0I7uB7fKnEpE#sqdp`b&-Uy3%Bqg^+0c!^&h! zFBB*EQz7km7wx!^D^?}0Kjh&2p@a2q4?@KL)N$b=H;OLxPE_j0ivC`MC(>gn;s?*3 zanA#oG+Kvbppc&&6-Y$J=&U+|_aY5J3VVh`bncNyMf#4IF)}!`YeE(dWuOe6K6hgr zJ$>Scug-9*#;^^^Ffu$@O9Vwttm-g|pI?kG>@={VFO>zfH-V~2tYS71c8tI6{d*)9 zX7*(4Yo$CZOxVRUBz@WU;ud(mE3gV(mKDA{+IH}rC{q3rp8zaaFy|c5mfQTsSkWb; z*B~nR6U}z=gq!3<&@01cN?!M^R1ly-+K$E`cI_N?+K;!poMyz3y(()|K`Tad+cIHL)krRD~$~IQ}$)8#A?$@4h?Gb58AV0Q)Z*RM{`iU zu(XDacbT?zUiyv>e%@_0=dwTEX%$-OD3rW*mOL1C+w_u3&w7jBxcV5kw`?8QRCHB8 zCCfjtsZf%tU-kUUn-67Ic=7VZbkE{+XV)bltMH{{!rZ{?80tGLSDk+Y-xGFKuV|=q8{#F^|;bTcxT%{?Rn}>CHk|&xCP#(VZM94;T6$ zz*8vXCO?BZyQ;13vetSCb;}J;joE-^Uv*!GhcMskLkTe%0|w_zo(KuIreKIzn$*+| zialYikmY$mu%|Rzxw}wOdBh4?zVR%s=497)rN*7v+voL!#uPxlBvk{D%|l?!e&B>q z&ymVl6AQq*}X^l{KmQDrAs-qDL>JUbq7(kXAUDkK5-2<;3>^395o?K`1C*w@q zHsokRZ4)GnyMQw&LmM z`X#VxxCBKkHNAhf4H8SznOC8h!V+)b&K!>-J}n3R;(p|VYI+`Sd4VPdsYJNLKmhW~Y&OO1$RER;PPKW|?Xd&Ry); z-XM=$o58V^+Rp>D3@^LDY$g{Y+)^FvC%`_(a<{+a&uNiG>lsZ%p2@3b3ak z(OGijIW1a|r!C?+cBRrtM18+e^0T#l(rEhmG>hK({F-Js9=hE28HCkU)wC23YsFIU z61Uy5k2hXiWVy!s-*|`2{Ych|TuG5wREmm1dZ*KkN57XJwR8#UJkyQ;Bjn&>; zK4o2GkIlh;#r^z$P}n!@Ph+4dj%13r2LMe_(2^~9picgI)Jwop0F{>skhj4cP+q%~ z@%f7d>#LR)XxM5s98*5y?v?#Kv;o^h%HSfobL)qMdV05X25UGDK8fv?nCjNBjoJpF zyNOlOlu(4($AAB>24m7oLX7Kw&=ldV&LUyIiIK!@OVCN;Dz7gL7;yOG8}8MM_Uu!- z%!~YD40AB=z3awSX0B-yL6Y|vrk)Q%NDLw=Bg5!*us~T5j2EjU8^l4M&akj{Hv~!6 zJ*v;kPe!kEC;Yq3eB#H=F=GXM&I$TA>~ALV`A-ELgJv7Xd}fZkkxEoSNd~4$(A=1B zuaFcr)byb!ur<_~u5(pC6bUq+IbGs40-#{mF=L{vj!pxRe&xLDmLdZs;VFY6$vFU7 zGaZuw_hQWhrR1lODR)HFK4r18pq^u99zhVoE^z-hEhS$j5Y8|J$OLj}B-s`%Ni`zrK~62#+MkOnsrQo~n4FjmmdZMb4YBv~4i_VdqKPWW zpakRN2fP4-vp|Ru5O8R?>g=?|3D;DOgY^27&IIJ8dmYbK5!_BP^o=(efFIIl(bDY8 zCx*YXGQ{@Aj|O(7+9$q(b4#A>OWToc#lN{1Ewpzs2^?Tn!`gxSRbZgM z)>RQ%fWK|a68|XMGH+QjiSC0yu9`aZ-J^>HygHJiyZ{O^*0-pFA5fI!-&))6fEo>O zDwh{Em$I%EeT}VqV-#VUr;C%gim#)uJaDo#2J4P89aWIEn?`a)AKhe=cx;x-rucrfqfhu~mItUriEs+z1ZB+!3m zpJS+^?oHBOXxckjOPto+|lD8g&X7$>8y7|8r zHW=UxEl+={i_CxPV_=%&p0h?Deoq9tceY0k%`IgPM^b#HrVmGfN1!11y`Td65{%=i z!G|**`S&-?Bm$Tf|AU*5)OOlzi(e_kH0EF8qi&xm^r%Qg87?#$mT-~83${!mfy9uL zN$4s0&{04V`XNA+E1sSAs{^2l3Ymik5cGy0hT&p3$zn_&FrKjIsKNUSWCN{?VJEVm ze<`py-Fjz9kNYi%^ZXWT!2{!3h*M2O$fwHlvOmo~R)!O^c-$p{5tzCnuRM6CP)cIE zb{Ady&4u=FeG-~s*XGGJtJRo1C_l%QqtM5cD-yxuGe1PVG1>_4T5@Er zIAo`TCqAZ{0EF-@Npu2j+2^osbfo5|P+lWo(}9ViDV21@2sR-X<3-*Ukat~z;*4}H zx-6^k&T)pzA}(59$aIs#F1NoKS4umomMcTR70?X`-_qh?go8nkcyK`Q045%-l)c_1 z!W$|-p{Dua7rm*(bLJ8M_r`GtaIn(KSC$RgdFuKsZ?0**t`)>H4*Bs~$V!`G#C&d4 z7^8YmM0lw|Ad-TQ`f(r%%xf1Ncd%8f^P7*wx=_|Cq#WEuZYuHM(f0zzzvOCd7<@pA zl`&k8X35^rnHXI@6iuBC!lMT@?5}&7)xnO7RC6-YwPi}Vl{e#&&T=rj{BdPCKwt4D>DWbFoPiKnpKYYKq`uK<)VPMeOB4+qqy8v@|$x_CPYg$L+L5==gRRowbOQL|V3 zhV9K3qOYh5ctv|LYW(Cky?b%-3){hh;V+gogH^WoX$|}6b)ndD+G{<8F+RS$JDp=T z?CTGvv` zI&mK@BNS^K%!0yzG8zfCvYqwd;_pZ`}Zte91(!=*r&-O5W-ouQByT)l3^~56m zIJFxH5jTN2?N&a9(Qq@u=|R-oI3WGNZFxYB&YTSy0FmACq;oce$YN42V;^TFUjaS@ zQxdm+6=9z(?dqFvnO(4w0aYPd$hmOvFb3*vt-gdqYRJK$4MS`3^tSB0hn3XU>h(K7 zSdcc+FVtxgH%Q}Oze2VL;m1`Hiu)mpM_N5;c&5fc+>YtM`Kw0etT=-9>{urvN;fFMhG|mep+*ncpjlB^TDofkbc)202n^!n z#-V|KZyPWCP(bUEm0gsF-``?txzG4_!z<+l_!@s`NO;tYxo58OL%-Ls`4?!-rJ8S98oyW(d8`Q2Ek9l-F6TG_`zWk4(M=T;aWL!$ z)I~hSAnVELc-LOMM3Ehr<0{}2ucnFpD;Ir0hN2M5Kum{_mB|Lr8kOz^vg@eTh+Vr> zN!Wp0Gmz+dFW~lL(d6P|dA%oP)ws3cQwFLak*qLY~l88BD-d*x_NX7vo;@a2fj#(D(YQd z6u8n(W*2~y4~Z~Tgav`)&}5;+Q~EixAC&U1hCnEqiP26_s_i~B?sfC_-Yh72lBBNJ zcT`u}HCBaS(It%8@cFM)5LM0^Q$kcZ-YHC>D8s>kbSVjBwZC_3J?MAX>T-Oyen(`x+y$#Ser%vdLlG1e$s3V>k19+lrN95BdFVJZ@61 zufC1a;kVK#A8R*5;a$O>vAE}S*Xr`7mEnp)Lr7^8Rl)$6)T>v%CBhIO9mLWszDiUO z;^c>}!-UU05u^lGLP4FqESidOBxgF)uQUX&e|BoEii&fFdM^2yGTtID>{j@y2eAEs zpi+c}Wq>IZVW5m^Ah1v#6ltia9Ypn*nu?xyY6)9kS8U*3d;J)EEbrOVt|Od;CtMSG zXHK64KaLu@HLN9E2hxD#JKj96d<+TaVMN1)L19EK79PM5knpv;!H#4YLgD%J)h2*o?`tkej=JBaSg=mMx^2@{X z4a48frGjP^Z`-rlxVFa7Z^Dnq0%g~pOqLYuQ)FtHQsBEUS76KRk&7!(jV2@? zNcqtc@>LrQ0B|Ip@N=Qf^w*K}!LwH4J?$C$@k5W?yY;k9B5~fWqN}(wM7Uk&=0NzD zNAyP%JKY#-!MB)CLDidC|4dlv`05CK`5oua`GNflDpgQOv~*n*!K6U*+3qsu6lrj2rzW%egXRBi>i|jl9F$AsnnP>iBA?CS~ zDx+`sC&PF7>(S*KjUsfi+fkJav4nHsr}sw6E1I1+3n@nhbdV3f)n8)lkxiIq2H*p<&0;C~}tb69v8CEZmA z5c{}m>UV>G#?y;21}5`ZLb~8x(cZ!R&O$MW$wlFO47#Wwn&sD7w}C2m?j)-Nm_K^% zFwj!A?$y4r$6zT1llg#%+Y{=0H1#(N{`7JL3sjN(!}@6#{c3Bgr@H=TLJ@yV2hM_i zJjtdeN@!-rtkccpXI=Hf$~oxHRFi&Hw-l8aq_npe>40AI9s!s2&uwN`$*J@Y5y zXKS1W_FHIhS)J~!;TMa;hFilEKwX-#89c)x6=UP)i^^B+ zufMv*=V92@OA9}0P-z?oq=YjGG|==}qIaesq4u}B)u#M*$4qPVqlTM0gK z%~Li_XCjaATGNcnq6b%FUlL_#+p?8hR?ui1spae@0C#eA3w@+198L+D_ChTquUo77 zr~2KuIE}P#EvVNjIN#y;y}&KdDJLS(52r07KLt@Akh87Q-5KGrey`A}h{gch5qk%4 z+Fi5&z=~w||C#%Mc5vqe;Ftqo1)6gK1kzwCfV=-$Bt1?8STW7HkOXRBDkyjF1KR~V zC;nY*$`e}xSY4ZQ0S1aZdL7b>L5U|r%buXg-vLz-pu%%lSv=sy#MoT z0kqT7Gll178%pdK=0k6u+%!W+O;t_1TH)6_e;VU4gkryvem2o{2eV)&(#hV7k#WcS zy5%=Wrpp3%UL(C8p3!MkaaZa_r@Ktecb2Bf{D}j*AdW<@t2qO8kbB>v-XpiGNh-(*=IGj!gs0LaLjXCXLb0U79_{-~aXmhGk0U zsXloQG|<=cAq}8SseMY5;mzFK5tI+P*aYgQYq|W=^@k zSK_5v-Y>nglGu$J+Lvb^dZ&zKoq9=cvQpd+{p4(Q@pZFO_3E-)-W>2A#ya*z;7sg_ zK-$#rDSE+Ra~ApoZI?ySYy-~+u2p-hy?If#J`o$dh|s^B?=s_gGbr!VHEf2-RTBp74yYTVRZTepF;Dl@?|0~Pj-B*kw z(-g%GET@jRHGYjiYtzbc$D+q{+@k9(ShK;mp^^@;M&3_C40&?P_3##YSkL!;F5zD_ zt0TJ8Y=x-E{3+X^W63F!+Z0WGR3vC7*quN?fe$o?NpF|I32iO|AnhA@MZzUJT*GPA z3e4C`qzQe#Y#?#KDp=;=>9y0w`PO3M>5OY-@A<<^)3$cHk^(&*?H=G*{#=rb^{2+> z?gPmex+`f6R84#%3E})7Exri=Fa!Sg!i3E5 wCtmWiCeBKV04)-!q@;oPL=`E)|0MgF6f3YY@wJpZkTkJJiXYr@|L@NK0zDwN8~^|S delta 6869 zcmZ8m2T)U8w@nCyDpg8Es-ZVU+J|&02?$6LLJLKZ-UaCiNN+-@A_ywI2%$?y2@sHu zbPy4wNeAij!uP*7^XAXY-I;rH?z;P|wa(rrbANbldsf*~lgAhAHMKAR0MAl9sLhFc z-BI%?!_;m-^%O-KRn+`(kLO=Iy)$oD$u!-X>F;IjEo@ugK0K)ZX~ziGGrv6C}JmQzfMc7M=olN~vl z*$u#EYyWLJO>PqL7&EMMn=OUSH@HcsCYZ`1TiL&h9?VJ&yg?x+?*vIAsQOu;v1#+syT`|zB5uX{ADD}+w#O21#$2Rqu zv@d1+CHDS~bVZ>w7&wAs5?-4s7*KB2WeV`>-y73l*x}F)>IjE-83wCK72X;vX^Vds z99RqJiiF?Gc_K%G_W6Q z*VVl8;;Vz_60%0zM#g#*{&wBjojeM2&?}D~?CFo&)J^!|GMa79bd__nEN4mA`1IoU zb2<6`RTM5uMA70`+{*gW*_dq$LIlNBl>1YC+O<)fpd8 zpm*=rS&k~y&-N|=xdfz{1^K=GHLn`RM`LS?E;%9n1(VuHHde0n=bJrD$^DZk*C*Ss z#bbAzrW<8!>^y3eI#5~Ezx1?!jH}E>6PT0(<>`GNDJdw>x+=x5^^cuETbq|!1Rc-m z8p;aHe}@hZiGlUqrH>PI*Qc?anu|6#n;2`+%8Ki?{`3ivXuJHVa49$X4#u(nh{1oP zYdWN-FZz*UMbg+o+*qfF&x3c*<>kcqThHpV2cnoC!5I>rTqLER z)WYv*(&P4Ayya$3{Nc|%(R4f|>sH`hD2gtIF9s%ekI1iH-p|InW9<65T5_}Oa#(*Y z3B{(kIZF=gjXW4KuU$u?|r|Ig}vnl z#Jwf5FajV96_~I}S_=SMh%)k(?P3!isfhTJuHNj^@s_D$rU+n$GglGKX#h31`qtml zGi_>Z!3&`d5A}ciyzM`R+Ckf<2EFiwk@@pL7Sn9e#$P!rXL^MEqe)rP!6BM+Jp-PH_Kqt?7dA5N`n*zgjBCv0$Mu?W6djlwJ^8pKprhd6O#Al;uJ_E;P`R}7s z^*4w3$|K4$wP@dZLWXLsYrRAw*eH@zu+TJWT0H+99!`1{IXHNistOGQ?;ScSxu{1M z4H0uQ((3Al#@|$u>%3_*|C#T}bZKSCTeSn!Pm^*RQRQ5%e$C;tHutivrk%~Cu$yQ6 zCADD$9!p4)X+N`Yg0WN`uyJsemKvZEi}3E98f0DJelU!WYwEVe185HmHl{YH%8)?;68E@i0405%cLd; zz7xJfZU=$$v(nO*2CLAO0m9D&pVy81QvZOd2h#nN3wtI%`bhI)2eDp%;f48#fD)-l zx1)T;`wq&M3Mdj!g^yX&JElIG8HI8~>@*5!j-U>>m5sx# zKt4X^@uq6>NVnihlbX?UOm4y#eqkJSNbart%8?XbJ<7<&*hZan6v|;nkZA3Hai8edZ*0as$;~BNN*7e(YbpmY8snixF+bfZd}WlfTwLz3 zabon0WR;-dC@5G)Nf-pJcC|;V<`Q=aa>gi-!?YAvX%lVpnUkERVarj`?BrZ;s?Xa# zP96A%51=+ijl^3yg@0-b|14%}RxpvB<46#X36cs{67SPIjtZyk2OLM)0krB#Xsy;$ zPzaK)QNAk&$En+&2{dvim@s|TO}Bl+yGWnSVSz;*FbvvVl%Lm_l>&3tv~R%p;7;bR zbZ*~DQg>7hk@|;|KqpvLrPmG(?h^0dndKsDJoHy8C-Pcn2(5ko{MuUqz#VGSBbbma zZ9T;(W1PIM_3l5D-6B50I<-?Wz3Rz6A|`CruhfH86-XXj!~K4!(rbnO{YAWk6GDp; zU#kg5kk}3voja_LU#ECTq)t=2k!;k<$s*rm^MNnBbE4+sOEbaK>@{tCPnOYg%cqT` zAcO*+R&9@Lk*o-6t*VMoQxALc@}IM+E2`DcYSplUEWTUK2;ZMQg!{0IAFix?dJN z3{;(?sTJ(5w~~G5pGs1f_0TdxF{U(>IQ>m7u4QqhO>4*^#^n~V`B^!N-3ZWl2$u)v zv&&$xW0B^4*SRv450|93{uAb@$ps~dAyyP+x`K2wgMb;j$el7uvIOhRfgOwr$MZ$2``-+WkZ zY+&SLdLw78Yp)9RB~_d4TL@1lhx>zM>|Nr#zUCl?nLp40`1i?TzjR==i6T1)Xm069licUe6A7GdhLz?HNd)5l0S z*)(xQk74I=m;n&K?6~cIY&BBG^PY(erCvVZMqT#uptLZ29vC#eQxcHvcR5N>UaAZ=jsAb|2mLCVF3C}Onl?|HpV&T7A(Y*w(z zZe?5$vvnQ@0v2)V7aiJo;aiu`A6rNLricEuD*R1Ugz4xn6xVe}`DIP$D6Tk=%h_{w zBjEJmsT@-ffrAm`vNzme;?4lAYSIfA-2R|}QqaOfyyi0&;`dY0J+<5!M)qvK5_u@1 zF~6uO@s_V=xVVV#b<9~q0LfcjAkMRKHY1?&;im^ob8G~ik&9R;E4f@cluMPtcL~c_ z_5dk?x>~=F*W_hZEtrbZ;44%e=S`z_?4if1!W{|u=@S6+#g%5PA$b_rE|B3_IRUWb z(D4CNEdh{_o&lWcbAdA&54t>d#qsL$rw;`{P4LTMRX5ucv>MC?d$_1y_Rzmng}=O> z?IIT!p@}S7Lpm^9UqFbno9sp?AolPdIi}l$*%I=52feQR(VS!t>TRs~Zg;MlLmIwK zVBgdy>Je6lS)*xG#nTw!_t)ibf5MVb4mts5gmR!R$&104Y03wdsj$9J$~w=2-GWe(Y73Sh1}aS6(;ko6}--dW}RUwCIg7G=hZq+bR&Z2<%LbKO%|*N^B-SgV{wNG8hYAhU=dS?{P^;Q9p`_d|yD`r4l0% z3M0FB<8CMdeAh1WhSkRXPzREZaeE@16%ir7KSX8b&xnA9z`#q?unKxTy;u3Wr(<)0 z_^@XS7W>L6e|kbp_DP%Z zP&<;bvfrutXl4zyIXydqY(>Ob4)oNV`kihS9@o5|Ia;woiL+idv5y};zsTLZn6!*+ zuSH&nyIlUM%=%kVZ!A<8!YuNvVcYJARI2=PzO2(fL?ScyRZ-lGU5L>m2`A{$Oi)mW z=ke;wZ>P(mM&!AKnW%kAbGyazOFae4%LfD5a zlu`T|Yy<`XBn1F~n*ad7&qd7J!P?8t*5J99qno|AsGq9~&Q#0brwnb|PUT}evHPU6 z)fbxlR!PLk`EY5z=C#^;cxKY(^x>)xENZu@IGUs3;{NAZ_l z^fL)NW%}IO=Zp=s39Vf;OL+L^-VYT=R0Hi#x8Kdy^kXR#LUt7#MeS_&yOaGSp7B-K z4;E=;b+_2x`t(Sdf3)m!dHh zliL?5BW_%D&Zb2@!ZPx|ilk$7zdTLIWq1CRM@ubttJyKhaOIU}hsl$vB4JAZCs`zt*HmTsF)R*Reu?uI$VFd6C`yO54{ zV`JZxyKvN~F}X0S8gRAxoBe%hvUxpPuzAQ@n&+)21r47kF$2erOM2b(j#ukD^zTc z_OTg$ljq(T@6Z<)sEQct;A}np5Rsxs8eYPH-9b3>nB!({q6MAWL$R2iPE{M@M%ny+ zkuo7A-2O=EQjWGhCpr6%hQBWkRa0cH&X@SRej{`-(pN3A?XnLu4!Pm-v1c?W-QF%X zbBp>#ljK!+NLS%!cRrD1obXWM#JWj0L-PE%#s}kCXk?SL!?R-5Izz*GAF-WahnzJk*?U$~xC9T*< zOSmv;joRIdT&uu9Y_Y2*{|4DlcRS{ANxP}%OPVi^nLNzWUsO0y3k%>TPz)u7NanvF z%F|mWCA{<59~LexEtX~KLtPBkc?HyVznRI9GMY3iJG+j@9?Ug*KWRSjQ8IkE1)JBxPB(*MJzBaT2-S7V9@t+s$zXqLt;g;&Q*K`g_?}+4@?Cd7pHO- z+%}~-B$XZJSW`mffi6R-|8oMc)U(MPGo zB|_WBsu3-T&UWp3>;5Og`WTtFH1m7%#-}*VCNPSP#gqW+{C9$nKs(L2*&%k8xReflF4S@Wxfv>E7CYx+IkeHS zV62-vTH;p*gA8oin}s+M&vai7PdG^GePkt?s|ebOgjHo*`4SCj7z=K zHhMy7dX9Wc2=z(QLdJRcM22)UM>b#mX*F@FfWQNyR$^>!OVwHPB4E9h63Otop009Q zVwR~i^!6-!skM=%*XD1E=?=DCF59QCTSfG}OD>=E9GOIkpo*~bS`}+$CK=V_vo9Hy^GNZfrWp-jjZju}7?iN@ zY`yEwGs!~yv*-#~VbW~s#ur?q@0UGP;+HcrzEQd zBaDL8w|VkD(doMj9WIoQlBS+?)LK=HJbuS6XFBnMry-Swx*h`*L5ZJWDfJhLO}T zBg|kMP4aVt-jaFGL*&4kO}8bPll{C{wy#yub|Gi-1l{N-y zrG!Xxv;KFJCJg`ggAQFvX_i(7I;C7n%L3<8WMp_*uL0sxPR006rG8%X#@hqk8p S%P0WpQW|CMfWJy!|MWjoI@g>4 diff --git a/summary_results.json b/summary_results.json index 18bb771cb..39a7bc9d6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.119, + "Performance_Result": 10.2605, "Performance_Units": "Samples/s" }, { @@ -1364,7 +1364,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.27277, + "Performance_Result": 6.22992, "Performance_Units": "Samples/s" }, { @@ -1508,7 +1508,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433963, + "Performance_Result": 0.43494, "Performance_Units": "Samples/s" }, { From d294bbc3c8137fd55d34f77f6142e42f9fe126cd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 00:09:43 +0000 Subject: [PATCH 69/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14561 -> 14558 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 325deb219..dc1a81969 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 00:07:04,540 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3270 INFO] --- -[2025-02-05 00:07:04,540 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 00:09:38,175 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 00:09:38,175 submission_checker1.py:3270 INFO] --- +[2025-02-05 00:09:38,175 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 00:09:38,175 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 00:09:38,175 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3286 INFO] --- -[2025-02-05 00:07:04,540 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 00:07:04,540 submission_checker1.py:3307 INFO] --- -[2025-02-05 00:07:04,540 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 00:07:05,582 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 00:09:38,175 submission_checker1.py:3286 INFO] --- +[2025-02-05 00:09:38,175 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 00:09:38,176 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 00:09:38,176 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 00:09:38,176 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 00:09:38,176 submission_checker1.py:3307 INFO] --- +[2025-02-05 00:09:38,176 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 00:09:39,511 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431775 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431458 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 502380c6c..5b484d908 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -610 +611 diff --git a/summary.csv b/summary.csv index cf0b160f5..5d333a83f 100644 --- a/summary.csv +++ b/summary.csv @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431775,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431458,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 31af17a4b..8d7c7adaf 100644 --- a/summary.json +++ b/summary.json @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431775, + "Result":0.431458, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ba7a4e168da133f4113370942fbfccc9a07caa74..f31bab4707dfaec82ffb848248e1325f58b42b88 100644 GIT binary patch delta 3383 zcmYLMdpOf=AO8(=j#>2N!JIZZOx|Y-&1uF{+bE~foSAtP5vu2cBto+D(xf8}w=E#Fcn&pjXYbz7|jxFxBCuzS~ z7ST#>H<-42OiBtoc}K-IzUFc^8i~3jIBzwa{Gx*(Qo9ZUJbnLii(TkCxriY6|v9d|0qZEN||Ay;S#nRgWL5?5Jy56d^zkNPhftuGcxuKhrCk>0~`Gd(Yor>eaUSFxY1PPnXYFPW&ka&~T_Dw&e* z(JYZEXqJ2(0n@4TSMHzhg7U2X1a?`AP=UJJiD*hPrybIZVYDOqI%({Vb9Mh1XU8mo z*2Z9yGla`*P4U5=mWoMO4yW(G*%+M=$3FgjexdAP&w;gDuOWTQR^Gm8lIJ$YDWDwbD6YXO*iz(ggk3*mU?kCLpsS$H+ji&E7p zpn^V+wOYpx?Tu0deVmi`8UeE$xz3B6Ywj(re0=2Df)tpqhxB06i|iHm2!RKtK#Q{c zJ_+Tmcalw}r;XekbR&4P$bVfW6VXE&AejZB5GB)j@^H27^k6)b)f;aZO@um(xohb{ z1>M8G@29)rpYc|gNxC05Io;f{Serm&AD$0Gkj0O^wWH)>n861ZP??FUhz1DVB|6dVV#n6voaa8kl1KaZLg@&oVbmK1H?I6ldUOQ=YDj!zOts83EB-?K*D2f>ZL5GvpSxe!rw9-q;}9^`+aX z`4Ynf`%0tWE2Rpx&J9MpoFwS+wG$2pJgT~HEbi7}It-5n-XOPOktxcOEm*X;GDXs0 zy#iYWxXhcavuY(X~Ce=ooU6~1z|K9qBT;!K;JzWU5pAU-=U7( z#WVP}8g1@4&WPD_bQlG~e8G5&7zz=?>;$3bSnh_I}fWaL=7htDzbeQPS6pe{f(?&t+C1>BP6wt0(Fhtk=1qQxR}HMmz<<=q=D0H1%xO;_r-fpQiW~QPpO(itplU5eupEqVa1uOd3!W`~o_EsxN?| z(d=`oOSRF#I`95I`d2V@)|+dO@6_TGBs~veMBZ5Z>7;cPNF6+09A>v9BBm}8eSCgY zw*$1BA8q!Y6D0b-Zenew{^x}5gJROo9@?Ox)cGz`*Wu|v?M=vSdwg#I_~;R)Qxt=e z6hy4#x6~1G3cq^US8bU?mL4ldN|<{SB_p_#^y-~SDG~`>#!=M3&1grBqM*C=YB`|} z{Y}fIqY8u!qno8OkML(EMzyt5_Tg3oRY6f~9dIFtD{J~EuY6O~)PnNTuA8-%&nZPO zSZbVLRp~!EYPMH<4C{n&bbzA{BX)%n*&?Yu0oWp`9p=g?)-AZlk)hO)xixZR z+%IHB)D7>PXnUC5%X*bex}`H@HO?7nbU6@~PLV^5FnvhZX=F>zB^1fa5{;z0i=IMQ z7Z4^Angdr#5u6{1AwwmG=9bBVm|~Rfq2x4HI8xxkN`W07Ki}i=VL5i_Yxz$+AJ|2T z6yo|Nxa-xU*englDmW@zLqsn~M6VwR644`Zp^9{u1$Pl(>G0eKa-;}Ql=%SCQ@WrA z^1Gt?{nJKcvH1GLG4Vc<-ft;o8z`T)sWUe6Q5%+=4PDZfCHfibyJ!qKknak7fD zXf0UIZ5>h@7LEM)l}3a3C1F%GK_iM`}EtGj)1NRL7sh^_BF`;Y`({bPgW{VLtcHGaJxG< z&NZJ#+nm65F)U`HySrWtxvd1=;N&L|r01!s#65fhb^-*Sj)#1ylf{+S;>{&Hei5zM zRR>FajMN4=RA9V?A~WwA@FT<9rEeB*k!^0a>Y9zbpziEQp}%{Rzp=+l7KhF!K+;ok zM*v8AFz9Zk_2yX!P%qnC;S6EgK^UAY4d@54`7zxoSU>Q3PtEujnbbAUcsmqZj^QS4 z8x_ZEl_`tC5lfYgQ#E&yN(Cb03I+ut<3g#r{7%$Q>Q<}wA9R9#1$4 zN_1gx2JA4L6(}M z1pYSkW#@#C&+O*fXuyH-*Vku-!gjVl-EVK`BhOWnvBLw~V_s3+9`{dd@eVb`)Awu6 z@K*mC8H)9kPl()%H!G6o@K@K{xf=s*eEwMGQ-7?NidvzDzyJHDcg>ZFwT1K!L%)Ji z>Qd9t^M>Yk!qv4GPlg7Vq0bA>f9nfhUy65mKC()$5&F|rzYoW+`tj>N zy|^E`)ja7}eQC?L@zT_mex+?);POYm9Z!WN556W^`99k#Y@pn9)vb8`{;e?cuB}PP z@g-Z{r^)h(GY77|{I;y4nYFgLc$(K|ef0_FEZy@u%cNoR_0rC^AM}QUIws)t-k5LRCM&Z;!$t4Y;9B}dvZoOK8Z{im{Hw8_l+ zIZKzvAwxGxTK+gv4|8nhaaK=rp+2xPIVuu*baKE3;I!=Epz9`(IZ5}i1SEO}8Dbh% zPaD*WIw059%+(~{J*=S48^3HU?CR_Iat0IQTR?jkkiGf$*VgzrHIKt(HrJTqp@t6( z&UP}Q4yT%A#zI395nwq<&rY{zXR2@GkbeNy-PQ)0??#028CMv}4f)Cjm#bxN z?4};!Z$1y4J@!NOcDMEqy+WYp<@2KAP4NbaeX8RB`{1y3%)KG@>~rQ;kV*Dkb32F^ u`?{ZK>?O%vLh}6@PBdS{ZI=r`8OpPhMTxoJghXG=u$v?>om1 zRg8+hI7XU)V5+Z?Af*~y`OYDvhIfL%unGmh@|!4rdxuK}h+{97VY-YyGL z5+YkNuEt-B{7he#A8r=4TY4=&mA#DoK6Y#gs!QK-$H11<+3`vDu8*RCDwg>Poq3;v z{D^h1DC&)Br#U=0B15nY3OE0h1D_c{(~mK(V(C)wxKav%F4R>~k1bHjaiWGQ=8y&3 z_X!`=t)t0p5f{9Faj{#|DP)eP5MeXcgw^g`oe@Ph#_ZBhr2eXW{Bxm-f%3KFw1lRm z=2*}HWg?8+Bh`2(9z|!0b(J#(RETnZwgqa0ZUk|&%o)>CZ4GT4lO1JMJ_}W2F-2J4 z^txR3735gWed-FWrxw>*<}pGI&#Vk;m~}X<#UJzcCOkMkzNVzpIgl=cr) zfBQmG_XX-pguGlJu{?}-17wAS?yCMCK50;2VXGg`+qP&GP|H46zkcxWmCIzrq>|(2 zn+2;2!{Gal$56V1rIq)-)CY7azTnv2e{x*ky;G7z)R$2I0KX~@5P;eIqVLU&E?+T2 zRt?^X@@HvrM%38}j`TlDyemvVA@JD57(h@;g_g+`V}q_oi6gyl{dect;L zO{5m~7#AsDVBlWSd(aa-B;Sl>NROaOR%k5hpPcdWV<-j3JpAbKn^yXW#u6b_J(>|? z87*tXVC{IzaoQO--15I>gWgr~Q*ZP?$fpK)_U#W*sr9s?qzP5nqllGIJbI8Io}LP~ zqXVW;6oPJUny?dIDED?@e-c5Y=lO9eLQ4d0_xFWy%A4G~OFY*j!{tA|pnhj^xYA!c zFk!ld7d`boieN60kmuxX4&KUokxUSyNWoLcFdr~jwrl=)3IXt(+L%SSK**>aG@gNS zU=Rm6aI|8Fw?3!(LcySZemTU{!_JKpT4l9np6`#G2*qctidP-gaSSXbRB&~Ws@ey> z$cSqe130-MNHIXVx^mc}0>WKjXOe-k=W|L|g$Mx>k@W7z`HtTfs|}hjmSnwU1RvFK ze!0utOYb=t74Lv!>nd9dQ=fRV2OE43$T;`I9YA`WjASpDr>R--cNdSMS_a=wxM|0u zh|qAQRcE)mMI|1+pfIJgT<1r%rn+MND<8!g*!Wm=Z7vU<(mpQb;BDjfBIv-q&~`t_ z@o$k%Ka;G>>3GK%l$+#CV=&bZmTruK`q8DSH0x4kw*=%m#-^W2yA3r^H3^v+X@Hw9 zA~ZcYuxlo_eb3JAbLqU~;>rP2T-g3mO!AZ9f-k`}P#0tY-f>C*ylVocvS4>iP*4`# zED_>T+9m+jV8Z)pt$%o8rv?sUz|@Z(VR6>Ie+>_tpY`xpXX~w!Pu@EQsGY_g=zi}+s0zqu7%tue01_7cPzw!eXp@smo7?--LNctFwdn(_b!3;@pg3j-;)o1ut6@THw6yQY&ES-XWJU894|W3)B#3VG_s8@Q_~H@08I z)cPfwJ4rD21TzP}W`R*Y#90P`WQGD;h8A+*zN(0?EGZvi^Jf5+U@Oo?C2!=ZLrH&x zB`pu))*H~+)ueadZF+AEE40QlKD%=^lUo%_vbna?ynAY%@+a_Hw(p?I^xcsJu{J4w zJQ>ysGGxLQp2w4c87}jO%1C^fww5R`rF8Bq!O+kC+hSpekvI%%+fuMnOSz3M|R{~^75VA zue){4;o=RWloc|Pt{b$%vbVZn3Z|ZhT{A^NPdD1c(riixyY&%;n1cS~p+Dq!UGb@N z6}ytf-d?FCPg3I~(udC@-8fPySVa&1HJ4e(#hUI{4Lv9a?kkGsk3vU*(GW1Qz-Wj@ zvsgj*H4AX*m^1y!EdtzO3DZ13t)w(Mm>T$4-L%>FO#h`1M-l|Q+2lKlCj9xHy!2NU zT*?yJH(xhsNg!{hfae5L5%8QE4Pt3jQfZ?An2!nQPiFt8PltnRjg%gVomo%tXy{YO ze9u-VxS-n*s9wW2+)s>EN_zKnLyv|7cbamOu@*xRYmnl{kYOx86F!82NlJ6Zq7q=) zA}ONo=m^DFP`u+@#SWy{eZg(V#EJc?%A zT>Vj8x(}_o8%;w@(3l2tYaHVmA>=$e3N>y#LVtFv%SNry$_{0i1wvcAGe!hq?`X`y z!@nF(NLKkV%#A2R^4X-F;S)_cQ}(f3QO}IxTa}NyaRycV9QfOvWqg*@%wY2(;F%yJ zTTN@-0M)6I@`4s$(+w*XMX7V8-$<>8csBdi&sBwCR+onvugvKm7(N^8F29#5%IrDv%n{;y6EA@UQ;T3?SQNCV5dfGN zXu31jHbKH(N&+qzTe?0wur2Bf7wuu|nDE|*cDi}PTuJ@wp71=}ddAS+r#8AxP!h5- zP;ZlEkG!BEkJE*wYA^s@4vkB|E$mMW6rpjMo0HpofMlOI?=ZW^L7Cp^UqwBJ8wjF_ zINflpEZU+Cm*!g=q;ngEoluh8fkPNtU!S?&KKpxlG&14gmcZI1D2QMHr0bs0ybTcH z4(NU*V)qfSe@Y`?K*VB$x@3-kDS4q8%4=$4O&)gvwfqLkBOG=;&7?Mjo_pOgnz_Gu!{WIi<}UAN^$)n=6{Er{eM7$EDMMrI zAAT&)E6lm3Uh1fqce8io7WdiRasJZ!2D;v$)9d~DubUs8zud3U@uXv&GvRL;Ql#ODDW9)=qCZKkwvy3+{dK zxx9C2^qAYPB=$3t#MgXId2e7PX1rwK&3e ziYNCfoB`9Bm_>y%Op7ZC2)YVQ{as1^@XJ*zHa^NcGmey*_wO#8u1Z{)-$P8+gN05A zFKD1MBcN;F_fLE7Y(v<<|5Ty>{c&-MFo1Aw(;+Qb=K+Xd{U3;^48;Hd diff --git a/summary_results.json b/summary_results.json index 39a7bc9d6..f5c3c0c5d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.431775, + "Performance_Result": 0.431458, "Performance_Units": "Samples/s" }, { From a5c79b3dbfabb56dfd79b2acf850f0736cc1f414 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 05:44:03 +0000 Subject: [PATCH 70/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14558 -> 14558 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index dc1a81969..4bf9f1916 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 00:09:38,175 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 00:09:38,175 submission_checker1.py:3270 INFO] --- -[2025-02-05 00:09:38,175 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3270 INFO] --- +[2025-02-05 05:43:58,707 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 00:09:38,175 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 00:09:38,175 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 00:09:38,175 submission_checker1.py:3286 INFO] --- -[2025-02-05 00:09:38,175 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 00:09:38,176 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 00:09:38,176 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 00:09:38,176 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 00:09:38,176 submission_checker1.py:3307 INFO] --- -[2025-02-05 00:09:38,176 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 00:09:39,511 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 05:43:58,707 submission_checker1.py:3286 INFO] --- +[2025-02-05 05:43:58,707 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 05:43:58,707 submission_checker1.py:3307 INFO] --- +[2025-02-05 05:43:58,707 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 05:43:59,878 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.414195 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40113 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5b484d908..282d20871 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -611 +612 diff --git a/summary.csv b/summary.csv index 5d333a83f..dca013945 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.414195,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40113,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8d7c7adaf..92eba53b3 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.414195, + "Result":0.40113, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index f31bab4707dfaec82ffb848248e1325f58b42b88..30dc6ff7ba1ea4b83b2f109ca56baaf7e34f3839 100644 GIT binary patch delta 2004 zcmV;_2P^pAao%yTtOp832ctxV1^@tJC6jRu8I#Wk7Jq3Zv{69H#_@tNcCbzAohTuV zKou=S5?;Rj(*oFZHwrHoQpv#)qks2wx8}!;nCX0Z2?ODYC#fGtU6D0qQQ(miyM8q9 zijNQH4O0}-%#K_;h$G$=UwJBi>HX7rj*}nhg7Zws9YkqYTx8j@ElH`f;Gvy1<0X&s zJ6@cGc7K+COy*L$Ot|fqMqwZ+vaCy??MGtep`9G!BlbMs;b*b43VD=OP7)s2Sss46 z@RzCB>$rY?ugFJ8xYrd=SiAq7siM^DlzmTp{+zzM7K&76ar~qBaOHMIxsXaodtQcd zmk2KR>{XDBjW%}Bd$!u$FHCap66|Hz4uK{1%NL8<8glBeU_c}@ZEF@(Q>C!Hq z3~MV&UKXcl{Z=#=u}SNO@uoq@^C+tK)KfdsQ{i~6xQ8YWu;CFlOx`s}c|6;o6gnsc z#eX30#xTZ;7Kl=TQDPGn*gO&vZC5fOB>GmelbcHCrXmLnh)nb)S%c zv2jL7KvBOE5sO|_$ zR1nK8Apx=65gmbN^o@|H^mx1Qn)}&eHETL?D1|}(H_TT-`PGsq-qepgp5$veKFxTVP1DsZ^wVNllun|=gH-&~k$*g5D*twjl=1~_5nB1*p12;WxLCFPXsSlb+&3o} zG@}Lv5DcwTLIS=``dES(B_xnhLIN4Z5)F8{FlrdZ=7`Z6^TFocX1R6^8-}<89XPpx zHX|Y^FpJ$1Zs8ZYaYX3mi>d7tZ!_TZiX|FcieU+&j*vj=2nnQ)hy_w-)=-Dd5p^}@0~1mQ8-}<8 zL>;sl5kZ0C-C~IW;=VKgLh6o@(*e{?%pK}vRnuj24|W!forSRjZyl#tqJg2*#}b4c zA%WNt5{MlU3&hT^o}A2NuK*HVknG2s>yqB7%aG9pM&EcGjPZ-Sl^{J4Q~2 zOm^1pWTzRXx#uSqCezrJ?SMP*T5(EfG?Nw#B!5H)Rl7k}BPuiqA>|oT_%8fE5<|Ty z|9=dRlGGs_t^?bE*peCB^K%TNe?~uH_-FH$206qUO@$m{4d#&0APxx);*f|3amea8 zB=!mrhcylc3~m%$qn07j3FZ)5ji}Jz8kEo&406c+Mh=gW)FI2zbzs|dD3cX!-|HWv z_kWH~7_l{^LlUt@Ye5pR29ro=5Q&5akx0a2LK2la5(x(uB(cW9V0-h8MA$MUI?+`~ zBD5M&R)|EbF%(Fma$FKuvsIL>8i8H9cro?;P_}*x%46hL59Kaz^nT_E!`p-{1M)^} zX=;!+tiiky8pIo+LA((f3-YGc@rE7XnSZaw!GLlA_at^4033#t12ioXJGh|iBT8dH zJh8@5A)e}=fM>~N^yIs~UG7Uf@FD^J0kSBBA=RuGPPMp`X%75Ehou3)A_M|*&0^61LeRaa)%}pLh9Dlif zv4*jO>cC9qM{^~+bzB7a%tkG80oD7Y_PbqGgTCVrgw z*=vn!a# zzp<0KAEiRTy}ST&(-4KEI@VKOXC?8nywXHr7H3%;Zayuz?ee7flo#EL<81SxD9QGC zRPR4fO9KRx5H}jLcMe1(1n`CMqm#cfAb(E7Fc5_AJVoStVkaR8tR$5H^+Y8kia^{f zd)r#XKe7#J-ySDz(^l$<<9cSlon5(}ZB>{2g7ebo2IUz+321JF)awSlFX!n!N_=2i zu+C^`&<;GBwKp}llp6=H&RB3kf=?nyd&+HtHX&GwvF965%xBS|qqs6ou@FnQ#($Ra zZ?*`LKv5UP%y- zyQp7zJ1nyA1Chi4bRYMqe3Vbm%Op8!3sO`mDN<4`NkwlgegOpVh47=3UpRkW#H>pI delta 2004 zcmV;_2P^pAao%yTtOp7^0WQ&n1^@tKC6jRu8I#Wk7JqGMqkxo+;{{{vV4KuCQ9>Gl zDq4s}c=`5E3t-dTD7;)qB?m{0{@v5vnjbS_rt{?`3WX<@@*s)3qSBN_fyHj(1@XKq zK0cf`EK%T@6MIgW#H=g6GA@4U{nL3)(jR=mSSI8SV%`-OS+;CT5_cCYa(FXYvN*rv zCu!tl`G3cBF7ai`9IrHrLP?cnLyDXr7Ap_!^bjA3?*}eBOWak&;;eF#ve3!$@cANG zalFIg~OWO>>2!o*ENUj7xLAg`q=3V)IF(iJsPcn-X5Q8*SBf#a~pU%*mWyCrzG(wCsT|o#M$T zZB@<7;ufvnism9VY27g1GzfVfMfIL~T1R>+9IqAk(BuI&Ji>;_y9OzbXB(7C2c@DI zJ&Sfw&+Y-$X*4RVucAZ#R>~#`b)(lR%oD`Nv&ijSDnoDL`M$n-7ZeCLIH!iPe{Pn zI3pyWs9y;QAo!e+02D3=2^fW!gaizsD}O=)NVYCmZ+e*H-rRKcZ4Mx zh~<`$fLQK`j=(eeMo2VzyczU(Gw}Bx$oG_xfMohuqJk%QMo7RD{7OhbN9S0ggN`mX z65<>25}Ruv+p8M$0bvd#*EQxtY;M4@MMwZpBO(YOxh1@SLf*Z_>Gv-Gz56;IUVn|Q zKb8w6p0AYVLAF@Unr;$FQJDXY@>NiNwPdM34Pu|A`C5)mGsd$iU(KR`7t10(ebSq< zl% z69Wi_)+r$Y-zI%5L5va-$S5I!jADrnyj++yjAC=dXpQ+`b8oX;yM_%z+<^g{+(4TV z5fqrkZV9*W3*9^-bo0g3af`Q^cf`4G#5Ho_^qjc$JIK^cVn3K4K;1ENI)8w=M{}1t z%d~Bqrp}_NvoKO%_C3WC9WKSN1W`vwAa#TUQb)uBsk3UR!{&&(8uNh#se=te+ySBv z+Kh;xK=E#|!~}8QS$`pQ$H?gb>L%6>b+V=#vb6_0o5s$@*nzi>Q!LTJQ0ij|!j6zY z><9_Oj)(6dW!O4zr3nx4K&&6)~yVxBgr$Z(? zdv~(aP0QNz6N@6#*p=;oJMda@N@#SG7z`wT!~j*hK~^IwGzcN(8B+Kz{67*yqbdJ? z43Co3Asns)+kn_oT8gdj0g?V4{Y1gPo40fbBG%{{1QBa6h=c}#NN5mK?aen5Vat%{#Lys# z&}u|kAri61R3VA#aY9VCfvl3-klYRSUP8Y7KU7Klf0f){@d2-0L@rwPs+0MV}Y?tfS1B0Se0uRD2v_-{4!F%zmQ#ish4U#m- zUaNeHV?e9W=|ze6u?%(<2R~Qr;|KSH(f;5XpFRufC!s6k_1j4Etjk`C3o5K$XX(t@ zjg!uUm`Ltk;$I{|cUUWZ6vdxF0B-`^* zz5f7EO9KRx5H}jLcMe1(1jww%HI}sKz1xCY}Jt m+}EF=`2({7G}8hJI{_}yg$4isW3$~hegOo?tj9N#UpRl#@~^f4 diff --git a/summary_results.json b/summary_results.json index f5c3c0c5d..4754c16c0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.414195, + "Performance_Result": 0.40113, "Performance_Units": "Tokens/s" }, { From 37a7e576810013af010fe504c81a61b1de28b3c1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 15:10:56 +0000 Subject: [PATCH 71/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14558 -> 14566 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4bf9f1916..372ea48ec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 05:43:58,707 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3270 INFO] --- -[2025-02-05 05:43:58,707 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 15:10:51,685 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 15:10:51,685 submission_checker1.py:3270 INFO] --- +[2025-02-05 15:10:51,686 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 15:10:51,686 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 15:10:51,686 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3286 INFO] --- -[2025-02-05 05:43:58,707 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 05:43:58,707 submission_checker1.py:3307 INFO] --- -[2025-02-05 05:43:58,707 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 05:43:59,878 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 15:10:51,686 submission_checker1.py:3286 INFO] --- +[2025-02-05 15:10:51,686 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 15:10:51,686 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 15:10:51,686 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 15:10:51,686 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 15:10:51,686 submission_checker1.py:3307 INFO] --- +[2025-02-05 15:10:51,686 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 15:10:52,828 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40113 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.436081 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.264988 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.375178 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2605 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.84918 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 282d20871..8da60bbdf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -612 +613 diff --git a/summary.csv b/summary.csv index dca013945..90ca18192 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2 MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40113,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.436081,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.264988,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.375178,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2605,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.84918,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 92eba53b3..9b5e72141 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436081, + "Result":0.264988, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2605, - "Accuracy":"acc: 76.000", + "Result":9.84918, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 30dc6ff7ba1ea4b83b2f109ca56baaf7e34f3839..dfbc4835682ac6e05e921da0ab58b7f1716a3976 100644 GIT binary patch delta 7041 zcmZXZcTiJN*RK;IO%OsiASDdg%eNfTJl%d$@(|tOfR{U8TfU%PKJWIGRe)H1eBJGNU|#L~ zCv*p!gUuNFYn8>-U8BeOLOkH*`rz6cDIin=D|b`4^S)bvsn%d_)~9d8rJA#?v$B?k zEUr$Xe{uhPb2YtU1xP~i(ALG_Z&2w+#li~U5*BGuZL<8<;+dht zx;<;dMh7k*VqVzy4xl>E_&_VRGYEc3sMIIXSBqzOUl3FOfgC^rqB~ zeH*U)RLC7}mHM{D!kOMVapP5vi)BvQ=eO8j!-fZ&(_e01QhRslxi4QS=_q*kt{2vd zqZM^!P<+RsGjwJ*e{@7Vas6kWl%mL`Ua$vWCyQ{ItcDg$a6dtHd8jXxb;JDu(xz*U zxmLow)5$Fl$4ctP%SNxzvhDC`HDhn>b)-+fC|Q%N8?lbduoL8v#N68Z+0hlM^+FY^ zHDT;v|5hSE+dJdzF~L;B?h4^A(|(=*MLF7`j61Rt^T)CM?HOwcJSEM#jYqim z^#floTU;5?!EBGSRCb!9Z?m!S&+(sZEb#x8KRy@1jwv4dw+NV}+$bRJ?VoMxyGgyQ zy~eeMy|(6Bd^YANp?EHvlf!x7J7g4Q!5O#K+dWa-A5?AjM+_OBV6GQ>=Vt6LtV(H% znQmQ^<4);D&`G()ZTNuW#{vu~XGYXmb1E|6LAtU0;4m?J#*h4K#c2X#@6dz24Cq|Veh*B#BWElv=nN5oD z$a5Nq7K?RH8CSfyRRH6+{0MM_35s52K|vna>8<&1Mld$<_-K3$YCKX8#IoFZ6Tmf? zoR!sYaK`4Nj>xc3Oy<(g`thzTzK?N~`;LPAU)d|=p4SX2qMU?^(!ZvWPQHh&lRQ)X z10EP_%mR>GFEjwTJ7X7fWnAu~3S@I2y^@{7ns(8ccoCE~mn805jQU-a>T(QAyX3)Q z%sfz6_lR#peDCAkAiJZ4@7VAlp7%{iJAS3)W~B^S(WmaSdIlXLlM<<5cGC6OMn}Od zCB6Gu-TqBDy=KMVs0S`S%0CF;#XbZMA2LHE_GnBWZrT-3@R}E= zhsMD`Vg$yjXo!Q}i3v=X1$SRJg6{@E>^gs(N#0d2Kux?rr7psI-l3Ae%=U&_lYe`- zSVUbSus-2E-a^icY5Uuk8JGJmYrpq}E6+ZUF+?UlUe#{H5cKxvXM~`usSY?}0?CvLOzVGq@*RVVcYu2JvyjCy?g)5H2(b$8i1I>SVysA3rEyY zMI5S#0d;sZ9^3Rp&({t8BqiWmx(}N@4<0bQ_G|SQDD>j1OZ| zhD8OblXP+oyDkE7KLEF$YAZI_Y|3;77nq;&I^@+pr^GX$6RPQ>#%vUbf&n4eH z*AMhOJN46|gvL5PbChM_ z=exDJ8!T5fV-m!Ms~V(&z=W`r;vC3lg7CJ2FaZ1xuTZiB;z$iC0);kY zjjHVXzI98B=EK$bieku9U9K7SeL{xDYl6IWH6~5o{(bkkyNOx@_hGNf2n{rS_r*Z} z>^2T;oNL*2U7!Nl^8IJjQNsYmf2sxocsUk&EDnjsD!$QLY)c=4OUESC-HNF#L0*`~ zkJ+-iq-Y^rSd-&xMqz?byktAf4uT_I0e{%gk1?UfkFB%LpYQVN%n-)((DcRIlh@{V zKlHB(GNOigy zMsIAS3C%Q&7Vl1WfEod34rf#X6`P&(>sS5QZS(`Sh@L^IDJ?XOk0#cOaScTmN2)8X zE3FwKVhooX%u9Khl7x9?)oG(0Qc{_~^w1a}6L=61<30-!_~(?24x5Armks5oxai&P9A=JgnD4Q z@h1sv2U+}6#c0!C51@%0*OvtPJ@Jla2&ixxAqc0qxXAY=(7b@|Nlh4xfic1zf zx{ZoWMcgCR3*$IHLSfIZ?_K#omy`#qPDBb@My^UQyakQ3k{HOMhW80hGZ#xs$xS+v zMO{)%YmNNrx-7oA4- zi`;PJXH}sAIq;_eY5IZCr^wkTLnRtaV`7@l5U_UDf&ErUo^kG>3QfoN&(b^!ON;?I ze!O$&{&f>C4P>LeWcn)h6)z!CxJDfY=jW^$-G@T)x7zn@A-E({B>MdraxlsOH?$u{ z9l{OcckZ9D{h*UNQm)&$G-N1z(l>P7e~2(VQCzyQze)3Pr`5v|PGV>*H!}<~xS{t6 zQ_O3X2n|<{b81<9|OMX>76szN7RS^f(@}&XA z`(G=!7Y%RPQd1&2Zm+Z%k(QM$vx@%5PnAui6-b{{aj*iNLmRJ88(3!l3Ma+*9s_!y zTbB#4v8tn!w)rMskCreA-%4MfbW(?BYGSpXdJaS-Nxk`A;#L$hNfI?xCN)xsa z0irt@mmA^8RPRaS4%&SW{a1CA3V}yo7XQdbc#kYRx*)ZxEXi$hVY!j3=JlE??E-`9SQ^s@ z2z5kz-meUWGMvMRah_AhHU(css-7E%n?$R2fkW=;a#B!hV?k#kOc zs%)1j;<8tjbOsm~f;!x4U$%wVP;50aE{X#{gW*i1j-xg64fwK3lM^L&VN0L&NmF@k zQuD0qyHCGZ1h+S0(W!Uov19Owu!QZNw7YLPMAKh*);&cR#EH0^R3%vi`lIMXFI9z^ z(1=j%PejTnV6fLXFYu@{RHI|xwpiQqM=;&Lq7dxY01?bRPv!(pU@gUI@2R0B)C42> z9F$qqf_)5oLE%EDiZ5hR2&#%u+OdTw5q@-mw8&?CM!?o8>{{svN#IyXQ%5a%0Kl+R zL_GT@ZZbNsM_d1h@=Y5Qf;g>N{K|PE-l{~|O3lWN409$bw+G@iW9yhQdRfy|Q5jK@ za_DbRsDq@^H%bJE6mfU5J4JlvaN{ngA0^EjBKy@aKT0p46Ks&Le-yYbA%Ogm;Xu)& z(!*|d+RG&mz9ljF^s<6fQ9e=*NCyK#26-;1<|#KL(kT`q*OEhc1a>GyHyX>xGB|ZLUWl?y(TptK>*F%51rQq(vVTTPdX{Z3!&!IP^m<2 zMAAK}vUlSQ_`ni2km_UZ!Q14rb=i?HzLyQ(s`3|0zMoR-Z=#M}Aa4=Cf|qT!su%_O z0+#w=Pmq?BP2>WmHG$fi!jIg=xpAoZude+M>Im-DdcdfNZo$cO^ zq2gEV&-uW?>e$$m zh{6MO(~^T~;=72v;6=zOBfk-F_t5Zrciw;b2>~*6&}y?%*||Vb?iM8aLxJkQT{qu2 z`a<)1ctTI!7_q7)&jY4WwtO}hgj=Bp0xGpaMm?)+k8tC2*gJL^C zPtvlUQ#}Lz!$#dP`jBl(>>Q6ifpzDl^;23+S#Cq)_qjr}{2LfKI&G4fL1>grY0ijt zGNm~at~}&MUB`T3E(aKpn{%7^7xd{Dn0fDqb2~y$PSeOReBQG+N~w1yE^aboj~UGd z$DiCP&s19bAn-hX<({+ql^3j+?rX9|N3yz5C4=p_ZK$dlZ3Y;5+Q*%!K3~L@3}1-t zxIpnQfvVKv5f$(?(ENNv1-Fsbr~K8&rKt!pqSstqndn+HfdN%)xZnH~XLs`7>k{?@ zQh~XJg&6lM(-YlE|KX;|2iV{qqc(r?3~gP=)4H9NwII?)e~>BP@7?7IcO*nSePzSc zelOQQoh#dEzRj=px{RDcW1|T7pjUdgC+RrEv{k&cUxw$JweHufZJEKknU1)Y2Lr#IIicukC~2?tI$wSSXe;#>H6BX?|E^<%(0ZYxD#=?RbcD2u=ukz%Qz|v7n1XnCbYNIjnwAare4zS{-Ftd45u;ILJ~?Gw+B6aHm~77% z>HGlA4cjx-Bu;P&R;M2;-@W6epkSr(gR|h{bIffmJKsq&@E7vulOHMQXUDH36x>>F z$tdWtZ?+`$75O5tLK0K)L@8g$-`m+DewnOO#W(R)A_i4&%`{eNd>DdJoFr13qdYvN zyF(GO!O7bCdg4xz@_LHDCtEJQ_~KU9_4j#Trn{_7E=dM!x<2jBhSS%i7)%W_S-IJi zvg?94$L`9TWgo-~-EE3Ve85mE_^c88&iVDAiHxJce9jd3)5HCzd$NOW z&!xD~a*XqR-GuT4SZ~SoS2mx}w^P164u!qQIa}_Dd;hyDv7tU_S7Zo1#pYYUc!f8K z2RCud*Id!NUp(o(z^#+HW?NubqcOt`(thd@SMs#_nKep` zmK1dBEZ2rMwEB$)x^Iil(aVtwFReQ5w|p`1h{;<%EwIR$NekL7!&dUzit7)E;e^IC zUwGasZ=uuL(qT3&LG0e9u&DCxgt;s&hiTJ&tefY={=r=1W5cE+f1IOb0DfyR&%7J# zmD(tP)2Rr3e17YS-_B_ZTcJ;7t#;=_>B0KhXKMm}!ImGH|1vqMwc}_sc0VpITBUP8vyzro=lf-jHv$#=*prSRhTxel73ej*ecvU#eU3+Rf$c2wT;!hXiK=^*rvA;JS|2b$mTFR zWtQDREY!hGua<@82&u4#L^^!eF7^_+qcU)sEFdTdc<8eQ-sJ8<)%`FMx;a*r52!m? z#ME8gY3iIE23{VVDtqmLZQJqjsKjfLU`k-z<#q zxM4+dslQGN2&{@{HIyu8t6CuBNudfFYYFzjYeOj8ZMPf2Pe`x=UL-&P*ax829 zqgy6FN)4>ZVe zGi>uYG{2U)%*=0u5xY*FhRg2&5(ct3w)tNn901Rw!$u0n$Y;CZ2`EpAB8C$W^bg@}9&3 z7KU^)+Xb_VhXZndfh4o*wsh34;O5sbVmpE#>)Lq>jNidreANV>=_sO`ji_H8SIQTM zx6r!~qs^bjb7 zyahLLk{WjgB&H2ncEi_tI!YxuEs_?AbfY=G=C3U2((JZRnAi_Y{l1HM#_Nl9je1mL z9nW5R0J-pU_S6ydoR^UReR-D8^rFZ3TEz+;;1m1rN)b$a>!t}e7XUgZGtczi`(B68 zFpwrmvhRBWR55ReuO2uiQgdyy(dic+PE zbm`KiiZA-jeDmJRk28~d?^(0<-fQiB=H4G3-#seqsmNd1-?FTs0{~;}Qz|i3u1h(l z4?+i}>(LUeEw>;-rm+xOp`V|gm@i;Yzt_*YYSUIuHl@6-nZNof?3=CP@34Dyv9(h5 zOEN>Gr+UN6@_x3za0Zoq&i+wg`PJg@t#>t6=1|hHk%edCUrsjX4<7!WzxY#q@kIPj zfJegGfSImM(^RoFG}%}r^-K32iYtSH@zu1UobZch;%V9>154$Hhd-8_(y3Lpw6FvD zvDrq?MVYj>Bp9qdD>;209PH`59Cz`R?JF2a<2t;2Rsnt4*1%*fym|2<$78>GJkaUz z(xv=LuVwv`A{4EWYGn=q7F#;E25dS!3b zE*>K!(=NRZxxLI6@{673eimg)_Lpt7{81m2Dxqo_w>NGhIW_-osP*fQ zEpBKTNwKj?%W%^J;ZDP?3{>%t@u}2$%1=I&wFtK2>Jz{DZE^YXYxBHX$mEl%#ZZ@1 z6DJ48@wCNJ@b#ugFGqr6o{9MrUFIyCVG_4EMNDW|RgKsDEap zLxB0>c)!c1p6)p;<%m??)gQ2T%4%c0*R3pQckZzCJ12@d!)UkJdIwC5chwaRicf-_EaF zbZPED+OFn-N4Y8UUDR6%ALfI&CH5(KM!u`kf`F5s_CAw^O4IQPp!_8Z);=xS?XR@u zozi;s!o1&6OlQ+q?0;te{C*HHD}CwJ!eK@)lzq5=uyn3(`yfq>FS#4V*X>krI%X&* zFMUgU=)kYiZa_Qv4B^+*a3#hpVlZrO60M^Tz_c0vl|nME62tX0T*)XDkZEc_AkzQfIp z&u(+$3qBfhHWV`jI@!0q^9o2W2#`ov=zAGMafjhg``^G11S92kkr^)z#=a-0ny%Hg z@a$)5MXatf>OH5<*4?3s2O5S(7d<4$YT zxYcR9_g$XL%%1wGu`U7J>Uv`x6bouxwA!=@z6hc{mw>&lpXh>ufMm~P-#S@e#0hTw zh3Ab%RGS=vT_K@a!PP_We8*K)mpI3y+W@BHlC1lclFpPABe(=Dn^xf7rUz zf6yu{eVPrPD>c(B!G7JJ*4e+<5@GnRtzC;k(%e&T^^%5!K@%w|;G*qZcJU8aZ4+c3 z0*tbFRG^|T5Q+%KHG_IZLz>z8+P$zXFDhONzw+Kf%L=1wwLx{5#xM&hcT(?w9hdvr zHJ$-}SmUepjl7SaBoFfEL4EvZKeyw-2kN5jLVZAailS9*obdtpy6-XldKE!Cs`%GF zhB)3h5oFa8x9sM}d?oU+7Sl*%X?^_Tc)V`OXnWzcyi8Eh-kaQJV_yZ)euqAx zm;bh?k@$8~95@9wBS0(T`k4dQfbw(#W&w0_;^FrRS`&&zVjM%r^^)lqh>Sq#`b0Hi zOGi!eIp6LwuLb}!0EjU7Vc*ho5Z@L8R&dx3`WsC8ZhJwgWI3oMc6gQcN? zdaql~H`h$Xw6$S?BfqXv&(?Vg@#o*q5geoH`TTG%T}uCVG6 za`qL!ha9I@uky80tig;jBqx{mJ11xEJq3OQbLKq-0X?~bNO}dSAR`(f!1KJBOApvU zJqp%(X40V7ZxvMAJu!^c{if%-o+B8bR4nu)HJ7SfTk7wQdnG(t)I29A@@_jD zg4rg40&aEpJAZ~^mtZenr+W}Xle&~##BkzB4m_E$=@>3CJf+4va`4Qg^0pW=g?;q9 zfsUq+lQqXJuCwYtZYrSB42`HS|>+Ha*Ev#rI5rs@zG)!zJs$FP#?W&B9a8v`w&Wq zBwA3|uXZ&?XDA}h`l+Bj^3 z81=C#l@&IXFm4o%#MVX|MVRI4N;_)Q;!A0mI04i~K0*IoDajm<8y}VsjBaA1f1*wm zie38Fb)~@`(oB>F$%~8nW4R!ZKX0{c(W!qNlGi3j?<9C=#w>xZ#QQ%TXTMiTb)Tp6E>#iKXMC-Q-4{W|0ke^Hpe_f~?N?;PYDY#|{e9-7``=i%!2 zlHZ_@HMA`4i7q_vmA8Rewp;cFI9yB+a&U1#3dt#<4~LVDP@ur!CM3fULYP=kKDZd| zaBh%w!7i66U@q!Uukat-grt`9CdANM0lFdYYAEXVg?#rz(I~^E2E*c81o6BdX5b)v z%&!UX@{Ix!;N=_TGQ`X=Vh>bagJ%x`qk4mqVYnDp;ute=+B4>CRakGnOrW(f^i1aC z4|x=WqtF{GFm9h0=kYbxk{x<$DNY5eESDn3!TcoeR0)Pp<7uZTnrHS8X~oeS`4R%e z#ywQcS7)lf^+{-gUSA~Ds8XeSO8zmf%;c1OP4vaY9NrLUt)qOxTrA{%87UtXDLm&c z*KQkod`#6Sh>LSYv>k+CUPyGMCbB@CLpY2;jYp;mX4jziKZnzB2K+(Zr68mh%GVlgKw zj8?5XBD};P5KhKPp%Z9A!hyuZ9qrU;?{hNP6v$WymyvXmnu&@(n#!mB>#pYdesQ8i zX~RtnL&lc&WsNv7^qfc~Zh1#iv(ytt=4qk~H{weszhU|5I z$&tfYWX%?1u?N5GOER7WPk$4BoE9}XCk~`VJpka4ogD3>1xFPG=)9rzyEaD(8a-M! z5dt6ZR87iZ<*h{~6KDHix1yrwi2nRR|MF>NNu!dUtW72q7e;w>{Bf2^lN2P3o}@tv zN=T0&5jZN6DgkuX?|J>T3}=eec1`g9HN4|*GL@HzF_q_$yO5{MTl$qEF`WIy0?|^X zFDG1+DQx=FW1ZyV)eI8h7%3>dhHgYy{AWNn;tBdu)gpEL#-XPPh-0i^gK5|^lgGAz zl2wWk6{30z5+*baJpW|i@>8Z~(`RH;@C|acmfzzeWGsPyys&ALXY$PWCHvA1AI(5-4hY^H0b*6xT2}mGf4-x;?K6?PuQGe zym#K(BS_zF-_TJ@iwH+J9>(KMZF5+G!n?Ze5cD8Olv)UQdTzl+c$>Ngi4WJ1^l!LU z;!$&i$zL#*>3mgZG9zV)Y9yB811dwCygDm;B##CZc`Mbp1db|npy6z^Fj2xVHXKN_ zPfJcAe`xJ7G5S%)psm7MFfQhuID}4oi*s&|h5To1(bU|TB&%C-hpjjzip{yinT{R% zKmo$50ik4L#N>4-!N?Hkj!sd4Yr30M;fP$O1VFDJ`cRxo0tqavyite3v z3~YxbhEkDpCWF%Pl1uIy1>Fux#b?dpT#TJC6`!>gj?53Ra<+|d3j8#y)>I#&-unj> z>e^?6xK(fA+QnR>ESu0Kc8LyOF|SjCwQ~%h;3TFhqWB|H=CPw}2^u+UoawB%g=fY} zBp%Sdu2Hn{y z+KyjsgQ)PVD9%~gu2cQ1WnZauoC(=P0fb1xT#dx7Zl;tjD2yneF2sOm2$k)raVx|2 zL*>mlC*}Uyo!oIMc$83f?ouc0+4MLkXPB76{$pk|er+7mffLfRTL=IAP0h|h+#T^w zys==7=C5VpkP$SxS6Cv@Eh-QZB7SiIG~09d@$H0#KzD>n4~Oo)+V zYX>Gz{r(|S^JJOFc`AVwzg03sOKXGn{j?!MZytkDk%;STV;BqdcU~saxD+0BGifLU zrH?-mgS!A{ywQ~FNU725W8NmjL?dfECVSqQJlv|m5s5C4J)Oibu;x_gC!_5h-;v9L zzkn9t*9cO!cPX^GX$Dd;w=_yj!M{^wjH!sdYB=@p!qTYuhHMr+7eo zI$tTJ$duuq-$Mp)h{k44kmGdT*6Q)%2OxB9s>^lRc3UWPc( z6&1eoxsbB8kMiwRT#L*bTBx+yQ_xU}C5VG${q6chij$QLTT+W;a%=q!rDd^mo?bDc zYMZC6IIjjNTp^cwlN1UH*}e3FlhIH-^)ev=BX922@Px>jMi?p(1=Yn8o8z0lKYc@y zlJ`c`YIp*2VM2(;e__J8`+ShMKz`Gi^Yv%ZbzT?KC0#Xcw-1a}H?L`*qlGG;*BJPk z6!I|Odx8ZaJ>Gj3(?+uB;-SUI3}EAoE?dPVyu#Ute;%Sy-AYZIZ$WLZ28_0kWf%tS zzyF@r`>-}uC$R{XbCBu^d9dW!(=|Jy_w#WUE-3&`wvJ?0;~RiWO{39+Q}!nn*AxoM zVG<-Q|A-bbR)7OPLpbpR0hnq)kgx07{E0Hx>05z4cqJjeulouT%c`w`SY1duK;|!=lC?)I(hg=$t2I;7>FP zwS}LN-(%1ny5=uK>+KF*2^GL76S6*$oWag$^MqcWJ#2qjGuC9F#;JsDm)U>y4>Wg zaX)j}*H)T+96oSC{H=Z3!eZs@;6JZRRt^uFvDQ|%ojOhTMJ6II@9z8~F4(Hvf zwf?C4>`UGIc4H&HrW65B&-}$RC~yP&N6mGoLrnt%6VZd#Qf3S{({EW@|EOK9FAgkE zN3{rAC66PwYDXrk>vLNCE5U84arXFk2eV&BhA>W($)UgdT#mNIUSdQGL~O&A66r>b-9Zse=HwH ze~=fkH{O!(KAtI!rn2|Eg68Tq_?fY3IRfq%N3 z9)Og8&$SR=ig~7}LhBRWVfZF*6H~U;AV@9qGpd3ncJz|}iP&&id6Oe+0r?n;SGzM1 z0ux~$oUKdz{E_1*sZY%Krz2O&$?j`(tSrq#C3(%BMP-C>lT^05xsfp%Q?FW#@ex-G zhWIMapU;nQYhssnt8>6(wQM}@SAKrpewf?qeyL~l$GhwaMIZdVrW5TST0zce(yq0= zt5IgG=8`shH}w_a&MPwcIifPj-=S>#vxSp<8fUXje5)av@<>tcs{TAvQ{0E3Q z{#ZFZi~9F5osvV%aL1u{2}u8Q4JMJbna~GIGfSM=VnZ_4{ZS7Wn9$#dRv0;w2VFRQ z0Kc2@8iV|+CQ_ZXfXMqjGvAxsa~__wF;Hn8QK|elg$GCX+Y3avOn(=k7GnzYF$_O0 zx(rm zb5Dm`c1~i!j8=~3`9-#~9y9qnT%Ird>b}x;>bXviYAU(g>e{WszPAN6tS@pSZ4HSC zRn`p^V&fMJ%hv5~ga~myHSFx6f}PaM2S|i2T9(VRzO|aSJjN8DyrzHO-=e@4O2OzD z>rqi;Lc6yWVUyQAWKy*!b4OolnBTJO##P%DM;RhM?7S|`Z!nJ3baLf^Il8!pKGNV1 zC+GU+X|f8xVWZ-g;(OonJkqW?zfLp%a+hU)iH)aSR*XPDh4 zQ@&jRg$A`M9~{9@RC_#tHNpR1?gOnP?UTSM3t&yS;0y#(qRT;h|G7wXnhIFcEjSYd zYNE@@_wIvQdD|!dU2M!1SplrCFF1n&h0*22d$m9-t}B^@&deLq{SL>sc^CRK%u)-snD5;Z@kXj z_>z@QpU~;>`^6IB?uYK#AvUIg$&!3mRNL-j*YdDO)etJ0JDr5YpZB5(dgS7@jV^w= z2W+1Sn4_X*E5Er|4{f%8Hy%Qd7Wo$UG6`?mo4;@*n(6sHJb`k!uUmGL;9F_n?n}6* zjGtT-N=C!V~rj)`_WRZ1l+@2#J zn5~Z8!n$yBEAFH3mPcQ{wy=%yxgz{Gdgkz>CsbwIE+m(5^ z5`f_2BrAcu29A|^AQ**j(12O9F8L}-$9XnB*D<5bsAYe8^&!mm!Cv>m;kp zisTM%VPCgMT13`1WYq(0%qj*Q9EPc6Qfnpd*C*X1Egv*_su-fp1rO+V4Hf4!!Bno@ z)cI^bl9kgSs#OWrsP94H_TB49EWpnq7>IaHYAIeQfPkK!u-Zd+ z4Kvw5N^X&JZ3Qo;1NSAGilLx0@|kKfF&4uA^&L$}!kAi#3jlmwM10(zIV0TNov|Qs z`kViEo&*2=p@V0z?GlQVW`h6rJoof-dGh@44lLIL7J|=E>^_tMODqn)`L8M<0MH=- d0Gj`92LN<%DfYz!IS>fj@_?J9`TpNq{{tpvYLEZ` diff --git a/summary_results.json b/summary_results.json index 4754c16c0..ef6eff4cf 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2605, + "Performance_Result": 9.84918, "Performance_Units": "Samples/s" }, { @@ -1220,7 +1220,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436081, + "Performance_Result": 0.264988, "Performance_Units": "Samples/s" }, { From fa4a67788ad5376e26963c3e07c87b6076c62d13 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 15:11:42 +0000 Subject: [PATCH 72/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14566 -> 14567 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 372ea48ec..d3235018d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 15:10:51,685 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 15:10:51,685 submission_checker1.py:3270 INFO] --- -[2025-02-05 15:10:51,686 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3270 INFO] --- +[2025-02-05 15:11:37,424 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3286 INFO] --- -[2025-02-05 15:10:51,686 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 15:10:51,686 submission_checker1.py:3307 INFO] --- -[2025-02-05 15:10:51,686 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 15:10:52,828 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 15:11:37,424 submission_checker1.py:3286 INFO] --- +[2025-02-05 15:11:37,424 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 15:11:37,424 submission_checker1.py:3307 INFO] --- +[2025-02-05 15:11:37,424 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 15:11:38,769 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 8da60bbdf..9af70b055 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -613 +614 diff --git a/summary.xlsx b/summary.xlsx index dfbc4835682ac6e05e921da0ab58b7f1716a3976..cb4aff6c0c52f8da93fe9be0bcc590afe00fb0cf 100644 GIT binary patch delta 296 zcmV+@0oVTKap!Tc#4-gP`^sb`lgTnFQ_I062-|_S{Xlmf%DSn8j&nV321JF)Y}?;te5E=B|b1MSZg%Ys0WW0%|pd4 z<;KCAlSVTme}`=W7kU1O6$B<&VE6z`?MOr?q~Igc{08aVY zhEch4Dj5a~v(~dXx%Xa9#=h^felc2O{v3ahQ9;((e;aLOI9U|dt->mnYm zV?M0oi8m6&;}G?kx6>lWJ`hO^K#y_H%E#jAxm}^AAs|JWk|HH#o0l{vv@E_3BX9aI tKy@1OZ{itn%zgU_nm@A&G+G1&Y7o4pB$H`4Q~?Q-zBnTW#5Di_006mRg}(p* From 12b2b681f70dccbd4a1bf56b4f7b0b6b04230832 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 17:37:56 +0000 Subject: [PATCH 73/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14567 -> 14565 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d3235018d..26bb39680 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 15:11:37,424 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3270 INFO] --- -[2025-02-05 15:11:37,424 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3270 INFO] --- +[2025-02-05 17:37:51,155 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3286 INFO] --- -[2025-02-05 15:11:37,424 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 15:11:37,424 submission_checker1.py:3307 INFO] --- -[2025-02-05 15:11:37,424 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 15:11:38,769 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 17:37:51,155 submission_checker1.py:3286 INFO] --- +[2025-02-05 17:37:51,155 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 17:37:51,155 submission_checker1.py:3307 INFO] --- +[2025-02-05 17:37:51,155 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 17:37:52,469 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0741 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358788 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.35302 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7134 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 9af70b055..11a42ac27 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -614 +615 diff --git a/summary.csv b/summary.csv index 90ca18192..92f442b6b 100644 --- a/summary.csv +++ b/summary.csv @@ -15,7 +15,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0741,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358788,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.35302,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7134,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 9b5e72141..079713e67 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.358788, + "Result":0.35302, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index cb4aff6c0c52f8da93fe9be0bcc590afe00fb0cf..b9648f4226ef7094870553234e36bf8f10c77eb5 100644 GIT binary patch delta 5489 zcmYjVc{G%5+hF&p3N!GDu$&#hy zjgU2jtP|O@#y8&YkMDc_xX*dcbIx^N*LDAXzw0^oyx*)}nI{j2@Dr#-6+ZxA$@h7b z@a(UAN@F?^{7C+eYl>B+jwi4gXPKP|4|ld5CdoS!HQ#;B`DjCR>8X`J4(l9N^>ssU zuN?0GnkoNtnJV8=vFPk{DJN8p%Hx^4wuvh}ocjAKGw^*ihUax|%dF2oD+5ilv3pJ1 zN1oeVK8M@W@Se1)oMwNTZ4>4~sk3rMEPl!WC4@X%QHZlYnDJ4ea#Pw}=I6h(4`_(S z4Q5KDnF$*d&tVm^2Jgu>{E+zgv=*rogAN7`|dQ>#o;zO>C78(`a<}WWD>c=ni z+{r!gxgS=mkrlQpCQ5Zd#&_~;q?4kiC%?F%w#j`stFv=p-|i_vew%;!_@jvplF7kJ>reFUGm;YpFY+?`77eZr zoR!LM^S%g^@LRt<{Wmw7H&Ft$$9GP)3inE2?+(rCYGu2Ls96JEc@4xbamsWH4;jID zUrpOnj67VNV`odZn0A_gomv|VHh@p3A6|nqEe;{zH$(K`OcX7&e#5f~5>aj4iSJbD z4_GuYH^WswHusS}e0t%Jf?J#8L^3hml*)~8&ebw*TiteuZTexq!<(vB#wKWK-w-ZL zGH%P5g>(^vx>nMAr<>nCbp@E&YJnxzRbI}eZUox`Z@7{>Xx=^BJ))6}}+cUlSaE$m*S4dW=eC0p*!e0d*`3qs1 z!tza!A+y+NMUUQw_gN#-C6v3I%iN_UK@_Uwm5Rm9UyGgt7G8QQV;$F`Di%j}8V*bM zq+ED9(qi}S)qhcX(0y1Da|vQw5^}L>>*(@a08U9hV8cY2kF1}t5%i(g6s)=Ctkq2H zo2GW~cwlEB(L~ju6$BjLB1Q(S^6>@_i#BiguSoD|j3gy7?od7Y~al4xiXSZ zJ*W(tu^g;jmY1$C#i=*c*^{etd>(5$35Ysm@z_C3ISES$X&r$}MV2aojuusy-FeHLw)fOR+?OQwdA8=JPJ98OTzyGSqS(V;#9PxA<*;fnPecFhUR#EUVWR= z)@1a!L(YCL-f8CWfEWfCtTVlu&fe_zHL_&M#k5=LT3qdtqDMv-F&jYq1csKG4A&0T zphB`Eq^(M+Wa~Xq4=Y!U>J0QJb+C;fD37e0d0poU+Zy zi#8OJ3uyRE#Cb_QBY7xhL)5tebW)wU&RD zCI58RJe0M&@%8y&%GY??CHp9@?04L&c&lWiB^u(4(q}>A3%#5KM-u3PG3%#xZql8w z8NO}c)7G&C+IMS#7hatgBQoldCxU{E*?Zk10H^x#1=K9-J%+dN!MhW-ia$Vyq&UER=dm~-)3!P(TU=f{c5YB zFqf#)d0&mZOl67Klw`a;m;)9Easzh(ar(<3!7y*IG{Lq8b(shO1iDxz=>lApmDp1= zus$yH)R}<~*WFW1EP<@ze8B?)PeG+gX@k?ur zp6UWJK_JJ@XC|4bs6*o=s))TcJIcJ|K zZI&l2rQQaM7&Mmo0wY;kwXGz9a|EHhH12G)1j1bKUWz%?4Knj=s-}_CNgP7eyWP1U zW%VlJu4dm!WT1_DM7|W{moB;Ge)Vwe`65}IZ%t#3uS4ga?u$Tc3UM_4O{Ssx`y)}w z8&~|{@lWwF4Uk}MMU%(8ul01brMl3%c7}YPB~FBUQ_b6CfR~ItS;Y!*FHz5M~D*M#tbNwQw4~y5Mf8%$6d>@A2z8v{W zgJR>WZJlFKH9wmrR?U;$(=+Sb)^A9cSTohS-X?jg_DKbdiS_YVqjUA)O&9Unil0ra ziQ!3K0BfQk)hL!uj(K_qIw|je! zfK5rI^K!r_AUbasK=EONHuudRmT?ka_5AI%D!Uxcun`_%C-A|Xf=!c{rP$Evv)s=FB7>P|4N)&bvN>?dGYX8Ptq zUIh#1@w%plhM;Ah&@Hq)797ag2YIAv?z(z;zC+GB>{{ip>Ly<} zG2T#Acv1^66y;9PVYv8Ja=(o8Q+CgvUaNv*hH*@~yqol7=TA^}%{sT2k}p~YE4rUK z(05)en0+*=@fi)gxR8WXPBwsh?Up6sQz;C=;UEe_aQMIL$sL)0>`-}3OR4j+Uk9MO z+NK?^InG(@)`%=ds@guE5!-(I4PuHKqL$=${c=!f+Am2>ExNE!?QOSa8Lxl2CNnTt zGja!(eEJdQlL;*<95A6ITa?Fzldx{dA-8bOyt2^en~;UGUqiFSZ$+JKyMkX}lo7$$ z=B!9?dQ4T;NeO&-h;1j6UYW9cp;3lmp3^oazLkDapCvkPuvM4#EfmdO%l$PJ!}<>K zh!vEEDvW{v7s9L(K?Q8~ih;vPrX!3NZDEn!q-VqgWQy`mQR>tsUoX*%YBJ1HTG^_365u|D!uJ*n`7%UI3&^+iWlnquwzce3oWg+NN3m+b}vT?QyxJI4Kvpr^~9qRr{ zCsa=ErfKuR@4-9brrb1VGrIw=OO@B0R0z@;>M~(avY=@4jR1@knc( zdqkoAp7b&%?bqhtE<))HFL};k{E;Vm)Y0tn?s{Xw*J<_m4Z)%x?L~#yWSz7M_mKQ| zF9t3w4q(`d_2r}U8W@UGg3#<ayE!eoOTt$RaXjXFPSUH&P1!^4hlH_jS!o=)(({1u^V zjK~xbhnqNt@*2vo*5;XJ9Wg&EaT!j3( zq7~KZn;QJ@>$Y3*0n0PD_}Ga09?|%`wqudOXm%a$q+krIHo{>SAfloeBEeytiJ%CE zNJVzR(xtCY*GzPY5*W?bZiVv~m5&P6?K}6LzBz4go>HCqOWOGR6Y(_%_~KSs(wI^{ zgO@vDh`~#Iq?N5~(~H6D=3+%CH|9THtR1pkQ}dsJezuMkd`65JRr3BWa<+=J#o||6 zu!nh-`MJ@>BsLyECsI^3^GEp6(*Ye$Jnx1-^3{KFj(S7(h+R5(G0lFVHmhfTr~ zbbcv4!ec<4*`?Y;Pe+*LgoIAS*)9x-y(j1_4AeMyuHjscSw$$^7Hi4&mQ2{lw&r1t z_R7g?Z`ECAw)k%f#4@Hp3lRHp4=G{;2yWVG2p~mncMiLf_k%=eU`HwmJm$+tk#U>) zfRaS8YmZIJ+Zns`y9IL(eN?MwEaUO$+BW|};S@e6edDgEh?xLe25cYD#eik3^C&&( zlVvy@OWn2>F8ef6G;nxk@PA_vMk-Q<*3MFTxQW6g13) zu8Bm*x2hw;(X0!aNM>|-W-!5+`lD5*`AjtqQYUai^fm9!c1Baz4{Oy}z1n zGgD44(i_9m+5k(dF<00w*Yzuia40MB3X&OG9v=+oQ)^pgphQ%_$ES72XTu_5TK>HG zmNB2Mva6Vg1FiO_!3GsYF2efh+=%fF_IESU_qAHQABbEtN%0TFyb;e7=F^p7ej!W& z$&8>hv|pKzxOigPrkptV9ik$3JY4>)~#@MMk7lwm$-`d=`e6KU?tU;Qtbrf8F^+Kdc&5=Hu2XcP*23SAnG ziO9Tvg8I;7%*Ns4-BESJ=XuLJ-IRb~b;>8W$*4LKaqn*cn*9`aSpWvl3dYv2=JDxJ zOBiaOqFg>cEoG<`X1PmGDh_oEaYG2b+y2}j`fl5v)>6SiPu%lHix+a&aXI`6zR^d# zK9sr?+1~$nGL>E3-u{@T0E1>JNPt1J)TdQsav*<=kxqp@KWUR|44Oty&S^r|lkTED zKb6jwL+Um0=C3xMZ{w@KtzNEPl*nv1GK+Eq8=db-e|N>66UD%L%F6-DLW-?ng`y=h zz>`Az+@dz1)lKV#j_jRdz~oR6R|}gNACsFG%-F|~#<3|yJ8F}EgY|=Dcbt2)`^J#Z zD1l9a_JCkAc1lhTDqxbwqOZ&DO<-wL1Y|N4zy$hn9-kPP2enUvbWA%8L9IaD7Fsn= zQGYL5mhxD)07r4N=$vNEnn82NMlub07Jv1Pu(3?oC~q-vW3Pd_FN(6@nBWyQczo6! zw!2SZ{BC4NzR`Yl$(obSyTfhY+4odIE;erY$zl6N^tdG>eQ7^T&7Wp{QqN96_N2s^&naIY9*fdm+~$g9)T=OnV3(IT~yCj zVBt5xCzfjimC2tJMe$wUtqNd+?Z2G4@8!P3%uZD^`*yy4qU&T|$KQuIRV9M<&O8W% zG?56lP6@v=m4a9yzjA$YT*|K9w$4EsTb_vVy6~-0e`QPJv3Byt4ZqRbEaK`<^nTGx zZbmBjP7Qc74|3hN1f-49rF&Z*=_v#*0i-&F+6{2fs0m%qYa2go@yBeNu%Z0^BcI0X zx9d384i9(^sBmxtZ%*S&UYiceX8^UX;JEHJ^2fMzT{>1z1HX;rk$VC3rhD|;bEey zQYWBcW0t!Z4b36Mxf`=yjdA@N*U|#2;js`q51095)xT%5&qp3)s4?x0)ljqdDmawS zGXK9DP^79YNS_?8ssZ{=Zc)`a=_&&N!9Ma2{X=}*ACfK9PLsvd&NB5Q$!kam`Gy+! l+_Br@AONsn0)SJ%|2>2!m1Mjc3dBxsR1;%uy>$HLe*mmjJbeHF delta 5475 zcmYkAdpwhW`2RO^wwRF`VbdJjR8tZe`H;;aInS9PIV2Pzvn4)`i7k_}L~=fi5(*!4 zPUI{przMASER5wVvER^Vb_wU@8}^uRCfl^6iv zNed8GBjb!ZoR8A*3Iv6u91kCtB7XW!A8;t}{p^=p_s!<+eer*#jb}Qq5N(Dc_kHVU z;w6`!{yzBPqIm4#Ku5&cT+_Q*rAN<8b=*0p1v486$y)pJ<|97qVMmaLHq80m^A7(l zhngFI?yZq?$r=5BowJX1)S62?Q4Psj>|?PQA@|5f4NETWUlAPmia3A0xOm=tBS^h5Q+${eoEjE*){!3Dq9N;| znt5-)tdlihv*rCRad`Tx(9iOHTffZjZx&Z-bdnNJC-&w^m6b$f_WyDI&Fb&*SF<;m zN+H#gu}N*sffm`qRjPDD#%Z$nJY&l2Iy9*IXz@q>IUS`VU67kH?V<|h8b&5axy@w#x&H2Z&?Ubs zSXbsJk-D=E?Owk56ey!3S|e_rPi#%bq?vqXqb+^nNeN4>e$XW`gD7`YjdW(?M;r5= z$rG}^$*8S|d+FJn<;3G`udI`+?t1Ze!_xJc4)gNyaktr8@%)7@sc{t6`FMW9AGqj- zaQHuRbMJO=`puDiNV)g^$)f1@KM?V8jH#|T(XLIpKC=egtvh#~P16UHTY`7@D<1s4 z7BANHNVKTKctSoyzuC^YHYxPgK{5<|^%MD&&6N}HSEj&wh8rTQvWa4UJBQ!g-Dgbs zj@_LJ=1(r4MfTD!lM$iq6j!EfO9k53gaHZ~8I zEq%{r)hQlhkri1!#RtQlXLWvS4LBVIh=g34^szLiJy{goH|uEg=R1nX(V-+@3!!||@7N|$&WBDFj4 z_f?H>fda+PEBnXI?GJ`0I+Bu_k)j)4$?P0`Waxq`pqUAh`Ng+>Y5HC~x0(9F0>E-Z zrDkXMSRIH3;xHMGI;qD@v%kM8PwEB_NnX^}`m1rG!e7yQ^f9CF3hXbON`d13^}TN@V4Q};H&7ii?dy42bSjEQliBqhA*F*S-}Jnj-+e=c6M zRc&h~ei~?Iwj;Boy7BH(C~+s5{4Fw6>U9&AD65;=tec^s{J?L@+^S7!>}(oDoS~A? z=qb0NYi^L;5!^&EZ`S#Rzv3Nu`8yRd(~Y4T(o=P*U`SZH8;;7=T3Ds~=xBM~3B&zA>;K1VLxLBMU-S7kP0E6Csob*qCvY$0;Y&afrgj+sQg zHLWUuNJN5ARB&}1%)@+-fHV`O7?|CcIR!we$R-lF5Cc&%BFagsZRkG!7rwp5d zTO(2T>mLDMCcln$)Vw9U`t{?aw}Ch7*QP?%!G~d?m`Ar4Oj}4)^Udi=dBj4R2gQ~O zbUyo_*+;f&0U@!zOjaIr=b_kspFb)DuR?;F!)3ds*Nv77p|-zWqJ5=r3pBo6?q4bU z&dix;J4Fs<>&>d+!>R)EJRX0b+JX^>uTa7339yCH-OvuM?y_P|5dhBGJV6$KF!@F8 zVJHDS%R@gJB~(5cNt7jNy`tlJoV-nWrb9a_gBt(|maeN1VJ!YsHC_-`kpAszC2(_lsvn?B5_vr=PO^=2++3*k6Pd)tM?!P{UNrw}^ ze`;d&MUkNO}@Ay9!tJa1eF7%H_N=c$)K?G9Jj#_{NnS;7^` zJnNl9H^2Vrg#OaIlwovoZO(jO^8{Qe_ytJ9o{tl69j!1ECh|}^ ze1L>$;{9YgD(aGc_-qli**>(S+)`Pvm9h0CZI#}rzh-T%P-}O}DBBDtjY-v8L8hsT z_9&Ia-^KGUfguS5-f!S*36BSq<# z*nL=~XNK=u1lR{@CfxCU>X3H_^e`vw{mfaT{-ke#-I2L${Q`7MQVF2q!RC|PK6&XrHV}a!aqNOr@~y1)qqnDBN?QJA%%;UQ9RfOa_E+V z2mtz`w0x=uX2%e#gg`W859w_NmHN`-WGMAP$|UwU(mDo4me#GuH^m6j&YM?^q5)iwpuq2%qCNnMR z{b=&}_~iOC@pUCw?uqDOH>mex6O1=BHK}$8DTjzwYef=a6xs>Me@m9}1fJ;OMd3HI*v5ENe`>^<0BPnY3_Lcf2zT*Jud-VDAJ}#8HK=JkVPNK0^R6YQW|>Vi8%1U zzx3u(ZQJ{BUQ1`zvD2(kr>RJCu;%j!F`_OKN;XzuFJ1TksG+6P|CI zx2?z9_U*%YDeYNsNbVi!PZm9kgj+_~Mt8OJg=f;#M+&t#BpEF$Q9HDWKt!Ku-E@Od zQtYr6uZOXHu~wA+%?O@wNu=zJo=La=uBHuEm^IG!TPy5!_p1c=QwK(i%a%5mdEc$Q zz3GW&2sKvNTSr)(GJj7hmUe!Gh{Pc~WF%@WRuKpa)f4#`b@C}PgW%yTl{ellh*Kc( zg!9(PaFog3pbg1Z6t9{Nedg?{RIL`RZ`6`DMA1 zpVik;^)gUBRZyJ_9jMy#{Bmp2z_J?;r=lb7y36ExA_(x=@ z8KOaRweJ45;555X3RhKr4p)^X;3x3p%XD~!X98lZtfW*khd!3k$9u-w($I6q13G__ zS}H8mVKIt8Ak}n7-C%H<9~_*jS@F5j*W)&1Bzok0j9NiBKb>};RK-Wu-Eo+4uSGs z#Ya9wqwnD^G~=S)y8DdXjTDZZQrUQ78s8eaFc$Ufq!oNv1HZL_jt^e>vENh z(^i!6%dOO1bn13E0q37NERSGfT36g)tyI4`5P@AMV*{w(w$tV zLsjTwviZ$pm=-v%T8}GHddKl56IQ1$n%bfn zkr%Vz@X}bX72b#AwY!4BVOsOaKQpw;DISBi`Ce_S9DKoCou{^!Y0jsg<*QL5%D~g}jypi_cGBH!hd!st#05Bqt@Ho>t=LU1-#2Rd3>`?$ON#YStBA!}t$PA=hIBqB{8R zeS+^H-xSkd^B_uis5V-;c>CDT?(yOUg`eB?f?#9v2z`w1nj1`)`ke_fVIRoY0N+-# z$_5$mU0q}HRm0R?fY59a?cl2_VW@Y5@8xyQzj0_dX1{jki^QJlTV2jpYCmFQojY2! z+M8sYP%oU(1x?q-XT-*6T6{$yJk)f*awaN4}2v{{3^!T?_h zzY3R8;Or4;p=II2sTzvvN#>R@ra}q?q&6lp`=w83{@vyX(u4{u{Mw(1St5LBMzIX{ zc-w+PM!TN$(rOCC0yeW+4Zha%YZj7H@ui>bU)AK`iU3(f#w{-$BMag@;`kJ96$GrsJ;~e)AYHx-DjPw3$SAo;9 zv0YK^9P`3>WQ_ExuuC!gQPu9g5+VPIM)alyI^2|Q@?JUaNgdh9L9bK?qf!kX*98-B zpV=!^h}>~k@=7%r-W@XAgN(vDaW)Zd(%WPn7Y;v~f}chZ4nLWj|KTtJYiVIR0c-Ww zA9#t4$o?EnvA+(xbL{2v#*4zEUX^L9|8~$mZy9k*qtP-F%echY=#+6lVyq{{_NAN1 z`toonfoEZKaf7fPnlE4cto5M`Bn)?c_l-oIsmRo2 z{Re-TH2=>HzY|ZOM+}U`;$ooQT&WPEq$^hq*tHid)jHzE^+AwQGQgeC#?8qjnXAn4 zIv4Oc(Bb!>9%YH%efdkEcT*8&kZj&uQ}xKFXp{u1T=F~pK9RWh|4&%Nw-^Nuj*ckF zXLfsLW5EMWV^_jDZCZlalYI5z59-(ES)q)jo=`iPU+Z(Depr}l`uvie`$k@H`mr3Z z>6SZnD(afrjg3l@eYes({29C9c5hY7deo&9UCh2@ud4Ue@3JB;Wj!LNf2lOoL6 zfp-)bG?#I{;PcXd7fW}4rhhn{^Z&(!4f8%sVrF! z78M!C$l7c$r3B_m9guJ|1o7Yc);L0CkgsLXP_z Date: Wed, 5 Feb 2025 19:30:42 +0000 Subject: [PATCH 74/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14565 -> 14566 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 26bb39680..bf6a942f0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 17:37:51,155 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3270 INFO] --- -[2025-02-05 17:37:51,155 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3270 INFO] --- +[2025-02-05 19:30:37,499 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3286 INFO] --- -[2025-02-05 17:37:51,155 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 17:37:51,155 submission_checker1.py:3307 INFO] --- -[2025-02-05 17:37:51,155 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 17:37:52,469 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 19:30:37,499 submission_checker1.py:3286 INFO] --- +[2025-02-05 19:30:37,499 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 19:30:37,499 submission_checker1.py:3307 INFO] --- +[2025-02-05 19:30:37,499 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 19:30:38,566 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.14067 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0741 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.5612 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.35302 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7134 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 11a42ac27..ac62e9337 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -615 +616 diff --git a/summary.csv b/summary.csv index 92f442b6b..c0b8694e4 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.14067,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0741,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.5612,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.35302,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7134,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 079713e67..c09b15dfc 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":49.0741, + "Result":49.5612, "Accuracy":"GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index b9648f4226ef7094870553234e36bf8f10c77eb5..81b96f09c554b03e299a20b1bda6ac353b7f26c6 100644 GIT binary patch delta 4622 zcmYM2c{r4N*vDtF#b690lPselnz5upma&8kLI^V?X&mVwON1~;S<5Vpb5c#&vd>s6 zB#Ia+Th?PKk*#dQDNDSM^S;;hK7ZW*JlFHR?)!Ux?%#d=o|)ip!5m*P{zlnY6-v2j^ly9c zuFv_c(16-Z`|6R&($|f0KL+wH)EpQYTJ&3=tvw&B^P` zSKHR>TJ8%yX1!+)tZ8ZB+^c_m3`w44!b2MG^3FHu_iT)=L{&aKw_CqSt5*-|zfc>v zU1Bx<(@yq}PD^z;wr9rm?9Bkyw+}^&sHQ_W!R-d#NcL#7(VMAjoG719BIRjswLNNg zUtaV4P#*bDQ1p-kcCb=EiuLnDAdgqF5-?>R9;qpd@3@;$aQBbntZjlHUQIMKS!XsV z{Q&lll$1IK39ciG@h-?vnJ|9vEABwQG~*Hcx~5;6ZWop*vCN!2p4S$C>qBSqeB;Y5 zDn)9qdACw;<+{;$!%gD`gR^p-chhaPBp3z~EtJthxzisf1MYju_DU{qK+{##cm2M5 zSz3D1k|RQY+bJOowNW7U53u+)G~)5ncpaa;4w;;g@=NfWRrmtU)pR}b==Sj3e?u=nU2aKk}F)?f7@o=dNTB#Rs%6x4#>A8V_b?7@dOZWzlYj{a5XF%!a zNe>+iFnoGd>C-eH3>5=4pYWBT=YzE7u{}Qgm@HX-3^N}T6v=ui$PmS{J)TlrU!SDy z{Zw?RVuBFH6L#O1N(Nf%AL-#`kHu1c7rfb?PhsM`ExFGFlM<;Gbg(o*z^xTlRTPb* zKND)seP)u41W9LQSm1i-x*(z@b?696?|Rjcc*SL#>T!K9dl9c6PI2Yl>gZ=5|4A8t zY~X%RRc+jJVmiw>Dyn!T_&%j^{-%kxb(4VsCb5wYTd~$|#{NNd{qZ%+uodEI#9vgyKzrJwyBs?;oD%nd50#r?BU0+4U)|S6O8br*9+g~~st)jM!l&7X^ zlh#tv|9wISdmg{tcu0Zm28!H}nyZ2|mp(JG6jo$Uzj#K1q*2g@EmdYg38((C|IWBR z>rz$p6vj>LaP{5^ND7N5~{WT8MqROevFq) zb~Z6I;@5Trt@1e~i;A$Eeb`M~kHGv8IdLP351$CWp}cYU}3_e*Cm_ zZATF$bmDOnm^UhpR%8Uqvlmt3Us)JDDp(0=_I?TSpp1LRGnu#qL-VLF#q>r}L`j*W zns9UVMkaPdJuC%ZRAo!5rLJu9pHTPpy-so;%BK6?IPZEZF)f9ZVGqwt5;y zw+>CFkTa!Lf&QxsbezFKRp|$~lJ8_^B$M6vH@$w9v&~!yZL@HawtlL!-&6OD8_={z zS$1F4tlR2)$XXB8!!-mKg)cD%f7S(HiI@P9`;zG=`{sY|={+5ND(NTXJYC)l z%*NU+Eja|ljA)7I)TTS=pcS3Bqm^91LX?dQbJzTS95%DDJS)L7Ob9gUJEy<1>^AQl zt4nbGE&+a;GpUc7*}Sm&_@U<7z)!nW-L-yHIr$Day=kadAg}Z~PoK>HgPL|S_2}B> zD3~jCgrE(bR$_wpqYfI=?|ZOP*s9Q8vu{ndGpUE(dw!H|Te#aXi^1?cb0Sf!9@~Yr ziS-`gFzpJu_n25^I(P8bE~|YbnEBICnkz9WU_EXk+>S+PJ*M~&pEb9eBp%esOutXJ((`#PKY#bA-8X@j zlxlC?Ilmz3quIViW?5y&=URy_cbxmI@;(J;SQ$8@?;%ol-tB+^-3w@hnGVz$%>?Uk z6RGD|SU@%hPu!RF-cLENJcGCeaT>ZCpq$u#`pFsM(KF}gPo4P9l;n4@&4Wty$ z{aZ^Dh68#?MT8jwDy7H-BStDtnB;bYy^&ON7Bs)$aj1|w3dR4Bq$9bB&OK|^QFyqC zjU9}ZzszHm)MVi_g}NdnyY#HD3>~#e+RE!yreze0TUio|cd2l1)RSEf%!?Zif>3ZV zOX8(*Zd+pt>ijupt^zhX{^f(-Arr#=AR4Bdy-2L_7ms({^~Mba(mg2?HE}l@9?1;l zNha~<_o8U5b7Fv|AD9LYL3x2IVL@CSA&5rIA$Vz=$Cja@I`R=9*`fMm+>|Xpe+Tx) z&a~c}J*x)8&yKw)DCtg-rfdt0s8&_jxFjv+^~%%iTlV$lx2SM!na^7sn3k1FB5aGS z-~vXikKjqiP5}hb&YR4|dm8nLp!lAib;YeGo*0fAj101oV@bZ6|24-vCwcI;8279s z5kqH+ixB-bsc!u}BMy2+bnk zL)g|AE2l$$&qviaHu3WsmPthJLPB6E6R9w86+2tkFUD?%8)N9+fVA4bRJaLO8)40) zInZr!+Sk*&7ccW;7=SdCBwj=Q&WVo@9FNME)#6&eg^D{>4pz5JXJ{=SBYFuj3;j2) z_9s(n7UfHW-Hh%QDbhYgxv4Tq;s?;#V3K$jD4X1^__@z_g9B+UuwOW?xM5=RPE0mL ztzpAT_l4IT&lso53f@=rn4{}#Yv#pA8{yRfGb`ZqGlR#?ktzA*Z_O%e!mNbUV4N^F z0o7KR;~XQ*4g92Mc42I9CEhG3Z1^&C9!aSO)To|BKdPsV$S^|db;AaUi#Su}>f{ne z5-oSi7WL{Zyl<`0fq8cpb3slzwV>pTnbjY0C2zG&R7QA$9qWJVmw~ABl3}o9{T_Pd zTLEb~k(O#P#2cFt_|10b?@NXPi#Lxi9wU-_X8eg0Y8t+^Hc_VrO|iW;N@~j-iUV#& z4q--uPRlXDBre|Aq_lhdGJl*4tNx0~csw&*PAC5q8LNPJBjUk{v~APr?lui|50kGh z=DA&|`SYitq`_zM>qP4wLFOt~F6bA(1S<)Tv`E@Eh z7xj6a1A9WK9MI1OJ2t7A#+qOP-e+hC|makS@b-QxW zVDqm!;DD(%cbvNX)Xmxulc*?QWr%|+(#xYk=B&4Gg+a;qJMEOXH&9@X`O;m(ueL-e zqs=ADdK^rw4d#`+H2v1M8&*PX8RMwhB4MGf5x3Z*qpk*A7p z+vKKqT=J(xH=EIKi|sR4vd^6PC$L6LAyB2qmaPJ`5U7$2*YbSonz3#s04&@QmC{*3 zRd!aHW^#+O_Hf<(TNOG}Fj8)E$K>JIX0tW*f6!+M6&bZ*rRfEa+W0X@G@Nv$>=%iKjaq3H(d)h%Ud_5~>is~~Z$>(lsm1^_@4$DpMCCs8ef3S8eU;mR98E3z*G^kSgv z@=P#h#LloezkB&I^0SD0lki261i-x;QpGS1{INb)b1A`wbUJp@#h(|AvOim`DY6?t zTb`nbI9I)Sk~Uh^S-o}dVsS8`auCgPo1&|#@i-1Pod7oe0v!N0{#B16;?A)?yxrd? zkJ=MH0@TP^;F*d0>%^4rX1fb8Y5c9TMvOA@9aF{BmeFD_<5%o zNEHrk1qKa1VrdA~#KFyhNi*G_v~);@@`&x{O3VQOYl)2V;jz`+O?wu>rRs&^ zu{IsU`T)iU`3I2jDTbedvv7mNvkL5pXxh<8M!P}|#o+#7aa->hP}YkhB$5g46H^*~ z09^!@hIfIA{LMfm_E2vm+?gdPbz6{69VN>TgYH{Q?S#)Ct#I2qP_RZ!HC0X%$#rxy z4&KZSWmx}sLZ1Vqi~*@Z;7H5@4+3~*kO@FBaJRCdv`UH>5TxewlZs zm)#d=@PtFTkgP#xwIbusC$1{nh`g!shae9M5XgrW%&vTG&b+Ehda$3L3XoqRSOv(h z@I41?8>0d*2mQ5W`$Wc8aeoO0Wp&vqHlOq=RUipzjU!Ia8F-Ec7Q=2FzW#kN?+_t! zGEP=MCoRqJ(*NEuDU9gY$2Zryx-0_pu)Ey)`N&BU_Wtm~bGz5Sid?R9zMW~5xvnF3 zFT1TqFq`em<9fY3Y~TH7+Fn^hqd#zEF37u3`DJ8v!^hJ-wxRRq!sbxvR2Wgc&QmtV z=$64~>A|5XSKg{W-ZQoV^}DPnJm&N;>%Hacilbpha(lD-dwv_)Uo^`PzJ|HK{#{x~ zDg7~!S}#KwaP6%YJaIWjIIlr{L`4}{2MypB$D3UwMM?IlNYutsKk3JAAG4dnP@OVL z|Nb2we!h0NSYwlawxIV^=ZOSF&zMlL?y0Ej+4F~$@mlI`z9&Cj1!YKoxY(vX^RA`z zgtQ0sGefBFafo|ffDqIB`=+!Mk9cEFwc?%=uC*_K!tvS32>=~T9J|6Fe?VGh0_tQS z(JP_$E3wA7LKd8GWzHmCOn#Plzk32?x0xfF7xKkf8zvn{vY(osh&+wjZtC6&sd=|^ zde`GsC$7TCA@TsiNnFd}hFACpeTNrw@N4vF2vXztD(F?j0_tOq?!+?_nLPIIj?&!|4suC5IT4VFXyLdb4xgJj7v2&F+prRrK}UlmJIP5pYQK`zVAQoIq&7%*YiC0-shh4o(q``srDD-khlxAuO|fwYJ<2* z#e$Bw@Rto}!IGlR*`R+`hTCV8neAKq{#*2*y{&KXzMT5Pj!>#=JNjgmmu7DAw7H?U z^0C@-to!avkXAlFx!Zew{zLb$zEsKxMK;G)!n|@0qn0;vU9t5+!p8>`N@~J(`S`DB zqFs)*`P}yGMYprKkW+2$AGabbzkl;D-~2d-+aCA&`y_f`TS;qkZtrLGQu_B@df~?W zM4&>xUv0Z9E=q5r+_L?D;o0LGsUt98&>e6etOgAOzv*jx^$Sw*OKzUso&0-%_|fw zzpPYuU9XptUrw%nGW?M1Fsgw)jTZbJF<+}%za1UhJ$!NV-P0rPM-qHr1pd7>-1?8l zdc*Xe=Z{?~E{AOxuH;V5%{xzjq9rb^d1+xY1f*;PKKK}Jkp{7j$L7Gnz0*fUT;25{uIL%d%i0ok!1$n3PAJpc}-77r%s1FcCXl;h+_M!M^^CZ!k7y76P%zn!y#_7cWRL zL--=#3SB7WXRQIZpMP7JcZdJcM;3DT#!XHgbu7>`|%5Ht0%);n<3e{?2R(Bhek zjY=~cyWc|9E=--1rwH}&Sy#2G_^(fQ_xexCGnV+PvKv5H7itz@XjEr|ohUvmllU#N zc%rM@h~f1fYiq3>0<2Iy35&mcw23}dhHz~8nv;2M;A2fPPJeqbV&oo>D+3PNv)F4VAf z6s}PT9PVqJP&)ToG=hEJ_BXThbWlKp2iH;ns3L~9+POkz=3|r|_0ui?^27BujXBy; zv*~$Wir7cexa&-bj1=NZuOJVs3c=n=!nOC>imu z!#g@Z$*zv2Dj~!B<}X~R}?&b#Oa)9L(YS?+gF<1HmOr~Zc{ zS^cT5OYRAL1uyu2Ic8ceQ4vQp-W|0%Ilzm*Km^^i;(Xi9PrMp4BfJH?-8_)M_*xV2 zD4^$5TyBfgq6uXJCVTcG)w-ug?2sTRQ-Gm8BPY zu9ewrm}L9HbF~U-+p!;EA_{fqCI)5U;xeaKStX-@kca621!SGMcE-FI7+kDPLW2s1 z<{_DEDF5RNh^|);9e*>*RT|>+QofM`O{JX2kGoS8Y>Or2b!o7;5i7+HFe1*uk+zZn zE<6eqqVgA@WD#~puV>joeIPUUryJXNJ!K&IrdN6vHT{9RIP!4y|jVqOh{6LaA zZ#0ZGov2X41~;}f27B}{k&nWi$%Kj2rxXj_m-|w3PI{s6)O$&-kVs>7>pMavrbuIX zItuA#A^cAEP;6kn9ZeB z&N%<#CKDoXBN@$kgB8bO!LIoiXo}G1hAj(w2|_R4 zkyFyxUJ$|Gl3;%n#Z$hhs(fwF`Je(xJXZ%!tWz*zAB!D=2?vCpbyN_}OsR21z+tf( z#t0vA9G-NRdwS!cSq0mx#q)_jaesi)AeP?2Jf%w`(o?@^J(EzK5SJxR1A*ReGi&_L zPl=~FDLV9L8G+h^@?n$GgM!YeA~SgVDV(wTJ8Ne`Y{mz`nc$1JRfl1Q=6R{SZ%gdN zn)j#Kb8BRZih$f2kMz*4+Tx|*QAn<#5eJT@avFawbR1CZC0bPozsaetHw6u1ypNZ$ zMxn+L1cxC`8H0Spp_$9N>Td`B?5eT9z*z%#|tUkSvr06I^ zQ+o-!K{pRr^RZadS<~u^w`ni|-&h>Nqd8^<=RpBw3n!2^78amo!RSqtDh3?RI|#XL zV0Y=)nfcc$&N1ie#&FtiEi%JAYjiJ2rT!*hdBYF#jV8oqbONIZzIYRsiv4ms)x2*D z`hE{MmK`8WVM7XSFfzT~Lb(l^eICm_>Kv)SUl2&q<)M>aX0bzN257kAZ~MlatzB8Ai&8O$8=B`m%DoSScc6w zYeYLalr|W-NnsQzp=olnu3X6yqoOg_Hmi$EP0;NP*cm}=5ajC~9D%`#)jCEli5Gx1 zP>=0KE^*+I`Q^9vySR8ehc6`v6uU0qSn!OaeV>>_tEgPG>Dc`_a#hBLpXz1nHXLxe z?wqF#bL&@(|g@DS4{1s(EdRI?eE*nK<&WfD( z2xLT{Ij_j<>;jdB5iG{;F^n`&*nb$0WK?2V`EGR1oq~MQH{$vbJRJ4Dr1x^_!1Jl~ zvrBl=Utz_?^=fEpPoasj(tq>GRv7{$MrbvPxeyB?wFFW4L z)xV)&PyrMyYDDuNHRvZ&yW+3BN;4=vo`*2`atiW8&nTfQI6Lxv3GH%f*z(L}VJ?E1 zUt%$hRn@FW6t@X~MkJck7~!!E5DXJo9KkWX=^$Jji=;aD(W<5Xd#lz+f-Ksg#HW1z zl;(cfntKoP-cwO`yR3%n9|~5V?#irsz!x{GGbS}kS-5=hqbyu9<6T_U8v!g_7Z+=z z`O*L4;(V>dH$DFj7~*=plAC0kFLz^GdA6Ro$q~|2#za4%yeqeoxFGCnw&(L5$1X-Z z5qi*b_Av)NS15R6Q6mNo`a0;(JgOE`AR~PrzsmbEy7-8?{O!#TT5bkL#!;1+7!&>y zVkb@aXVU~h<4PhNj=S{-#Mjdpln{$IA!qW}q+l{Ic#|I*x6z=Uf-2+Dd*kdX;^K!= zTo;C=U*b&`h8sQnSFzr+I2NopwJXw{1%YyD>jln&oQVO2#ob-VHFo=dT_Bm|0-Zqe z8>X_P3vhJ9ORp|Qc+E`&~k|^VNWwFdP3Yxm@6ymAV6njWyN_zoT-99H5SX!yk&$f5LK26ADbEZ z&l-dgD>R`^v*bP%V&C9xMK=#H(H$C4|NUWc*n$BW1+!)tC?iz6bP=&A&IJQyc2o@| z5^t6NrAx8nNCOtqEOtoh-^DF{5vCtT^q%C%sk^z`Ryj7et)7?W=PkDT3I3(~2<|B+ z&g4kFZ?^yD@fOGLM4w92IVw%Y!Ex@=V{%p$ihGNr1Uh_e5I3#Uq9(G)|5=LrXl9u$KHhni=&o?Ah zU_wlsl7b^ceMr5M|F9J=lLDojcn1mRN3)mWyNB+O-b&@-ge}v!WT`jt){}=-RG?zk z#T;fx?m#>TO&w6o1>+e98{yFKJgECC$i(Kg1*i*X-b85^tD7+smB@FHWmvM0ea|;m zw~yFeXj_;pcGP={U=qk9LVPlVOqPqOcQ9ElMm+me!b&k_LZ}jOVz%b*NEzDmd=`>0 zzk@sG3)|iyv({ii{8Qs6r=58jLVx((dj{{RspJ}W$C>Dy;48piPfbZ>&?f zHhLZAm7<-S_L}Z9ai2a~yN#*7kXAZh-;Epb^lI=5-5VTuHuh`j4^zELrQLJ(_U7&n z=hfy4F^{U$w!D`T%frc6SIf5}wWL(rr&8{?IJLB$ShDjg@*X8M=cVo6qU;AS>A!MD zlD*?Azw9{=wGaOe>>gYx>}HGd^AOFDcF~@8@t;LJNEQAbT73LnJTl zW3`vz7r}>?o5D3oZ`7r5^uR7Pu=&(q zF_3m5-qkbhCq*NI6Y`@dsL;Fm+!g0SWh;k6NdYH6wwbML%HA=~Tz3kYxWXZ$d)M>_ zbKX**88$|9Yn^mAul zHw2Ty2i#2e_;J^~KQ}vHe%xE?gMW5UrKoUjIXGQ`c#dp=Y9?~#HN@w>S?Y|P2V`~oI> zEsv%+`~Uy3k<<17nUU^m>w`X#I Date: Wed, 5 Feb 2025 21:24:50 +0000 Subject: [PATCH 75/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 14566 -> 14568 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index bf6a942f0..422511988 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 19:30:37,499 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3270 INFO] --- -[2025-02-05 19:30:37,499 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3270 INFO] --- +[2025-02-05 21:24:45,587 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3286 INFO] --- -[2025-02-05 19:30:37,499 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 19:30:37,499 submission_checker1.py:3307 INFO] --- -[2025-02-05 19:30:37,499 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 19:30:38,566 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 21:24:45,587 submission_checker1.py:3286 INFO] --- +[2025-02-05 21:24:45,587 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 21:24:45,587 submission_checker1.py:3307 INFO] --- +[2025-02-05 21:24:45,587 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 21:24:46,632 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.5612 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.35302 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7134 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72452 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -70,7 +70,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431711 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0278 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0164 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ac62e9337..e110e8daa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -616 +617 diff --git a/summary.csv b/summary.csv index c0b8694e4..4b8255bb8 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.5612,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.35302,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7134,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72452,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -48,7 +48,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431711,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0278,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0164,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c09b15dfc..3a689a14b 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7134, + "Result":1.72452, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1479,7 +1479,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0278, + "Result":23.0164, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 81b96f09c554b03e299a20b1bda6ac353b7f26c6..8bbed4196db334fbe873e8caaafd415625c085b8 100644 GIT binary patch delta 2642 zcmYM0c{mi>8^>ou%ouYO85&zN$P#HGTVuJlAzBS?mWDJ+SEY+7&14%xH_R|@KgE!- zG-SpS6~fqt82i>HF3OS^OQzIs+~@au?)l^WqsP%lMf7G1^!ccg4D2o)yIYK%I~A z9s1`3^YA2Vbo_79QPzgw-{|gf6kP5wUk;FE!)d1sk-)$wx{lzkiv{TNM# zo(4HR;s#!i`2;SjbszWy-UYZHG@zk5P&CH{VGD(W2Lk4>xHh=j#Rx-)f~E5g<6!&0 zQKxuyB9+qjzm{==(rvRUgKrkqJ{a0RKZKVlYcZx3)bs6lRgum+i1OS>y!1tk+7_V9 zCYq`Q@QL|+4;cB6glWTR^|vIraI)>}0i}7XV=~=gYtgR?g@ix*o-B&o7ATA@W&M4t zTu;LUbwxXKaJ)JbXfDXH0Tc;m1Em1Rb9)Hb**na;s3VAZmTzhVU!+q>nW*#T2}KU( zxw@~0Lc)T?$}* zIz@u%S@+U;{KSjROzQ%*9CXwB14`d4k5Tzy)lshsTM2*O2%9F1!x6M@MVAz6!g=Cy z8X{IDV3`Y}IIph&5iyDi1^`X*9(6EWp(-c{;psP)CpDcJk8tlLxwZNT2^W8ynysd< zNuAhNm!s#N!QW>>BZG0;wGJm20=1qIk5l6jT>u_67=WvwDFl2Hk?_FA{-QONUJ?sK zpvq@>DK`rFJ=PAe#8{l8G;mtdYtF_&WY6`v?67{)ZHmZ8AdXdW!^DNn z*UpCS?O|b(lfDmqkf7dIpcFyU-j1dnrgU36P`XUG5GaL?YXF^tUie?3d0wfjAn81> z_g5aaiees)xDM}bX8j3?(rR z{}K6hp~Ij~COo8FvE0EWgqq;fKPH(krj28g`9cqC0iPBZNPs1)-j>Rx65vO@l@c`c z1N#!C1H4_gwuc^5NJ4uHCS$p~Uf#tO8O=^1s7(Z#ZGJ%$-_D5mB@J;N8SsS*3lOY- z02;=~DR3xwiASLScw`qjN>8?)RIponNAtZh{HUzcm!lzVVbwPl3d;x?MBRH!m%XWF z1e#%fK^fm}W!PV0ktTqz#3Hs&D3ex^4x?%X$Vq3e3LiiX*m#OPWzjL$v z)i+5{QnSit{U|O^JYCVrdch4&b*fgoI`^FZmK#Nv@uQJhkJzo-c}l#zqPp$nFP(oZ z0R3WAC3)6~t<<6Fk`|BHRoe&FR``?{k@($jzc25#=(I&{^v@8Sd@dpTO27B5!H5p2 zt1ZiSX)hkJCnUIiU(*m10#ii~W_nbGEHH;B{1T`H@bFK#0!#jfSF)RnxYdi+B5N|0 z=`C$a+1OOYZyw)cU6pr(LhE>0si9R<9m|dAvv5zEWj z&tBt8P+Mt;lgNNAE{yEyqLbl3cgPsMo&fB~cQxP~eAB9q*SZC`F{uz8%Gu_0fZSNW{A zLp054y2gEJDVUv&)_qy`+hAhlWNvA8&PG}FWK!Af#nqJ5&^75xHZ3X37Y6#Gyxnew zZ<3DW!kAslt94Jm3ZHj%4X5<@Tlgrc<)HliXR4=bIJe4k$o2YuSwrXNtNTVOYaZ2; zt{UAwtRW=NOikBh_X(3jN3!C!1eaImNhe1JmdQ^<{$%8je$sLmckT7dP;t$;Um0F} zt_uH_lrf|+TD2ZoE@rxVy4B_j_xTb#^y1lI)G58&__=p9eqxN%jQezz~L?nQA3LI{*QJ7}QQN+D$dc zU7>sv#IGA$a*yhbUPpVzFJgTwLRp?!%!T+vE)Hcfbf0#z|Jd;dkczo$5lauA9W2J9 zf_HAk-mv2+2D>C{d%9iETgQltN0#>{KPXBba5T_6Ap^a)SfH$;OH1@>Dq!G6r_Ex2bOf`*CeEGep$X}h4Ezmr}mOT2yg zAi$j8OMy(Idgw6!I`z_sNHyu{&V30>us78?t%8_{^QKS3va2eaR98a2UJ{9IN>!ml zxyiuCay>gLvGq^S+$m29uT8XgDXaTVcEaOa?^4}EW#l)ClNJCum6ItP6X7uRj%RXa zQp~%jB2LUA1+zg7Z`SwzE;hzStFHV$i8PsM*n$jcRLUMgg8%)`F$~PSfKH4!Gb`Xb zMx&Xne2_i>5Ovx3dPMZ);Oh+FkzEXJb5+o=A>)go5<}_;V$ZhR7$5-P2?79q0sOlt hcWO)-eWp9NFEMr>LO}n2M7aZH)SK^>Zacia^Irm}+~WWM delta 2616 zcmYM0dpy(oAIHBl<}#NdYOY&tav4dIfa8CB_)?GF5?tk{JzdV=ljR!pYP-Gdc0n*&-49$d=_Kh#qz@xWjp28 z>|0dY4riG5HdFTuF5_idr+ilHw}hEvVh41bZ={9cC_OR@c>26cTj zm7>p~D#@ZtAXKF%KnY!hntdL&$(QJ<@>o8M+O#JzA1P|a8FXyATC~w}0@J(%4dOTG z>C-+7{Wk*2Hj{*r^Q(|6$Hxylsa>wD?{gG1C%VcQ$nz5^F#Ue{(~Imx3Z#5wVR?FI zHPbOBaUux1q}C3Ku>yS~Sb-$?godE|mQ z!_-#Kuex{K(e*zW_xjB2^(MvALmM}(>j2i0YNC9e^+=JR52mgS?-eSlJS0sb&JTKj z_=JbOKY#eWp)Y-8p5gn_z$C1NlxjZiVDu+xzrhOM6F0A|;z)(Q)#gB`!vs5fo@gdW zDuc{z53!%g1(NY1OuUbuQjQJDRp_){96vC#+Bu9tSw@rV#CYXr z3%@57uLF zD3~iA2SpXtUj&@EeSLC39wBEi;~~@-iIlwJlN8>Yj72-C*6DZ&#>D$>?Rclu)6*Qh z*~qE_?wvg!)`H0GCECgz%)PW=Ru0 z$)V_P{z%$(&lixAC9$Llx2lue)P*v!IHSvJ_DV6Yy+o?GGIW-4vB$CKlHx9N_jZ|c z8ikmoRf8e@x7~H2)nG_E!Zh$VpTZ4CSZE`&Yf_K5Ag$k6Tz4q1*V}BO_1cAoV{`J< zioBs$TIoHG>&E{>qiL*A%q83bBnC+v*?y`+wl`Ze8fl^ZZBKHBLEfmPZn_?#AyPL# zj{D+Y`Rs;7%TKwQ))ckA!G{-LatOYqAF;+~eFH|(Xu4_7To955o&rKat{M}7rKuN6 z75tO&?T<1SoaA43F_P!~BG4)K$D+#KA8?uws#V?lj&r~7Ew*aa^joiSuJjLXA5$b= zHmRtb?{i#l=Yjl2WXyy7f@eJUw;M1Dp-W=st1iJ)j-zD5h&v}bu!{-8jjX|%j(f78 zJ{-1WTU=)>O5~MZ@_v?}i zfMivW!KSj98mdb$?isRzKP@LPn6&ceS=S6p*gjnRRpvjoHWFwGajfAO$VtV zoWMzQ*qyX15{1}F_gMi(7NlCauzq5CtzyfA>qc&BLm3Vn!?#BgnNE?XmtvA|Ys4D~ zNt7CF87OE7{VynJI7z_US9weYsj-OMmXO{lQ1IG*F0ly_*bd9Xn}WG<>)F9ubuLB4 zI1?GtT1Q*LR2)Gub|0Rz4@1^Si5jxx6aTYr*U)h>mk+ zZV_{Y=aP!fLVcLX&nb(MakjaprQ{ww4-~f@pi=E|qa1aFEV>4wj(7spFdTt<9?)Plgf~}u&n0O|W2B~( zJaiXu)?>ud!3N5FIohp>_Dk;Fl{kp986nGXF9rJ zYBwV6nk77!#Oa@yU+l5Tb_sj7B5vfoUGZDKj>yuekSHGaHmI?iJra&3!5=m=B-Hl8 z+~``mjv;S)ZiMSoZ|(Zi1N~J)^gw^rPx%n{G(AK;@awjF`9614`0%2BOz{(URL@Zw zPnRlVI-_E>XcIUURV$ys)QbM_{*H!Q%Bu`b>(YV({MjxIob=ZY_Qmwu=hvS+IT2}T z^kcYvDdiQ_=-R~c{ZrjvYRJ+2cgH*TMIY0?S~AcqUD6mP9;W@SygpRFw_Dz?`#Z0i$@@5#U-jxl!C1YE@Au4;=a`>=j=8O? z@NVJFQbp2Dt{VKR#>do10-?L@t2i#H<%gGlW7;j_65;U7E0gtJeQx6qvevY4l9BSc zWDLD!R=V?mQ2upC+LHu@NU~`b8p6>1G=${k!5R`~k)~I#*SHn|t7?xKd@8+q`wms%BVB2qNpgaE}9^2t`(jniMn1AEx)VEAdHb>m} zeTetth^Kb$x6)(@t+=zytanO6_)#SE-`5=5*un>Lf}LZ5hfK2{S~w^K8v;Nw<4}A; z3L`Y0{gdS$HpWr|GRn5I)YIH;8VdmcPYD1}2LAmrGF#XgmIR0_yW8>rtlxb1&;J2^ C=%UvE diff --git a/summary_results.json b/summary_results.json index 7b83a5164..e6b08e560 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7134, + "Performance_Result": 1.72452, "Performance_Units": "Samples/s" }, { @@ -1292,7 +1292,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0278, + "Performance_Result": 23.0164, "Performance_Units": "Samples/s" }, { From ffa018b10ebff1b3128b06d3efee4395af5d6080 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 21:27:55 +0000 Subject: [PATCH 76/86] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 14568 -> 14569 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 422511988..b49786ec0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 21:24:45,587 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3270 INFO] --- -[2025-02-05 21:24:45,587 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3270 INFO] --- +[2025-02-05 21:27:50,171 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3286 INFO] --- -[2025-02-05 21:24:45,587 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 21:24:45,587 submission_checker1.py:3307 INFO] --- -[2025-02-05 21:24:45,587 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 21:24:46,632 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 21:27:50,171 submission_checker1.py:3286 INFO] --- +[2025-02-05 21:27:50,171 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 21:27:50,171 submission_checker1.py:3307 INFO] --- +[2025-02-05 21:27:50,171 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 21:27:51,481 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.264988 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.375178 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.310163 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.84918 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.15 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e110e8daa..212abf5f3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -617 +618 diff --git a/summary.csv b/summary.csv index 4b8255bb8..6c1bfd58c 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.264988,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.375178,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.310163,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.84918,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.15,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3a689a14b..878d46ab7 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.375178, + "Result":0.310163, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.84918, - "Accuracy":"acc: 75.000", + "Result":10.15, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8bbed4196db334fbe873e8caaafd415625c085b8..4c5309a0b727bbe94ffc957e90de2f74b285b6df 100644 GIT binary patch delta 2237 zcmV;u2txPhap`feXAT6a?<)wiYYxu|f2v5e914(jC_tL*YavIi9QENOFichS~4;UIJImh@krKUHZfSzZv9 z@g|mWl*X=4mJantqe~(mO&^eqvQqsaA;dC3{b3ovlI;KlSo+)_Rspf}xj$M8e>R8E zlF?D5Fv~*t$&&2S1YCmhqbpW^!nur}b$@nCEua&=Z^0%959l5G$KT>3O1Rsp&6c|ck!I)~7bkx@XqgI+z-3mJr+f^@}R&r=wD z&be_*Ri6_r_6@R8WF7kkX@~2hvp5;F3I(B`F{f3N@gpLCQ{y-ge&1iA%D&7z_>yl5 zn4&_UsNvcKw%DhXV=KWsKC+e^!v6Y|v2$R#LlPYJiK1K9*I%o<)pqCj$2^YEjAS%S zlar1ns~toVKMlfUa?*KxxR9I?~(s?Dc^W*flV@es~k0$h_^UT=vNKvT& zOk$79>69dYTr*Cy*kk8j$E%VW-DRAt;3dpCn!urgiya5;9sB_6f}+R+nakCPa`9NXFe! zm)?!XQJ9dr_>Y`qA)$(ZIPl_W)TFpNZ?Ks2Mpx&5=o^tIGUARyXL%5^@AZ}4G)l9d zy|yMXBXBqEu|#KoSl zc#w|Si|TZ8GAg6hJKoi!->GIbah0QTh>>Aishqq z#XFmXK;k^2Z~gD+iKI#K_r;d5;hIl^H~+jkzdA#gCNGlw zh)%iRPC7h`X^|bB)3cnVJiAGNMz3f|40UgR&`g)HY|5yL@)_Q;R8v3T$(%`?vza-s?x9grJ<+yvyp znN@NdkUM}}a7HQDgfm6gp3jP>R&f3bPMQ^AqSdlW*%$+DgKdhby|IR0Nvr^~crg5* zV0(M9tZh=n>j-6UWfz1sMA&TN33vAva9fslrECsNA;s1csi9t%{sRm78d;$x-JGM| z{a;;V+m?tL^2#Sm|mrXYY$ zu6uoTKOA)L`n@Bhaj!V0VW^sA+m2zm2whyAFSJ-UWy>{HLsuP*mszop_wODrd)g7Q zG?@p3hR~Y;A0geAUEa-Bf@>~tW!=(jgdXmG_HGBm`(A%IIJ>#O?iEj~qWc?S*69Y8 zRfHxqc7E{)EAK8&LRAZKJep6}q{4;^TWwX=Onc2%LzA)PIxAZZey?B)H`tfUgPgH37aRz}FP`l3+s0j-v`BLmeM4d4`-UJxia3i zEmHRwX~zu%Q68dGCF-1Mj=V z+s_^ti`T5SZGXS-VQsI4OuHlw5ir98e+A6&ivq?vkWW_o?r(eS`6*q0iQBQxR>Zc>VGy-koerRj7+9=m80q$15cSFRgnEeN(*bLYz#m%+taA0P` zCyfJweIc`kZ$!I-z57~nz!s_TE7Tzi*r(Ax;m5-l%#YyOub8bMw(HaOYi28mf9;ZN zw`%Ty)cm>@-k&@dRR z_thWG4jc6EPwc}507|u}X)f7+fBlyv$VIkms!vF>KA27B38xz-L>Q1RT6a{PYhAx~ z^cyFd;twB|eJm>b*bO^x0gkGi73?3QDr5kSh`iUN)B60tw3xe!1Jk{%(c7@bkX3Pf z`!@M=R9O{=f{Ur8j%92*Yg0>0wk*qD4J|Pb2uHOLy6oK!uY0##VhJDe7B}gZ<44hm z3#~M7C08hAr~d&^O9KRx-!~ewf+|A@1i5PMqLaxoAAiki!!Qtp?|llRyDixvZ4tHu zZON&DLS0&N6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V`32{t(G|)wf)dc&2&uOf z`e>HvBT9T=TCmn=s8A0cE$X|HTgr`tH)kxkAi*aQq&?-fLc0(w#n|&5DCV>1&{5nN zr&x%k+kaxq_&3`EE(p2D3IY=>Fnj=}b|j(`Qt%OJe_T5N1jo>V0v$Zgvm9T%0i5!a z4TEy!R5J7yCaq_2bnm?!ja}De-E6SN{5k$wy}l2trcxhH#R1i&;FLRH!MM7_*F`*B z%Y0bH6K^Dlr#|WvZ>L3$eISw;fF9$XmCxDpOC_^JbwP@GN{W=sn?~cPyzq|5dZ)H0000000000fPuJ^fH*z@43ob& LBL>Gc00000Dr_x> delta 2220 zcmV;d2vhgzap-ZdXAT5~9hhjdYYxu|f3{=Qc6!;kPu=Sp#HE7V99m>0xW&*537J!`rIEae+8RE zXvyd(QkZ2S{A5XXX#y@m`Oy_CKjB=)&$>Umr54bM-n4*pyPX)l6`ez9$;c?6-9fJ&>4gl!PC>e2ujeTY zKIh!HrK-<~7W)R-D6+0gR)_1Ovp5;F3I)G!@xfG+@gpLCbK*D8qKqwwcYLsxOkjWg%CR|Oxd{-HeM+TU*4JOFyVZ7c`ePo2Xhu>R zB=K2Glhqa?v6uKkJUMGUJzh%f7NX3JeK$;Ea@JZ9+WOJ??UYi6_@gmBYrQZwJy8_u zy^zSIaxx`=ahAB6Q6{q3hSIIxu6gZJQBNUkW}SQalmL**e5g#Syh@68WTY}ASri8 zReC=jhe1rL;y*Hy281dC;=qljVV&aoqQ+v*8(m+2pl?K;$cQ@*oo9Z)zE@XvlQ2nt zl7#@4jiP`t(dzS=8-(sC2m>ZQw4jVcRp}k~wP5o@ns|iLMA(1&yMX9LtGfCq7@2f3g!!e%)Z~l3Gaea=i$Yqjx1oa=Ek)b+jb=i_yy`o%4((JiCdHMhmnehPtL@)_Qa34ws`O;%~IDBa-s|z9grJ<+yvyp znU!)IkhcN3;EY1931_$w;CU6^2ycC)+O~@Qk!kr-O2#DRRi3;B_;kUO)q5Gvux!YG znnXPh8`8+7MA_lYU_>6guE+AH)^Oe$PMYOmqE)g=*%|{)gKdkcgRzERNh}Amcrg5* zUj-6UWfz1sMA&rV3U~Joa66WFrDzUJA;mTmsis~R{sRm723esd-Cm&H z!(Uxw+m?)@bU#Sm|mx*&i~ zu6uL+FdTI6`@IvSaj&#Z!%#KLw%dl~Aar?svD9MSlr6_p4P9+(yv&M)ynp|6)zeOp zrO7-PG=$y+_yp;;?C@^36x?uuBkPuCBlLLxvv)TbKJ@y-!TIgOO)r0172RKAvrad# ztRgg_QTrE^zwAEH+&9pacH8dGpPJ3;u!S5Ap;aXcUSL!?4 zAkyMWZUP5T0Tqu#TetvI!h+bX>C#6eWi_>JGG%|)h|7Bv1NW$qAf=P;CmR8jlL{zJ zf2ADYYXW>tfUgPgH37aRz}FP`5`RL9jwHX+vY5Y)D`me?>3P^Jl9D){4`-UJIWpe0 zEmk#T1#!0p9RwXafu%Q68`QUxEBk#Mz z+s^?Q%h#-?ZGXS-ac!@JOtU185ir9ee+A6&ivq?vl26wA9&UT&dI{ZpiQBW>>|W36 zz}{C+pO7hkt4Jxy>t|bhX$0EF!_YQpHBqh`0^F~B?}vz0G5e27u^zBdirZ`3@xaWQ zPZ~!A`$A?l--z}Fd-t{Eh%Hj>SEyqauur3X!jH!yjmBgNR8vy+1bPi(KY69~cfk?Tdmp?eGHokf zvuU;~R0BD-J5Q1RTX$5RYgNB- z^jjyI;twBIeJm>a*eyHn0FKI>73?3QGGqXahAQ%g&>EX!UGEin&>wrU}C)w>(s^zOLC57oCLD!a&+&UxihF(9c2^!f^|Zx5@kt>B2Y|oq4p*E zsMql$ikzp4(?%;OQ3npKs)vFZ!nB1qs|{E$z(oP19ATzJ2k#BR*s%jh>XP74LELF8 zsSl;ye`7=0H{Aoy({za?c*?1#xCh2&BBB$Lvk7T_Y%>5j!_a^PB0jdhl2(v)FwpD?zi*UGx zI==}g-U;AOUC{5mofaAUfJY$!8RMRn&)M@!CA~&fp5}`<&Es@Y=NZZ8WWL Date: Wed, 5 Feb 2025 21:56:04 +0000 Subject: [PATCH 77/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 14569 -> 14570 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b49786ec0..eb5788a04 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 21:27:50,171 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3270 INFO] --- -[2025-02-05 21:27:50,171 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3270 INFO] --- +[2025-02-05 21:55:59,007 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3286 INFO] --- -[2025-02-05 21:27:50,171 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 21:27:50,171 submission_checker1.py:3307 INFO] --- -[2025-02-05 21:27:50,171 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 21:27:51,481 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 21:55:59,007 submission_checker1.py:3286 INFO] --- +[2025-02-05 21:55:59,007 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 21:55:59,007 submission_checker1.py:3307 INFO] --- +[2025-02-05 21:55:59,007 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 21:56:00,245 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.264988 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.310163 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412266 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -51,7 +51,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431458 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.43471 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | | 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | | 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -61,7 +61,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 37 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411879 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409259 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.15 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 212abf5f3..39f541d2b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -618 +619 diff --git a/summary.csv b/summary.csv index 6c1bfd58c..7e3d42d28 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.264988,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.310163,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412266,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -29,7 +29,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431458,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.43471,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -39,7 +39,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411879,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409259,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.15,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 878d46ab7..72c2d2986 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.310163, + "Result":0.412266, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -909,7 +909,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431458, + "Result":0.43471, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1209,7 +1209,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.411879, + "Result":0.409259, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4c5309a0b727bbe94ffc957e90de2f74b285b6df..3476784985b6e9fcac557c8f9c6fb9db392edb5c 100644 GIT binary patch delta 4775 zcmYjVc|25m_?|JD>?YeaWHMvMPDLRxwxO{_Lu0axFp{ElFS4Jh$PBU#qY|cUWeH=+ zR`!sKkflUSTstAet^CIQ{qg&qKi>1lIiKe|@B2L8_kBNS;nw1eU)|$@3Wo zwPuN)Oo)(Ap=fU$QTWFLU_BU?dE*S>O(SV2wxiD@Y#V`7>(c}6=iEKL@|k5HOaTpI z9KD9LkW1f5Qp8?vt!gJkn(HuShVtJeJO%2{w1O4WLNadSagC?RDKx!;CI-!T?=+o{ zG?`4J&Weg1S!2CIqY@)BaP3RC@hHyx73l%aJc9dby!QoY;vxGZ zL$AYkDjWX#@YJuX>(DrdbhP`t;c^;nbXx93W?an9u`5wE`FbAmHp4KrLP8#U7gIXQ1du25zq=``KZH8iS4fdT&JjQCb=M4elh^tb#* zF|$w{liF`SNPR<ysVMcN=pq1JON2gp+zU)&=hI9dAhE@sMpv8O<|8)YU}#= zE|0o~TAA*y6u)=?nTS_R&#;>@7eDu}oSK5xui%)6+3rt7?iV-k=MPPXf)|Eg&H7;* z$979(&9kzms|R&J{i;kc>RfCvzuovX0H^Uswo0dVN0PNC4U(X2X?s|kszk>dSmHSJ zBhA`dQ}hY2o!yS+g66u7VTk7rF3_kx{&NnZLX?FkvmB46Mb)n;7hRSpCRy>t`NGX7 zPNW2}o}>CW^={3|lgAFDbF1{82IKhxDDZBD9=yCIl-AgP0&NMtJm|#g5diS2x&nB0 zOYqbN_AgOISt7aCv@FUOwZCRkLeT)DqZ+fuCd`ZJLB%USu0H>PoWttXGNw4kTsBTVDbye;%Rkc?UPS?*%s@CsC#vyU9ki_ceNh zm#Bss)M`6$Oa}3Crqju$iuy>W+ctJ4#3cmMo(zyX)L({v;i!ty!Ym%I@#SxkT529 z-?FPdBxM$I*E?qF(Dj;}F{LlPrIZUMKF$uNy5x%Ma;j90X0x?)*;==(=`Q z(4UsjE(~1$4{iL*trY7I5cyFDZec-nvpfL>m`;fst8Cck2@DdKA;}ZC5}f;Qnrvfw z35G^$2zBpf0+yf9BTuW8(uoSDoSVKx81n$Be7E{U z0HfgEc@Uf_O{p7{4HkoS%V%Q+X{CJY-|5DAJRD#_B{_wMrbILKBmKxz4c{8>XIzak zh`(5Zng7f8BcJn^9b3X^qhub$ zOOpgi_3%zDN(rK!*?g<8g_uICjcd~{*jBm9p#B*ZC$>wGSfssoMpwZDJr$My%s_n# zfQlKhboqos=jLH1k;pu4V`Wb`OWRn5fGwm+80$w8kEnqH_Zm?I(1IT%i$c*G?$ZOqCOg}&C#0cy_9D&vE<})s2yGyQh@1g+r$d$Yh1*5D1 zE24Dse7KzdiTXXQGBe_Pkve>C2r8#cF4^W}Dq7oE(-SU#5XDWkB7=_!O>|MX2^4W!Ex{Bar#B}+hm6#%?5M*Yk3vH(~AM| zr^1w81jgenE@5r~!U6FlXaF~kMZ=tOT-#OUUt$~Pf5(c}zGEgN^3F29p^ePo2$XN6cQp`8dhM*#rlE5aQQOIa*W3(rn%CO;^{{9G8!qd9o zKwkxZfC~M%_4em4T;|r|TYL9sdA(c2WvHQW9|xG(5W4t0^)xg~ucs*an(m--&}W4~ zk&F0T9IELAV664d>LM*VxfgVi=1sNLLQ;n21S$l&3hbon_opG5Y)o!0D3k4y6xddo zw>&&5nTa*xZPz$WJs}Jc))l4FhUC$aX^6ofy z8gTyfHgUY@Epl5LMZ*m)dmz<%5n?>5mn;a~TB%m4bn|~6tkVrksL|YUb-j7%4l_Yc z=foO|t2jjYl5YsdLv=e$hxPNwq8mm~XnHWM8sRUKV&yvCa<&R#=9-KTGgOzbE{ly$ z?g5p>dQ+|0zOaCWp)eA7YS3p(x*P~&VZ366>(UO`x>lH$y?HS{i2QSqo=Y?1EhyPN3KTqOA1SiAD@9uYv*vrDx6@JpgN0Q%p%Qnd@Z_SJn$l zocRo9=rP{No&!5Gphxk5zgn!rUVU24;6#+(eITIVcs`DS2;JE@CYzVJh7fYbc2!s}#{kHt4@ESGk{eb+Hd zf}SVRg=$K^W*0xuyT?>UZRo~wp(1*?i<2~Wagy)BLoK1wRoKc{$z-8Dfk2IihVN5T zOpHTg$cBT0($VC|h9If_oglK>C3n7i!JE2(o?bWk#`R(%Djh7<>Y%jS>lR?C*10ww=bwH2 z>%!G-@0KRb4Y!Cp+(8z2!G#z~Y+|dO)>IkMry#6d#W3CoDXYA{x!N$!Ax;6Df?6*d zvAPodCD?8Clg4Qhw@LWlZ$F6eBWYlcNDqkSr-T3v_^g%-@IAAes9>!cTU%J%=}h%g z{6+#}E@*?4g3141G#v42(VV6=Un0Ucf!cQ&D zEXBS~q{zKav3oend$BQM49#PPK(*YLk8BZ_PO(ckOBb$rOPjcy?}BT%obUR{W;>$h zhk{7Fxc75Qq;Ek7-)Od#V>{WY+a;hOJLE?@+@PitqGR&wg1hhwO1jByj~+zQwbPsr zL6`=Q^{Qhbn7fHgDN#YpAP0$@Io-_hfKR2bj6oXrP9cck&@)@122uuygZ(8S_b?7I zD7+xF``Wd}^VQsoKVdP|Vcc%#k(tSe19{a0IlJ;%oAdY0KiYvgF{MvG2$9PGDr>n2IpL>O6k#&go0# zZ`Y!#X#0U2a`y-U> z@3}=oL{&ER58O+}uUBl`03ymMq=uI``}PfM4D}tT#9r z9vh5hq%yduQW8}GJM`o^X^insOAQ!r$(O8w_;jrzEEe-rL)Jo6it+Tmnvb-diam&it6`)63$_XGj@`Nr9}Iuh@;LUw=vq$ zzJfQ2vgKrIJP-a%t4+-Ir0m@aukhp#n3kt^Yln%1V^$4q_t`h=Ax{by44G4aN?|q! zES>p?mWc1WlDAv2!Z#&kCMcG6ThCf6VmqvSaKcYIU9eE%jc&kDn3&H_3;p9*klUb! z#r3Y7>9Vb*e`~L)e+W>8KG!^>@HfxarxDB>_10WtUJADUc;BXZIg29dZDNet9SOy}yy6cA?{^<_f-4m@8It7RP zzn@8(x=vs`Z|DJ6hl@mu#c>x~p{^ zL1PzuId%gr|Ma0siWyVqOv%>PubI24Tn5Xg37t8wGOQEyxpIh=-Ush+6T7Cu9R~6u zB~EbQy+rK6Fkht>vyVT*7zr&dgC}6jz)CdY@NVwlXwZaQ|MJr7_Yi zJ5FL!`}H2_%zz+%M_|nMu5t1{>c#QmFBay%%x7*+!g)4^gY*$Ie;u15Ad4eNgiEj< z0z%csVe7*u0kYcWOk)GsP7flld5nm|6H&$&J|JI7uP@g{QG2oieozTh@0S zV*Z)3-{VSBD`gh4u&<900SGawow8J(HR_adX6v)ZT^;1b`kKyoBL7L#ahcls1Fea= zKYa$&lVvUt}zRchPmRxhGK_Zis+I-uf6W3p!2gJh%C%Zpayw=u7b{P9IsuAj< zii*YBSBNhv3?DtTI}EgRT;(U$cFcR`{#>*}D;AG!4tO-$rYaO`n|zKLn(LX_el$*g z-5H!(qWJ!G!E*lX+=VAvB8vYAjF`mCpl8e97gN1iSJ_L71F1Agjf%zXbfpH>^>$|L|+lGDGpYy+&Q`W%RlszqW_lB zN`(=Q!lW9|I-fvuQizbU(DFqEy5Fhc{i3 zXEkX4nRykL>aD*qt=qL)LiP^h6ZhV)5)B5uWnX!AY#48B0YWpNi$<0M&Fs^L2wpQ@ zN{}1Fy0KP}TVT_`Km)!OuX&v-b+kdeR&+xDcTe0XvpG`HT!zy~dm@1UhEntmIdO*8 z_puq?f3>NbTk&}MPAItWuf;(D3;Q?b8b}7U^4M952QnTB#ZGz(0a+lWMcH1W2O}~` zm=Q_M0{H~fYM2u^WJ?^=ZT`7TIoTDnE~kQcwI@wie!m-PseP-47M6KRJj(ufuoy*^ zb2DhK@{Wv%*9X9YhD^3C?B#i(azHLiQ${W(SpX!M6sIrO3#fwZ4B|&F!qgs=k8l;c z;wvW9UCh{ARwFQud>oF-?Jv_nGj*L`2rF9u>zMvWR8$cTD? zeccGvBzrsFX7y*1W(&kgi?Oy4wEpf%y&HHo$(>}g_Osf-$?z?9=G{q*51pzrZ_^O` zw6dsLKqF|5%gk9OR6=@9;6i*X%6>B%{@*JA*hx9O{+wuuEfEyF!@p1tX)4IY7_f?! z%vR^xLt@~X&E>j2VOPGJtt32{w=U01yHq5dE?hEN$$1vz3!BbH#EsrERJfg}3^-;8 zfNGG?S%5u028JBjZxjCwj$fAU^K;%xX>bkAL9rk#uCUDWi&M{zRo3e!4SP%O4KJ5ZY$6HAvVnv(U21|LU@Zf=;4%StDw>aMMZ{=oF`_I%8w}UU9BeczkCMjyV8O09 z&$8%Bh9?U0vO3ELjYWh0QzLLj>1_*M0wX0BWyywsM<`?1-C!h2o)#$6j03^AzBDY! zTn1@!s{uJqxILp%<-#{WJU+nvA4N_ao7W3qS2n1DDN&FPCv_Xms~k=+ydH|WxlVrtWqgGvEs+W9^U&+(CM z8UMKK6_x1P7`D!H+oRN@tqAEvhd4iI%-bZ>AJRsxwY~02Z{!0Mn3q|d6NYy`c*)v1 zlLv}BcrE(Lz~%nMqZ76CZ|QsleyiD{Cc3XSA1jIQLUip6cso#lxe8p&>L3(h4QAQ7 zv2d5LdR`ZAaE)h6Gt677-$eaz&34i1sj8vP3J*9vWl9}3_vgk&>Kmfs=FqNbwCd&n ztjPU2IJR-5k14a@!Iru^<7#}&U2Y$N|r@WvsXyGsXT* z;f}~&pwHqPV4QQ@HjhLy0=?0N^1XM-#0G&n?7T=wA{X)phQbo*f;vT8se*1zHN%jo9SSYLnXx=;O zha5j18@Hq^VA&|Rl+8;d6EzV8#o!Av3Bl7YC999%)m4LH>DyFmkKi_?8MerXy+J1} z97l5bbWmeo0XQ{}stb!RBu#%YHAhvrsV)$Fh=P*Io_V@Sg~|&pcGl0W`f-_qzHvAW zEBH&^sKd{PU^@2(oL5%`Y6Yi)RoWxsO=z-oZVKA&lzab_3%TDxFEK|CK-Y!M{ry9J z?y5nMdY=b0PYwSLS9zW_o^FNLEB7u6hyfPXG8kk8l1U092St|O^dyq7G$@@Hg=WU# zwN7P$VpXfWFR`hIjiax1HPqRr@Q>!lU)nbavUX0~)C~!$Z{p`-Gb-oKwM;xh)I>Bs zw%qav4sStWj$C0)z$%Erc7I&d8eFzrqixTs@?Uqdlsa|y+JTjR2FI|?Oj+xvA8TT1 z{=@*hJTZ;0fdF0%&`>VECdtW$%e;ipPB~9!_CpZxSyU6m4K@vUaWgpXUly|l*K7x4 z>}AyHOo|1jF5F40pA8oAHq5Wpspq;-Xm^b-{sxUt{$GxA3wMzNCPL(LX5R&hSu-cj zz;1Cro*N}JyMBB?;`Cs0`v;TMu>DO%Jwj(Z$h71S$;6-SB z4U2mVR1?0l;hgQ)`0S7Xm5a@e@F$*ISIFg0DLbZ3PLwcCpv~#YG&uz~LT+*ZT<_iQ zzb>^9_gkiT2R%wML_aJsx>+{+=x8ydw!V>Zt8UHSu6-%YyMQVnN*ua#Pgujd;9TVB zEwUJ|VMXNg!po0wEP3uQL-9_WjG=_!->RP_|1E{^8aY2J2A43~qXI^o>cdYvp5~tY zBPud~ba{-lc`Yw6NlqyOe|%eH*GP^+n*}bb|``bNnJbRy$$=DF#%6rqB>Llp`b(tq(2}CJTil4u&6t$T14E%3!F=r>} zZ}$OFv4PyBQV1reXHeN?34{O4DSXFIj)6-9$D-(=H!E_ztbR+Va7w?Zz@4I#T;cIJ zxBAVH!t#(Ub5V<8R93?$-Lm(AI2Hm&X`n6B9dY3!P!T#fQUGNCcG3Mt`8P)2&Y)LicQg;QTgTsZZ0DjJmA zGw;d>GNpCj!puLT#)_zLFQ?H@(N9|v6{P8UyRx1wtQt-7`;GFpw5{7?OLh}X(dU1Y z2v-y8g|C6XB`}=F5ls?5Nh?J`@rY`y<%%o3hF}^Q!QBZei}V2D%cHnl7Pmrd!A%aDgEwFT&k^Ze#wog+|GSjp)gSVdv}NDE zA0}ih9`3bVJ%9INcba%K`Eck59W_->Kj4|{3{Y(-KnDOb74@vRG9tt2GH1I@m$^H) zxl_7L+%|+M-Db-BZR^&PwdGYJ?qWMUavWCE5*H9g_Gmd|Nu4?}B9X}!EeAb~yD%dHY3`?NZ^OwCXg1xTpCoqw z{ci+uP~5Y@?ebNj-V|aeb@F=qx3Ks@h+%<953)#atqjX*VY%iC4~7lH;aC)A%@u|L zyjr2g`-c~-!Hs2e1-+GGd-U>MqjZ4Se&i<{{!*yUO8U9$)vD5yyy18GpnTf->>q_j zjJPa&ZJvCAW`Xlq(z#^*dm^YfafK)V#l=>c^;J)|?KbIa0{((BF=Lec zjg8UhE)m7oC)jBZuHs|9Mc zpGD{8Z`)}IUM{2q7o;}xk!(M*?~&S|r>P*!q^Lb1NW1o#uu}eiV)421^FO&Gn{P$w zWMnme(v?q86xAY1zFNC)7g=;GN-iU-=#%c|urpn}%^(N5cy&H-n91;LFRB+p5DQ!W za(n8hvw1p)bF4s;Dro>bdtCDMfF1;dHB-`#^GaS=m3O4om3qWos}%NKss$!Wg8*iy zsI<)NR{Cm$jdbFR)#pi_ee0x(|xrT72R6;?POK zq3rvXl{zi>#1qd!yOrYIZx=e~*T(4EvJo3qmzh0%x_hLU9{10-jV7cur+Jh{MmA%M zu2({!m=V#7a1_sN33E9rWj8mJNW=n0w14$vK}qPfdHRsG3ftAYMxhzl;f_CMUpNEL zgUgr`Op?O(q3Cey^7mrKof780uJPE<8}Q32oW@ZvGP2Lm$pi?6+wGhSq>SCAgI&03 z&=bBQ%t0eMPzY6s@a#^*-#J4)g%M|FBd zO?BJWTtV~0(tkAC=jVeZ|EYHzd{Hq&D@;k6wj)}Xw!*mbb60>o8pfAYI z__5EQ5EU9bwR2;;{=;5RZD;3jOt-VTy?}74h_myr%DJk7(2`VQgN#%1klS))--tp* zRc1qoul&<1qCLNU&Q+!K^+ewtNe(}1zrVd4VmUIfK`h>KCc=*ULpC}~cM?|NQsCkW zK|~ohL(SdQjD${)M_on?rzJ3Xw%BS&z>Q4`3OlOeO*W(KQJ_-o1 zs2MF#`NKG$-6zm-DU7#woF%_XVE+zv@uE?vi+$9VB2vF97J4nlg>r4l+27qd_f)?_6j*A$utJD@a!R3~oB(77pXE9a zf#|3KV))V<1`*Nht~l>p4}P$lc${^iXPkBR%GpAWAd+cz|F z735fonzn+6vZoU@T$-zETSoLX9Xms9TlpR!$+_qyz6ZJbubzkjLZk~nQPkjq^N2GzLpTF zstKMVz;gG?+c1S?wazmn)}F F{s*#k$;SWy diff --git a/summary_results.json b/summary_results.json index 13bab06a1..086bad1c4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.431458, + "Performance_Result": 0.43471, "Performance_Units": "Samples/s" }, { @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.310163, + "Performance_Result": 0.412266, "Performance_Units": "Samples/s" }, { @@ -1652,7 +1652,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.411879, + "Performance_Result": 0.409259, "Performance_Units": "Samples/s" }, { From 1dd90fa63ba9b14d6d971339bde3c9fa21ca9dc7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Feb 2025 22:03:18 +0000 Subject: [PATCH 78/86] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 14570 -> 14572 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index eb5788a04..1b53c2274 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 21:55:59,007 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3270 INFO] --- -[2025-02-05 21:55:59,007 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-05 22:03:13,857 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-05 22:03:13,857 submission_checker1.py:3270 INFO] --- +[2025-02-05 22:03:13,858 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-05 22:03:13,858 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-05 22:03:13,858 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3286 INFO] --- -[2025-02-05 21:55:59,007 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 21:55:59,007 submission_checker1.py:3307 INFO] --- -[2025-02-05 21:55:59,007 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 21:56:00,245 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-05 22:03:13,858 submission_checker1.py:3286 INFO] --- +[2025-02-05 22:03:13,858 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-05 22:03:13,858 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-05 22:03:13,858 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-05 22:03:13,858 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-05 22:03:13,858 submission_checker1.py:3307 INFO] --- +[2025-02-05 22:03:13,858 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-05 22:03:14,932 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.5612 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.35302 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72452 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74836 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409259 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.15 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2328 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -74,7 +74,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22992 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31453 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 39f541d2b..60ac764e9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -619 +620 diff --git a/summary.csv b/summary.csv index 7e3d42d28..b1ce3b836 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.5612,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.35302,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72452,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74836,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409259,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.15,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2328,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -52,7 +52,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22992,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31453,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 72c2d2986..0bce6e343 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72452, + "Result":1.74836, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.15, - "Accuracy":"acc: 75.800", + "Result":10.2328, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -1599,7 +1599,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.22992, + "Result":6.31453, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 3476784985b6e9fcac557c8f9c6fb9db392edb5c..c1318b9e7b55af395673aaf310addbb97e3b2d6d 100644 GIT binary patch delta 6681 zcmZ8mWmpt#+uo&1VrdXq)}Ua*`97 z9#A3iU8`pgXIOX(`<_4G&6E~j?9 zG=t_T{esDJt&B^Nmp9uRB*dAIu@=jmG{0v3A3?C0H#!i!?KvpKIzSnMmoBfa(iyRg ziIF<80MXQ~aH3<1a%@!Mih@)i*sb5fw#dGx%gHgBngtY4%OZJr-{F~g+=msjbE-6{ zdSWI$v(8913?fc)L?E5!AbsUQQGZy_=y1$-=ev<4B;9TIO@q{-`@^LUKlu(BHMZfG zue5||K4H~;cWlbi{n?h2+bc}`Z*4XluaJM-Z_j@0xgH$Oof1yIX@f{}2np57iz8n) z!M!|9|M=-Y-KRsg4{th_{n$2@6RKS}ns#_^l7GKeUV9*JW_=uc;`J>idn6#gPN+G% zYPaH9*;dFyHlYvnbJ}rOk!O>gUHJ=qwdrnT`xJGxe(41~GELhjyW5U4j}giT-v)G| zo33YkdYQ6OKL!shgFt40<5*_r zTd_%8cN+<;ECUpyXt;iDSrdWp(U3Bu)dz(^as2KeD!cimn0uGA(_NbFaf3GRI@(x< z29ve=K2WPcQ-())j^3b)t`pK|vO@JaOE($IBEIjqDQ$cETPg&ey8e?xCFCo&tE%Vd z=wH)(r@JY(gkbf8lkYoXF&@+*jS_m)9Ri9aD{Womo)yckAadg77-Hd~MGNF4)&9EeiD^#PeD^jnR1NX&3N9!OIk(AGKr zEn?_p8e{E5l~wlOZ4kA9sU`r;;m+ghH~HLlN#`!d9r_PlH~QcAPgFD+9u%#azg1Qm zgbV@Vp!hIbB2z;-h9PAF>q&#c5i&sB@Fxg^)6f+Xc5iSunIczmVV8KWzGT6Tt-g$~ z{x@H@QA2u?!RCG+(rrAiMm06{=Y991;R4w%!*emUXW0Na(h%vcIzxODtSi=mc+A{BYQv!x9#<{Pe)wwiEtakG z$gxa_UJjG`mW)bWm02*LpZf1D0z-hhOhwE1L}M7L6B$V6Z38z0dP|$j?i_)NmqeHgzK2c+V`FtgwyzP(PVxuK~&^I?M#W-#~_XiUE4U zi3a;AFwP-spi{u$DIJbj1-EyAT*T0WsGJTMfuA3oWYQrDV9~L(CAUuG&gD!BnRQz> zZgfCmB+x#*f@EA##aw8IMs!*qXb~90#SBijN11#EH#8)gY!vaLtIL!!3>EAj5{l>p z++j-&>~1g{p}v$Xv7>xOb3z~QvL^hy;D!v(HJ#67pP)J<(^HW6g9(k?Vv zf@X*pf%a1;*`}b5H6hxfcpRHL&Ich((kXb-M+V4~tBavikftq6p&)nESg5LNoo5LY zi8Zgh6JqG#?&MP{mQ4OATB>+L8eZ~`iik$wIj(f*@YgugfpkNWYotHok4ROj!-IwY z%o@M0D#m(|2r{!h%5I*hQA4(e^^Iwi9nzK9S~!U?jWNh%MEFG2!FPj3TGOh9R1n4v zXkLU0oMrpu2ATVKe*!H=#(!kAiqqAFcqSz&@HOlnsRJ{dH;0bu zOZl3ss3<0YUGXjZHaGo^U$r;?hwNg3-!Z6(QK7t7M}I2)b;lSlkUP#i{j3+;uZ$W$ z0#sLaogyY@G6L8d!+i%IvGmhk1n;BlW0Rp-Ku?+xz~7idjoEIBGa-admoXwP2-lUJ zhPHTJ;oJtuH7^h-e40-Xs$14>8@)wiqFFYziG){U#9!v>`z#1f>D`NOAvcaTFP@Zv zmzb-F4D+L%ktp*%5tK77a!0mZu0f1|9;XYB&g6r;pS0S0({-msQX5nBNDf*52}J;J zyIkBV)V<$CsTXTsFn4q3QYNbx**6SirFfY zRM9G-Ha937ZY4D-XAZEEdIXSTuQXUXPH#(}q+I~}i!1RL7bxL?qxhvG99I_t5+R{v zG3L(!d0UM@I`QhM_VuLS&C0+jdvn$bK=P|beISGr`al6_%M4C-mBxo&BJYBYfbf%l z6wG%1iJrhD>vdN+h$eGD0-=#E!{4vkZAUk0ByKA-%!iJ`>t9LMi+Q9D)as^Fo~jjT zLO8TJl5m*UoSc0k3fDHw-56H@fyw1J4kKX0$vb&7G6@{yq+YS`h|>8G#$}4&Y4}l* zfKlkjcdrV3kiMFNg|x^n1aR9|=0U9*4<)&Jktjqy=PnL&3J=1$Cen>(%e;!&ry_WI zDTYh)-ARCg8Nc+uX)1I?SU#L0XO4^y_wKhGH(&}Ld*`9~q<~hi^-iOnJ=>M=^XT28 zN%myj9!hmp)9uS7Y7wT@%X$i7b?!)tOM6_3%qI(SW=MYV|KM$f{fkHNTFn@C5-B<# zI2-UnldXWZYx{gFU7MCMf^tZW))b0Kry@5+TqIE|FKQz#6=I1t|u1?$TE-9`cL z@ht9OsG99vvg>uZymyn`{JqlPa-LNh77c4&*V}>Zo$m>cJKW7ohjzlW}J|V zeNt{4OwsdpF<=n9DLSfJEJ&TC+@2ZEHKNWB=Vr)~9N_s~j|bAQ3abUy5u1tl&Ubf4 zxILV@Q)plE9UWaUzq^1sQY-gMg9%4!8q5YZBUCEazfvy0K4rp&t4!_Myfz7Z2#tJ7 zjwnPprCOZH~(Ev?4V2cBy8zaLACexd1*M{AJo zW3)J0+3HCG9I)AC!p5rT&*9K^q@j^?L|f7@d^%eqzLN#%YQW>I6rZ2Sa|O;p>Am`} zQ?jMNVpfy7YB7_#5<(Z!%oU5E8hbS7*DAga>6({%625!N*29$v8XSXAebqyh(>cuT9GeN4cuajZ`7pDqsGrK{<>@idJ?U#TW`w&G#vN|3O z+VvM?T1ad+hU9)SU66^oTlVz7Lf-o;WEU*H$z>d?yp zwS)iTOuN5woVG1E{n%{iGQ91ZcS!Y=&gM)p<$bg$1ljNlNDimOmB~}0HaT3;feeV- zPAcT80WY=FzU6;>tn)5kUJB=FOgNY1=tQVz!zWW*@>V$?s zkY49N@}ajb(UB;{^kj<0SN2KvD8*gTAQRwFVV%NU_`-iDx;1mJ3#GU<^VZz)SFqJ) z3+6OR*hvz3Z0r$IY4<5$f}c2|PfQQT8s`PD9QWR4jq2|e>9$58`|Uy)aP(7L!q~|f z{-uBDj%=a{Nv$rN+FBi$*YkKkO?>>81v8946 zRNy1gQENWN(UD10fRShi6fSPcDf8W_4-O`5$9YL19gM?wer-8l4d=M+oS14c@VoW& zaY4-Z6f%h4)SMb)21c~}$SU^PdYG=xsj>4A1{LbX@uyC-g%hoMV~sxwUa->3-!0i1 zOyABx!134g^qH+WWqmL!-v~3BJ^#F~ncuh&$xnD@#qehrveM63ZXcz{*y51pcq-7L z{{v%NqrB13$EBygVe>jYH9(zYfg&luga|orZ~hCABbO8XL_s1%I*~9B*J7k&gJGn9 zH5^XzGofaFDS#|yJc!$Lh%7?k8=c=P#H0IFW72(WniSMr{cH@F4-Z=#}P4wZ3Rc@!-&- zmBfDW;p(vY1*Pf)DXMA4MMs&X%y8%+&GZjBIlEIFp!u+SzwBx8WbF#R_wobs-;*sd z(fx;TA#2X_Lb#FwOc08W7mx^J=lWf6PM8L}Pw>UxP6*e{KPCSNgIC`z1o^3GlE*}S z{cB?c;(u)nkfu$daEegC@v?J_vOPrMT1OGe2Jn|gH>mT2= zgPG$xg`{8*2`zbhg#_F1WMnG``qjb z^;($;g)+B>UKwlO8ZB1QYkr7hs4`D%yu{C$z4`fQW>r?6<@5Nb=|}q_@&XJj;$Kg{ z1_y_FoNcb}J-;q%v%C^BzwemR@xfv(LQl@>x<;zav1#Gxr%;vh(cjNDAqr==R88uT z5+E|8Jd4PP2mlc00|4j%0D!-Xke8E>h}kAj~#?W zi5DCHXz*Icl7+o_Imcb!pRb=#LvV1O+_Fafd#>(P9&x%1w8uu5ZmIG8joDI7tIQ|) z4ZWCWVh)O+f@W{}x76`neN@;X`0lUoN(kgz>K|@<9X6oplvx3X8ji9LY@!3n{$kH~ zY8=PP)UpRU9a-LKX_tyIWP4y#?=#+((#C5W>|E|;>9a?%^vusfBxB46H5)SvnqA7K zl}zvcNttxxq;WPY`vjAExKSpF*6p>6FJO0mS4d4M%hG{JGF(se=r(;aUj`!&c#@6O zXE1vVR*SqMcf-7Lbt%d`htTd1 z#>T!W!f@o2F)57o5pc6{#!*y~WJ!;j=xOLyng`31lD8k{Fr#M(tlj_!al3s0db2s1 zRC+&yv11CWD6%2s#vgDh@x75|GZjfLf@u?=s2zsU3ozMr@t;}zjIF0hnCcX^`7-BL z+X1KJdN9pB-s=KY5@H=&Yz2+u$-hV&5;I_Tf@Pj@&=n+D&^%HWLVxN}u{Ca!E*=rA z5>V(rnykd;Y3qYY*}uQN@jFpTk-ELY^7ieS=wc*qJEcEJD`%WQ;16RjsgSyZeQZxH zT31bzH{qdur61jS1e5z=;|a6drUSIeOJCHAjGLh^+a;Zzm8-NE8ZKFWq3(J7E8>^b zh;zJ1hQ2h#(jZrCgqlW~!4uV<+%mWF=FKs6k%t#tuQI&$_3nDO{-V~)E~({NS7DYS z0!sql5e}A!3anFt-SIt#n4^$SGsa#Bb|3S1GAs{}$WU_M3^s;vmDpv+^6ReP1z+-G zW`fwqVxCvFf0Rwk93FHqT*Q91R-x&v{d8Pr^kSJzCu`m2jd7#KA``xLgllX?SHnvy zIE=XIk?@}a>!6^9a#s!B9g-jJ4vdlF4)fkv4Zkyn7v|}HHBOW;zW!MxZAGai;|)If z1&e6~_fl@r^0k%4npA6;i@`QG-=o7Bb17na(+)*v*DtZh-`c#MbR7FA7%K0p&-GZo zjpbY6_fZ7oJw3bJ36P1PWVc*SfI?|} z&mizUclR8@c{ad`cGVd#SQ}MGdMN&P(D?p#$=y|FV6Y&nj_?rK0(j5RJ%>5hM=Hf7 zKtD`3nsg>OJABaF4mgMDW2Ew_9NR1X3>#g|8WyvFXPsoNQa&%T6dqx_b+*4F> z02E?pIlqX%N_oB*PcJl$t^l?K9E6oViP61&qI&d=_}jAC3`pZ`S7DlB|MlI{z<6St6Zo;a5xp30Lh zR<&y+sO005!RsPuDCn%e>{tbCcadAtX0_7PEr=~LREFJMWUsU_vhv*BBU|WZJLI&t z%jy!;_o}#l^6AtxS`b;@@F$jMG!y&?{aBQ#`u7aSD5dksf$m-}Awc6@x!e5(kPR6&v(5-FVX*;Y|9ra}yC(fqSqcCIpXe@jwFY@I_6c4cCC(!lDwjCVKfAAtpV zRfj%d?j(=Y)3z$+|&PM3e4Z|)!9JPyv23V z^o}5!Bu1i@rhRXTvM%VYlC37kk>qyz54D9D@-AU_a;`b1A7!_|$C~&ADc>bonj-m? zNeD#n&&E?mLO~dw+bXbskot_2CM8DU;0R{Qk+VGfy#JiwvK2>-Y<=KP|x_97ftD zKXvIPg%QVEuUHd02Kh zT~jI@Usg>lb2tXGakd+}rhQ9TuA_XTEsLM<&Rt&dn7HYV7&HEVf7A#_ngih}(UK6> z|21=*pZ~j|!d6rIBo%>9DMV7zz^^HaQrxV6IpzWa0FUti0Gj{55VuE#b)S;03o!XAiWAG9qGM_ zROtvpgiDnwRgpK|_nrBEeCNkAv-df(X79DuQ)bTQll3QM51FZt4rUYU65N>;!Bh#k z%x1>EO*Lrd{G7IB_mu7NmgT6^mZS5BNB!=9j>y78$<^7d-sP4pXt`3%_0*_E9R#-w zPi1ki_3tJ3n;DFxF8h_0fv#~C!RU!JvE;k_I(i8@Ect-w#}JF$t)K!X=N#>x;>qu` z&Y`mf0SfDZFH33q+x~0?ALiY(k1ea^DZy|bXl{_p7%kreA5tuzxy|3?`*ve2(x)V5 zekG)G_IIOHaH>+A-F*K`mWoBEcUe}SS3=EFjP*PCvIlPQIkiVxIcAFrZ|lM5ql1Ow zunK#ZME<=GWWE~bX|{$?C!%G=I^jkFcN>B0yNe^8ckCUV@<^p)nt&`Sl2C(Q%OM=a ziXj}@t2)g>bZ(M(zT`DWX94xM+QDLp{>gqw?Zz9JIJ{DRO9Ec~^ags6zfyT zO$d-6+@-uOgDKb>l7D5RBlqs(6MhA2zS_x>=+}d(4U8vuc3CzBqaf!KUz&oBDjS}S zWx4nEUYI4L`TC4hHxu!bi~R3WBg2mbAB5JN_naRBYJ>BK%z&LeR5nsNdw+M2d}{6l zD5YK(PwoQ5yT*-n3yOn7eU;PGm-opjvTAn!MlG+APrqOr(RP1ijMREw71q67IXTrV zuB%UX^^KgL{Ia`e^JK0{Le7bEV0Pn88rgL%e|T_bJ@Da4K!rl8QA>I#iYe*l&q0q( zZ>7}qS3?`;MX`I!4#9PW=QXn`A}gC}Kr!0<$#TkNbDQ%FCrppxzL{dFc8;@k-1pye zMlmD6+cSO=w^O^UE(0%6(_gh!fhf6;2mp$oia7t%G$h#-VRnvS8R&&y!G$Z z_oVQMZEQy9MnRKEY{j4lpf??|{AF=`Jf(`md4FTA;&^J$ySE5!2EvNJ2UxMD9XzvU zd=)n%N?u(lZOm-P$NPIzlM-H?w6w+Q z(VL(lDH1DgCBm1^c-9rrmc5)V*`v@EYv727L`&%1y{Lc_Cm>byw8?}^tqKPVggLOW z@sZq`+^(UjzvGd%w_1JF*9?JjmP{lj^B!MfX#Ed~A_w+jv_5U53%AbP)i|HbLg7Jj zyK6PL)!YFOt>^%iSl^j}+3Q!8ZepaB0W+@54^C4c*Y{e zi^9}umWH|rpKoiFU}eGZ(8jc>Ih|sHPw|iAM}@~f4wC6T3qDq3FDd9Z``m_y$;303 zA>~_z9}snxo#IrqL7b}8^(COP!eAN4f3)Y%owzkHvDk&iK8#xJOW6V64VAF!`hP_-5x?nY7*E#twMCgbmq14MmhSS`=&!`wLB=#*R52lk8alHtBPpEA;# z?;Rt}=A~4GxSgw(IJA2Ig25q;jx+%`5SVh%9`aZbH2x==T^-Z+#JKtbr}b$f}Y^WS=3xNqu4{Rs;8oLL@8>9S?TA_M)>J;rJ#t8(xu4Z)XOMJ7Dicf*Z@d zlLKnY!_Coe^0ynlc1hUMF!R+iE&hTalEEG`Z!lA=wN&(;Zdkcltqc?>NA&V zTI+8|cg-e4>?7E6P42iDNaC@HaVRJqEE$b6tE_p`RlG8d{RHtgtC!4KZ;}?#+LtBq=$R>aC0ggu^HFY-Y6K zIk}?qFj%gFx`ZQlrh>Yp8KMBsuC5${x+D$qK5Y~>1FgA3(y(YH^G}$N>aL051k#zKx9OD`5bHq+aQ za%5;d>RXi<%?*F73XC$reNoEc1ri(^nh9-QHB)tbCDOqj;@@m)-eW^x`VCSB+COy1 zj6~Ar&tut&_?6>yHFW0bDTWelq4@%X2_-tivt&f*oY|-}tB1rLIdCTS;U7p7gRm;B zH~Y0Ak<|&2x!m^im?WcR%LMN(flJW(le`4ebbF5Nz7t`e5n>{n7Iy_<1d_1%1<=B77tfYmviJ6`Wx9a^Y=!A6>zUdnmlOKbNmH=(LjZfo~We_tV3RCE)~i{u(~lwEVneHYigqOJ#uzw?el z>RO3D0hqj^&@^6@I28u9%&_g068eB>Sp5^hQu~P%9Ya+c4<$#!j>Wj6zg>}CXh|NB zPBVS00A_vO-@@-&)T-6^Yxa-psx>D-!oRWJuB&{B!2@Xwz#HJ2{2ms0tyt3Yf&W|}`MClbS^-bJ4o=Sey&9R6 z;U)L|;atX?8N(`|?P;xEKkmvqnZp)AojbxEmvNVwqKpeT4*Hba0bI`L)pvN2N-l&4 zT8y(jRS}ZqNTay7S0Zyxqf}rmoUvV3qu#M}MUoM*^0hXvXo$c$OEWD|5fj2y-ElDT zD7Lr2;qMu}Ly)xGj9wvV8ZJ1SCQV2MOEHe|)LjseO-lI>Pw1!me|XP_bEG)~+qn(X zeFVOm^{i-nDkGH3U1qy7wf07|ISIMT9hxX>1Uw8%gGR_epz{59c>HD_o)GEFEYeHW zEnh4_WVuttHqGhg0&{uiZJpy|zZRKU%AD8&_33h@o^8LkQcF7iFLfArGxPVQbvsc8#>LUZY5@vZJ7BeijNx*hFxE=tC+7&sYQ{E?7nI-z>vq{XO1B zQWDKZbrnt)yx-tfnj~{gJLrb9rMSzZw0dulIm+kWsk)dcfDm3=FEf#`<7d^BhuHFy z!18_hYU=sMeXhlQt{l7Bl{Zsh@n53%%vSGbq3KoOVO}(}sxlC-c-0V) zQN~aQJ?JeWtp_(qQTn#K(1gZEfwh+_cOQV;0xBIdRtR1Q%xUL>NPk=KrZ%j)Q{{T1EA&(QA9s({7|GcGI0?J%74`h^39gZ*Wc=@QEq?*f{ z4sZK*jpNsCwTKRnYX8sqT~4ojJeHlTnW2OMR=uddcibgHpL03vmNDXxhkIaX2rG*; zNa&PL0K&{d$z-SrW;%;{IVlW!cR0|n%X-M^(I4}b>i7d{Q_0432@mNAbB7s#S%$E{ zs#hKnzLbB9qJa#gQ-B1P=@UcX@`g6{+9?aRMH66Ww5Nyqd&VQ}R8%_?6NNl7#S{Vgv;$xSuk= zaZ%KsZKqlz}S@6Vl+rIdtTMk+!m@ec`*W z2yjdmDEq#>`2Oitq#>KAtN>@cAWj(`#z=!!k%3Smpfd4rw~VSabBJ8-u&Zv+&7AtR zbsrzTaGDsl@K{u*=Z}@RWLvlnXu__pi z#Q6CjX(gG;g3&Y*l(?mkI11|TvCIrQ@3^Snws$|`?$(ch3xBgrxKAGvZ}a%%tMG|T z6HpLeO$~cT9gn)5u4f%F??hnF&Ykbj+>KI zl(7N^wydJg0-XA}pDEfrz0RRT+*SFcR60eH6e8IS;H|MbMWi|wG}DMp+k^q}0g{6~Hl48f(XY@taUj)DSn@}J68Nj^%q zN&cex%C}__kqO@g5T;^-Juv1MJ*+RZ^bMW4rzI9V7(6D9I7)ePdZ}xYW z%b>x-aF`H@GrqH%BUL??^_TJQ=lYzNsazY&lVe@0$i2I*a1^r%;dXW&b*HB6-zN5) z=Wi3keGyi#PF2TRZ*Tuk{Fbs zI+u|(1SP>2E!!Xf`sWILCjx#U%Co4~|WwtQ|E7e`VgdIlgiZZbF$im&$)UG_V+)(Q}m?@ZkdMowo*$-jC=)IG5&#pGr)B%zoV8tJQrSjz42BiKRsMHhA z15;;ZTr03h*cX8TR_5Q8prPOFvpzL{RymIMq<@{QnOV3vbD?IcAkj&>?KYx<#Ml1k zG^zZVL!A~Y-ytE}W@6I5xMAFP<+w*BW;hyq-7p~`My!}$n^lde9s6`NAh?$JmO1;g z0o4_KHp)`8YXAGsk52mBKhp$O`*!fbfjs9`U;Mz4NF?~&}KaPEig=IzJly=B2W4X1; zo@KB1buEFFRo*ypd2)=_!vJW-_CeCzb^XeWB@0mhohQAoA0148W)SgY*M+eO2IfU~ z`mD(wRXgm5kWRx!?@a-x4Y<``aN##Son7f={=}H+sAM`!)7>|ZI&a;78!fc;^u>m} zIZD*RqPkvX=XFoV%iH?9If?P=b#+_$Pu_~%d!xbv>|bJO3Rd!yk$e-bv8tEgBBx3W zQ`lBRg1LiKB4384M0XHcKY06SlWZm|agob8K3%s{!_>(t*`8p2*o2SyChtn;6 zq{rw3cn%vTm%D!|Y*sCWz0G0|`~Czo#^WNF>&+fe9wBPOulK>mvU~BZGX{R^SrGhn z`X6$4RNY~3Y;C#!j>Le%(B-Gu^nCn8UfkCUOYLtgp7IiJ<7~ZjB`3oaY#Da4=}W^u zBFsE&y4rfHs3b+4;&-Kt=c$5EKP8L<0)u7lKyQk6h#>E1Z;tjf`LlzzKx6t)WC6>i<2K3{ldKvv z0Jq0ud;4E&pV>9+`{11Ie8V>!N^QEJo=N#aIHM0iAwFt+-rJ{5BDqgWmGbWwrZ~^W z+^?gxvrW5Fg;sR#zhR+e$^R-)3myI4z&D8BUvmIbyYKYC#I^?VWC` zqj%s=Ow}Ip&VVAmIEWo`R7ey`;3I=|;em3~3bphUH;ONuB&WP$O#v zD+x@e*5o_M!>$?^cuG)fcK?dIDkcmvd6EiE;px_E`hgNvn=H!$RR?T`R718Kx@Lz2 zxS}RYYCM3J?fV|3;f4)x=1W&QAk?3CqNuu+iN>g3Tb@CiCz5smVYd9MyZul~>%8R< zjwlHf_ew~v7l<@pr}Fq7`p6Banvs~41#h96{j@rZUY}jen_`@j>Kds2;pOn4L;Mxw zc?hiXrKfho^QK?cPg^mp{iY%#^-5zV_~+L`I^n}txk`@4 z1qYaUe{LIZc2I-#s|#J^rh&6$c&|>4Tahuc67l-(zjOt>V%l6pIcvXrX>N8czCU@z zQ^+iWq!1aYUsFuCJ;5+!_*GS**jNlpG zU-uNXlyI(emcr6rcw2~`N*Iur24>sxIBFmseeeF!w6lK7HY#?hI+Qr39Qx~%FBwiW zk1c+4=Id)(r_y`#lzJl!#qlYgT_^MPznVX!evXuoFS6e<<|ufNGRYF)v!3u1RS0uo z@I>*JVQ^7Y$Xo4(;fM3Q&&!>Hh-0E{Sfl-TrHPKJWKtA{|uxxbBohylq{W(!=pCB^pua?sI0DA zZ^mNj!N#vzSD7&{DVjpLTOyyS4rV4qn`PP;t=E4mzx9{KsU2vv?E2bb*C7JoAZ%oi z8RGKLl>z$kfRxKmQhHoQmbkek>CIFgzl^gydzh)he4@B};q_B{fKsa~S*~+K{_ppT ztfCDVnLtn!{=XhwrpN*Qme8%J0e+A`tE3E`O3+dgR7Xp#p&}f&Slv`=>4; TObG#IO6X7$q3ONx_s#zRmJ?Xz diff --git a/summary_results.json b/summary_results.json index 086bad1c4..4c04f6ef8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.15, + "Performance_Result": 10.2328, "Performance_Units": "Samples/s" }, { @@ -1256,7 +1256,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72452, + "Performance_Result": 1.74836, "Performance_Units": "Samples/s" }, { @@ -1364,7 +1364,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.22992, + "Performance_Result": 6.31453, "Performance_Units": "Samples/s" }, { From edb9729dba6957349dd15c1dd8f8d09ba5ade107 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Feb 2025 10:11:08 +0000 Subject: [PATCH 79/86] Added results summary --- .../README.md | 262 ++++----- .../summary.html | 337 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 263 ++++----- .../summary.html | 338 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 263 ++++----- .../summary.html | 338 ++++++----- .../README.md | 263 ++++----- .../summary.html | 338 ++++++----- .../README.md | 215 +++++++ .../summary.html | 533 +++++++++++++++++ .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 274 ++++----- .../summary.html | 349 +++++++----- .../README.md | 219 +++++++ .../summary.html | 537 ++++++++++++++++++ .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 219 +++++++ .../summary.html | 537 ++++++++++++++++++ .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 263 ++++----- .../summary.html | 338 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 219 +++++++ .../summary.html | 537 ++++++++++++++++++ .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 263 ++++----- .../summary.html | 338 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- .../README.md | 272 ++++----- .../summary.html | 347 ++++++----- 92 files changed, 17502 insertions(+), 11406 deletions(-) create mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/README.md create mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/summary.html create mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/README.md create mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/summary.html create mode 100644 open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md create mode 100644 open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html create mode 100644 open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md create mode 100644 open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html diff --git a/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md b/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md index 21db7c87c..e491083d0 100644 --- a/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md +++ b/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml

-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,185 +27,189 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: closed
+
+
+
+Code +Result Logs +Measurements +Compliance +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiB
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiBhost_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkTensorRT
operating_systemUbuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)
other_software_stackPython: 3.8.10, GCC-9.4.0
sw_notesAutomated by MLCommons CM v3.2.5.
-
-
- -
- +
+ +
+

Results Table

+ + + + + + + - - - - - - -
Model Accuracy TargetServerOffline
MetricPerformanceServerOfflineMetricPerformanceMetricPerformance
resnetacc: 75.6954
+Metric +Performance +resnetacc: 75.6954Queries/s 73015.3Samples/s 80458.4
diff --git a/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html b/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html index 3cdd88876..af4be1239 100644 --- a/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html +++ b/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,185 +336,189 @@

MLCommons - gh_ubuntu_x86

-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: closed
+
+
+
+Code +Result Logs +Measurements +Compliance +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiB
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiBhost_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkTensorRT
operating_systemUbuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)
other_software_stackPython: 3.8.10, GCC-9.4.0
sw_notesAutomated by MLCommons CM v3.2.5.
-
-
- -
- +
+ +
+

Results Table

+ + + + + + + - - - - - - -
Model Accuracy TargetServerOffline
MetricPerformanceServerOfflineMetricPerformanceMetricPerformance
resnetacc: 75.6954
+Metric +Performance +resnetacc: 75.6954Queries/s 73015.3Samples/s 80458.4
diff --git a/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md b/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md index 4e6b63007..536afeada 100644 --- a/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md +++ b/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: Community
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktvm-onnx v1.19.2
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.9.21, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 80.000Samples/s 1.73704
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 80.0Samples/s 1.73704
diff --git a/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html b/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html index 0625c5d6c..02e1b2ac8 100644 --- a/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html +++ b/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

Community - default

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: Community
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktvm-onnx v1.19.2
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.9.21, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 80.000Samples/s 1.73704
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 80.0Samples/s 1.73704
diff --git a/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/README.md b/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/README.md index 8ae6d7ed4..c5568940e 100644 --- a/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/README.md +++ b/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,185 +27,188 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node1
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node1accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, LLVM-15.0.6
sw_notesAutomated by MLCommons CM v2.3.9.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006Samples/s 0.374837
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544, FID_SCORE: 235.69504Samples/s 0.37484
diff --git a/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/summary.html b/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/summary.html index 9f562f93f..50868df17 100644 --- a/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/summary.html +++ b/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,185 +336,188 @@

MLCommons - 3b07702db56d

-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node1
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node1accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, LLVM-15.0.6
sw_notesAutomated by MLCommons CM v2.3.9.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006Samples/s 0.374837
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544, FID_SCORE: 235.69504Samples/s 0.37484
diff --git a/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/README.md b/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/README.md index f107ac1d4..0484fac2e 100644 --- a/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/README.md +++ b/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.10.8, GCC-13.3.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 49.593Samples/s 0.432023
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.43471
diff --git a/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html b/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html index a291df97d..52b910c6c 100644 --- a/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html +++ b/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - default

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.10.8, GCC-13.3.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 49.593Samples/s 0.432023
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.43471
diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/README.md b/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/README.md index d438385b2..aad1f4523 100644 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/README.md +++ b/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,185 +27,188 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
llama2-70b-99ROUGE1: 43.9869ROUGE1: 43.9869, ROUGE2: 21.8148ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0Tokens/s 0.387999
mixtral-8x7bROUGE1: 45.1429ROUGE1: 45.1429, ROUGE2: 23.1191ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234, mbxp_accuracy: 59.5584TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0Tokens/s 0.0356089
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +llama2-70b-99ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021, ROUGE2: 37.9679, ROUGEL: 39.3617, TOKENS_PER_SAMPLE: 610.0Tokens/s 0.388mixtral-8x7bROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234, mbxp_accuracy: 59.5584TOKENS_PER_SAMPLE: 342.0, gsm8k_accuracy: 100.0, mbxp_accuracy: 0.0Tokens/s 0.03561
diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/summary.html b/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/summary.html index 80b7c54f6..8b9292039 100644 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/summary.html +++ b/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,185 +336,188 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
llama2-70b-99ROUGE1: 43.9869ROUGE1: 43.9869, ROUGE2: 21.8148ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0Tokens/s 0.387999
mixtral-8x7bROUGE1: 45.1429ROUGE1: 45.1429, ROUGE2: 23.1191ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234ROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234, mbxp_accuracy: 59.5584TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0Tokens/s 0.0356089
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +llama2-70b-99ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021, ROUGE2: 37.9679, ROUGEL: 39.3617, TOKENS_PER_SAMPLE: 610.0Tokens/s 0.388mixtral-8x7bROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234, mbxp_accuracy: 59.5584TOKENS_PER_SAMPLE: 342.0, gsm8k_accuracy: 100.0, mbxp_accuracy: 0.0Tokens/s 0.03561
diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/README.md b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/README.md index c3e02d236..177ba2598 100644 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/README.md +++ b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,185 +27,188 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
llama2-70b-99ROUGE1: 43.9869ROUGE1: 43.9869, ROUGE2: 21.8148ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0Tokens/s 0.357886
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +llama2-70b-99ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021, ROUGE2: 37.9679, ROUGEL: 39.3617, TOKENS_PER_SAMPLE: 610.0Tokens/s 0.35789
diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/summary.html b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/summary.html index 755883533..773a5b61c 100644 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/summary.html +++ b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,185 +336,188 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
llama2-70b-99ROUGE1: 43.9869ROUGE1: 43.9869, ROUGE2: 21.8148ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0Tokens/s 0.357886
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +llama2-70b-99ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021, ROUGE2: 37.9679, ROUGEL: 39.3617, TOKENS_PER_SAMPLE: 610.0Tokens/s 0.35789
diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/README.md b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/README.md new file mode 100644 index 000000000..17ab84802 --- /dev/null +++ b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/README.md @@ -0,0 +1,215 @@ +See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/blob/refs/heads/main/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/summary.html) +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_action

+
+
+
+
+MLPerf Inference Category: +datacenter +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameN/A
accelerators_per_node0
accelerator_memory_capacityN/A
accelerator_host_interconnectN/A
accelerator_frequency
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
host_processor_core_count24
host_processor_frequency4800.0000
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_systemUbuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + +
ModelAccuracy TargetServerOffline
AccuracyMetricPerformanceAccuracyMetricPerformance
llama2-70b-99ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021, ROUGE2: 37.9679, ROUGEL: 39.3617, TOKENS_PER_SAMPLE: 610.0Tokens/s 0.40113
mixtral-8x7bROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234, mbxp_accuracy: 59.5584ROUGE1: 70.1571, ROUGE2: 41.2698, ROUGEL: 45.0262, TOKENS_PER_SAMPLE: 245.3, gsm8k_accuracy: 100.0, mbxp_accuracy: 0.0Tokens/s 0.01026
+
+
+
+ diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/summary.html b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/summary.html new file mode 100644 index 000000000..4866cb9c7 --- /dev/null +++ b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/summary.html @@ -0,0 +1,533 @@ + + + +MLPerf Inference v5.0 + + + + + +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_action

+
+
+
+
+MLPerf Inference Category: +datacenter +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameN/A
accelerators_per_node0
accelerator_memory_capacityN/A
accelerator_host_interconnectN/A
accelerator_frequency
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
host_processor_core_count24
host_processor_frequency4800.0000
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_systemUbuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + +
ModelAccuracy TargetServerOffline
AccuracyMetricPerformanceAccuracyMetricPerformance
llama2-70b-99ROUGE1: 43.9869, ROUGE2: 21.8148, ROUGEL: 28.33, TOKENS_PER_SAMPLE: 265.005ROUGE1: 61.7021, ROUGE2: 37.9679, ROUGEL: 39.3617, TOKENS_PER_SAMPLE: 610.0Tokens/s 0.40113
mixtral-8x7bROUGE1: 45.1429, ROUGE2: 23.1191, ROUGEL: 30.1562, TOKENS_PER_SAMPLE: 130.356, gsm8k_accuracy: 72.9234, mbxp_accuracy: 59.5584ROUGE1: 70.1571, ROUGE2: 41.2698, ROUGEL: 45.0262, TOKENS_PER_SAMPLE: 245.3, gsm8k_accuracy: 100.0, mbxp_accuracy: 0.0Tokens/s 0.01026
+
+
+
+ + +
+
+
© 2025 MLCommons.
+
+
+ + + diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/README.md index e14c0cf1d..e3e9a2e63 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/README.md +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 47.0771 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 47.0771 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/summary.html index e6df4c258..347df4ead 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/summary.html +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 47.0771 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 47.0771 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/README.md index 73edc30c6..fe277a35e 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/README.md +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64019775390625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64019775390625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264Tokens/s 42.3041 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.352784 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264.0Tokens/s 42.3041 N/A stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.35278 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/summary.html index 7f3069003..c127d15aa 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/summary.html +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64019775390625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64019775390625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264Tokens/s 42.3041 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.352784 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264.0Tokens/s 42.3041 N/A stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.35278 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md index 4777571bb..aed16051a 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v2.3.9.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 46.0817 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006Samples/s 0.346565 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 46.0817 N/A stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544, FID_SCORE: 235.69504Samples/s 0.34657 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/summary.html index 59238f20a..fd431f663 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/summary.html +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v2.3.9.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 46.0817 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006Samples/s 0.346565 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 46.0817 N/A stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109CLIP_SCORE: 15.18544, FID_SCORE: 235.69504Samples/s 0.34657 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/README.md index 5ad34abbc..91ed1a81d 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/README.md +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node1
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node1accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v2.3.4.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 50.3446 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 50.3446 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/summary.html index c061a663b..6bbc63268 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/summary.html +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node1
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node1accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v2.3.4.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 50.3446 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 50.3446 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/README.md index 02eedf7ba..86436b664 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/README.md +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 42.4133 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 42.4133 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/summary.html index 64eb48441..750e7331e 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/summary.html +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemUbuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264Tokens/s 42.4133 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2ROUGE1: 32.2581, ROUGE2: 6.6667, ROUGEL: 22.5806, GEN_LEN: 264.0Tokens/s 42.4133 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/README.md index 0070d7a95..cad832637 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/README.md +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64019775390625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node1accelerator_memory_capacity23.54595947265625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2610000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameAMD Ryzen 9 7950X 16-Core Processorhost_processors_per_node1host_processor_core_count16host_processor_frequency5881.0000host_memory_capacity128Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 512 KiB (16 instances), L1i cache: 512 KiB (16 instances), L2 cache: 16 MiB (16 instances), L3 cache: 64 MiB (2 instances)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

-
frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4
sw_notes
+frameworkpytorch v2.5.1operating_systemUbuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264Tokens/s 48.3215 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.358711 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264.0Tokens/s 48.3215 N/A stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.35269 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/summary.html index 142036640..1dd766796 100644 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/summary.html +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_action

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64019775390625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node1accelerator_memory_capacity23.54595947265625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2610000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameAMD Ryzen 9 7950X 16-Core Processorhost_processors_per_node1host_processor_core_count16host_processor_frequency5881.0000host_memory_capacity128Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 512 KiB (16 instances), L1i cache: 512 KiB (16 instances), L2 cache: 16 MiB (16 instances), L3 cache: 64 MiB (2 instances)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

-
frameworkpytorch v2.5.1
operating_systemUbuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4
sw_notes
+frameworkpytorch v2.5.1operating_systemUbuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566ROUGE1: 42.5566, ROUGE2: 19.9223ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264Tokens/s 48.3215 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863CLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.358711 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264.0Tokens/s 48.3215 N/A stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.35269 N/A
diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/README.md b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/README.md new file mode 100644 index 000000000..dc657dad3 --- /dev/null +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/README.md @@ -0,0 +1,219 @@ +See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/blob/refs/heads/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/summary.html) +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_action

+
+
+
+
+MLPerf Inference Category: +edge +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerators_per_node2
accelerator_memory_capacity23.64019775390625 GB
accelerator_host_interconnectN/A
accelerator_frequency2520000 MHz
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
host_processor_core_count24
host_processor_frequency4800.0000
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_systemUbuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + + + + + +
ModelAccuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264.0Tokens/s 49.5612 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.35302 N/A
+
+
+
+ diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/summary.html b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/summary.html new file mode 100644 index 000000000..2e8470940 --- /dev/null +++ b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/summary.html @@ -0,0 +1,537 @@ + + + +MLPerf Inference v5.0 + + + + + +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_action

+
+
+
+
+MLPerf Inference Category: +edge +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerators_per_node2
accelerator_memory_capacity23.64019775390625 GB
accelerator_host_interconnectN/A
accelerator_frequency2520000 MHz
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
host_processor_core_count24
host_processor_frequency4800.0000
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB (24 instances), L1i cache: 768 KiB (24 instances), L2 cache: 48 MiB (24 instances), L3 cache: 45 MiB (1 instance)
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_systemUbuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)
other_software_stackPython: 3.10.12, GCC-11.4.0, Using Docker , CUDA 12.4
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + + + + + +
ModelAccuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
gptj-99ROUGE1: 42.5566, ROUGE2: 19.9223, ROUGEL: 29.6882, GEN_LEN: 3615190.2GEN_LEN: 264.0Tokens/s 49.5612 N/A
stable-diffusion-xlCLIP_SCORE: 31.6863, FID_SCORE: 23.0109Samples/s 0.35302 N/A
+
+
+
+ + +
+
+
© 2025 MLCommons.
+
+
+ + + diff --git a/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md index ad62cf1ba..e4a8291e7 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 9.45726
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 8.08903
diff --git a/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html index 7a9c1b599..ed5906f74 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 9.45726
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 8.08903
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md index ef03ea3ad..e852040da 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkdeepsparse v1.8.0
operating_system (darwin-23.6.0)
other_software_stackPython: 3.11.9, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
bert-99F1: 89.9653F1: 80.0Samples/s 7.5726 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +bert-99F1: 89.9653F1: 80.0Samples/s 8.14067 N/A
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html index 9bfc7a53e..6dba47fe6 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkdeepsparse v1.8.0
operating_system (darwin-23.6.0)
other_software_stackPython: 3.11.9, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
bert-99F1: 89.9653F1: 80.0Samples/s 7.5726 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +bert-99F1: 89.9653F1: 80.0Samples/s 8.14067 N/A
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md index 2b32b2cbe..7fc9ed573 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.7, GCC-13.3.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 20.9527
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.323181
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 20.9527retinanetmAP: 37.1745Samples/s 0.32318
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html index 17384e65f..dc9f9efa4 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.7, GCC-13.3.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 20.9527
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.323181
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 20.9527retinanetmAP: 37.1745Samples/s 0.32318
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md index e43512a0a..5827c0adb 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 19.9943
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.379074
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 18.9092retinanetmAP: 37.1745Samples/s 0.41227
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html index 630cad71d..ee890a4d0 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 19.9943
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.379074
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 18.9092retinanetmAP: 37.1745Samples/s 0.41227
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md index 4e37f2a3c..7bb011a0b 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.7, GCC-13.3.0
sw_notesAutomated by MLCommons CM v3.2.3.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.350957
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.35096
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html index bc95bb486..873f6e2b4 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.7, GCC-13.3.0
sw_notesAutomated by MLCommons CM v3.2.3.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.350957
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.35096
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md index 4e3e4179d..8686c8be1 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.7, GCC-13.3.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.277839
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.27784
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html index f8989dd04..b0a47b840 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.7, GCC-13.3.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.277839
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.27784
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md index 600f036c0..44ebba402 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.431004
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.44005
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html index c884be026..84a4ca4b0 100644 --- a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_macos-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_core_count3
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count3host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.431004
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.44005
diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md new file mode 100644 index 000000000..1103e504f --- /dev/null +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md @@ -0,0 +1,219 @@ +See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/blob/refs/heads/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html) +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_macos-latest_x86

+
+
+
+
+MLPerf Inference Category: +edge +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameN/A
accelerators_per_node0
accelerator_memory_capacityN/A
accelerator_host_interconnectN/A
accelerator_frequency
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameundefined
host_processors_per_node1
host_processor_core_count3
host_processor_frequencyundefined
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + + + + + +
ModelAccuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745Samples/s 0.26499
+
+
+
+ diff --git a/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html new file mode 100644 index 000000000..967d107da --- /dev/null +++ b/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html @@ -0,0 +1,537 @@ + + + +MLPerf Inference v5.0 + + + + + +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_macos-latest_x86

+
+
+
+
+MLPerf Inference Category: +edge +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameN/A
accelerators_per_node0
accelerator_memory_capacityN/A
accelerator_host_interconnectN/A
accelerator_frequency
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameundefined
host_processors_per_node1
host_processor_core_count3
host_processor_frequencyundefined
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: 196608, L2 cache: 12582912, L3 cache:
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_system (darwin-23.6.0)
other_software_stackPython: 3.12.8, LLVM-15.0.0
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + + + + + +
ModelAccuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745Samples/s 0.26499
+
+
+
+ + +
+
+
© 2025 MLCommons.
+
+
+ + + diff --git a/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md index 60401e77f..c226e6336 100644 --- a/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktvm-onnx v1.19.2
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.9.21, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 80.000Samples/s 1.71387
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 80.0Samples/s 1.74836
diff --git a/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html index ae224048f..2839664ba 100644 --- a/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktvm-onnx v1.19.2
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.9.21, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 80.000Samples/s 1.71387
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 80.0Samples/s 1.74836
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md index bc8d163a9..07a56bbcd 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 22.9898
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.429956
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 23.0164retinanetmAP: 37.1745Samples/s 0.43171
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html index 2ecad043a..de33d625d 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 22.9898
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.429956
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 23.0164retinanetmAP: 37.1745Samples/s 0.43171
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md index bd03c3fcf..1ec53df3a 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkdeepsparse v1.8.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.11.11, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
bert-99F1: 89.9653F1: 80.0Samples/s 6.25372 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +bert-99F1: 89.9653F1: 80.0Samples/s 6.31453 N/A
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html index 682e2d1be..3c32c99d2 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkdeepsparse v1.8.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.11.11, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
bert-99F1: 89.9653F1: 80.0Samples/s 6.25372 N/A
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +bert-99F1: 89.9653F1: 80.0Samples/s 6.31453 N/A
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md index 213f2a37c..f67c368d5 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, LLVM-14.0.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 23.227
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.433265
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 23.227retinanetmAP: 37.1745Samples/s 0.43327
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html index f5df9354b..84be9b256 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, LLVM-14.0.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 23.227
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.433265
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 23.227retinanetmAP: 37.1745Samples/s 0.43327
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md index f49a09e0b..8e2f66cdc 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 23.3041
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.436753
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 23.009retinanetmAP: 37.1745Samples/s 0.43494
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html index bb15b6d71..4985f57ac 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 23.3041
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.436753
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 23.009retinanetmAP: 37.1745Samples/s 0.43494
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/README.md index dff64532b..3fbe7df5d 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,185 +27,188 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, GCC-13.3.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
rgatacc: 0.7213 N/A acc: 75.400Samples/s 10.07
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +rgatacc: 0.7213 N/A acc: 75.6Samples/s 10.2328
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/summary.html index bac4ccbf8..149be1c4c 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,185 +336,188 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, GCC-13.3.0
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
rgatacc: 0.7213 N/A acc: 75.400Samples/s 10.07
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +rgatacc: 0.7213 N/A acc: 75.6Samples/s 10.2328
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md index 5988e76d3..aba101dc3 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.3.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.409818
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.40982
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html index cf0e9c2ba..05b3b66c7 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.3.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.409818
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.40982
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md index bd4a5f72a..80eab0e29 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.412839
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.41284
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html index 22c1c82df..f31543c1f 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, GCC-11.4.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.412839
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.41284
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md index 708a7422f..8faf04bc3 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.412174
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.41275
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html index 1780ebaeb..f420d5b9b 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.412174
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.41275
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md new file mode 100644 index 000000000..8a2b6397f --- /dev/null +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/README.md @@ -0,0 +1,219 @@ +See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/blob/refs/heads/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html) +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_ubuntu-latest_x86

+
+
+
+
+MLPerf Inference Category: +edge +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameN/A
accelerators_per_node0
accelerator_memory_capacityN/A
accelerator_host_interconnectN/A
accelerator_frequency
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
host_processor_core_count2
host_processor_frequencyundefined
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + + + + + +
ModelAccuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745Samples/s 0.40926
+
+
+
+ diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html new file mode 100644 index 000000000..3c9688afd --- /dev/null +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/summary.html @@ -0,0 +1,537 @@ + + + +MLPerf Inference v5.0 + + + + + +
+
+ +
+
+
+
+
+
+
+

MLPerf Inference v5.0

+

Copyright 2019 - 2025 MLCommons

+February 2025 +
+
+
+

MLCommons - gh_ubuntu-latest_x86

+
+
+
+
+MLPerf Inference Category: +edge +
+
+Availability: +available +
+
+
+
+Submitted by: +MLCommons +
+
+MLPerf Inference Division: +open +
+
+
+ +
+
+
+Code +Result Logs +Measurements +
+
+
+
+
+
+ +
+ +

Accelerator Details

+
+ +
accelerator_model_nameN/A
accelerators_per_node0
accelerator_memory_capacityN/A
accelerator_host_interconnectN/A
accelerator_frequency
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_configurationN/A
accelerator_on-chip_memories
+
+ +
+ +

Processor and Memory Details

+
+ +
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
host_processor_core_count2
host_processor_frequencyundefined
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_interconnect
+
+
+
+ +
+ +

Other Hardware Details

+
+ +
coolingair
hw_notes
+
+ +
+ +

Network and Interconnect Details

+
+ +
host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
+
+
+
+ +
+ +

Software Details

+
+ +
frameworkpytorch v2.6.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
+
+
+
+ +
+ +

Results Table

+
+ + + + + + + + + + + + + + + + + + +
ModelAccuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745Samples/s 0.40926
+
+
+
+ + +
+
+
© 2025 MLCommons.
+
+
+ + + diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md index 591047a75..6265b83ef 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.17.0
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, LLVM-14.0.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.0188
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.0188
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html index 260f13bc5..538232576 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity15G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity15Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.17.0
operating_systemUbuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)
other_software_stackPython: 3.12.7, LLVM-14.0.0
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.0188
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.0188
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md index b4aae9d75..965dd56c2 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.18.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.1187
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.0425
diff --git a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html index a7fb15235..85113b876 100644 --- a/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_ubuntu-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity16G
host_memory_configurationundefined
host_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)
host_processor_core_count2
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameAMD EPYC 7763 64-Core Processor
host_processors_per_node1
+host_processor_model_nameAMD EPYC 7763 64-Core Processorhost_processors_per_node1host_processor_core_count2host_processor_frequencyundefinedhost_memory_capacity16Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 64 KiB (2 instances), L1i cache: 64 KiB (2 instances), L2 cache: 1 MiB (2 instances), L3 cache: 32 MiB (1 instance)host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.18.0
operating_systemUbuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)
other_software_stackPython: 3.12.8, LLVM-18.1.3
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.1187
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.0425
diff --git a/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md b/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md index c904be933..fc2c865ac 100644 --- a/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md +++ b/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,185 +27,188 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiB
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiBhost_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkTensorRT
operating_systemUbuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)
other_software_stackPython: 3.8.10, GCC-9.4.0
sw_notesAutomated by MLCommons CM v3.2.4.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.078Queries/s 75016.1acc: 76.078Samples/s 80229.7
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.078Queries/s 75016.1acc: 76.078Samples/s 80229.7
diff --git a/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html b/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html index 51466e02b..37ba0146d 100644 --- a/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html +++ b/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,185 +336,188 @@

MLCommons - gh_ubuntu_x86

-MLPerf Inference Category: +MLPerf Inference Category: datacenter
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency2520000 MHz
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacity23.64703369140625 GB
accelerator_memory_configurationN/A
accelerator_model_nameNVIDIA GeForce RTX 4090
accelerator_on-chip_memories
accelerators_per_node2
-
-
- -
- +accelerator_model_nameNVIDIA GeForce RTX 4090accelerators_per_node2accelerator_memory_capacity23.64703369140625 GBaccelerator_host_interconnectN/Aaccelerator_frequency2520000 MHzaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacity192G
host_memory_configurationundefined
host_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiB
host_processor_core_count24
host_processor_frequency4800.0000
host_processor_interconnect
host_processor_model_nameIntel(R) Xeon(R) w7-2495X
host_processors_per_node1
+host_processor_model_nameIntel(R) Xeon(R) w7-2495Xhost_processors_per_node1host_processor_core_count24host_processor_frequency4800.0000host_memory_capacity192Ghost_memory_configurationundefinedhost_processor_cachesL1d cache: 1.1 MiB, L1i cache: 768 KiB, L2 cache: 48 MiB, L3 cache: 45 MiBhost_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkTensorRT
operating_systemUbuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)
other_software_stackPython: 3.8.10, GCC-9.4.0
sw_notesAutomated by MLCommons CM v3.2.4.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - -
Model Accuracy TargetServerOfflineAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.078Queries/s 75016.1acc: 76.078Samples/s 80229.7
+Server +Offline + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.078Queries/s 75016.1acc: 76.078Samples/s 80229.7
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/README.md b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/README.md index d9f4ed1d5..2eb9cd7fc 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.349311
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.34931
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html index 5bd0e924c..4068f4817 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.349311
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.34931
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/README.md b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/README.md index 8874c5db0..4bcd2e1c7 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.336027
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.33603
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html index 906692c20..59c560ad0 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.336027
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.33603
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/README.md b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/README.md index b18107b7d..60b78edfb 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.3.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.327316
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.32732
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/summary.html b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/summary.html index 4da576b37..41094950b 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.4.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.3.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.327316
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.32732
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/README.md b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/README.md index baf0f0ddb..6ce703bbb 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.329647
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.32965
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/summary.html b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/summary.html index 0487c9b26..de7007ccc 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.329647
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.32965
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/README.md b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/README.md index ae533d030..a4e0b85d8 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.313782
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.31378
diff --git a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/summary.html b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/summary.html index 0140f2302..f859d5c8d 100644 --- a/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkpytorch v2.5.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
retinanetmAP: 37.1745mAP: 76.951Samples/s 0.313782
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +retinanetmAP: 37.1745Samples/s 0.31378
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md index c36c8e0b4..87c0d6274 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.4.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 17.596
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 17.596
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html index 77f50f1c5..7c1aa4fc8 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.4.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 17.596
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 17.596
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md index 233f651fc..39f5f2ce4 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 17.3279
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 17.3279
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html index d826a22af..6bbdae6f2 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.20.1
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 17.3279
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 17.3279
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md index 3ca2be482..1ec7ab430 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.17.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.7501
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.7501
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html index 75a7b4dab..ea3e9841d 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.17.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.7, -
sw_notesAutomated by MLCommons CM v3.2.6.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.7501
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.7501
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md index 61a2be856..cf00878c5 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.18.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.5168
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.5168
diff --git a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html index cc5285126..7486cee2f 100644 --- a/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows-latest_x86

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.18.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.8, -
sw_notes
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.5168
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.5168
diff --git a/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/README.md b/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/README.md index 828002f1f..bf11f14af 100644 --- a/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/README.md +++ b/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.6, -
sw_notesAutomated by MLCommons CM v2.4.0.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 17.8036
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 17.8036
diff --git a/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html b/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html index 5537af8da..ad708733f 100644 --- a/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworkonnxruntime v1.19.2
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.6, -
sw_notesAutomated by MLCommons CM v2.4.0.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 17.8036
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 17.8036
diff --git a/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/README.md b/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/README.md index 7746946e1..e38e02c04 100644 --- a/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/README.md +++ b/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/README.md @@ -14,11 +14,11 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -27,190 +27,192 @@ See the HTML preview [here](https://htmlpreview.github.io/?https://github.com/ml
-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.17.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.6, -
sw_notesAutomated by MLCommons CM v2.4.0.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.8155
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.8155
diff --git a/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/summary.html b/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/summary.html index 871af13fe..5e13f87d5 100644 --- a/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/summary.html +++ b/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/summary.html @@ -26,6 +26,55 @@ overflow-x: hidden; text-align: left; } +.details-cell { + text-transform: capitalize; +} + +a.button:hover { + color: white; + background-color: gray; + text-decoration: underline; +} +a.button { +display: inline-block; /* Allows background-color to apply */ + text-decoration: none; /* Removes default underline */ + + } + .button-container { + display: flex; + width: 100%; + max-width: 800px; /* Optional: Limit the container width */ + margin: 0 auto; + } + + .button-container a { + flex: 1; /* Distribute space evenly */ + text-decoration: none; + } + + .button { + padding: 15px 0; + font-size: 16px; + color: #ffffff; + background-color: #007bff; + border: none; + border-radius: 8px; + cursor: pointer; + transition: all 0.3s ease; + text-align: center; + display: inline-block; + } + + + .button:active { + background-color: #003f8a; + transform: scale(0.95); + } + + /* Add spacing between buttons */ + .button:not(:last-child) { + margin-right: 10px; + } .topbar { /* position: fixed; */ @@ -37,7 +86,7 @@ border-bottom: 1px solid rgb(255 255 255 / 10%); } .topbar-container { - max-width: 1300px; + max-width: 1350px; padding: 0 2rem; margin: 0 auto; } @@ -95,6 +144,15 @@ font-weight: bold; margin-right: 6px; } + .titlebar { + display: flex; + justify-content: space-between; + align-items: center; + } + + .date-right { + font-size: 18px; + } .table-half { width: 100%; @@ -136,7 +194,9 @@ .table { border-collapse: collapse; width: 100%; + table-layout: fixed; color: #212529; + word-break: break-word; } .table-striped tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); @@ -154,6 +214,10 @@ .table-full .table:not(.results-table) th, .table-full .table:not(.results-table) td { width: 50%; } + a.button { + color: black; + background-color: azure; + } .table th:not(:last-child), .table td:not(:last-child) { position: relative; } @@ -169,10 +233,15 @@ transform: translatey(-50%); } .th-parent, - .results-table tr:nth-child(3) td:nth-child(-n+2) { - background-color: #b6bfff; + .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; text-align: center; } + /* Apply styling to the first two columns of rows starting from the third row */ +.results-table th, .results-table tr:nth-child(n+3) td:nth-child(-n+2) { + background-color: #dddee3; + text-align: center; +} .footer { border-top: 1px solid rgb(255 255 255 / 10%); @@ -254,11 +323,11 @@
-

MLPerf Inference v5.0

Copyright 2019 - 2025 MLCommons

+February 2025
@@ -267,190 +336,192 @@

MLCommons - gh_windows

-MLPerf Inference Category: +MLPerf Inference Category: edge
-Availability: -Available as of February 2025 +Availability: +available
-Submitted by: +Submitted by: MLCommons
-MLPerf Inference Division: +MLPerf Inference Division: open
+
+
+
+Code +Result Logs +Measurements +
-
+
+ +
-
- -
-

Accelerator Details

-
accelerator_frequency
accelerator_host_interconnectN/A
accelerator_interconnectN/A
accelerator_interconnect_topology
accelerator_memory_capacityN/A
accelerator_memory_configurationN/A
accelerator_model_nameN/A
accelerator_on-chip_memories
accelerators_per_node0
-
-
- -
- +accelerator_model_nameN/Aaccelerators_per_node0accelerator_memory_capacityN/Aaccelerator_host_interconnectN/Aaccelerator_frequencyaccelerator_interconnectN/Aaccelerator_interconnect_topologyaccelerator_memory_configurationN/Aaccelerator_on-chip_memories
+
+ +
+

Processor and Memory Details

-
host_memory_capacityundefined
host_memory_configurationundefined
host_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache:
host_processor_core_count1
host_processor_frequencyundefined
host_processor_interconnect
host_processor_model_nameundefined
host_processors_per_node1
+host_processor_model_nameundefinedhost_processors_per_node1host_processor_core_count1host_processor_frequencyundefinedhost_memory_capacityundefinedhost_memory_configurationundefinedhost_processor_cachesL1d cache: , L1i cache: , L2 cache: , L3 cache: host_processor_interconnect
-
-
- -
- +
+ +
+

Other Hardware Details

coolingair
hw_notes
-
-
- -
- +
+ +
+

Network and Interconnect Details

host_network_card_count1
host_networkingGig Ethernet
host_networking_topologyN/A
-
-
- -
- +
+ +
+

Software Details

frameworktf v2.17.0
operating_systemWindows-2022Server-10.0.20348-SP0
other_software_stackPython: 3.12.6, -
sw_notesAutomated by MLCommons CM v2.4.0.
-
-
- -
- +
+ +
+

Results Table

- - - - - - - - - - - - -
Model Accuracy TargetOfflineSingleStreamMultiStream
AccuracyMetricPerformanceAccuracyMetricPerformanceAccuracyMetricPerformance
resnetacc: 75.6954acc: 76.000Samples/s 21.8155
+Offline +SingleStream +MultiStream + + +Accuracy +Metric +Performance +Accuracy +Metric +Performance +Accuracy +Metric +Performance +resnetacc: 75.6954acc: 76.0Samples/s 21.8155
From 07fb97ffcd79b181dbcaa5e0ebe554503e2e3d6d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Feb 2025 10:13:20 +0000 Subject: [PATCH 80/86] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 14572 -> 14571 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1b53c2274..d2036c6a9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-05 22:03:13,857 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-05 22:03:13,857 submission_checker1.py:3270 INFO] --- -[2025-02-05 22:03:13,858 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3270 INFO] --- +[2025-02-06 10:13:15,336 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3286 INFO] --- -[2025-02-05 22:03:13,858 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-05 22:03:13,858 submission_checker1.py:3307 INFO] --- -[2025-02-05 22:03:13,858 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-05 22:03:14,932 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 10:13:15,336 submission_checker1.py:3286 INFO] --- +[2025-02-06 10:13:15,336 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 10:13:15,336 submission_checker1.py:3307 INFO] --- +[2025-02-06 10:13:15,336 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 10:13:16,480 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 7e1aa3219..ff39c77bd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -621 +622 diff --git a/summary.xlsx b/summary.xlsx index c1318b9e7b55af395673aaf310addbb97e3b2d6d..b67f0f7e2526453ed3a5eb6a00fd816706ad64b0 100644 GIT binary patch delta 338 zcmV-Y0j>V*aqDrg$ub3=C~0LRlg=_8e@|-yF%ZS?{S-^?yGd4*5_VT;i>C^OT3UJw z$*7^5KO|{azkTDXtHPcce>5~zz1bbjc7qo21Zz^%%(;kt402ZvJknE zytM{uw1)AAq?Vi0FirVnDiI-}V40Ftnh-#DHl*@x>d#>oD5TD_2gf;@-ii z^(+qVT~LFu>$;+w^wyN0;;-fFd%tR~%;8i5P*X|4ga;n2Zz_CUq{Fp5#AQ11T7i7p zCw=7Yw8*g!WR?Q3W8AayIeC6rBrQ-=l5(1pGAFYYC5%oOoqiui-t=F9`ZVNj;u&zv keftTTf3puXS_B22C~0LRlWsUv0S=SHI3othH2?qr0E6429smFU delta 339 zcmV-Z0j&P(aqMxh$ub2p1NYA*lg=_8f6Z#cFc5_AeF~$yE!l2L5w-(u$*F-tQ(AHp z(k2nLevs5UdHc#v;w0cxkE@yec6L!U->WwH1?Q#FHOe!B642ZTskb%ySS`{=l=#53 zV6D+mqXT#}Z|*8?DK`$@oU!171fN8Z_LSQi?Lx2=W6yV>n9rg^M{#4EVj-4pe~T^S z-)sxGAmjln2u!fRa1TuFNJJ;3;3LxhxOM;tj-dqwI(VFCIlg!UIOQiB2Ib1BWH?xu zw4TM$z4vl7c3qcs Date: Thu, 6 Feb 2025 11:14:12 +0000 Subject: [PATCH 81/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14571 -> 14571 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d2036c6a9..6eb314163 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 10:13:15,336 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3270 INFO] --- -[2025-02-06 10:13:15,336 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3270 INFO] --- +[2025-02-06 11:14:07,403 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3286 INFO] --- -[2025-02-06 10:13:15,336 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 10:13:15,336 submission_checker1.py:3307 INFO] --- -[2025-02-06 10:13:15,336 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 10:13:16,480 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 11:14:07,403 submission_checker1.py:3286 INFO] --- +[2025-02-06 11:14:07,403 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 11:14:07,403 submission_checker1.py:3307 INFO] --- +[2025-02-06 11:14:07,403 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 11:14:08,500 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40113 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.406358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.264988 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ff39c77bd..379854108 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -622 +623 diff --git a/summary.csv b/summary.csv index b1ce3b836..56dd9a50e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40113,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.406358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.264988,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0bce6e343..447cee6b6 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.40113, + "Result":0.406358, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index b67f0f7e2526453ed3a5eb6a00fd816706ad64b0..85800e2db8b80bc74ec94aba8c9b435d34762c4e 100644 GIT binary patch delta 2049 zcmV+c2>$o$aqDrgtOp9Xx2w>E1^@tKC6jRu8I#Wk7JqGMqkxo+;{{{vV4KuCQ9>Gl zDq4s}c=`5E3t-dTD7;)qB?m{0{@v5vnjbS_rt{?`3WX<@@*s)3qSBN_fyHj(1@XKq zK0cf`EK%T@6MIgW#H=g6GA@4U{nL3)(jR=mSSI8SV%`-OS+;CT5_cCYa(FXYvN*rv zCu!tl`G3cBF7ai`9IrHrLP?cnLyDXr7Ap_!^bjA3?*}eBOWak&;;eF#ve3!$@cANG zalFIg~OWO>>2!o*ENUj7xLAg`q=3V)IF(iJsPcn-X5Q8*SBf#a~pU%*mWyCrzG(wCsT|o#M$T zZB@<7;ufvnism9VY27g1GzfVfMfIL~T1R>+9IqAk(BuI&Ji>;_y9OzbXB(7C2c@DI zJ&Sfw&+Y-$X*4RVucAZ#R>~#`b)(lR%oD`Nv&ijSDnoDL`M$n-7ZeCLIH!iPe{Pn zI3pyWs9y;QAo!e+02D3=2^fW!gaizsD}O=)NVYCmZ+e*H-rRKcZ4Mx zh~<`$fLQK`j=(eeMo2VzyczU(Gw}Bx$oG_xfMohuqJk%QMo7RD{7OhbN9S0ggN`mX z65<>25}Ruv+p8M$0bvd#*EQxtY;M4@MMwZpBO(YOxh1@SLf*Z_>Gv-Gz56;IUVn|Q zKb8w6p0AYVLAF@Unr;$FQJDXY@>NiNwPdM34Pu|A`C5)mGsd$iU(KR`7t10(ebSq< zl% z69Wi_)+r$Y-zI%5L5va-$S5I!jADrnyj++yjAC=dXpQ+`b8oX;yM_%z+<^g{+(4TV z5fqrkZV9*W3*9^-bo0g3af`Q^cf`4G#5Ho_^qjc$JIK^cVn3K4K;1ENI)8w=M{}1t z%d~Bqrp}_NvoKO%_C3WC9WKSN1W`vwAa#TUQb)uBsk3UR!{&&(8uNh#se=te+ySBv z+Kh;xK=E#|!~}8QS$`pQ$H?gb>L%6>b+V=#vb6_0o5s$@*nzi>Q!LTJQ0ij|!j6zY z><9_Oj)(6dW!O4zr3nx4K&&6)~yVxBgr$Z(? zdv~(aP0QNz6N@6#*p=;oJMda@N@#SG7z`wT!~j*hK~^IwGzcN(8B+Kz{67*yqbdJ? z43Co3Asns)+kn`T4NbT9fJpz2exl&t&09JI5o>e}f`~O3L_&i=Bs2&_A|3>yQU@Zj zS%5&SaWGKePQf*584{gf5TVtG3Joqo35}^h5S70X#A76N$U1Z#*mfbR$f~~Y{ZFBP zH;+yfVsFTRBw~%;f+S)MCXvt}5(y0=k%-5FB&u~J5)N!gVvU2r_U0Rjuw_VeVrY;= zXf>j&5Q$i0s*ptWxFoJ-t2kRVLZ@``Vj26PZ2cCL$H=c9%3a>*4b2k;?-RC6$Q!Yz zsYBkd2J=Q}5O0JA@kVHD$eUKj8+L$yXTBN-1Ihv1l-O|qa2Qe!(6mTw;exV{D2)m6 z#2QnBcxrzFo+X#@Q{V+oxiRs;VxFYqhu^^Q82KD9M~>HlRdb}N%Zkxd}8oh+M;5j;Jx_ODI8&$21y!Z zuT?(9F`!lG^rFQ3SO&X_gP$w*@q_!pXn$~xPoD+#lh76N`fa3n)@3in1r=7WvvlU{ z#!2Tv%!QEoc>(06DGF(IuBW^r&XQ$$rHjHW$+9Hcd|EKaV`=dzFS?&3+2%u0lI{7Z z-hZ=M4sav|$s~YdC6mrF9)HbF!!Qtp?>t51dty6FiBiluX#+RUw>oA#W!CAE(w`p z4UtP8Io<(i zja}Cj-K4js{1ksKUf%jub7l6Y5`daY3MM@8Xnj-R^CBItHX7rj*}nhg7Zws9YkqYTx8j@ElH`f;Gvy1<0X&s zJ6@cGc7K+COy*L$Ot|fqMqwZ+vaCy??MGtep`9G!BlbMs;b*b43VD=OP7)s2Sss46 z@RzCB>$rY?ugFJ8xYrd=SiAq7siM^DlzmTp{+zzM7K&76ar~qBaOHMIxsXaodtQcd zmk2KR>{XDBjW%}Bd$!u$FHCap66|Hz4uK{1%NL8<8glBeU_c}@ZEF@(Q>C!Hq z3~MV&UKXcl{Z=#=u}SNO@uoq@^C+tK)KfdsQ{i~6xQ8YWu;CFlOx`s}c|6;o6gnsc z#eX30#xTZ;7Kl=TQDPGn*gO&vZC5fOB>GmelbcHCrXmLnh)nb)S%c zv2jL7KvBOE5sO|_$ zR1nK8Apx=65gmbN^o@|H^mx1Qn)}&eHETL?D1|}(H_TT-`PGsq-qepgp5$veKFxTVP1DsZ^wVNllun|=gH-&~k$*g5D*twjl=1~_5nB1*p12;WxLCFPXsSlb+&3o} zG@}Lv5DcwTLIS=``dES(B_xnhLIN4Z5)F8{FlrdZ=7`Z6^TFocX1R6^8-}<89XPpx zHX|Y^FpJ$1Zs8ZYaYX3mi>d7tZ!_TZiX|FcieU+&j*vj=2nnQ)hy_w-)=-Dd5p^}@0~1mQ8-}<8 zL>;sl5kZ0C-C~IW;=VKgLh6o@(*e{?%pK}vRnuj24|W!forSRjZyl#tqJg2*#}b4c zA%WNt5{MlU3&hT^o}A2NuK*HVknG2s>yqB7%aG9pM&EcGjPZ-Sl^{J4Q~2 zOm^1pWTzRXx#uSqCezrJ?SMP*T5(EfG?Nw#B!5H)Rl7k}BPuiqA>|oT_%8fE5<|Ty z|9=dRlGGs_t^?bE*peCB^K%TNe?~uH_-FH$206qUO@$m{4d#&0APxx);*f|3amea8 zB=!mrhcylc3~m%$qn07j3FZ)5ji}Jz8kEo&406c+Mh=gW)FI2zbzs|dD3cX!-|HWv z_kWH~7_l{^LlUt@Ye5pR29ro=5Q&5akx0a2LK2la5(x(uB(cW9V0-h8MA$MUI?+`~ zBD5M&R)|EbF%(Fma$FKuvsIL>8i8H9cro?;P_}*x%46hL59Kaz^nT_E!`p-{1M)^} zX=;!+tiiky8pIo+LA((f3-YGc@rE7XnSZaw!GLlA_at^4033#t12ioXJGh|iBT8dH zJh8@5A)e}=fM>~N^yIs~UG7Uf@FD^J0kSBBA=RuGPPMp`X%75Ehou3)A_M|*&0^61LeRaa)%}pLh9Dlif zv4*jO>cC9qM{^~+bzB7a%tkG80oD7Y_PbqGgTCVrgw z*=vn!a# zzp<0KAEiRTy}ST&(-4KEI@VKOXC?8nywXHr7H3%;Zayuz?ee7flo#EL<81SxD9QGC zRPR4fO9KRx9XA@Yb`DY`1)eBrWh9f$G9G_VYXUJ4#qa$TOYXZ#R+JKUS7?i;3WZu) zdJD;@p_@M>X;;5}xf_V+gp^`Hx?kV+04XrEpuxm|X+iPD8^P-^+R!UkPOakJ z!Kn2t4(?r0gR$$nqMP*Al%L|S_Nx_5%9<6UGd|jl&wLHXSI`LY8 zeA*{{Wh4Or08jz|01*HH00000000000DytJlY=;DTxq{D diff --git a/summary_results.json b/summary_results.json index 4c04f6ef8..88f96cda8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.40113, + "Performance_Result": 0.406358, "Performance_Units": "Tokens/s" }, { From 3a22927cc6027980c24f7d732618199a29f14d37 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Feb 2025 16:29:19 +0000 Subject: [PATCH 82/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14571 -> 14572 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6eb314163..0b7534a3f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 11:14:07,403 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3270 INFO] --- -[2025-02-06 11:14:07,403 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3270 INFO] --- +[2025-02-06 16:29:14,183 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3286 INFO] --- -[2025-02-06 11:14:07,403 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 11:14:07,403 submission_checker1.py:3307 INFO] --- -[2025-02-06 11:14:07,403 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 11:14:08,500 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 16:29:14,183 submission_checker1.py:3286 INFO] --- +[2025-02-06 16:29:14,183 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 16:29:14,183 submission_checker1.py:3307 INFO] --- +[2025-02-06 16:29:14,183 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 16:29:15,280 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409259 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2328 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0506 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 379854108..d27f55212 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -623 +624 diff --git a/summary.csv b/summary.csv index 56dd9a50e..3c39263af 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409259,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2328,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0506,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 447cee6b6..b5da46171 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2328, - "Accuracy":"acc: 75.600", + "Result":10.0506, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 85800e2db8b80bc74ec94aba8c9b435d34762c4e..df8fff6f20ce669680c630a1bea143baadba98f6 100644 GIT binary patch delta 2035 zcmV;tS z`qBCAlv0NHqcJ^ey)ZUCQ55REkjSNSG9@wBjFU8S8UHt(D0G?<*QYOpuqafps#;3q z25}2{NfxuSmSsCFl*Pe68R-@hzNXbVr9tPEb#4=%gr^GYoGQ7X5Dq*NyJ?VATJ1o#OhU#$wJJU0@s0G5rSfHBeP^O+lj?kETYCO)*Fj6_xG9rv|h^Fx|= zgwjOVfBL(C=y@b0smsL0uCREJjM=N33SPZSm}tZWVVr-AAntO5t1?qoB~?|ms@!*( z%fp5{O7X&k5?{H!=_XN>#8jEQ46_jry=*vdTT(m={J@na(@c6KF=yU?lzcLFc@z!3 zBpwG7fnDM1kgT?>t^gIQN2`i=HVJ{md4%8k-_bKk;{5NcEn&kkp9OFJd3|wxj;_dM zl6nO7AD@4bp*m`H*^*~Po+tSUopQgOwRjfOJUd#Q^Nb}tyNQoR3$!AJy0>Vi%UCvL zRNbD}5YABYgYh`aYmGD)QaTLX$ikv*AsH03>+Ph8-Uyd#vP#(+ z15JZ%i>ZULhF?i62eWuE{GVV4d$Fi(QpoEFWp8B{gf&Fibm9tk_YQD7mUpFS4oo4% zHWPoTrd}8R0}J^ES)nG~UZCE?UtMI|mX0h#>Lw{c@YP#52NhYnMQd)=^7Gufb5*y~CC^mzZXcQ+V5 z^!mfW`R&6^FMnDU-Cwg#H?XWCG@()Z7mu*w?&2g=H5W&t`E)}nY`C=5R%Ok!H*7UD z8CyQ;JTQOJaJKG@A;!19UDf7m@Ud1EP7B0Y)upoA8y7UoASxs%5Oxai8 zHRAFf#lSr(BuFWT)&u>42J{CS&>v_(f1m;Vfd=#k8o(C|o-XA8UkvcY0ACF7#QwtjOL{2>_?iG;6X0tC zd`*C_3Gg)qzQmu9q9e)gv@GVY<4W0YRC*pZi=-ru=fjyMYmSU}ZA;b&>z-$c5QSbL z8G6&{8%mCl@_r1L%{QWb z!QOo>Ibw@c`xWY#1?*iT+*{atW`^xg-riA>vy*KC^Y3e`Z4?GDv2 zt1r2A_JwTV4)_qVvCQ5Tskzkts^j{s!&X1kSbfD4W=xQxM5q}IR{QFYW`{NU_b2w@ z0sy64R5yQ@>@WT$F>;XYnCdgqtPf_BS9jsSFwN(#;=uGEYxE|pF=SO7-@Z-$+N!LIL&4?L zQpYkj+Z$6$OSUY_UJorX4~VvEA#~Nd8{YKpxWp3@KICuHt;Ua{5tmwF-de6u$~v=U zDq{!)Fx95olgu(7f6Gq8Fc3xed`0Aa;v|I%tR$5Hbwwp4fv~JYFDE7dL7@R$a$(bZM1?C9l@bhbyF}y zn6~h0wE^n|xF~>>Bg~X&@4X=yJGKW&T@oBBh+AzX^`W#oe{3lGraQoSnl7;fPdW7z zpMbHMh-g4^HX+TAZ8`vF7#fhEyu(?N;j`C+m2S47Q!boR_@jYY>nKj{gA zbk>kR!=Ia%w{F!~sMDz!psL`EFbmXcTNU`S2#0H^^P6zutpNUb4EoI5ut?trJPH9w zA9qwf7EjOhB^p(En%~7~9;eGXTax^NWVc_Zkyrg^pd5z$n>Ygc+}EF=`UA5LG@Jtk zn9%t(SCg+dR{>~~2RJ$b+LJ;!CL1u-rrIR|002+|000pH0000000000004l2yOV=B RJ^>Gt!Z;%a$~6E00056!*gXIM delta 2034 zcmViX=NQXYjG!PwR! zH;7xvOR|`sv}~u-LRlRAlhJM=5o=nf$1FHK=BGD_Ps3x4pB`(upb!o`8oOzbROOFx zz*$w;rz{J3RhoZN7E?(&ps8?2ReCobhe1rM;y*H)29#+M;=qljVV&aYyvAa|8(p2F zZ&aPANH`9iWq!cFS66nEFiC&XR|zZ|MFHos)n_v|2;ET-23&sVm2nzXrMJS@lFj#N z;!(yDY5(c(5~AnPkftt|7rWBpK{DnqZYp{8HsP`n=ahd7GLpC}46e#76)PBHy(;%z z?uxJxj?%m^Vbs@duDeMTB{9<`&%^Q%#HPY;CoPf1EYFVC=~>1Tk=@isqgS*fhPpRsW~fB9 zR8-ww)DXc?@q_U=%WI7?mr^V{+AU6TI1<0i{E9DL#Zv%448HImbm(Fl4!1F4+7T(4PI}RrP$g+JU zrDK|kDo@`6eEKSq)q5V!h-#|3!aR?d%E)C@+v3b%L=n8M$MdIFaNY_|ndM=kSF%dm z7z0g%ZHlSAv4&qsEC;hiF#Mljdwa2{ZAvKW2;*;M7lbus*mUAbclQ=>Tb6gNXbvnX z#nyilsis~R{sRm78d<3(-JGM|{a;<=IJSXoQ|TrtMex;II0qG3yFqJh)Qa=mxN}vz z^L0Q}L+wb{!CtqDDc>q}K>(dx_xkF7IOyK>dq+qYUTIsViFMm?+NRw>=;G>pp(Tc; z+8qm<25#%3%*utlfA@IV(~pp?t0EY5gx-Gy&))4| zc;D*}2WL0;*S-8{EW5vEonaCcBQ#-A`i&YG>J zt`fV`UfF7jdxhAf)>a~v#@05-w1iTaApD8}m54+~x&TwkgV?R<(nmDqHMMOr<$pZa zh|7DF0QabrAf+5y5A+8*&>!ePf1m^Xfe!QsI?x~J0AC_`x|9Qa3BZ>Cd7^XtYXN*MfUgDc zwE(^rz}J%aQh!2=jwHX+vRJ&1Yh}Mt>3R4pQj$2H4`;fncT}=%Te3n}^*jrNDD(=+ z(3?)*P;!J+^dmt;WdNY!)*;k2jFW2HjTJA`iKzj#dGCFH^#kv_#oNyw7>n1erfq+} z?_q7PgiNy}4iPZJ0|m_Rivq?zkWW_o?r(eKdI?*9iQBQt|bjX$0EF{m|BEHBqh`0^F^9?}kXQoc#x-SP$4J#m%+ta9~!=CyfJweIc`&Z$!I- zz57~nz!s^0_AAsO3)rX8KHL@Y0vwe&E7?CrWyk;;8F{Zsr~UbXX+C$A2c~;jV>DrnsbYD2`!@M&V-?Fo!Nt_l zAS$sf+G|rwTeWT5Sq&|T2#7Ye5xVT%4zGK+LSjoF@;B+0<44(u3%xLJCD$0^r?X}% zV+aJvB!FX+%`zW<%}&EG5QOhMMdW*8JB5O*B$WX5L?tAOK-?^Q+gijwvJGk99ye*z zR_cl4dS<_!UAdZVb({Ty56YSv(So21jIdIfb&cMa^ZWs2A#x*mYYo(B2La8Rn@Tt) ztcO=`9r&mqWC>&f6Rt*^7#+hnhz)2SisUd!Tv@MqOr>9cW5>leUjr@)nPLr*OCCAi z19LYL(E%yNh;%=`?Ez9?XhDOC0n>uwlQ)9b;c7##oH@0MI|o;-M{#s-gBp!p*A?BQ zx2F6Qe=c6$`c-pf_NNknno0^LJn(3JQ{nR>9j@geF4Bot3gqK1=~vzkiwu1rvlM_0 zagWN!*}Ti~bW(ABOy!cmxc&FF!%^2eS_}oC5`y zsF?; Date: Thu, 6 Feb 2025 17:37:57 +0000 Subject: [PATCH 83/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14572 -> 14573 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0b7534a3f..2eafadbad 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 16:29:14,183 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3270 INFO] --- -[2025-02-06 16:29:14,183 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 17:37:52,060 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 17:37:52,060 submission_checker1.py:3270 INFO] --- +[2025-02-06 17:37:52,060 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 17:37:52,060 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 17:37:52,060 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3286 INFO] --- -[2025-02-06 16:29:14,183 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 16:29:14,183 submission_checker1.py:3307 INFO] --- -[2025-02-06 16:29:14,183 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 16:29:15,280 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 17:37:52,060 submission_checker1.py:3286 INFO] --- +[2025-02-06 17:37:52,061 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 17:37:52,061 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 17:37:52,061 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 17:37:52,061 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 17:37:52,061 submission_checker1.py:3307 INFO] --- +[2025-02-06 17:37:52,061 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 17:37:53,466 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.5612 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.35302 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352003 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74836 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index d27f55212..67ac2bbd7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -624 +625 diff --git a/summary.csv b/summary.csv index 3c39263af..8bd0d6ff2 100644 --- a/summary.csv +++ b/summary.csv @@ -15,7 +15,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.5612,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.35302,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352003,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74836,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index b5da46171..749a18b5d 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.35302, + "Result":0.352003, "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index df8fff6f20ce669680c630a1bea143baadba98f6..8ea5e8a0c4b7c69b2d175f55cc405f460277c529 100644 GIT binary patch delta 3293 zcmYM1dpy(oAIJAKx6n41Fmm67rWi?X%}B;M2RUXgA(>l4YBYt->=<2KHl#yMF3H{8 zDz}tda*9*^ha^?iL_-`Dr^U5Z|gt_z0jA%=bp zY?TLrhz(JYdI@U6z3bX9%bb|UN?a{*r55&Rr>k3Pa#%(1l2$ID$YV^yKipV@+gH)! z(ci{B?^c$4xF~4|S36PBI8uGGC8=k~sqCY2X`p{7T6sv(@@8_(Z?(9^D{q=}st$LD zc@nSbh1ug&<~Mez1+L2<(@a$i*8C;`o*v&i53&cii=yi6A;NKb`7l^T0#>#X#1I&e zo1Vu)?jnneQ{&WaAE?jks_k5=A{#u;FFB}~MjIK7x_y1^TLr~sNQu92*#Xe*%!je&$F32?5M#4SVf?)_ zcA-?>oL&S@_ab#VSx$~Vrf-OFDQgK#o0BCtmyOO+ovTavZtnH(bfje63F10tnl&IEcz}Vy6aW=jh7r(JBCRg*pp@u##mi$iRKDUpUU> z?_Cn+L}!PSI7C6cPX@V5{s239vfGQhf5X)L8hgOR_pZ~pGI2S?H@$3#wh^>L`(aHyEU<081AndbyE{cKsGJob6?qbb$u>jSKD6}{w1ox`4_8>kX zQ7YA^0(5(>UPR$Y)o9C1-RH(GU^K{Ibsm&;y?zwX)rZw;YcY}A_Tg6Ps{L`((cmTV_V=aw}`qI~n z55^`weRQ1@LydJlQYhbE4$Q||8rPgW0NLwUY^r@cf4`7<3aqU)jmCfj;ghTgn5eJ7 zN+|QuMjFQ3!s3U_&emi<>rrVq<2F5j+!HwLt{gw~DBAfnN8a%B-g>tnHC5wq=2Een zMwZJHh@*p-{?|+xT3PdJos5yAE<;8@5=w+`%-eWd9Ql_wm-K@tT71%zw81c$xI7+r z-Wkb}Z`bFuFT*vA%io4^X6y_){o*=APVZ$w8-qx{*SR6di&>GBi#a%(3$;$CxziW7 z=P$Rkb%?FEuCn{#X8|btkcu*Jntf8m5ctZDQSqS06T7oL)N*VxaoHJ;Efy#RXHO<0 z^AC_GlSuoiei52E?;b+{G{-^~l#OBvk!P8EO)V7ZOf>}KX1&&0dO6`;oXq%Y(YQzb zv%-N__3iGyL*X|!!>f|T7HqQH%;+le$hr>#p0RBdyqkJbt{ zq{1ypj^~~Q+aMT94-41xuEyNqB>&SauyWY4>J5Fl6Q>*7$xD>ZQMgRd%f&)>6OD!L zCfZ$tHa5FnCju&SnKdq4$2j*wR8wPi3*>zq+Y594UDd>FoOZ!5d)dG%Xe6$Iv{@Ai zJ^{JZ*|wUAgRq)spx3}fsZ=7dev4Yj@(SUqcO`n`PJnwm7xe0!wK4x37(O?07Y$X7iQbsKTkRiJ?RvCA@{)*~!#mw5j{>G!k!pcfLo ztNB+wI0w%2QL{~~O|j@l^8!NY?V(DXny*}l{Z8-)up()yb&_doQtg*Zl+b!tHV4hZtdDobRR<;5%G_f;X^we&u}L;$MDB$B z&ca6_D(aQ6Vv|ui@E|x(lLty35BXdt!m{;Gsu64W|AUTHp1cesk5ooI7@FdD44p$R zN}m~t+*zod68zeizFz175l;&%+I4LW?8>AvGM|9139}}geFBuwe&r1>D@D`~``MNh zbuP@8!-BnyDE)2vFMMlEO@`5#VSIMQ=LVy{P7TgZL@x1)RkAMJn#Wt}AzM;exln1| zVhv=ERwhY8d9ifFZ{R!72Un zYkjjOq6sq^-e-ba9JzwZ!e&3eLU8m;iE)(7k{P)|5YG(q`2=FyJZ5^(qDXuLvZ18S zk3K3CFB_N|%^ zw{=|s#`*G7J!_wM(nI|f<*}>2_}ek5mbuEBPFF2a2r4n0$$)i(qM1aHeX~52HfM5) zASU`}*$Tn+QXx{OxbrN|OrCDb# ztg1xT7S<^!iq-CcPdY$Ttx8l!;cHhOk$NJB-RI-~+2$A?iBFGc!G(Ns8`DzimhCgq zT}jWS@8Txyu(SgMj#gd{Q#@T!O0P{9(*tCcu&>3Ks`qb#{5r88ZP^SO!P`+z)srM* z?X-0KleB@CujKtty?!qdhN@?o@vF`Dq8 zU(5nMlYp&ruLIBp+Ko*K+Fp=$FgFof9-kZN%A2C!R~UEk?U;L8a?0{PLXetE9T(~= z-L%6x5^0-VXbkd}s~pZC;We}x^sAta^Hsyfb zy8TP5zpU}&-ti6}R%&`*wRK?e-fY6TYV*KKf6MKuV(<1Rze7I-w!I>o`YDd7`*z$J zJdiwd!OFVUaHnBqt$wp;{boA8Cncos<;}PV{<6W}m9ze}S6>pFCN5JWHa3@*Lu>tf zulW!5HSj6?<=~BmqSu=<0aUc=cFUdTo8)ht#cwbDQyw?^eM6JBKiB@gQ{C(-G*!tE z|3IC%DPdN<{a59axJ1)DPDa_a*Th6T$~3%jv5CL+ZE9tB;MEI(OLY6^ux8&O-AmNg zXA;Hn9Ri=|^TWXnGk@*%$1|q~NIPEyw3Y7~4UXDLFwyyzRx$Pl6lM2UN_LbehyB5itH&&=i1xt=8zN<0|Xa^~YWnN(1l2dCtDtfjL zqFL@tw-bx5tuC?2Zl?C8)JJ)PhsMZBNJ@Ws*+Xz|Fro=v8S-kk^V*PKjk_8f8n(c}p z5T)sN*+y}C)NK=%v^~>;JA}sHI)Xzv#jZgWY?Z(>uslGiFJ(q@`s$J-t~QBgv@z;g zR>Pf&kkE%f?F*ZS>Kxh=dj^~;zAD`c@CiYw4yagOPmW5xi(iQA=jGNK;zB)x(se>{ zc(wVJEqd|E#i4W)DcvRFB+#d`ZkjJE36voZ&?SQ)c%347`RaJ9JvE>dfo^0=bF^%k zR(W|ERo$jleg2^GR!nWA?!!W^-uE=d(BFnPGM6^0FS`uvUH^DayLaFEd?vB+5$M>& zLP+xSk?T}wV^Z`Jd$nH0b=o3>@&^l109@MB58BZpdG>r$21yk0qGE68y6AhmkUsjH z&UySnclu0(fopX8YL$ezDE*P48Vmxiz&7e6@!QXW%&~&PLbQ);ub-K zV-9#kaU(e^vRhF*B6+gMt6|@YiA4sd-`JzfX;d|G3Rz>T8_IS4*yg~6R%ahe$0hi0 zF@)G^c}Z9iP%YmNDx%mvxjm>Hpq%Yog51HH?O*{jdXO;UE5Z2jn!9>t@d;9C+ z=wCLm_mj^Y9hQcqTuqAf?L9N-u1XuYcadb{b_WTxlsOGo#oOc>v+s- zUXU%1Qk@jk%NRKxOpy^#gcczy<5u3frvLHg5`XbT${akAD=Effv7^)r1Zat3=lmKc z4x({5^i62Vdplh|k@{el&u+F{GbOCB-bE{Nffar^A{TFUsm95tVJ2d8KDNE%f#~x6 z-#Pt|KtPsbsHO@ya~#$5f!Ca9wNvy@c79K*uQ_QvK|FOp_&`-Za^rW=Jl zIKCqOWTgsS3(B51j*`ho53w7%#nhnEUEQ(@j|k)P)W;skFBq|f7}o_|FUrvSu<>_? z41!zwV@*eG^u7_`p zp@zaE>C;2hZSYdsT-24@7 z{MVS{y;}o=|6U;Lt2c#yBL(`9*YkN`TtE|Uc*?(w28T_7NrDb_BB*{P^r%>~dPv`xKlpua>eq-M#z*k<59i+MLB%#& z-m-U}AgS$5WPI)LiP;gZtUAu!5W(vb5 zk0sNXNy(-#Y!-~;NI+ldUF^%G@&7yM+{l4WhRxhaD>?WZ&3Cqiy_#wiVoqQ7yck~{ z_?SARE1ukA;wnrHwhmO<(QTjjF?zdEe4z%$s|e$l5g^s2Rer_aw;Pj1lDOl7m-x&Z zwp|LZd5eDOkY69H=XC0)SAKZahlnuq?=!Af9q&Dhtbyhz_~7vU%+#4EPi9K?FsPpy z#1QJNmEED;^)#AjyKst@Dj_wZ$7bH$bl6!nFm^57MDyPJyg_~T;BcI-7g?U|tr^-f zUBHc+JDTSRbNYA`i7@OI=I1CjcN7VBY^=M?z?V#P>+;QcH05>`5si?(IUP5M9}E-4 zo_HgAz4_9{gSxQ~>!_R8e)Cn?Sb$IT?yaPx+U#u!$_-xCyYDv54yd#H(9omh>h{6E zERUW;b(9#F$)=LT#wg@(pdzax<|(XMJUUDChPXEh{Ra&x`8JljFG=F_wYsslRH?MV zxPxAOQpwmiN->$C_69Wr=pHCRMqaBIs=|JFxW&#^B7o7>jRL`6r;B4(3M^}PCC$0V zAS*3*rs-q@qDT^xhY%Jqda6OW$dZ1>WP9GY&q-x3?$#L7q4nw|TKO=#7)t0+(7FLt zIg1Y#X89BpEX>l0w@2>eJSUnY${h|~B-KZk3K!~=3N|w3`W*l+mpRd@oPjALSuL--TH;SS4nkinta)^$p-4!q&aT#fW z`Q@(+vGS31CQVbqk{95wy}xU=7}7d2+EB#1=6d^v1@A(OEcvm57Yea65k(Tk4b3i)zDDhs<2yqeJzbD4nF#QYGX+0{qGxv5%x2TocXt&f z*CxilU!9(H)_m%pD6RYc-w<*g3N{Xe3}&kL@DvO-W6GcMhfL-!j)6M_ChA0hx=O{U zV1(~)PUk|oI8J?d_q@z{=V@L*>ugn|)$-P0o|@HTv$x!)waWQy>8Z47h+>Qbp-YH! z`wD~cIYEe<$OGRfin}m@CI5Eof^I&LdaFuT`maHTez3Jtye$@~k4V3&XO^h0^}S?G zG#6TxSXFYA9BC(I1x?0bd!y zSy;~S2kq3x4FccnQ6^ZIc~}t+S5hw8ct|a18~8?syH!Cw6yk9N`T|qP62M?w6kp&E zDPb!!0Eblja0IfbOR0h5ioSMs)6%k>$Idy$g-rVV!g{LG1VFW>3RAy>~xMkJCCf zqKyS!u8V?8_0|Pf6MS4g#-g_dqqY_dyM4V46Ck{VpPtU!#Md>-goha_HRP;D0k~*% z>+{GY##-&!;NuteizsV76*bdWtFLIZgo{6B-+wy0SE%tw%|5%q${4XRdu~Y5Zq{|pti0AJ z!8}*FTy$Zo4zzVyM)2%)z2k&9MhbA$i1DyeaT#3IQaeR0oahZzujogU*&B(JR^JMi z=Vox}{wo_r7P|y*UY3*!z)kWd5b*#1fpAdfo`56g4|5DK!Fg bf60bhr~vmlsTNj%9H+ Date: Thu, 6 Feb 2025 19:32:10 +0000 Subject: [PATCH 84/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14573 -> 14573 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2eafadbad..59937d188 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 17:37:52,060 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 17:37:52,060 submission_checker1.py:3270 INFO] --- -[2025-02-06 17:37:52,060 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3270 INFO] --- +[2025-02-06 19:32:05,202 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-06 17:37:52,060 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-06 17:37:52,060 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 17:37:52,060 submission_checker1.py:3286 INFO] --- -[2025-02-06 17:37:52,061 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 17:37:52,061 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 17:37:52,061 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 17:37:52,061 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 17:37:52,061 submission_checker1.py:3307 INFO] --- -[2025-02-06 17:37:52,061 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 17:37:53,466 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 19:32:05,202 submission_checker1.py:3286 INFO] --- +[2025-02-06 19:32:05,202 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 19:32:05,202 submission_checker1.py:3307 INFO] --- +[2025-02-06 19:32:05,202 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 19:32:06,222 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.14067 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.5612 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1296 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352003 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74836 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 67ac2bbd7..104eee38f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -625 +626 diff --git a/summary.csv b/summary.csv index 8bd0d6ff2..895d564cf 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference- MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.14067,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.5612,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1296,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352003,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74836,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 749a18b5d..bd014c033 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":49.5612, + "Result":49.1296, "Accuracy":"GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 8ea5e8a0c4b7c69b2d175f55cc405f460277c529..121f4f0702fdd028d8d9d47627ea6c602c80903d 100644 GIT binary patch delta 3551 zcmXAsdpy(oAICS#%ypxPbJ`el$ws9@avfzX5n^*)A(s&42+@YclFJO!uZv4AKCFf4&~i_v`(BzrOG9QutCh=Yq_x=H`acx*(a0VpPld zk&W?J=-vD4Ps;`F&P-*oqmt>&0Z<{sk2U}*A#C1Lbp*|?lTTEF`15Ni1aR?Tuu(53 ztTek#7<8+uP_wA1t| zE_|c8r&hxJQ4QXe7&3=&(=$rBJtrHnm>D`eP3^XQxY}yday}CVo7@`frD_eFm_>VYaW{H)~TZG}`d~Fn%?n9Lr{);u|8g<9_40P zK^1v)nA95fSI>z*cv)2Ez7v?DN>&rJ=L8P2^9JJ?6@$9xOJ{>3;V=6!)D`7`EuVh z3(Wy8wwQJDj~rb`gYk$a{2w}Un`xU6))|);Jcb?&ZNlRq!7Op~T~jhpynb9Ge;ckW z$ktIpiVCuK+L)*ASM#)*w6VEgCA3T=;z7TJ!wAQAk^R2+W~8C{ozEB5kSc(hkvWEr zff`cHaS#mG)1PLQzpG1I<)*1PfoQhVGd&4HL$JoGrBw!sjCnin0<2AKWj%b2;lU9 z1^}2ITMqTAsSjV6fIZn)K(&uQ2#KB_*p=FC#9`K%8=%-Zr zsGVG5#vNHuTiq`J_iADNl;!YHh_5mm4DF>eEvWfR0Ar%*q&4y34x^rgqRDSP%KE32 zw7f8=`=Aj#v~2du0|RyCNp9fGc|LkGK=qc@@aj1Oo6&jvUKyW8S<#b*E}r~yxv?vgqBraIp4<} z2zo{yb&)~08wMjSJu(RkMu3_pp!OW(38>-!R=Hv7o-7ZNh~o|b1F5x(oQ%}xcon*5 zL_ijEa-fS1^)wyuTI#9wL~eui;>u;PAYb7rB&1ZP-uE*n{$w|PnUhFgcGS1}=a78=@r(|k=SYjK zf>OjW_r|L!{p2OCSK42E+btI1&2fP88bCP)x&~0Xa9_ds3t3l5;DXV)FN$GT3)HDV z&4JS6e<#zIMfOIZoN5QwuJGTw-{mWZ~hGJjYLHH2tvor*lfQ ztl$Mmm@pRQ0(!STZw!q<1@sEeL26}Mb{6}Ro+NDGuKW*Ud-_^5*HitLl+h)2Ro-Yw zLC>_zZ8pU2e72S2!ns0{tqF!o4}cO(aF76&bR5Vo|5c|1=#FVgH*FZAqjWr&q49a# zvSr#vZVTyD`)uus;aitm-NA0gW2Dz!rlm`uOaU&lK8mEpR)CA68DIr?KZxObSQ#{= zxxx_vJs6xm^lsUlcoGW!kb`t{@Unpo?W*c_uG}zA0yfzTOZG-kbDl5W9JV@kPbABb zgGnG_cqj!{>&FNJtX2wfjRfUjTEBzXrZqroadB=L;3_dUy-0z1v>$rQL#aU6{yomA zc42LoZvKZSF5IUox(1%g74@&8kCr78XHt8nf@9n9tV9`cbS}6WfXnR~0qe$^yR6b^daDak{*;pDQ8&AECuJVmTOTBsN zy?xdre?btAs_*MiIb@d&xJI}RT3}KUdWw`4j?_WmCFSz~`x8xXbtmwC*?(DVP)#9W z(Mc|$wW^-^WFqMCz=kZ)k`%|$>k6gr<(vB^(oCYA)f{yXPX>vVSnWNt6AICjqiA&T zk@nSdN(I2%Jm_ExB!qYTmz}*3?FaUv`SI`1Ap?V@N)tkxKU}#rTeqHPJElM{Rv@@7 zMQH>gV=~ko8LSw%WN$wQ@Q($K0sLbHV^Nghxo;ea_qCOuN8 z`l00T`Ve0f8>~O>V%%Oj`0<=P(=LC&$6^l1uXI&qBK|RIK1n5^)dw4SHBV$_Ezib< z`&yL9A@TlW)w{oLXWc5)@UwaHMEGkuzXK+((%Tnu>4)DY*bF zukW5HT<-;u40T`->Gz|8c^}iMdtXw8r?@Ph=$ZXrr-wp{_Dwyim+|kGtFf~2Prl(c zJPvrN*FFw-$qMO$I17uBF5t3t@^OVqWI*pHt<^G&!hI-78Tcfw=YZG5;4K4S#(ut@6nZ)J)g$sOlW6wNf}UJO=c6vBnb5pwOy1 z^=K*+Nt!t@VFUmtyS{0p5c>-*%qjWv=M_etn$$orK z0k@l4^d?QPfnEKa#LF^Xw%vu)(LpH$+1vs$>ow>WkXf#cKJev*S5YL<+0nTb(sERR zdQnk(8VV6jFepAe`|a|0%4FWd;myB)n7Idi-!~DFH84|DYN^XmOZPwZ7O5ku49ImP z(?Ry^%qVVZSC$=YIW;Z`UIB^{YDf4`9b$I`32n3+?-Dh{!L{{FjDEFXS4_5mIv)(;YW)ynwID3k$3{^ko3WL67-;7k~C%EaD0Gh`-r~?k0N3L9CYy& z9v4KwrC_T2HXl!$XZKi4tqeyJ{R-Yk`FJaj68e5X-}1*jJZ3jnM=8eyQ}<`r)}CZ} zd+7#sl2`1<>;`{~xrKMRJc|_atQ%qyk1EgbqUz=Yp5RJNnqp1v+C%$4M8$ZyZ~0Gt z@LN1FcK}(HB0ZR1b71Yw>crWnu7l6Kbgn^ew=H-F~#X zCa!G-95hq5-`mi?RY?jM33z>RZDo@;nV(x%v5`L$v-B#s`JUaEpOKmHlbf*{k7wTH zy}Z!hD*dcdrZ;zd>)#FVsP`(io(pwgf_jHTdJ**+y zaGg0~bp40+=^4yBr;LjKZAV6)X?|OwzbQIbG9=%3I6;p87FPCHKIW3kyoqfr#f>u- z@W=s<(phX}x%Z_dy87dqR5~;#(0oo?s2B^|9^~|IAy(ha^VH#QFa31`{H> zFfv6no+_@OXbN&VE;l5;|2w1JtQrYQ@CmSpmqE>u(+~UuI=$JOWYK(CTPa2|Ypl+& z<(pUfi|R#gs6NT(=RRt+_diKzoyy=3XJ`wKVSPvr#lv%EQ599~XIlCGrh#0SkMhRN z$FEeoO%XOy<{9CGechkuo5_VODm-KL6SfL=Kb6e=Mb+18I(zQ^lytbCv~@QMy6?)= a^#mcj4sk>Y^7|)F$G`;q+p$nX>i+=|^MW1# delta 3551 zcmXX}c{~&RAK%QCYjcEE?oDVqM$#Ouk&LGY$qY*+%h4z`jIi03p>oWT9yEpIo@fGf4*Pu&-?p+e?Fh@Lc~G@_ne9}G5AM7L#WDmIjT{3 zPWH?(lnyczBZOsIsq#%iQlg#Z1xk=qyzH!!dusNOsi929pB%3)1wf&FZL$U$rCyY! zq!{EfQCZ6y-xiBZ^{Y^hKS8_@9Dnb+;ma}?*1-~Y8*C}c5$Te;rlW?@1kD@d0yX)} zkI)Y{mY|MR^f=_tG4DH-B_A$R<;|fwM=BbJtB*D(b`QFgebFurI2DZ49@MnC$%wpK zixpmZ&C99U(-q=Pyk-*Oh}|Yw*`OA5p97Mn3@T@1&mVd2W-}S2wO0E z4KD*5ozV&rDNj2D^Jh)Mu*R3E9~o+D^iflDm|IzMK+3Et-nDFGhU!|y-k(%(Jkn?P zh=QmUD3++a)VGFgwRsb&Ej(&{#sVAP^&8 z<^~Vg3i*@cO8(U;XH9f3oxqX6Q`-fA%sI)Oe79LiAph|w+1HO* z@bKlL2N6b`Di5t?TbSFrrFWL@V6ULgfa{1qF(w(Zefq zx9YtJe%)mq{z;oPel}TO4coxFT4i~~S^jfN6}(IGz@9Tg?Wt79<1Zy>b0`>7Lv|L00BHosnyyTsssQe@S&@s>HMGGq z)I2wFRpD)H$##5gl_kEmRED%A@86SGssmFDUR9d5psJHRtfHuK#I9^FogDiNY<9YH zGYX;M>dj(i{0;DC5ov$a3&9xz&qzE_e>7-Ldv9bB{47h^5~WFJ>A;vb>kXFD%JJ`F zl*T^a9rLPxR@6UH-{wge486G;TE)Ok-E-@2F7fZ9k)S%8rg*jG#sZ@(sH$Z zbT6$1WKNaz?0o*&IeQpW>tWGyUTWlR4&xI~Z0odU+Y|h9BgQzYgP)+7qY;ZY$;E&- zgN*@i2HR7QHafG+l>(Hx&Fm{$M!WW?Y)_8d9HZ}JvV71N-&Kvz#26L~Wi6Qb1P;eE zkXEb0K}S@gI~sQcs};l8$H9u-$L9>F296f!xie677HJj~EH2Q+0ftvh?PRlcgZblU z94yZa>5eG7u_r(0rLGrpI^XkrZO@HP)%7%gTiO>Um)vzOrKGzPQ`bmwmOre-rQl`J zuzV39**=A}6#$7>Ne8kdMnTG()Xopnms~K(#=&;?0;*^BS^g?t_T`J|)cYF#ZrR3t z?=|we65*-@L(^~Ej5qts95Bg5+IT0D2A}Abr)!of_gZ+ zm5Tz;csy`&q)e>s9$!R-8!>203T?3^T1b;Cs{>Xcc85A*ssrN>E8Sy1;YGf=+@u;b ztaik4WA2Lt>cWI%(J&B3&_F0(pATS+1xXUYJ}gOuOvC>h9j-hY3uFveUU)D#C2Ai$ z4;Lz)84ll=tDO>`=uKNL@=}pa3Awwe+7if}MP+6@0bG-KO;UTs2#NgKD?YZGh~M&O z*-&)rAOsE@@+PeG>c_XZmdKj)y=R7S*%jX!_P#wnFf$%L&oADVdFhq_XJZ0yPTm{} zd|{2s7K2h?uEZyjIuxIDeA{W!ngaQ=Bp0T5`*|+E%`Ux2vH=`mKGYEOrG9gcAB>sndXctH#GqSn7 z{qM1g?7&dwV2_&>cZW=`#rxF_EE8>Y2HFv8FORfe+vCnxVC)X_Lf^Ilw`cen1Y%NVjNY=K@#3xpe4N-RT9iWankN!F}ZSoCG=QYCpcP zOEU&j7dK|achS^hmac^nP+7Um0p@SCTmDOhtgy|IjVG}`mAF9JkgV!gsc`Ay0 zKdR)ok_M4|z`8v)p1Ej`W2@ z7Z&r}Tv}K&Ht4IzsDW0OYOjUy zVp=YJ6K>K0L)+Q!Z0qAR#W$9w^w@W@ya2ZGTbuV$P5yI`e+TBv;VdQ%@9V6#-J2w3 z=Q4ljtD>0?LH_>N9)J6af6|P4L0R~NX!v4st|mmWsZ_=hBOgkW=|Td`@*%XjQJUnT zU(5izCxM4udz^rsfKA(!ARWaiyK)mS<#D+I?))kGeT^|ULi_BSlH)e_VdCUm>X?LI z>8b<9nMhmhM55uh+|{sV@vp$ufD__Y&JR6%_buM>3%PT zIq;g2FMou-*E}EHef6-~Bdhz@Z-6}S-U$wos&n`I3k_rD$>yOEpgd%!?B3N|=tim2 za~6pKdlQQ!_h^ULZi_^~D2-a35NyyvcKulNnG(`ciX4t;t*cEBg-yqU!e>dlcQ3_> zLb*HUKmWNTih0N1vq!7xebw4d;k}vo^VQY?m8Wdhr;2^sp8N&=8qhjHw)EE=)g`pw z9@xp4zGQ1xYrfI2xKzJ-cll--t~)8H_vOu)Fwuh9zm+qmYExelo5o|QVJoZi3&FMi zglne;dK*L((ZacvhP$s;r%zLn+t-_KKVK#Pvx5#7-B6m?=8(>v=pfG-U zBF5AAlKM!GYEweSb5zWazO)apNze4kTkBC#9(AZbW3h~oiG%mCHI5YJeZstei^pp| z9Mw4JwzcdA&i$^sbz_CD!a4a-&39E+r`rHpG8vbcx72i64@jTwRnaeZr8~$()K-_+ zXY+cJ>M!_$21m(>a0<1ryzC*U=N#-VkUx=Zd59+f7Mux2)0b3&ut~N4U|cgl$^TJ= z%X>Fj<7%x4vR8q1<)$!7EclGwP9Y9uMKz9+!?&^ zH08V0+NkNQ`h`xz8T&~OdGuh!p1zrR5wXR|>-MK*>wypCx&dfzAP=@2_wq)o@UhhN i-jDxwDNYvr*ib(~ZGc!pLH~W?%n#TB|5Gg7n)*K^?TXL< diff --git a/summary_results.json b/summary_results.json index ef32a38e5..5c1f46de5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -824,7 +824,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 49.5612, + "Performance_Result": 49.1296, "Performance_Units": "Tokens/s" }, { From 09ab1aadccc1ada818c163b6d55ab52565161a6e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Feb 2025 21:17:00 +0000 Subject: [PATCH 85/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 14573 -> 14572 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 59937d188..1db60d240 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 19:32:05,202 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3270 INFO] --- -[2025-02-06 19:32:05,202 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 21:16:54,602 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 21:16:54,602 submission_checker1.py:3270 INFO] --- +[2025-02-06 21:16:54,602 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 21:16:54,602 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 21:16:54,603 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3286 INFO] --- -[2025-02-06 19:32:05,202 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 19:32:05,202 submission_checker1.py:3307 INFO] --- -[2025-02-06 19:32:05,202 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 19:32:06,222 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 21:16:54,603 submission_checker1.py:3286 INFO] --- +[2025-02-06 21:16:54,603 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 21:16:54,603 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 21:16:54,603 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 21:16:54,603 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 21:16:54,603 submission_checker1.py:3307 INFO] --- +[2025-02-06 21:16:54,603 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 21:16:55,718 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.264988 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412266 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.9092 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3387 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 104eee38f..d8560d5c6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -626 +627 diff --git a/summary.csv b/summary.csv index 895d564cf..78e60eeaa 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-re MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.264988,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412266,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.9092,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.3387,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bd014c033..4f8b5a55c 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.9092, + "Result":17.3387, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 121f4f0702fdd028d8d9d47627ea6c602c80903d..5a33c49ea392829b516d5715ffa1684754e92557 100644 GIT binary patch delta 1294 zcmV+p1@ZdraqMxhW)1~X11bHwvuh4}34huRV-nYi)8Ptfv`FMP<_}0lS?LOq5V9HI z3b72ZNhXAB`eY$i0onA~Liay>U{khabQEAycI%ODc$SpC`nqD(=cjE&EE&ZHBXei0Z zC?M6cnaB?jJrY7TyYSjHPznm9^h*9o(a7~Tedm)=I32s&&X zOJorxZ5@zz-GwcCJgVJ@UB4pFZdg3Y1FuBT(bD2!p_a6GNCq1&;?>GCB&}=mC+3bVE8relM4;*zI{LozJ}gv!#a5 zu}wGWEIw*F{ac2E_+j zGCGRX3s?v~Xp#^ROAAS2enBOR3jsCW*vm&X+HvH6cwiDM2dKmy(b589 zp|-SuNC-gTJfIjnAZW?hC}0>sw;t(+RE9`DamCV4D3#Ar z3@TY%5S7CEiOUzAn^bBCl|&!W()wYcwzPgo2&oLPepm*mBoj&^Nu^Kwp%s8ipZ9bB z!v`wC`=O)APopedKU`jL; zElnU6YD*J{gb>pJ6NqI1Q)buzralvhRsc+WCQu9}5VT};6u8P|CJNj^V+&pY|$<)Sv675`-2^t^Aa z6-R9KYo}j)RI4{w^C4c0H=B;e%VOGt%h$8{WUyS0r{}fBf7V@IwqA7p($T8e@a*{d zGOsPaUFCcA{OeWmx~vvkuZ_N*y&dhCU2-|_SB2Y|oq4p*Es5kK=ikzp4(?%;O(Fq({R}TdWx(A%6=?Y8mf0R>CaR-deL_`CU zvk7T_ZPNod!_a^PjK0!k-MxT1RnmADx(tZQCa8qPK?p8UEV7 zzW1xfLUpHNfU1Hs!Yoj)ZB^jwA{?%v&TqqscLMm+Dd;nA!y@NC;86%b&T&WObMgFA zZ%~z|`7%!PI9=6w8Ah@dSuVf3kvIJppd5zWO&kH|+_#^g`UkTRG#LX0QUfXdx|1$9 zTLDOuiZ?s~-;>8TK>=u!2{<|d+LJ^$CIo5*B@mNtI2HlBlZ7}w0S}YJI3otiH2?qr E0Am$n8~^|S delta 1312 zcmV+*1>gGYaqV%iW)20pxwX~1vuh4}34fAC8kH_mH`#Q!LK-a+`HlGll2KN=LL`K2 z2Dm~j18kBBA)7u~h*dx~eYVj34z3omiFPw7JMCO$>=C>Q-|!;6U!4L z$-uo==Ka9E*NP3EO;CT-S3@dUWzb#Nvd5#^jo9@o^6ZAilRWTB1RX6c9u{gzi-&}e z*8q!$Wq?<OXdc}O>;GvxPj*^1qsr_%Y%`#)Q1 z_#E4Glg{F!rb8}>BaUZG+#M|i9~Np!!H0wZPnMB*vW&!&O$hLeeTEOK0C@T;bYoC_ zpe3WDNWFlC;DaU!0kO1@BsQ+t1bUR%f8rNZvbYdXJWdKw*Apq5<0kH}Qs?P(8!2^PpjEw?@0d(t;Zb)T_^b=Ps{e)8a zEXAOb#RX9*te?1i(YZ;bc2G(55iPAB7HUiDhlG&I0PBZkfJ!o#dlsPuV1 z_dk4~61*Qeiu^RnLVxlXF7-$VK@CxXN^7bV)F1E}w34L-VOlnUiVK4za&1r<8waLD zL($R%VxhJ)fk+534KRUN1~6rY4PfdsfoKK5)Mo<4U;;r)Mn{3GTxJ3RrU6_1&_aOe zmdHWh@zQonWuW7DV_zTDNXq*L+|b!vT;%!k)nGZ;-<{<1^MAaPUtTV1!&&i9)=kg* z)>?7IR=;-o#YeSzgEb%G#dx#nXuK?@J-B>5n@8tTl{C; zuP^i3^4nFuSI@s*6|c)`vGv;M>)G4UUcI;$znYI{^YQZA&FE)Ewd>2l{$w`#bv>!; z>G(g_`LDkWe_?68_+{Ao?O!$hTWzmaEd4H8tZqHZ>JMyM`@2_z^ZaNqKOavQwafgh zSZLH}*K70f`NiFzm$R$&pW=G$(`>n%P4517kq<`seD$~D;GWH9%e%j<4#}6<{O|Pv z-TyzcNFsp{1Z|}zp_9!rD}UJ`p%u0RZON&DLS0&V6VfITwSJJ)I)3{qPU0lsQ;(~e z{dRUyx!kKZ`32{t(G|)wf)dc&2&uOfdT&1&{5nNr&x%k+hWW3H`@X(2w7kSfqw}W7(M_~I}*_e zDfo!AKdv1Bf@5ewfes$$S&lE>08aVIhC#VhQ9;(+Q>aLOI9U|e0|>mnYmWj?Iqi8m6&V;}X2x6>lWJ`hO^K#y_H%E#>K zxmlsQAjLc-MM@S;{uw}L1ts&Z!^oTd3s9Yg{F`_N9CP1(g8C1$4>TDA1-ZGk)w`1| zH(LQolZiJx0pOFyH$eeulL$CE0o#*9I3^oyr6!>z0RR9{0ssIJ00000000000001h WfxMH1I6eUolfyV82Fo=700030EO8S6 diff --git a/summary_results.json b/summary_results.json index 5c1f46de5..4f96ac191 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 18.9092, + "Performance_Result": 17.3387, "Performance_Units": "Samples/s" }, { From 56493a9cf651021f7f4386f810b3efa4371a98c2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Feb 2025 21:21:35 +0000 Subject: [PATCH 86/86] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 14572 -> 14569 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1db60d240..0c630a279 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 21:16:54,602 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 21:16:54,602 submission_checker1.py:3270 INFO] --- -[2025-02-06 21:16:54,602 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3270 INFO] --- +[2025-02-06 21:21:30,151 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 -[2025-02-06 21:16:54,602 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 -[2025-02-06 21:16:54,603 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 21:16:54,603 submission_checker1.py:3286 INFO] --- -[2025-02-06 21:16:54,603 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 21:16:54,603 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 21:16:54,603 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 21:16:54,603 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 21:16:54,603 submission_checker1.py:3307 INFO] --- -[2025-02-06 21:16:54,603 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 21:16:55,718 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 21:21:30,151 submission_checker1.py:3286 INFO] --- +[2025-02-06 21:21:30,151 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 21:21:30,151 submission_checker1.py:3307 INFO] --- +[2025-02-06 21:21:30,152 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 21:21:31,321 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -62,7 +62,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409259 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0506 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0454 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d8560d5c6..01d65e21e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -627 +628 diff --git a/summary.csv b/summary.csv index 78e60eeaa..9a6e83b29 100644 --- a/summary.csv +++ b/summary.csv @@ -40,7 +40,7 @@ MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntim MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409259,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0506,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0454,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4f8b5a55c..4f24c2064 100644 --- a/summary.json +++ b/summary.json @@ -1239,8 +1239,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0506, - "Accuracy":"acc: 75.200", + "Result":10.0454, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 5a33c49ea392829b516d5715ffa1684754e92557..8e08ce4ea2dcbf3268e8641c843414eaf8d98a4a 100644 GIT binary patch delta 2009 zcmV;~2PXLJap`fe?IQ)5YrN-Glky`Xe^cW)5PsiZp~}9@Jou7t3Yelops3;61h&|x zlw&KwJ3g|O9K!zkm9cYRxkC~h_KBif*4JOFyVZ8*_{Ths(2Qg>Op}w2CaWDp5q=t+}lEqv24nyzCEuYoS_ti<8fZp8fh-1bQF29hb3DbNv>O0 z#^Ic0UL-kldnA~i{^G`qx%SA9xUqQfD9tm^7jmKs937AwfZPP+!kJZa8<0DITyRDy z*Mu`%3-F=}uZ6chQf*ts!N{}%DI;T&@hVT=0(|<)lhuD7&aiCAf0{&n9~;uhqeR)_ z%wR+wyq?dBr&e(O3Qn38VWQQtO4%3#ZG&x!slBm=UrDS0vv@H4pJ01?v8-)U#OnxU zZ)F#RHAL8K;t6;67I0gZccp9&Od-YA6RDwIm;M6_`5IZFCf%H)-u+))WZRaGEJNz1 z8A0&XTRI1oS-U}Ne{R(B^W3;|UAyyjfMs2Fh3jCiTg4D>m8KwoPOf`>bw3<*@A|zX zq;an}reUa>W!sKnxd>fcoiDUlH)YE;RYO-Djh9)mkoWH%FMHY%vNV|ogND$X03RXU zmR;V>R)T9TaAn=nY=j=}e)et$!~0%;I5@kxzwQ-JtD^fGf7a;+mQ{o%GMQ2`Z? zL|eE3Gs42eYv|HPBx4P=Z8BwlH;Aiy6a)9DkRX*DS`YLG8qgnTK!2bC{ecGb2O7{H zXaHX zfUgddK`0h~H2}T_z*j8kl^oz}0(?z?uL zB|q1&p#Zgc?|roc@4Lm@&mI_y*Q~Z}f4}cxZLfuYOuHlw5ir981(O!PgeWx zZ+q z{RgGk4A?5g&9&`tU}nQ7jRS&xA+v^WM7x5$`&x3q7OC+o)FBJlr_ny)$HN!QkKo#` zn5`gxw(HaOYi28m?UHP_YVLs6LfPC2?JK#pT{s)i*th`F>?WjEW3mEjC@DJvy@W}S zzR>2o-~j2p3tk(Uwh^!0G}{)cg&f-+s%2JRa_j60*}@(0A!KWry)9CEsr^;Q^;w54 zf2gtiiYLsNAVr1HFc_@&)gR3c8}#o_?860r07|u}X)f7+{g))jMYe0IPe`*qm`&yh zryC|j7?3VncT}BgUB7np8z-9L4>r~lWB`qbyw{}D`uxDO zn7fJt)4ialKWo$ZYQ%g&>EX!UEEin%WN3{^T?A;C+ zuY0##VhJDeH|dt+N70B2tu${XS14ttvt}w|2nE$QH6bLE&N3c<%WA_g5JmU>3ZuO( z*`}cw+kv)Z)j**xE!_!el89OlB(;wJzVahZ0$CNbk?x%{cLtT~qimyJuuf=Ip)?^V z0>v~JYG0xEW)nZ4$a$(bZMA|5_2AIDzAKq2n6~h0wE^n|xF~>>E10R!!Fy9+?AQS$ zbxCljAnvr4)Q78o-D5-9H{Aoy2w7nXo^t9b9)Ph~h-gM~wjj-qZASoS7+R2^yu)dd z;2y? zwl4AIB`mI`&TqqtcLMlhAM_h<^B_|l@F)Z*rno2NBY%1%ZZ@dSNOm8SEGDZaON(%F z@-?))sy_qe+~nWH6JW}H{R!$nvkx?!0|l9DyysPuus2r$W|IgwIsw>|LpUZI)i*UE rBmn>bPyzq|5dZ)H0000000000fPuM_gg8C{43on+BL>Gc00000`dQJ} delta 2021 zcmV6cP9;pG2uYb;Q2rE)SQe=*mL zlQeP}|2Lf|bea;^r!R!CC{(bjT1w;waSM4#7PGUKWjigD#lb%r=@t^crqwy6LFbfp zZWEt`rwZ$wD!HH#4m=XOX^>RqPjSF#RoEvq3t3f~5*iahIv^=`M^$=19*03ps^UK~ zk_Ln-0^-1preU4p`l7~S&Kq4{f1qzfp2&zh4xMLyz`j>kc9Sqkf0Bg&mW`r-G12Pt znHz-eC_qAa2Lz;Mm(nQ#Q`n!PWc_bvM%f!X5uy~M+*{hohUcF10 zXv76!oQxpua)YZfQ&uHaRkfhC52}!h{lExxMKoQIy0~nY;|Me-RJ8Y&dUQ zQalU%z?CM`OnM|SXWoC5d@^=<6b-#39tRVFUE%7GthTJK02Ql8tBQ9v34z3Ugx~t# z(KAWn{O_wRVZ$+>1#kX&eQ|w`uE=GQdIa?!pOK+DYIWI?XGNYT`3aqJzn!&s7SlXC zTAlNZB|N){k46i$B8IxRe`u!5STL zqhQWbHmAW8+POf`%{V*JK@B6(I zq;apbO~X(%%eLEwgg}i_Nbk);Nkfq5y7&L_51o#B$ zw(Rh3wiMiOfg|gdW+U`?|Fd^D7(Vp+!@>FO!%Z)LS{2=2f3r?Eu&g39p;7x6kFetI z;v`fx7e}M{bVDj^xU|()WzDoVY&A3)TTXjztHJLTY~fm4F<0t4+aS{7N^XKF^Txhj z#Us%cF2IzqAa-lI^btu}O>LV@+21wd@*c&&Jt`zfDTmer{ecGb2O7{HXh46U0sVmn z^amQi7Ym*)8sz|A4DiJOUkvcY0ACF7#Q=gxUP3ot;`S^zyVtWiu=mx|CuGXsDpE@F`q>s=8iBU)FtiO?O_b|~0QW23`ypag z%>JWNtOsnA;`Z8hJTSB7lg1IjzK~hXH==#P-hC}OVvAIN`xWY#1?2vs_z<$O%-$8Lxzzru=hW=xQxM5q}IR{QFYW`{NU_b2w@0sy64R5zFGFa9Mla**wq>NC=;4`!2D z%;^R(5eB5o)*Y4STGekH{nm-5_``=)AB&1UcFWE?fTJ>J1^dUS3>iQpA|EvAv_3yD z&F8M-!1N$%^d_t^WK|sBzD@qxs;r7b!R6Fa$1*lE+Z$6$OSUY_UJorX4~VvEA#~Nd z8{YKpxWp1ZmQ%`XRjLk$u1Cp}|X?|_f z131IbfCS|o&XNpYydJD{vkkp+<&?so49r?badID>n2c@PChelPhWr`++P=Q`tHwfg zr(%Grf-}M_P_J!O;OinBuA$Cv!-;nS_|qxqGjGEp=RV+32tdwpN9A+z{8A-vP?e|o zGEVb2UDbI;vK3h_zq^q){THAdhTKgY0q5MepP>2&vkWwx0|l7S`88LQtv6QzXp;pv zIsw{~KsY8FY6c|`B>?~cPyzq|5dZ)H0000000000fPuS{fjB+^50k(+BL>Pf00000 DKk(0b diff --git a/summary_results.json b/summary_results.json index 4f96ac191..9e5a52842 100644 --- a/summary_results.json +++ b/summary_results.json @@ -334,7 +334,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0506, + "Performance_Result": 10.0454, "Performance_Units": "Samples/s" }, {