Updated results summary

gateoverflow-admin · gateoverflow-admin · commit 77dc916f55c9 · 2024-10-12T20:34:33.000Z
diff --git a/README.md b/README.md
@@ -1,20 +1,20 @@
 Please download [summary.xlsx](summary.xlsx) to view the most recent results. 
  ```
-[2024-10-12 09:45:45,039 submission_checker1.py:2936 INFO] Results=22, NoResults=0, Power Results=0
-[2024-10-12 09:45:45,039 submission_checker1.py:2943 INFO] ---
-[2024-10-12 09:45:45,039 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2936 INFO] Results=22, NoResults=0, Power Results=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2943 INFO] ---
+[2024-10-12 20:34:30,922 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0
 
-[2024-10-12 09:45:45,039 submission_checker1.py:2949 INFO] Open Results=22, Open Power Results=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2949 INFO] Open Results=22, Open Power Results=0
 
-[2024-10-12 09:45:45,040 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0
 
-[2024-10-12 09:45:45,040 submission_checker1.py:2959 INFO] ---
-[2024-10-12 09:45:45,040 submission_checker1.py:2961 INFO] Systems=18, Power Systems=0
-[2024-10-12 09:45:45,040 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0
-[2024-10-12 09:45:45,040 submission_checker1.py:2967 INFO] Open Systems=18, Open Power Systems=0
-[2024-10-12 09:45:45,040 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0
-[2024-10-12 09:45:45,040 submission_checker1.py:2977 INFO] ---
-[2024-10-12 09:45:45,040 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK
+[2024-10-12 20:34:30,922 submission_checker1.py:2959 INFO] ---
+[2024-10-12 20:34:30,922 submission_checker1.py:2961 INFO] Systems=18, Power Systems=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2967 INFO] Open Systems=18, Open Power Systems=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0
+[2024-10-12 20:34:30,922 submission_checker1.py:2977 INFO] ---
+[2024-10-12 20:34:30,923 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK
 INFO:root:       ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-submission-checker/customize.py
 
 ```
@@ -42,4 +42,4 @@ INFO:root:       ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4ml
 | 18 | MLCommons      | available      | open       | edge         | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50            | resnet              | Offline    | 17.0661   | acc: 76.000                                                    |                 1 | undefined                       |                          1 |                           1 | nan                      |                       0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0               | Automated by MLCommons CM v3.0.4. |            1 |        0 | v4.1      |          0 | False       | Samples/s | fp32                |
 | 19 | MLCommons      | available      | open       | edge         | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config          | resnet50            | resnet              | Offline    | 21.8653   | acc: 76.000                                                    |                 1 | undefined                       |                          1 |                           1 | nan                      |                       0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline          | tf v2.17.0          | Windows-2022Server-10.0.20348-SP0               | Automated by MLCommons CM v3.0.4. |            1 |        0 | v4.1      |          0 | False       | Samples/s | fp32                |
 | 20 | MLCommons      | available      | open       | edge         | gh_action             | gh_action-reference-gpu-pytorch_v2.4.1-cu124                           | stable-diffusion-xl | stable-diffusion-xl | Offline    |  0.346736 | CLIP_SCORE: 15.18544016778469  FID_SCORE: 235.69504308101006   |                 1 | Intel(R) Xeon(R) w7-2495X       |                          1 |                          24 | NVIDIA GeForce RTX 4090  |                       2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline                | pytorch v2.4.1      | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. |            1 |        0 | v4.1      |          0 | False       | Samples/s | fp32                |
-| 21 | MLCommons      | available      | open       | edge         | gh_action             | gh_action-reference-gpu-pytorch_v2.4.1-cu124                           | gptj-99             | gptj-99             | Offline    | 52.9041   | ROUGE1: 32.2581  ROUGE2: 6.6667  ROUGEL: 22.5806  GEN_LEN: 264 |                 1 | Intel(R) Xeon(R) w7-2495X       |                          1 |                          24 | NVIDIA GeForce RTX 4090  |                       2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline                            | pytorch v2.4.1      | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. |            1 |        0 | v4.1      |          0 | False       | Tokens/s  | fp32                |
+| 21 | MLCommons      | available      | open       | edge         | gh_action             | gh_action-reference-gpu-pytorch_v2.4.1-cu124                           | gptj-99             | gptj-99             | Offline    | 51.9542   | ROUGE1: 32.2581  ROUGE2: 6.6667  ROUGEL: 22.5806  GEN_LEN: 264 |                 1 | Intel(R) Xeon(R) w7-2495X       |                          1 |                          24 | NVIDIA GeForce RTX 4090  |                       2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline                            | pytorch v2.4.1      | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. |            1 |        0 | v4.1      |          0 | False       | Tokens/s  | fp32                |
diff --git a/summary.csv b/summary.csv
@@ -20,4 +20,4 @@ MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-
 MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.0661,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.0.4. ",1,0,v4.1,0,False,Samples/s,"fp32"
 MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8653,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.0.4. ",1,0,v4.1,0,False,Samples/s,"fp32"
 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346736,CLIP_SCORE: 15.18544016778469  FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32"
-MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,52.9041,ROUGE1: 32.2581  ROUGE2: 6.6667  ROUGEL: 22.5806  GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Tokens/s,"fp32"
+MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,51.9542,ROUGE1: 32.2581  ROUGE2: 6.6667  ROUGEL: 22.5806  GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Tokens/s,"fp32"
diff --git a/summary.json b/summary.json
@@ -639,7 +639,7 @@
         "Model":"gptj-99",
         "MlperfModel":"gptj-99",
         "Scenario":"Offline",
-        "Result":52.9041,
+        "Result":51.9542,
         "Accuracy":"ROUGE1: 32.2581  ROUGE2: 6.6667  ROUGEL: 22.5806  GEN_LEN: 264",
         "number_of_nodes":1,
         "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X",
diff --git a/summary.xlsx b/summary.xlsx
diff --git a/summary_results.json b/summary_results.json
@@ -68,7 +68,7 @@
     "Code": "https://github.com/GATEOverflow/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code",
     "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99",
     "ColorKey": "availableMLCommons",
-    "Performance_Result": 52.9041,
+    "Performance_Result": 51.9542,
     "Performance_Units": "Tokens/s"
   },
   {

Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,7 @@`
`68`	`68`	`"Code": "https://github.com/GATEOverflow/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code",`
`69`	`69`	`"Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99",`
`70`	`70`	`"ColorKey": "availableMLCommons",`
`71`		`- "Performance_Result": 52.9041,`
	`71`	`+ "Performance_Result": 51.9542,`
`72`	`72`	`"Performance_Units": "Tokens/s"`
`73`	`73`	`},`
`74`	`74`	`{`