Skip to content

Commit 5dba7aa

Browse files
arjunsuresharjunsuresh
committed
Results from self hosted Github actions - NVIDIARTX4090
1 parent 9d3e060 commit 5dba7aa

File tree

9 files changed

+308
-199
lines changed

9 files changed

+308
-199
lines changed

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
2-
|---------|------------|-----------------------------------|--------------|-------------------|
3-
| gptj-99 | offline | (32.2581, 6.6667, 22.5806, 264.0) | 0.767 | - |
1+
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
2+
|---------------------|------------|-----------------------|--------------|-------------------|
3+
| stable-diffusion-xl | offline | (15.18544, 235.69504) | 0.347 | - |

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pip install -U cmind
1818

1919
cm rm cache -f
2020

21-
cm pull repo gateoverflow@cm4mlops --checkout=594c4c74415af3b29642fe52e1732bf62623f201
21+
cm pull repo gateoverflow@cm4mlops --checkout=052dd0de3e25a4f2918932bd6c6bca95388b08bd
2222

2323
cm run script \
2424
--tags=app,mlperf,inference,generic,_reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline \
@@ -99,4 +99,4 @@ Model Precision: fp32
9999
`FID_SCORE`: `235.69504`, Required accuracy for closed division `>= 23.01086` and `<= 23.95008`
100100

101101
### Performance Results
102-
`Samples per second`: `0.345412`
102+
`Samples per second`: `0.346736`

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline/cm-version-info.json

Lines changed: 134 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,16 @@
6060
"parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_sdxl,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline )"
6161
}
6262
},
63+
{
64+
"detect,os": {
65+
"script_uid": "863735b7db8c44fc",
66+
"script_alias": "detect-os",
67+
"script_tags": "detect-os,detect,os,info",
68+
"script_variations": "",
69+
"version": "",
70+
"parent": "get-cuda,46d133d9ef92422d ( toolkit )"
71+
}
72+
},
6373
{
6474
"get,cuda,_toolkit": {
6575
"script_uid": "46d133d9ef92422d",
@@ -80,6 +90,56 @@
8090
"parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )"
8191
}
8292
},
93+
{
94+
"detect,os": {
95+
"script_uid": "863735b7db8c44fc",
96+
"script_alias": "detect-os",
97+
"script_tags": "detect-os,detect,os,info",
98+
"script_variations": "",
99+
"version": "",
100+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
101+
}
102+
},
103+
{
104+
"detect,os": {
105+
"script_uid": "863735b7db8c44fc",
106+
"script_alias": "detect-os",
107+
"script_tags": "detect-os,detect,os,info",
108+
"script_variations": "",
109+
"version": "",
110+
"parent": "detect-cpu,586c8a43320142f7"
111+
}
112+
},
113+
{
114+
"detect,cpu": {
115+
"script_uid": "586c8a43320142f7",
116+
"script_alias": "detect-cpu",
117+
"script_tags": "detect,cpu,detect-cpu,info",
118+
"script_variations": "",
119+
"version": "",
120+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
121+
}
122+
},
123+
{
124+
"get,python3": {
125+
"script_uid": "d0b5dd74373f4a62",
126+
"script_alias": "get-python3",
127+
"script_tags": "get,python,python3,get-python,get-python3",
128+
"script_variations": "",
129+
"version": "3.10.12",
130+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
131+
}
132+
},
133+
{
134+
"get,generic-python-lib,_pip": {
135+
"script_uid": "94b62a682bc44791",
136+
"script_alias": "get-generic-python-lib",
137+
"script_tags": "get,install,generic,generic-python-lib",
138+
"script_variations": "pip",
139+
"version": "22.0.2",
140+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
141+
}
142+
},
83143
{
84144
"get,generic-python-lib,_package.pycuda": {
85145
"script_uid": "94b62a682bc44791",
@@ -90,6 +150,56 @@
90150
"parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )"
91151
}
92152
},
153+
{
154+
"detect,os": {
155+
"script_uid": "863735b7db8c44fc",
156+
"script_alias": "detect-os",
157+
"script_tags": "detect-os,detect,os,info",
158+
"script_variations": "",
159+
"version": "",
160+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
161+
}
162+
},
163+
{
164+
"detect,os": {
165+
"script_uid": "863735b7db8c44fc",
166+
"script_alias": "detect-os",
167+
"script_tags": "detect-os,detect,os,info",
168+
"script_variations": "",
169+
"version": "",
170+
"parent": "detect-cpu,586c8a43320142f7"
171+
}
172+
},
173+
{
174+
"detect,cpu": {
175+
"script_uid": "586c8a43320142f7",
176+
"script_alias": "detect-cpu",
177+
"script_tags": "detect,cpu,detect-cpu,info",
178+
"script_variations": "",
179+
"version": "",
180+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
181+
}
182+
},
183+
{
184+
"get,python3": {
185+
"script_uid": "d0b5dd74373f4a62",
186+
"script_alias": "get-python3",
187+
"script_tags": "get,python,python3,get-python,get-python3",
188+
"script_variations": "",
189+
"version": "3.10.12",
190+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
191+
}
192+
},
193+
{
194+
"get,generic-python-lib,_pip": {
195+
"script_uid": "94b62a682bc44791",
196+
"script_alias": "get-generic-python-lib",
197+
"script_tags": "get,install,generic,generic-python-lib",
198+
"script_variations": "pip",
199+
"version": "22.0.2",
200+
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
201+
}
202+
},
93203
{
94204
"get,generic-python-lib,_package.numpy": {
95205
"script_uid": "94b62a682bc44791",
@@ -117,7 +227,7 @@
117227
"script_tags": "detect-os,detect,os,info",
118228
"script_variations": "",
119229
"version": "",
120-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
230+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
121231
}
122232
},
123233
{
@@ -137,7 +247,7 @@
137247
"script_tags": "detect,cpu,detect-cpu,info",
138248
"script_variations": "",
139249
"version": "",
140-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
250+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
141251
}
142252
},
143253
{
@@ -147,7 +257,7 @@
147257
"script_tags": "get,sys-utils-cm",
148258
"script_variations": "",
149259
"version": "",
150-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
260+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
151261
}
152262
},
153263
{
@@ -157,7 +267,7 @@
157267
"script_tags": "get,python,python3,get-python,get-python3",
158268
"script_variations": "",
159269
"version": "3.10.12",
160-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
270+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
161271
}
162272
},
163273
{
@@ -167,7 +277,7 @@
167277
"script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d",
168278
"script_variations": "cudnn",
169279
"version": "12.4",
170-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
280+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
171281
}
172282
},
173283
{
@@ -177,7 +287,7 @@
177287
"script_tags": "get,install,generic,generic-python-lib",
178288
"script_variations": "torch_cuda",
179289
"version": "2.4.1",
180-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
290+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
181291
}
182292
},
183293
{
@@ -187,7 +297,7 @@
187297
"script_tags": "get,install,generic,generic-python-lib",
188298
"script_variations": "torchvision_cuda",
189299
"version": "0.19.1",
190-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
300+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
191301
}
192302
},
193303
{
@@ -197,7 +307,7 @@
197307
"script_tags": "get,raw,ml-model,stable-diffusion,sdxl,text-to-image",
198308
"script_variations": "pytorch",
199309
"version": "",
200-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
310+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
201311
}
202312
},
203313
{
@@ -207,7 +317,7 @@
207317
"script_tags": "get,dataset,coco2014,object-detection,original",
208318
"script_variations": "validation",
209319
"version": "",
210-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
320+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
211321
}
212322
},
213323
{
@@ -287,7 +397,7 @@
287397
"script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
288398
"script_variations": "",
289399
"version": "",
290-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
400+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
291401
}
292402
},
293403
{
@@ -297,7 +407,7 @@
297407
"script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
298408
"script_variations": "",
299409
"version": "master",
300-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
410+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
301411
}
302412
},
303413
{
@@ -307,7 +417,7 @@
307417
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
308418
"script_variations": "",
309419
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
310-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
420+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
311421
}
312422
},
313423
{
@@ -317,7 +427,7 @@
317427
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
318428
"script_variations": "",
319429
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
320-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
430+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
321431
}
322432
},
323433
{
@@ -327,7 +437,7 @@
327437
"script_tags": "get,install,generic,generic-python-lib",
328438
"script_variations": "package.psutil",
329439
"version": "6.0.0",
330-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
440+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
331441
}
332442
},
333443
{
@@ -337,7 +447,7 @@
337447
"script_tags": "get,install,generic,generic-python-lib",
338448
"script_variations": "package.diffusers",
339449
"version": "0.30.3",
340-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
450+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
341451
}
342452
},
343453
{
@@ -347,7 +457,7 @@
347457
"script_tags": "get,install,generic,generic-python-lib",
348458
"script_variations": "package.transformers",
349459
"version": "4.45.2",
350-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
460+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
351461
}
352462
},
353463
{
@@ -357,7 +467,7 @@
357467
"script_tags": "get,install,generic,generic-python-lib",
358468
"script_variations": "package.torchvision",
359469
"version": "0.19.1",
360-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
470+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
361471
}
362472
},
363473
{
@@ -367,7 +477,7 @@
367477
"script_tags": "get,install,generic,generic-python-lib",
368478
"script_variations": "package.accelerate",
369479
"version": "1.0.0",
370-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
480+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
371481
}
372482
},
373483
{
@@ -377,7 +487,7 @@
377487
"script_tags": "get,install,generic,generic-python-lib",
378488
"script_variations": "package.torchmetrics",
379489
"version": "1.4.2",
380-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
490+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
381491
}
382492
},
383493
{
@@ -387,7 +497,7 @@
387497
"script_tags": "get,install,generic,generic-python-lib",
388498
"script_variations": "package.torch-fidelity",
389499
"version": "0.3.0",
390-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
500+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
391501
}
392502
},
393503
{
@@ -397,7 +507,7 @@
397507
"script_tags": "get,install,generic,generic-python-lib",
398508
"script_variations": "package.open_clip_torch",
399509
"version": "2.26.1",
400-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
510+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
401511
}
402512
},
403513
{
@@ -407,7 +517,7 @@
407517
"script_tags": "get,install,generic,generic-python-lib",
408518
"script_variations": "package.opencv-python",
409519
"version": "4.10.0.84",
410-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
520+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
411521
}
412522
},
413523
{
@@ -417,7 +527,7 @@
417527
"script_tags": "get,install,generic,generic-python-lib",
418528
"script_variations": "package.scipy",
419529
"version": "1.10.1",
420-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
530+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
421531
}
422532
},
423533
{
@@ -457,7 +567,7 @@
457567
"script_tags": "mlperf,benchmark-mlperf",
458568
"script_variations": "",
459569
"version": "",
460-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( sdxl,_offline,_pytorch,_cuda,_float16 )"
570+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( offline,_sdxl,_cuda,_pytorch,_float16 )"
461571
}
462572
}
463573
]

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline/os_info.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
],
2727
"CM_HOST_PLATFORM_FLAVOR": "x86_64",
2828
"CM_HOST_PYTHON_BITS": "64",
29-
"CM_HOST_SYSTEM_NAME": "0c75048abad5"
29+
"CM_HOST_SYSTEM_NAME": "550f589179ea"
3030
}

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline/pip_freeze.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
"giturlparse": "0.12.0",
1313
"huggingface-hub": "0.25.2",
1414
"idna": "3.10",
15-
"ijson": "3.3.0",
1615
"importlib_metadata": "8.5.0",
1716
"Jinja2": "3.1.4",
1817
"lightning-utilities": "0.11.7",

‎open/MLCommons/model_mapping.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"gptj-99": "gptj-99"
2+
"stable-diffusion-xl": "stable-diffusion-xl"
33
}

0 commit comments

Comments
 (0)