Skip to content

Commit 5e1742f

Browse files
arjunsuresharjunsuresh
committed
Results from self hosted Github actions - NVIDIARTX4090
1 parent 506efb7 commit 5e1742f

File tree

10 files changed

+8485
-303
lines changed

10 files changed

+8485
-303
lines changed

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
22
|---------|------------|-----------------------------------|--------------|-------------------|
3-
| gptj-99 | offline | (32.2581, 6.6667, 22.5806, 264.0) | 0.733 | - |
3+
| gptj-99 | offline | (32.2581, 6.6667, 22.5806, 264.0) | 0.767 | - |

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pip install -U cmind
1818

1919
cm rm cache -f
2020

21-
cm pull repo gateoverflow@cm4mlops --checkout=c1d6e4dda20deaaa03e232843676f30fe220113a
21+
cm pull repo gateoverflow@cm4mlops --checkout=052dd0de3e25a4f2918932bd6c6bca95388b08bd
2222

2323
cm run script \
2424
--tags=app,mlperf,inference,generic,_reference,_gptj-99,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline \
@@ -102,4 +102,4 @@ Model Precision: fp32
102102
`GEN_LEN`: `264.0`, Required accuracy for closed division `>= 3615190.2`
103103

104104
### Performance Results
105-
`Samples per second`: `0.733042`
105+
`Samples per second`: `0.766727`

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/cm-version-info.json

Lines changed: 20 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,6 @@
6060
"parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline )"
6161
}
6262
},
63-
{
64-
"detect,os": {
65-
"script_uid": "863735b7db8c44fc",
66-
"script_alias": "detect-os",
67-
"script_tags": "detect-os,detect,os,info",
68-
"script_variations": "",
69-
"version": "",
70-
"parent": "get-cuda,46d133d9ef92422d ( toolkit )"
71-
}
72-
},
7363
{
7464
"get,cuda,_toolkit": {
7565
"script_uid": "46d133d9ef92422d",
@@ -90,56 +80,6 @@
9080
"parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )"
9181
}
9282
},
93-
{
94-
"detect,os": {
95-
"script_uid": "863735b7db8c44fc",
96-
"script_alias": "detect-os",
97-
"script_tags": "detect-os,detect,os,info",
98-
"script_variations": "",
99-
"version": "",
100-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
101-
}
102-
},
103-
{
104-
"detect,os": {
105-
"script_uid": "863735b7db8c44fc",
106-
"script_alias": "detect-os",
107-
"script_tags": "detect-os,detect,os,info",
108-
"script_variations": "",
109-
"version": "",
110-
"parent": "detect-cpu,586c8a43320142f7"
111-
}
112-
},
113-
{
114-
"detect,cpu": {
115-
"script_uid": "586c8a43320142f7",
116-
"script_alias": "detect-cpu",
117-
"script_tags": "detect,cpu,detect-cpu,info",
118-
"script_variations": "",
119-
"version": "",
120-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
121-
}
122-
},
123-
{
124-
"get,python3": {
125-
"script_uid": "d0b5dd74373f4a62",
126-
"script_alias": "get-python3",
127-
"script_tags": "get,python,python3,get-python,get-python3",
128-
"script_variations": "",
129-
"version": "3.10.12",
130-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
131-
}
132-
},
133-
{
134-
"get,generic-python-lib,_pip": {
135-
"script_uid": "94b62a682bc44791",
136-
"script_alias": "get-generic-python-lib",
137-
"script_tags": "get,install,generic,generic-python-lib",
138-
"script_variations": "pip",
139-
"version": "22.0.2",
140-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )"
141-
}
142-
},
14383
{
14484
"get,generic-python-lib,_package.pycuda": {
14585
"script_uid": "94b62a682bc44791",
@@ -150,63 +90,13 @@
15090
"parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )"
15191
}
15292
},
153-
{
154-
"detect,os": {
155-
"script_uid": "863735b7db8c44fc",
156-
"script_alias": "detect-os",
157-
"script_tags": "detect-os,detect,os,info",
158-
"script_variations": "",
159-
"version": "",
160-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
161-
}
162-
},
163-
{
164-
"detect,os": {
165-
"script_uid": "863735b7db8c44fc",
166-
"script_alias": "detect-os",
167-
"script_tags": "detect-os,detect,os,info",
168-
"script_variations": "",
169-
"version": "",
170-
"parent": "detect-cpu,586c8a43320142f7"
171-
}
172-
},
173-
{
174-
"detect,cpu": {
175-
"script_uid": "586c8a43320142f7",
176-
"script_alias": "detect-cpu",
177-
"script_tags": "detect,cpu,detect-cpu,info",
178-
"script_variations": "",
179-
"version": "",
180-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
181-
}
182-
},
183-
{
184-
"get,python3": {
185-
"script_uid": "d0b5dd74373f4a62",
186-
"script_alias": "get-python3",
187-
"script_tags": "get,python,python3,get-python,get-python3",
188-
"script_variations": "",
189-
"version": "3.10.12",
190-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
191-
}
192-
},
193-
{
194-
"get,generic-python-lib,_pip": {
195-
"script_uid": "94b62a682bc44791",
196-
"script_alias": "get-generic-python-lib",
197-
"script_tags": "get,install,generic,generic-python-lib",
198-
"script_variations": "pip",
199-
"version": "22.0.2",
200-
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )"
201-
}
202-
},
20393
{
20494
"get,generic-python-lib,_package.numpy": {
20595
"script_uid": "94b62a682bc44791",
20696
"script_alias": "get-generic-python-lib",
20797
"script_tags": "get,install,generic,generic-python-lib",
20898
"script_variations": "package.numpy",
209-
"version": "2.1.2",
99+
"version": "1.26.4",
210100
"parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )"
211101
}
212102
},
@@ -227,7 +117,7 @@
227117
"script_tags": "detect-os,detect,os,info",
228118
"script_variations": "",
229119
"version": "",
230-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
120+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
231121
}
232122
},
233123
{
@@ -247,7 +137,7 @@
247137
"script_tags": "detect,cpu,detect-cpu,info",
248138
"script_variations": "",
249139
"version": "",
250-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
140+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
251141
}
252142
},
253143
{
@@ -257,7 +147,7 @@
257147
"script_tags": "get,sys-utils-cm",
258148
"script_variations": "",
259149
"version": "",
260-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
150+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
261151
}
262152
},
263153
{
@@ -267,7 +157,7 @@
267157
"script_tags": "get,python,python3,get-python,get-python3",
268158
"script_variations": "",
269159
"version": "3.10.12",
270-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
160+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
271161
}
272162
},
273163
{
@@ -277,7 +167,7 @@
277167
"script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d",
278168
"script_variations": "cudnn",
279169
"version": "12.4",
280-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
170+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
281171
}
282172
},
283173
{
@@ -287,7 +177,7 @@
287177
"script_tags": "get,install,generic,generic-python-lib",
288178
"script_variations": "torch_cuda",
289179
"version": "2.4.1",
290-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
180+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
291181
}
292182
},
293183
{
@@ -297,7 +187,7 @@
297187
"script_tags": "get,install,generic,generic-python-lib",
298188
"script_variations": "torchvision_cuda",
299189
"version": "0.19.1",
300-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
190+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
301191
}
302192
},
303193
{
@@ -307,7 +197,7 @@
307197
"script_tags": "get,install,generic,generic-python-lib",
308198
"script_variations": "transformers",
309199
"version": "4.45.2",
310-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
200+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
311201
}
312202
},
313203
{
@@ -317,7 +207,7 @@
317207
"script_tags": "get,raw,ml-model,gptj,gpt-j,large-language-model",
318208
"script_variations": "pytorch",
319209
"version": "",
320-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
210+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
321211
}
322212
},
323213
{
@@ -327,7 +217,7 @@
327217
"script_tags": "get,dataset,gpt-j,cnndm,cnn-dailymail,original",
328218
"script_variations": "validation",
329219
"version": "",
330-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
220+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
331221
}
332222
},
333223
{
@@ -407,7 +297,7 @@
407297
"script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
408298
"script_variations": "",
409299
"version": "",
410-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
300+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
411301
}
412302
},
413303
{
@@ -417,7 +307,7 @@
417307
"script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
418308
"script_variations": "",
419309
"version": "master",
420-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
310+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
421311
}
422312
},
423313
{
@@ -427,7 +317,7 @@
427317
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
428318
"script_variations": "",
429319
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
430-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
320+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
431321
}
432322
},
433323
{
@@ -437,7 +327,7 @@
437327
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
438328
"script_variations": "",
439329
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
440-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
330+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
441331
}
442332
},
443333
{
@@ -447,7 +337,7 @@
447337
"script_tags": "get,install,generic,generic-python-lib",
448338
"script_variations": "package.psutil",
449339
"version": "6.0.0",
450-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
340+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
451341
}
452342
},
453343
{
@@ -457,7 +347,7 @@
457347
"script_tags": "get,install,generic,generic-python-lib",
458348
"script_variations": "package.datasets",
459349
"version": "3.0.1",
460-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
350+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
461351
}
462352
},
463353
{
@@ -467,7 +357,7 @@
467357
"script_tags": "get,install,generic,generic-python-lib",
468358
"script_variations": "package.attrs",
469359
"version": "24.2.0",
470-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
360+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
471361
}
472362
},
473363
{
@@ -477,7 +367,7 @@
477367
"script_tags": "get,install,generic,generic-python-lib",
478368
"script_variations": "package.accelerate",
479369
"version": "1.0.0",
480-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
370+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
481371
}
482372
},
483373
{
@@ -517,7 +407,7 @@
517407
"script_tags": "mlperf,benchmark-mlperf",
518408
"script_variations": "",
519409
"version": "",
520-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
410+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_cuda,_gptj-99,_float16 )"
521411
}
522412
}
523413
]

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/os_info.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
],
2727
"CM_HOST_PLATFORM_FLAVOR": "x86_64",
2828
"CM_HOST_PYTHON_BITS": "64",
29-
"CM_HOST_SYSTEM_NAME": "3388739f9c2d"
29+
"CM_HOST_SYSTEM_NAME": "83769595a6e0"
3030
}

0 commit comments

Comments
 (0)