Skip to content

Commit fc5f00b

Browse files
arjunsuresharjunsuresh
committed
Results from self hosted Github actions - NVIDIARTX4090
1 parent 925099e commit fc5f00b

File tree

9 files changed

+201
-201
lines changed

9 files changed

+201
-201
lines changed

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
2-
|---------------------|------------|-----------------------|--------------|-------------------|
3-
| stable-diffusion-xl | offline | (15.18544, 235.69504) | 0.345 | - |
1+
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
2+
|---------|------------|-----------------------------------|--------------|-------------------|
3+
| gptj-99 | offline | (32.2581, 6.6667, 22.5806, 264.0) | 0.733 | - |

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/README.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pip install -U cmind
1818

1919
cm rm cache -f
2020

21-
cm pull repo gateoverflow@cm4mlops --checkout=0efa8f35d073e68543317b04ee7e5487f88af5cc
21+
cm pull repo gateoverflow@cm4mlops --checkout=c1d6e4dda20deaaa03e232843676f30fe220113a
2222

2323
cm run script \
2424
--tags=app,mlperf,inference,generic,_reference,_gptj-99,_pytorch,_cuda,_test,_r4.1-dev_default,_float16,_offline \
@@ -30,9 +30,9 @@ cm run script \
3030
--env.CM_MLPERF_BACKEND=pytorch \
3131
--env.GPTJ_BEAM_SIZE=1 \
3232
--env.CM_MLPERF_CLEAN_ALL=True \
33-
--env.CM_CUSTOM_SYSTEM_NVIDIA=yes \
3433
--env.CM_MLPERF_DEVICE=cuda \
3534
--env.CM_MLPERF_USE_DOCKER=True \
35+
--env.CM_GET_PLATFORM_DETAILS=yes \
3636
--env.CM_HW_NAME=gh_action \
3737
--env.CM_MLPERF_MODEL_PRECISION=float16 \
3838
--env.OUTPUT_BASE_DIR=/home/arjun/gh_action_results \
@@ -50,7 +50,7 @@ cm run script \
5050
--env.CM_MLPERF_LOADGEN_ALL_MODES=yes \
5151
--env.CM_MLPERF_INFERENCE_VERSION=4.1-dev \
5252
--env.CM_RUN_MLPERF_INFERENCE_APP_DEFAULTS=r4.1-dev_default \
53-
--env.CM_MLPERF_LAST_RELEASE=v4.1 \
53+
--env.CM_MLPERF_LAST_RELEASE=v4.0 \
5454
--env.CM_TMP_CURRENT_PATH=/home/arjun/actions-runner/_work/cm4mlops/cm4mlops \
5555
--env.CM_TMP_PIP_VERSION_STRING= \
5656
--env.CM_MODEL=gptj-99 \
@@ -77,7 +77,7 @@ cm run script \
7777
--dump_version_info=True \
7878
--env.OUTPUT_BASE_DIR=/home/arjun/gh_action_results \
7979
--env.CM_MLPERF_INFERENCE_SUBMISSION_DIR=/home/arjun/gh_action_submissions \
80-
--env.GPTJ_CHECKPOINT_PATH=/home/cmuser/CM/repos/local/cache/98e466fd3f2c42cc/checkpoint/checkpoint-final
80+
--env.GPTJ_CHECKPOINT_PATH=/home/cmuser/CM/repos/local/cache/f6ab729f2dca49d9/checkpoint/checkpoint-final
8181
```
8282
*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts),
8383
you should simply reload gateoverflow@cm4mlops without checkout and clean CM cache as follows:*
@@ -102,4 +102,4 @@ Model Precision: fp32
102102
`GEN_LEN`: `264.0`, Required accuracy for closed division `>= 3615190.2`
103103

104104
### Performance Results
105-
`Samples per second`: `47.1875`
105+
`Samples per second`: `0.733042`

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/cm-version-info.json

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@
227227
"script_tags": "detect-os,detect,os,info",
228228
"script_variations": "",
229229
"version": "",
230-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
230+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
231231
}
232232
},
233233
{
@@ -247,7 +247,7 @@
247247
"script_tags": "detect,cpu,detect-cpu,info",
248248
"script_variations": "",
249249
"version": "",
250-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
250+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
251251
}
252252
},
253253
{
@@ -257,7 +257,7 @@
257257
"script_tags": "get,sys-utils-cm",
258258
"script_variations": "",
259259
"version": "",
260-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
260+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
261261
}
262262
},
263263
{
@@ -267,7 +267,7 @@
267267
"script_tags": "get,python,python3,get-python,get-python3",
268268
"script_variations": "",
269269
"version": "3.10.12",
270-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
270+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
271271
}
272272
},
273273
{
@@ -277,7 +277,7 @@
277277
"script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d",
278278
"script_variations": "cudnn",
279279
"version": "12.4",
280-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
280+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
281281
}
282282
},
283283
{
@@ -287,7 +287,7 @@
287287
"script_tags": "get,install,generic,generic-python-lib",
288288
"script_variations": "torch_cuda",
289289
"version": "2.4.1",
290-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
290+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
291291
}
292292
},
293293
{
@@ -297,7 +297,7 @@
297297
"script_tags": "get,install,generic,generic-python-lib",
298298
"script_variations": "torchvision_cuda",
299299
"version": "0.19.1",
300-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
300+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
301301
}
302302
},
303303
{
@@ -307,7 +307,7 @@
307307
"script_tags": "get,install,generic,generic-python-lib",
308308
"script_variations": "transformers",
309309
"version": "4.45.2",
310-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
310+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
311311
}
312312
},
313313
{
@@ -317,7 +317,7 @@
317317
"script_tags": "get,raw,ml-model,gptj,gpt-j,large-language-model",
318318
"script_variations": "pytorch",
319319
"version": "",
320-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
320+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
321321
}
322322
},
323323
{
@@ -327,7 +327,7 @@
327327
"script_tags": "get,dataset,gpt-j,cnndm,cnn-dailymail,original",
328328
"script_variations": "validation",
329329
"version": "",
330-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
330+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
331331
}
332332
},
333333
{
@@ -407,7 +407,7 @@
407407
"script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
408408
"script_variations": "",
409409
"version": "",
410-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
410+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
411411
}
412412
},
413413
{
@@ -417,7 +417,7 @@
417417
"script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
418418
"script_variations": "",
419419
"version": "master",
420-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
420+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
421421
}
422422
},
423423
{
@@ -427,7 +427,7 @@
427427
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
428428
"script_variations": "",
429429
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
430-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
430+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
431431
}
432432
},
433433
{
@@ -437,7 +437,7 @@
437437
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
438438
"script_variations": "",
439439
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
440-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
440+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
441441
}
442442
},
443443
{
@@ -447,7 +447,7 @@
447447
"script_tags": "get,install,generic,generic-python-lib",
448448
"script_variations": "package.psutil",
449449
"version": "6.0.0",
450-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
450+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
451451
}
452452
},
453453
{
@@ -457,7 +457,7 @@
457457
"script_tags": "get,install,generic,generic-python-lib",
458458
"script_variations": "package.datasets",
459459
"version": "3.0.1",
460-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
460+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
461461
}
462462
},
463463
{
@@ -467,7 +467,7 @@
467467
"script_tags": "get,install,generic,generic-python-lib",
468468
"script_variations": "package.attrs",
469469
"version": "24.2.0",
470-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
470+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
471471
}
472472
},
473473
{
@@ -477,7 +477,7 @@
477477
"script_tags": "get,install,generic,generic-python-lib",
478478
"script_variations": "package.accelerate",
479479
"version": "1.0.0",
480-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
480+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
481481
}
482482
},
483483
{
@@ -517,7 +517,7 @@
517517
"script_tags": "mlperf,benchmark-mlperf",
518518
"script_variations": "",
519519
"version": "",
520-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_offline,_cuda,_pytorch,_float16 )"
520+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( pytorch,_offline,_gptj-99,_cuda,_float16 )"
521521
}
522522
}
523523
]

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/os_info.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
],
2727
"CM_HOST_PLATFORM_FLAVOR": "x86_64",
2828
"CM_HOST_PYTHON_BITS": "64",
29-
"CM_HOST_SYSTEM_NAME": "369d6e494394"
29+
"CM_HOST_SYSTEM_NAME": "3388739f9c2d"
3030
}

‎open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline/pip_freeze.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"pip_freeze": {
33
"accelerate": "1.0.0",
44
"aiohappyeyeballs": "2.4.3",
5-
"aiohttp": "3.10.9",
5+
"aiohttp": "3.10.10",
66
"aiosignal": "1.3.1",
77
"async-timeout": "4.0.3",
88
"attrs": "24.2.0",
@@ -24,7 +24,7 @@
2424
"mpmath": "1.3.0",
2525
"multidict": "6.1.0",
2626
"multiprocess": "0.70.16",
27-
"networkx": "3.3",
27+
"networkx": "3.4",
2828
"numpy": "1.26.4",
2929
"nvidia-cublas-cu12": "12.1.3.1",
3030
"nvidia-cuda-cupti-cu12": "12.1.105",
@@ -60,7 +60,7 @@
6060
"six": "1.16.0",
6161
"sympy": "1.13.3",
6262
"tabulate": "0.9.0",
63-
"tokenizers": "0.20.0",
63+
"tokenizers": "0.20.1",
6464
"torch": "2.4.1",
6565
"torchvision": "0.19.1",
6666
"tqdm": "4.66.5",

‎open/MLCommons/model_mapping.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"retinanet": "retinanet"
2+
"gptj-99": "gptj-99"
33
}

0 commit comments

Comments
 (0)