Skip to content

Commit e8c5721

Browse files
gateoverflow-admingateoverflow-admin
committed
Results from Bert GH action on ubuntu-latest
1 parent 1a07cb7 commit e8c5721

File tree

9 files changed

+197
-197
lines changed

9 files changed

+197
-197
lines changed

‎open/MLCommons/measurements/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
22
|---------|------------|------------|--------------|-------------------|
3-
| bert-99 | offline | 80 | 6.209 | - |
3+
| bert-99 | offline | 80 | 6.359 | - |

‎open/MLCommons/measurements/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,4 @@ Model Precision: fp32
5454
`F1`: `80.0`, Required accuracy for closed division `>= 89.96526`
5555

5656
### Performance Results
57-
`Samples per second`: `6.2088`
57+
`Samples per second`: `6.35852`

‎open/MLCommons/measurements/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline/cm-version-info.json

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@
197197
"script_tags": "detect-os,detect,os,info",
198198
"script_variations": "",
199199
"version": "",
200-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
200+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
201201
}
202202
},
203203
{
@@ -217,7 +217,7 @@
217217
"script_tags": "detect,cpu,detect-cpu,info",
218218
"script_variations": "",
219219
"version": "",
220-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
220+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
221221
}
222222
},
223223
{
@@ -237,7 +237,7 @@
237237
"script_tags": "get,python,python3,get-python,get-python3",
238238
"script_variations": "",
239239
"version": "3.11.10",
240-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
240+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
241241
}
242242
},
243243
{
@@ -297,7 +297,7 @@
297297
"script_tags": "get,install,generic,generic-python-lib",
298298
"script_variations": "torch",
299299
"version": "2.5.0",
300-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
300+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
301301
}
302302
},
303303
{
@@ -357,7 +357,7 @@
357357
"script_tags": "get,install,generic,generic-python-lib",
358358
"script_variations": "transformers",
359359
"version": "4.45.2",
360-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
360+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
361361
}
362362
},
363363
{
@@ -387,15 +387,15 @@
387387
"script_tags": "detect-os,detect,os,info",
388388
"script_variations": "",
389389
"version": "",
390-
"parent": "extract-file,3f0b76219d004817 ( path./home/runner/CM/repos/local/cache/4a89c433acce4d3a/oBERT-Large_95sparse_block4_qat.onnx.tar.xz )"
390+
"parent": "extract-file,3f0b76219d004817 ( path./home/runner/CM/repos/local/cache/24d95f3f6ac145bb/oBERT-Large_95sparse_block4_qat.onnx.tar.xz )"
391391
}
392392
},
393393
{
394-
"extract,file,_path./home/runner/CM/repos/local/cache/4a89c433acce4d3a/oBERT-Large_95sparse_block4_qat.onnx.tar.xz": {
394+
"extract,file,_path./home/runner/CM/repos/local/cache/24d95f3f6ac145bb/oBERT-Large_95sparse_block4_qat.onnx.tar.xz": {
395395
"script_uid": "3f0b76219d004817",
396396
"script_alias": "extract-file",
397397
"script_tags": "extract,file",
398-
"script_variations": "path./home/runner/CM/repos/local/cache/4a89c433acce4d3a/oBERT-Large_95sparse_block4_qat.onnx.tar.xz",
398+
"script_variations": "path./home/runner/CM/repos/local/cache/24d95f3f6ac145bb/oBERT-Large_95sparse_block4_qat.onnx.tar.xz",
399399
"version": "",
400400
"parent": "download-and-extract,c67e81a4ce2649f5 ( url.https://github.com/mlcommons/inference_results_v2.1/raw/master/open/NeuralMagic/code/bert/deepsparse/models/oBERT-Large_95sparse_block4_qat.onnx.tar.xz )"
401401
}
@@ -427,7 +427,7 @@
427427
"script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing",
428428
"script_variations": "int8,deepsparse",
429429
"version": "",
430-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
430+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
431431
}
432432
},
433433
{
@@ -437,7 +437,7 @@
437437
"script_tags": "get,dataset,squad,language-processing,validation,original",
438438
"script_variations": "",
439439
"version": "1.1",
440-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
440+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
441441
}
442442
},
443443
{
@@ -447,7 +447,7 @@
447447
"script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab",
448448
"script_variations": "",
449449
"version": "",
450-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
450+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
451451
}
452452
},
453453
{
@@ -527,7 +527,7 @@
527527
"script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
528528
"script_variations": "",
529529
"version": "",
530-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
530+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
531531
}
532532
},
533533
{
@@ -626,7 +626,7 @@
626626
"script_alias": "get-cmake",
627627
"script_tags": "get,cmake,get-cmake",
628628
"script_variations": "",
629-
"version": "3.30.4",
629+
"version": "3.30.5",
630630
"parent": "get-mlperf-inference-loadgen,64c3d98d0ba04950"
631631
}
632632
},
@@ -827,7 +827,7 @@
827827
"script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
828828
"script_variations": "",
829829
"version": "master",
830-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
830+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
831831
}
832832
},
833833
{
@@ -837,7 +837,7 @@
837837
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
838838
"script_variations": "deeplearningexamples",
839839
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
840-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
840+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
841841
}
842842
},
843843
{
@@ -887,7 +887,7 @@
887887
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
888888
"script_variations": "",
889889
"version": "deepsparse",
890-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
890+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
891891
}
892892
},
893893
{
@@ -947,7 +947,7 @@
947947
"script_tags": "get,install,generic,generic-python-lib",
948948
"script_variations": "package.psutil",
949949
"version": "6.0.0",
950-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
950+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
951951
}
952952
},
953953
{
@@ -1007,7 +1007,7 @@
10071007
"script_tags": "get,install,generic,generic-python-lib",
10081008
"script_variations": "deepsparse",
10091009
"version": "1.8.0",
1010-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
1010+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
10111011
}
10121012
},
10131013
{
@@ -1067,7 +1067,7 @@
10671067
"script_tags": "get,install,generic,generic-python-lib",
10681068
"script_variations": "package.pydantic",
10691069
"version": "2.9.2",
1070-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
1070+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
10711071
}
10721072
},
10731073
{
@@ -1127,7 +1127,7 @@
11271127
"script_tags": "get,install,generic,generic-python-lib",
11281128
"script_variations": "tokenization",
11291129
"version": "1.0.7",
1130-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
1130+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
11311131
}
11321132
},
11331133
{
@@ -1187,7 +1187,7 @@
11871187
"script_tags": "get,install,generic,generic-python-lib",
11881188
"script_variations": "six",
11891189
"version": "1.16.0",
1190-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
1190+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
11911191
}
11921192
},
11931193
{
@@ -1247,7 +1247,7 @@
12471247
"script_tags": "get,install,generic,generic-python-lib",
12481248
"script_variations": "package.absl-py",
12491249
"version": "2.1.0",
1250-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
1250+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
12511251
}
12521252
},
12531253
{
@@ -1287,7 +1287,7 @@
12871287
"script_tags": "mlperf,benchmark-mlperf",
12881288
"script_variations": "",
12891289
"version": "",
1290-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( int8,_offline,_bert-99,_deepsparse,_cpu )"
1290+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( deepsparse,_bert-99,_int8,_cpu,_offline )"
12911291
}
12921292
}
12931293
]

‎open/MLCommons/measurements/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline/os_info.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
],
2727
"CM_HOST_PLATFORM_FLAVOR": "x86_64",
2828
"CM_HOST_PYTHON_BITS": "64",
29-
"CM_HOST_SYSTEM_NAME": "fv-az1208-573"
29+
"CM_HOST_SYSTEM_NAME": "fv-az888-80"
3030
}

‎open/MLCommons/model_mapping.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"resnet50": "resnet50"
2+
"bert-99": "bert-99"
33
}

‎open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline/accuracy/accuracy.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{"exact_match": 80.0, "f1": 80.0}
22
Reading examples...
3-
No cached features at '/home/runner/CM/repos/local/cache/5b339e2b164c42f4/inference/language/bert/eval_features.pickle'... converting from examples...
3+
No cached features at '/home/runner/CM/repos/local/cache/7c7685aa065b4373/inference/language/bert/eval_features.pickle'... converting from examples...
44
Creating tokenizer...
55
Converting examples to features...
6-
Caching features at '/home/runner/CM/repos/local/cache/5b339e2b164c42f4/inference/language/bert/eval_features.pickle'...
6+
Caching features at '/home/runner/CM/repos/local/cache/7c7685aa065b4373/inference/language/bert/eval_features.pickle'...
77
Loading LoadGen logs...
88
Post-processing predictions...
9-
Writing predictions to: /home/runner/CM/repos/local/cache/b867703bc7194528/test_results/gh_ubuntu-latest_x86-reference-cpu-deepsparse-vdefault-default_config/bert-99/offline/accuracy/predictions.json
9+
Writing predictions to: /home/runner/CM/repos/local/cache/099f2dae2cc343d9/test_results/gh_ubuntu-latest_x86-reference-cpu-deepsparse-vdefault-default_config/bert-99/offline/accuracy/predictions.json
1010
Evaluating predictions...
1111
hash=7308e7ffe3ae4e8361a04852551f8b640b45974e4e8e87e64ff5f1b688ba3500

0 commit comments

Comments
 (0)