Skip to content

Commit b055a67

Browse files
gateoverflow-admingateoverflow-admin
committed
Results from Retinanet GH action on macos-latest
1 parent 83ec55c commit b055a67

File tree

11 files changed

+221
-222
lines changed

11 files changed

+221
-222
lines changed

‎open/MLCommons/measurements/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
2-
|----------|------------|------------|--------------|-------------------|
3-
| resnet50 | offline | 76 | 19.85 | - |
1+
| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
2+
|-----------|------------|------------|--------------|-------------------|
3+
| retinanet | offline | 76.951 | 0.247 | - |

‎open/MLCommons/measurements/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ This experiment is generated using the [MLCommons Collective Mind automation fra
44

55
## Host platform
66

7-
* OS version: macOS-14.7-arm64-arm-64bit
7+
* OS version: macOS-14.6.1-arm64-arm-64bit
88
* CPU version: arm
9-
* Python version: 3.12.7 (v3.12.7:0b05ead877f, Sep 30 2024, 23:18:00) [Clang 13.0.0 (clang-1300.0.29.30)]
9+
* Python version: 3.12.6 (v3.12.6:a4a2d2b0d85, Sep 6 2024, 16:08:03) [Clang 13.0.0 (clang-1300.0.29.30)]
1010
* MLCommons CM version: 3.0.2
1111

1212
## CM Run Command
@@ -55,4 +55,4 @@ Model Precision: fp32
5555
`mAP`: `76.951`, Required accuracy for closed division `>= 37.1745`
5656

5757
### Performance Results
58-
`Samples per second`: `0.390447`
58+
`Samples per second`: `0.247035`

‎open/MLCommons/measurements/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline/cm-version-info.json

Lines changed: 41 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
"script_alias": "get-python3",
2727
"script_tags": "get,python,python3,get-python,get-python3",
2828
"script_variations": "",
29-
"version": "3.12.7",
29+
"version": "3.12.6",
3030
"parent": "app-mlperf-inference,d775cac873ee4231 ( python,_retinanet,_onnxruntime,_cpu,_test,_r4.1-dev_default,_offline )"
3131
}
3232
},
@@ -67,7 +67,7 @@
6767
"script_tags": "detect-os,detect,os,info",
6868
"script_variations": "",
6969
"version": "",
70-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
70+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
7171
}
7272
},
7373
{
@@ -87,7 +87,7 @@
8787
"script_tags": "detect,cpu,detect-cpu,info",
8888
"script_variations": "",
8989
"version": "",
90-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
90+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
9191
}
9292
},
9393
{
@@ -106,8 +106,8 @@
106106
"script_alias": "get-python3",
107107
"script_tags": "get,python,python3,get-python,get-python3",
108108
"script_variations": "",
109-
"version": "3.12.7",
110-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
109+
"version": "3.12.6",
110+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
111111
}
112112
},
113113
{
@@ -146,7 +146,7 @@
146146
"script_alias": "get-python3",
147147
"script_tags": "get,python,python3,get-python,get-python3",
148148
"script_variations": "",
149-
"version": "3.12.7",
149+
"version": "3.12.6",
150150
"parent": "get-generic-python-lib,94b62a682bc44791 ( onnxruntime )"
151151
}
152152
},
@@ -167,7 +167,7 @@
167167
"script_tags": "get,install,generic,generic-python-lib",
168168
"script_variations": "onnxruntime",
169169
"version": "1.19.2",
170-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
170+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
171171
}
172172
},
173173
{
@@ -237,17 +237,17 @@
237237
"script_tags": "dae,file,download-and-extract",
238238
"script_variations": "url.https://zenodo.org/record/6617879/files/resnext50_32x4d_fpn.onnx",
239239
"version": "",
240-
"parent": "get-ml-model-retinanet,427bc5665e4541c2 ( fp32,_onnx )"
240+
"parent": "get-ml-model-retinanet,427bc5665e4541c2 ( onnx,_fp32 )"
241241
}
242242
},
243243
{
244-
"get,ml-model,object-detection,retinanet,raw,_fp32,_onnx": {
244+
"get,ml-model,object-detection,retinanet,raw,_onnx,_fp32": {
245245
"script_uid": "427bc5665e4541c2",
246246
"script_alias": "get-ml-model-retinanet",
247247
"script_tags": "get,ml-model,raw,resnext50,retinanet,object-detection",
248-
"script_variations": "fp32,onnx",
248+
"script_variations": "onnx,fp32",
249249
"version": "",
250-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
250+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
251251
}
252252
},
253253
{
@@ -256,7 +256,7 @@
256256
"script_alias": "get-python3",
257257
"script_tags": "get,python,python3,get-python,get-python3",
258258
"script_variations": "",
259-
"version": "3.12.7",
259+
"version": "3.12.6",
260260
"parent": "get-preprocessed-dataset-openimages,9842f1be8cba4c7b ( validation,_NCHW )"
261261
}
262262
},
@@ -276,7 +276,7 @@
276276
"script_alias": "get-python3",
277277
"script_tags": "get,python,python3,get-python,get-python3",
278278
"script_variations": "",
279-
"version": "3.12.7",
279+
"version": "3.12.6",
280280
"parent": "get-dataset-openimages,0a9d49b644cf4142 ( validation,_50,_default-annotations )"
281281
}
282282
},
@@ -316,7 +316,7 @@
316316
"script_alias": "get-python3",
317317
"script_tags": "get,python,python3,get-python,get-python3",
318318
"script_variations": "",
319-
"version": "3.12.7",
319+
"version": "3.12.6",
320320
"parent": "get-generic-python-lib,94b62a682bc44791 ( requests )"
321321
}
322322
},
@@ -386,7 +386,7 @@
386386
"script_alias": "get-python3",
387387
"script_tags": "get,python,python3,get-python,get-python3",
388388
"script_variations": "",
389-
"version": "3.12.7",
389+
"version": "3.12.6",
390390
"parent": "get-generic-python-lib,94b62a682bc44791 ( boto3 )"
391391
}
392392
},
@@ -446,7 +446,7 @@
446446
"script_alias": "get-python3",
447447
"script_tags": "get,python,python3,get-python,get-python3",
448448
"script_variations": "",
449-
"version": "3.12.7",
449+
"version": "3.12.6",
450450
"parent": "get-generic-python-lib,94b62a682bc44791 ( tqdm )"
451451
}
452452
},
@@ -506,7 +506,7 @@
506506
"script_alias": "get-python3",
507507
"script_tags": "get,python,python3,get-python,get-python3",
508508
"script_variations": "",
509-
"version": "3.12.7",
509+
"version": "3.12.6",
510510
"parent": "get-generic-python-lib,94b62a682bc44791 ( numpy )"
511511
}
512512
},
@@ -566,7 +566,7 @@
566566
"script_alias": "get-python3",
567567
"script_tags": "get,python,python3,get-python,get-python3",
568568
"script_variations": "",
569-
"version": "3.12.7",
569+
"version": "3.12.6",
570570
"parent": "get-generic-python-lib,94b62a682bc44791 ( opencv-python )"
571571
}
572572
},
@@ -626,7 +626,7 @@
626626
"script_alias": "get-python3",
627627
"script_tags": "get,python,python3,get-python,get-python3",
628628
"script_variations": "",
629-
"version": "3.12.7",
629+
"version": "3.12.6",
630630
"parent": "get-generic-python-lib,94b62a682bc44791 ( pandas )"
631631
}
632632
},
@@ -686,7 +686,7 @@
686686
"script_alias": "get-python3",
687687
"script_tags": "get,python,python3,get-python,get-python3",
688688
"script_variations": "",
689-
"version": "3.12.7",
689+
"version": "3.12.6",
690690
"parent": "get-generic-python-lib,94b62a682bc44791 ( pycocotools )"
691691
}
692692
},
@@ -786,7 +786,7 @@
786786
"script_alias": "get-python3",
787787
"script_tags": "get,python,python3,get-python,get-python3",
788788
"script_variations": "",
789-
"version": "3.12.7",
789+
"version": "3.12.6",
790790
"parent": "get-generic-python-lib,94b62a682bc44791 ( pillow )"
791791
}
792792
},
@@ -846,7 +846,7 @@
846846
"script_alias": "get-python3",
847847
"script_tags": "get,python,python3,get-python,get-python3",
848848
"script_variations": "",
849-
"version": "3.12.7",
849+
"version": "3.12.6",
850850
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.ujson )"
851851
}
852852
},
@@ -906,7 +906,7 @@
906906
"script_alias": "get-python3",
907907
"script_tags": "get,python,python3,get-python,get-python3",
908908
"script_variations": "",
909-
"version": "3.12.7",
909+
"version": "3.12.6",
910910
"parent": "get-generic-python-lib,94b62a682bc44791 ( numpy )"
911911
}
912912
},
@@ -947,7 +947,7 @@
947947
"script_tags": "get,dataset,openimages,open-images,object-detection,preprocessed",
948948
"script_variations": "validation,NCHW",
949949
"version": "",
950-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
950+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
951951
}
952952
},
953953
{
@@ -986,7 +986,7 @@
986986
"script_alias": "get-python3",
987987
"script_tags": "get,python,python3,get-python,get-python3",
988988
"script_variations": "",
989-
"version": "3.12.7",
989+
"version": "3.12.6",
990990
"parent": "generate-mlperf-inference-user-conf,3af4475745964b93"
991991
}
992992
},
@@ -1027,7 +1027,7 @@
10271027
"script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
10281028
"script_variations": "",
10291029
"version": "",
1030-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1030+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
10311031
}
10321032
},
10331033
{
@@ -1046,7 +1046,7 @@
10461046
"script_alias": "get-python3",
10471047
"script_tags": "get,python,python3,get-python,get-python3",
10481048
"script_variations": "",
1049-
"version": "3.12.7",
1049+
"version": "3.12.6",
10501050
"parent": "get-mlperf-inference-loadgen,64c3d98d0ba04950"
10511051
}
10521052
},
@@ -1066,7 +1066,7 @@
10661066
"script_alias": "get-python3",
10671067
"script_tags": "get,python,python3,get-python,get-python3",
10681068
"script_variations": "",
1069-
"version": "3.12.7",
1069+
"version": "3.12.6",
10701070
"parent": "get-mlperf-inference-src,4b57186581024797"
10711071
}
10721072
},
@@ -1126,7 +1126,7 @@
11261126
"script_alias": "get-cmake",
11271127
"script_tags": "get,cmake,get-cmake",
11281128
"script_variations": "",
1129-
"version": "3.30.4",
1129+
"version": "3.30.3",
11301130
"parent": "get-mlperf-inference-loadgen,64c3d98d0ba04950"
11311131
}
11321132
},
@@ -1166,7 +1166,7 @@
11661166
"script_alias": "get-python3",
11671167
"script_tags": "get,python,python3,get-python,get-python3",
11681168
"script_variations": "",
1169-
"version": "3.12.7",
1169+
"version": "3.12.6",
11701170
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.wheel )"
11711171
}
11721172
},
@@ -1236,7 +1236,7 @@
12361236
"script_alias": "get-python3",
12371237
"script_tags": "get,python,python3,get-python,get-python3",
12381238
"script_variations": "",
1239-
"version": "3.12.7",
1239+
"version": "3.12.6",
12401240
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.pybind11 )"
12411241
}
12421242
},
@@ -1296,7 +1296,7 @@
12961296
"script_alias": "get-python3",
12971297
"script_tags": "get,python,python3,get-python,get-python3",
12981298
"script_variations": "",
1299-
"version": "3.12.7",
1299+
"version": "3.12.6",
13001300
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.setuptools )"
13011301
}
13021302
},
@@ -1327,7 +1327,7 @@
13271327
"script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
13281328
"script_variations": "",
13291329
"version": "master",
1330-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1330+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
13311331
}
13321332
},
13331333
{
@@ -1337,7 +1337,7 @@
13371337
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
13381338
"script_variations": "",
13391339
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
1340-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1340+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
13411341
}
13421342
},
13431343
{
@@ -1347,7 +1347,7 @@
13471347
"script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
13481348
"script_variations": "",
13491349
"version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054",
1350-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1350+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
13511351
}
13521352
},
13531353
{
@@ -1386,7 +1386,7 @@
13861386
"script_alias": "get-python3",
13871387
"script_tags": "get,python,python3,get-python,get-python3",
13881388
"script_variations": "",
1389-
"version": "3.12.7",
1389+
"version": "3.12.6",
13901390
"parent": "get-generic-python-lib,94b62a682bc44791 ( package.psutil )"
13911391
}
13921392
},
@@ -1407,7 +1407,7 @@
14071407
"script_tags": "get,install,generic,generic-python-lib",
14081408
"script_variations": "package.psutil",
14091409
"version": "6.0.0",
1410-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1410+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
14111411
}
14121412
},
14131413
{
@@ -1417,7 +1417,7 @@
14171417
"script_tags": "get,install,generic,generic-python-lib",
14181418
"script_variations": "opencv-python",
14191419
"version": "4.10.0.84",
1420-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1420+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
14211421
}
14221422
},
14231423
{
@@ -1427,7 +1427,7 @@
14271427
"script_tags": "get,install,generic,generic-python-lib",
14281428
"script_variations": "numpy",
14291429
"version": "1.26.4",
1430-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1430+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
14311431
}
14321432
},
14331433
{
@@ -1437,7 +1437,7 @@
14371437
"script_tags": "get,install,generic,generic-python-lib",
14381438
"script_variations": "pycocotools",
14391439
"version": "2.0.8",
1440-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1440+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
14411441
}
14421442
},
14431443
{
@@ -1477,7 +1477,7 @@
14771477
"script_tags": "mlperf,benchmark-mlperf",
14781478
"script_variations": "",
14791479
"version": "",
1480-
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_onnxruntime,_retinanet,_offline,_fp32 )"
1480+
"parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( retinanet,_onnxruntime,_cpu,_offline,_fp32 )"
14811481
}
14821482
}
14831483
]

‎open/MLCommons/measurements/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline/os_info.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@
1010
"CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD": "brew update",
1111
"CM_HOST_PLATFORM_FLAVOR": "arm64",
1212
"CM_HOST_PYTHON_BITS": "64",
13-
"CM_HOST_SYSTEM_NAME": "Mac-1728411641016.local"
13+
"CM_HOST_SYSTEM_NAME": "Mac-1728412083383.local"
1414
}

‎open/MLCommons/model_mapping.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"resnet50": "resnet50"
2+
"retinanet": "retinanet"
33
}

0 commit comments

Comments
 (0)