Skip to content

Commit 7ee9cf4

Browse files
anandhu-enganandhu-eng
authored
Added support for automotive pointpainting benchmark (#172)
* initial commit for pointpillar * rename benchmark name + add deps * [Automated Commit] Format Codebase * added scipy dependency * Revert "added scipy dependency" This reverts commit 3b3a0e8. * add scipy dependency * version max as string * changes for registering to docker cache * added opencv dependency * add more dependencies * support custom query count for pointpainting * fix version check ffmpeg * version dlted * remove scipy dependencu * fixed scipy version * version change - test commit * integrated accuracy script * add extra loadgen options * fix for accuracy * added starting weights filename * commit for valid perf run * submission checker version defaulted to 5.0 * fix version bug * commit for compliance * add path to new env * add test 4 for pointpainting * commit to test numpy fix * add dep name for scipy * fix name keys
1 parent 1dd7264 commit 7ee9cf4

File tree

12 files changed

+211
-3
lines changed

12 files changed

+211
-3
lines changed

‎script/app-mlperf-inference-mlcommons-python/customize.py

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def preprocess(i):
6868
str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE'])
6969

7070
if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get('MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and (
71-
env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL']) and env.get('MLC_MLPERF_RUN_STYLE', '') != "valid":
71+
env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL'] or 'pointpainting' in env['MLC_MODEL']) and (env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" or 'pointpainting' in env['MLC_MODEL']):
7272
env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \
7373
env['MLC_MLPERF_LOADGEN_QUERY_COUNT']
7474

@@ -524,6 +524,28 @@ def get_run_cmd_reference(
524524
cmd = cmd.replace("--count", "--total-sample-count")
525525
cmd = cmd.replace("--max-batchsize", "--batch-size")
526526

527+
elif "pointpainting" in env['MLC_MODEL']:
528+
env['RUN_DIR'] = os.path.join(
529+
env['MLC_MLPERF_INFERENCE_SOURCE'],
530+
"automotive",
531+
"3d-object-detection")
532+
533+
cmd = env['MLC_PYTHON_BIN_WITH_PATH'] + " main.py " \
534+
" --dataset waymo" + \
535+
" --dataset-path " + env['MLC_DATASET_WAYMO_PATH'] + \
536+
" --lidar-path " + env['MLC_ML_MODEL_POINT_PILLARS_PATH'] + \
537+
" --segmentor-path " + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] + \
538+
" --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + \
539+
" --output " + env['MLC_MLPERF_OUTPUT_DIR'] + \
540+
" --dtype " + env['MLC_MLPERF_MODEL_PRECISION'].replace("float", "fp") + \
541+
scenario_extra_options + \
542+
env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + mode_extra_options
543+
544+
if env.get('MLC_MLPERF_POINTPAINTING_TIME', '') != '':
545+
cmd += f" --time {env['MLC_MLPERF_POINTPAINTING_TIME']}"
546+
547+
print(cmd)
548+
527549
if env.get('MLC_NETWORK_LOADGEN', '') in ["lon", "sut"]:
528550
cmd = cmd + " " + "--network " + env['MLC_NETWORK_LOADGEN']
529551
if env.get('MLC_NETWORK_LOADGEN_SUT_SERVERS', []):

‎script/app-mlperf-inference-mlcommons-python/meta.yaml

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ input_mapping:
6666
multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY
6767
network: MLC_NETWORK_LOADGEN
6868
sut_servers: MLC_NETWORK_LOADGEN_SUT_SERVERS
69+
pointpillars_checkpoint_path: MLC_ML_MODEL_POINT_PILLARS_PATH
70+
deeplab_resnet50_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH
71+
waymo_path: MLC_DATASET_WAYMO_PATH
6972

7073
# Duplicate CM environment variables to the ones used in native apps
7174
env_key_mappings:
@@ -491,7 +494,6 @@ deps:
491494
- tags: get,ml-model,llama3
492495
names:
493496
- llama3-405b-model
494-
- llama3-402b-model
495497
enable_if_env:
496498
MLC_MODEL:
497499
- llama3_1-405b
@@ -502,6 +504,26 @@ deps:
502504
MLC_RUN_STATE_DOCKER:
503505
- "yes"
504506

507+
## pointpainting
508+
- tags: get,ml-model,pointpillars
509+
names:
510+
- pointpillars-model
511+
enable_if_env:
512+
MLC_MODEL:
513+
- pointpainting
514+
skip_if_env:
515+
MLC_RUN_STATE_DOCKER:
516+
- "yes"
517+
- tags: get,ml-model,resnet50-deeplab
518+
enable_if_env:
519+
MLC_MODEL:
520+
- pointpainting
521+
skip_if_env:
522+
MLC_RUN_STATE_DOCKER:
523+
- "yes"
524+
names:
525+
- resnet50-deeplab-model
526+
505527
########################################################################
506528
# Install datasets
507529

@@ -641,6 +663,17 @@ deps:
641663
MLC_USE_DATASET_FROM_HOST:
642664
- "yes"
643665

666+
## waymo for pointpillats
667+
- tags: get,dataset,waymo
668+
enable_if_env:
669+
MLC_MODEL:
670+
- pointpainting
671+
skip_if_env:
672+
MLC_RUN_STATE_DOCKER:
673+
- "yes"
674+
names:
675+
- waymo-dataset
676+
644677
## llama3_1 dataset
645678
- tags: get,dataset,mlperf,inference,llama3,_validation
646679
names:
@@ -1299,6 +1332,45 @@ variations:
12991332
MLC_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<MLC_TORCH_VERSION>>>+cpu.html"
13001333
MLC_TMP_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL_DGL: "https://data.dgl.ai/wheels/torch-<<<MLC_TORCH_VERSION_MAJOR_MINOR>>>/repo.html"
13011334

1335+
pointpainting:
1336+
group: models
1337+
env:
1338+
MLC_MODEL: pointpainting
1339+
MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://github.com/mlcommons/mlperf-automations/tree/dev/script/get-ml-model-pointpainting,https://github.com/mlcommons/mlperf-automations/tree/dev/script/get-ml-model-resnet50-deeplab"
1340+
adr:
1341+
pytorch:
1342+
version_max: "2.2.2"
1343+
torchvision:
1344+
version_max: "0.17.2"
1345+
deps:
1346+
- tags: get,generic-python-lib,_package.shapely
1347+
- tags: get,generic-python-lib,_package.numba
1348+
- tags: get,generic-python-lib,_package.open3d
1349+
- tags: get,generic-python-lib,_package.numpy
1350+
version_max: "1.26.4"
1351+
names:
1352+
- numpy
1353+
- tags: get,generic-python-lib,_package.numpy
1354+
version_max: "2.0.2"
1355+
names:
1356+
- numpy-upgrade
1357+
- tags: get,generic-python-lib,_package.numpy
1358+
version_max: "1.26.4"
1359+
names:
1360+
- numpy-downgrade
1361+
- tags: get,generic-python-lib,_package.tensorboard
1362+
- tags: get,generic-python-lib,_package.onnxruntime
1363+
- tags: get,generic-python-lib,_package.opencv-python
1364+
- tags: get,generic-python-lib,_package.scikit-image
1365+
- tags: get,generic-python-lib,_package.scipy
1366+
version_max: "1.11.2"
1367+
names:
1368+
- scipy
1369+
- tags: get,generic-python-lib,_package.ninja
1370+
- tags: get,generic-sys-util,_ffmpeg
1371+
- tags: get,generic-sys-util,_libsm6
1372+
- tags: get,generic-sys-util,_libxext6
1373+
13021374
llama3_1-405b:
13031375
group: models
13041376
env:

‎script/app-mlperf-inference/customize.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,13 @@ def postprocess(i):
129129
accuracy_log_file_option_name = " --mlperf-accuracy-file "
130130
datatype_option = " --dtype " + env['MLC_IMAGENET_ACCURACY_DTYPE']
131131

132+
elif model == "pointpainting":
133+
accuracy_filename = "accuracy-waymo.py"
134+
accuracy_filepath = os.path.join(
135+
env['MLC_MLPERF_INFERENCE_POINTPAINTING_PATH'], accuracy_filename)
136+
accuracy_log_file_option_name = " --mlperf-accuracy-file "
137+
datatype_option = ""
138+
132139
elif model == "retinanet":
133140
accuracy_filename = "accuracy-openimages.py"
134141
accuracy_filepath = os.path.join(env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools",

‎script/app-mlperf-inference/meta.yaml

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,10 @@ variations:
261261
reference,rgat:
262262
default_variations:
263263
backend: pytorch
264+
265+
reference,pointpainting:
266+
default_variations:
267+
backend: pytorch
264268

265269
reference,sdxl_:
266270
default_variations:
@@ -823,6 +827,51 @@ variations:
823827
- igbh-original
824828
- igbh-dataset
825829

830+
pointpainting:
831+
group:
832+
model
833+
add_deps_recursive:
834+
mlperf-inference-implementation:
835+
tags: _pointpainting
836+
env:
837+
MLC_MODEL:
838+
pointpainting
839+
docker:
840+
deps:
841+
- tags: get,dataset,waymo
842+
enable_if_env:
843+
MLC_USE_DATASET_FROM_HOST:
844+
- 'yes'
845+
names:
846+
- waymo-dataset
847+
- tags: get,ml-model,pointpillars
848+
enable_if_env:
849+
MLC_USE_DATASET_FROM_HOST:
850+
- 'yes'
851+
names:
852+
- pointpillars-model
853+
- tags: get,ml-model,resnet50-deeplab
854+
enable_if_env:
855+
MLC_USE_DATASET_FROM_HOST:
856+
- 'yes'
857+
names:
858+
- resnet50-deeplab-model
859+
posthook_deps:
860+
- enable_if_env:
861+
MLC_MLPERF_LOADGEN_MODE:
862+
- accuracy
863+
- all
864+
MLC_MLPERF_ACCURACY_RESULTS_DIR:
865+
- 'on'
866+
skip_if_env:
867+
MLC_MLPERF_IMPLEMENTATION:
868+
- nvidia
869+
names:
870+
- mlperf-accuracy-script
871+
- waymo-accuracy-script
872+
tags: run,accuracy,mlperf,_waymo
873+
874+
826875
llama3_1-405b:
827876
group:
828877
model
@@ -1904,6 +1953,9 @@ docker:
19041953
- "${{ MLC_DATASET_KITS19_PREPROCESSED_PATH }}:${{ MLC_DATASET_KITS19_PREPROCESSED_PATH }}"
19051954
- "${{ MLC_DATASET_IGBH_PATH }}:${{ MLC_DATASET_IGBH_PATH }}"
19061955
- "${{ MLC_ML_MODEL_RGAT_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_RGAT_CHECKPOINT_PATH }}"
1956+
- "${{ MLC_DATASET_WAYMO_PATH }}:${{ MLC_DATASET_WAYMO_PATH }}"
1957+
- "${{ MLC_ML_MODEL_POINT_PILLARS_PATH }}:${{ MLC_ML_MODEL_POINT_PILLARS_PATH }}"
1958+
- "${{ MLC_ML_MODEL_DPLAB_RESNET50_PATH }}:${{ MLC_ML_MODEL_DPLAB_RESNET50_PATH }}"
19071959
skip_run_cmd: 'no'
19081960
shm_size: '32gb'
19091961
interactive: True

‎script/get-generic-sys-util/meta.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,43 @@ variations:
9595
brew: ''
9696
dnf: dmidecode
9797
yum: dmidecode
98+
ffmpeg:
99+
env:
100+
MLC_SYS_UTIL_NAME: ffmpeg
101+
MLC_SYS_UTIL_VERSION_CMD: ffmpeg -version # tbd: regular expression for version
102+
MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0
103+
new_env_keys:
104+
- MLC_FFMPEG_VERSION
105+
state:
106+
ffmpeg: # tbd: complete for other flavours of linux
107+
apt: ffmpeg
108+
brew: ''
109+
dnf: ''
110+
yum: ''
111+
libsm6:
112+
env:
113+
MLC_SYS_UTIL_NAME: libsm6 # tbd: regular expression for version as well as whether its installed?
114+
MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0
115+
new_env_keys:
116+
- MLC_LIBSM6_VERSION
117+
state:
118+
libsm6: # tbd: complete for other flavours of linux
119+
apt: libsm6
120+
brew: ''
121+
dnf: ''
122+
yum: ''
123+
libxext6:
124+
env:
125+
MLC_SYS_UTIL_NAME: libxext6 # tbd: regular expression for version as well as whether its installed?
126+
MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0
127+
new_env_keys:
128+
- MLC_LIBEXT6_VERSION
129+
state:
130+
libxext6: # tbd: complete for other flavours of linux
131+
apt: libxext6
132+
brew: ''
133+
dnf: ''
134+
yum: ''
98135
g++-11:
99136
env:
100137
MLC_GENERIC_SYS_UTIL_IGNORE_VERSION_DETECTION_FAILURE: 'yes'

‎script/get-mlperf-inference-src/customize.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ def postprocess(i):
114114
inference_root, 'graph', 'R-GAT')
115115
env['MLC_MLPERF_INFERENCE_3DUNET_PATH'] = os.path.join(
116116
inference_root, 'vision', 'medical_imaging', '3d-unet-kits19')
117+
env['MLC_MLPERF_INFERENCE_POINTPAINTING_PATH'] = os.path.join(
118+
inference_root, 'automotive', '3d-object-detection')
117119

118120
env['MLC_GET_DEPENDENT_CACHED_PATH'] = inference_root
119121

‎script/get-mlperf-inference-src/meta.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ new_env_keys:
3030
- MLC_MLPERF_INFERENCE_VERSION
3131
- MLC_MLPERF_INFERENCE_VISION_PATH
3232
- MLC_MLPERF_LAST_RELEASE
33+
- MLC_MLPERF_INFERENCE_POINTPAINTING_PATH
3334
- +PYTHONPATH
3435
prehook_deps:
3536
- env:

‎script/process-mlperf-accuracy/customize.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,10 @@ def preprocess(i):
202202
CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "language", "llama3.1-405b", "evaluate-accuracy.py") + "' --checkpoint-path '" + env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(
203203
result_dir, "mlperf_log_accuracy.json") + "' --dtype '" + env['MLC_ACCURACY_DTYPE'] + "' --dataset-file '" + env['MLC_DATASET_LLAMA3_PATH'] + "' > '" + out_file + "'"
204204

205+
elif dataset == "waymo":
206+
CMD = env['MLC_PYTHON_BIN_WITH_PATH'] + " '" + os.path.join(env['MLC_MLPERF_INFERENCE_SOURCE'], "automotive", "3d-object-detection", "accuracy_waymo.py") + "' --mlperf-accuracy-file '" + os.path.join(
207+
result_dir, "mlperf_log_accuracy.json") + "' --waymo-dir '" + env['MLC_DATASET_WAYMO_PATH'] + "' > '" + out_file + "'"
208+
205209
else:
206210
return {'return': 1, 'error': 'Unsupported dataset'}
207211

‎script/process-mlperf-accuracy/meta.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,3 +269,7 @@ variations:
269269
env:
270270
MLC_DATASET: dataset_llama3
271271
group: dataset
272+
waymo:
273+
env:
274+
MLC_DATASET: waymo
275+
group: dataset

‎script/run-mlperf-inference-app/customize.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ def preprocess(i):
128128
test_list.remove("TEST01")
129129
# test_list.remove("TEST05")
130130

131+
if "pointpainting" in env["MLC_MODEL"].lower():
132+
test_list.append("TEST04")
133+
131134
if "llama2" in env['MLC_MODEL'].lower(
132135
) or "mixtral-8x7b" in env['MLC_MODEL']:
133136
test_list.append("TEST06")

0 commit comments

Comments
 (0)