Fixes to run-all scripts (#234)

* Fix execution_mode in run-all script

Author: arjunsuresh

script/run-all-mlperf-models/run-bert-macos.sh

@@ -44,22 +44,22 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 submission_cmd_scenario='mlcr generate-run-cmds,inference,_submission  --scenario=$scenario \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 readme_cmd_single='mlcr generate-run-cmds,inference,_populate-readme \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 readme_cmd='mlcr generate-run-cmds,inference,_populate-readme,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 # run "$MLC_RUN_CMD"
 run_test "onnxruntime" "100" "reference" "cpu" "$find_performance_cmd"

script/run-all-mlperf-models/run-bert.sh

@@ -44,23 +44,14 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--category=$category --division=$division  --quiet  \
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 submission_cmd_scenario='mlcr generate-run-cmds,inference,_submission  --scenario=$scenario \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--category=$category --division=$division  --quiet  \
+--skip_submission_generation=yes --execution_mode=valid $power'
 
-readme_cmd_single='mlcr generate-run-cmds,inference,_populate-readme \
---model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
-
-readme_cmd='mlcr generate-run-cmds,inference,_populate-readme,_all-scenarios \
---model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
 
 # run "$MLC_RUN_CMD"
 run_test "onnxruntime" "20" "reference" "cpu" "$find_performance_cmd"

script/run-all-mlperf-models/run-cpp-implementation.sh

@@ -48,7 +48,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=Offline \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -58,7 +58,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=Offline \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -68,7 +68,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=SingleStream \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -78,7 +78,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=SingleStream \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -103,7 +103,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --model=resnet50 --implementation=cpp --device=cuda --backend=onnxruntime \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -113,7 +113,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=Offline \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -125,7 +125,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=SingleStream \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -135,7 +135,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=SingleStream \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -147,7 +147,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=MultiStream \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"
@@ -157,7 +157,7 @@ run "mlcr generate-run-cmds,inference,_submission \
 --scenario=MultiStream \
 --category=edge --division=$division  --quiet \
 --adr.compiler.tags=gcc \
---execution-mode=valid \
+--execution_mode=valid \
 --skip_submission_generation=yes \
 ${POWER} \
 --results_dir=$HOME/results_dir"

script/run-all-mlperf-models/run-nvidia-4090.sh

@@ -50,7 +50,7 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 
 
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
---model=$model --execution-mode=valid --implementation=$implementation --device=$device --backend=$backend --results_dir=$HOME/results_dir \
+--model=$model --execution_mode=valid --implementation=$implementation --device=$device --backend=$backend --results_dir=$HOME/results_dir \
 --category=$category --division=$division --skip_submission_generation=yes --quiet $power'
 
 #run_model "bert-99.9" "10" "${submission_cmd} --offline_target_qps=1680 --server_target_qps=1520"

script/run-all-mlperf-models/run-resnet50-macos.sh

@@ -43,28 +43,20 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--category=$category --division=$division  --quiet \
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 submission_cmd_scenario='mlcr generate-run-cmds,inference,_submission  --scenario=$scenario \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--category=$category --division=$division  --quiet  \
+--skip_submission_generation=yes --execution_mode=valid $power'
 
-readme_cmd_single='mlcr generate-run-cmds,inference,_populate-readme \
---model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
-
-readme_cmd='mlcr generate-run-cmds,inference,_populate-readme,_all-scenarios \
---model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
 
 # run "$MLC_RUN_CMD"
-run_test "onnxruntime" "6000" "reference" "cpu" "$find_performance_cmd --rerun"
-run_test "tf" "6000" "reference" "cpu" "$find_performance_cmd --rerun"
+#run_test "onnxruntime" "6000" "reference" "cpu" "$find_performance_cmd --rerun"
+#run_test "tf" "6000" "reference" "cpu" "$find_performance_cmd --rerun"
 
 run_test "onnxruntime" "100" "reference" "cpu" "$submission_cmd"
+
 run_test "tf" "100" "reference" "cpu" "$submission_cmd"
 

script/run-all-mlperf-models/run-resnet50.sh

@@ -44,22 +44,22 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 submission_cmd_scenario='mlcr generate-run-cmds,inference,_submission  --scenario=$scenario \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 readme_cmd_single='mlcr generate-run-cmds,inference,_populate-readme \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 readme_cmd='mlcr generate-run-cmds,inference,_populate-readme,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 # run "$MLC_RUN_CMD"
 run_test "onnxruntime" "200" "reference" "cpu" "$find_performance_cmd"

script/run-all-mlperf-models/run-retinanet-sh

@@ -43,23 +43,14 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--category=$category --division=$division  --quiet  \
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 submission_cmd_scenario='mlcr generate-run-cmds,inference,_submission  --scenario=$scenario \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--category=$category --division=$division  --quiet  \
+--skip_submission_generation=yes --execution_mode=valid $power'
 
-readme_cmd_single='mlcr generate-run-cmds,inference,_populate-readme \
---model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
-
-readme_cmd='mlcr generate-run-cmds,inference,_populate-readme,_all-scenarios \
---model=$model --implementation=$implementation --device=$device --backend=$backend \
---category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
 
 # run "$MLC_RUN_CMD"
 run_test "onnxruntime" "50" "reference" "cpu" "$find_performance_cmd"

script/run-all-mlperf-models/template.sh

@@ -47,20 +47,20 @@ find_performance_cmd='mlcr generate-run-cmds,inference,_find-performance \
 submission_cmd='mlcr generate-run-cmds,inference,_submission,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 submission_cmd_scenario='mlcr generate-run-cmds,inference,_submission  --scenario=$scenario \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 readme_cmd_single='mlcr generate-run-cmds,inference,_populate-readme \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'
 
 readme_cmd='mlcr generate-run-cmds,inference,_populate-readme,_all-scenarios \
 --model=$model --implementation=$implementation --device=$device --backend=$backend \
 --category=$category --division=$division  --quiet --results_dir=$HOME/results_dir \
---skip_submission_generation=yes --execution-mode=valid $power'
+--skip_submission_generation=yes --execution_mode=valid $power'