Fixes for automatic mlperf inference submission upload (#186)

* Fix automatic submission via CLI (thanks to Pablo)

Author: arjunsuresh

script/generate-docs-for-all-scripts.cmd

@@ -96,8 +96,6 @@ def preprocess(i):
 def postprocess(i):
 
     env = i['env']
-    if env.get('MLC_TAR_SUBMISSION_DIR', ''):
-        env['MLC_TAR_INPUT_DIR'] = env['MLC_MLPERF_INFERENCE_SUBMISSION_DIR']
 
     x = env.get('MLPERF_INFERENCE_SUBMISSION_TAR_FILE', '')
     if x != '':

script/run-mlperf-inference-submission-checker/customize.py

@@ -72,16 +72,20 @@ post_deps:
   names:
   - push-to-github
   tags: publish-results,github
-- enable_if_env:
+- enable_if_any_env:
     MLC_TAR_SUBMISSION_DIR:
     - 'yes'
+    MLC_MLPERF_SUBMITTER_ID:
+    - 'yes'
   tags: run,tar
+  env:
+    MLC_TAR_INPUT_DIR: <<<MLC_MLPERF_INFERENCE_SUBMISSION_DIR>>>
 - enable_if_env:
-    MLC_SUBMITTER_ID:
+    MLC_MLPERF_SUBMITTER_ID:
     - 'yes'
   tags: submit,mlperf,results,_inference
   env:
-    MLC_MLPERF_SUBMISSION_FILE: <<<MLPERF_INFERENCE_SUBMISSION_TAR_FILE>>>
+    MLC_MLPERF_SUBMISSION_FILE: <<<MLC_TAR_OUTFILE>>>
 tags:
 - run
 - mlc

script/run-mlperf-inference-submission-checker/meta.yaml

@@ -15,8 +15,14 @@ def preprocess(i):
     benchmark = env['MLC_MLPERF_BENCHMARK']
     submitter_id = env['MLC_MLPERF_SUBMITTER_ID']
     file_path = env['MLC_MLPERF_SUBMISSION_FILE']
-
-    r = get_signed_url(server, benchmark, submitter_id, file_path)
+    submitter_name = env.get('MLC_MLPERF_SUBMITTER', '')
+
+    r = get_signed_url(
+        server,
+        benchmark,
+        submitter_id,
+        submitter_name,
+        file_path)
     if r['return'] > 0:
         return r
 
@@ -37,7 +43,7 @@ def preprocess(i):
     return {'return': 0}
 
 
-def get_signed_url(server, benchmark, submitter_id, file_path):
+def get_signed_url(server, benchmark, submitter_id, submitter_name, file_path):
     # Define the URL
     url = f"{server}/index/url"
 
@@ -72,15 +78,19 @@ def get_signed_url(server, benchmark, submitter_id, file_path):
                 "error": f"An error occurred in connecting to the server: {e}"}
 
     response_json = response.json()
-    # print(response_json)
     # response = json.loads(response_json)
     try:
         signed_url = response_json['signed_url']
         submission_id = response_json['submission_id']
+        submitter_name_from_id = response_json['submitter_name']
     except Exception as e:
         return {
             "return": 1, "error": f"An error occurred while processing the response: {e}"}
 
+    if submitter_name != '':
+        if submitter_name != submitter_name_from_id:
+            return {'return': 1, 'error': f"""Submitter name mismatch, given = {submitter_name}, name mapped to ID = {submitter_name_from_id}"""}
+
     return {'return': 0, 'signed_url': signed_url,
             'submission_id': submission_id}
 

script/submit-mlperf-results/customize.py

@@ -18,6 +18,7 @@ def preprocess(i):
     input_dirname = os.path.basename(input_dir)
     if output_file == "":
         output_file = input_dirname + ".tar.gz"
+    env['MLC_TAR_OUTFILE'] = output_file
     from pathlib import Path
     input_path = Path(input_dir)
     cd_dir = input_path.parent.absolute()

script/tar-my-folder/customize.py

@@ -9,6 +9,8 @@ input_mapping:
   input_dir: MLC_TAR_INPUT_DIR
   outfile: MLC_TAR_OUTFILE
   output_dir: MLC_TAR_OUTPUT_DIR
+new_env_keys:
+  - MLC_TAR_OUTFILE
 tags:
 - run
 - tar