Fix docker working with MLC (#143)

* Fixes for docker deps

* Fix update_meta_if_env working for docker

* Added backup url for R50 model

* Update test-mlperf-inference-resnet50.yml

* Fix README text in script module

Author: arjunsuresh

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -1,7 +1,7 @@
 name: MLPerf inference ResNet50
 
 on:
-  pull_request:
+  pull_request_target:
     branches: [ "main", "dev" ]
     paths:
       - '.github/workflows/test-mlperf-inference-resnet50.yml'
@@ -72,14 +72,8 @@ jobs:
         git config --global user.email "mlcommons-bot@users.noreply.github.com"
         git clone -b auto-update https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 tmp
         cd tmp
-        RE=$PAT
-        echo $RE
-        echo $PAT
-        lop=$RE
-        echo $lop
-        echo ${{ env.PAT }}
         echo "hello" >> README.md
         git commit -a -m "Test commit"
-        git push https://${PAT}@github.com/mlcommons/mlperf_inference_test_submissions_v5.0 HEAD:${{ github.ref_name }}
+        git push https://git:${PAT}@github.com/mlcommons/mlperf_inference_test_submissions_v5.0 HEAD:${{ github.ref_name }}
 
         # mlcr --tags=push,github,mlperf,inference,submission --env.MLC_GITHUB_PAT=pat --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

automation/script/docker.py

@@ -31,11 +31,11 @@ def dockerfile(self_module, input_params):
         return {'return': 1, 'error': 'No scripts were found'}
 
     # Step 3: Process Dockerfile-related configurations
-    environment_vars = input_params.get('env', {})
+    env = input_params.get('env', {})
     state_data = input_params.get('state', {})
     constant_vars = input_params.get('const', {})
     constant_state = input_params.get('const_state', {})
-    dockerfile_environment_vars = input_params.get('dockerfile_env', {})
+    dockerfile_env = input_params.get('dockerfile_env', {})
     tag_values = input_params.get('tags', '').split(",")
     variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")]
 
@@ -55,13 +55,13 @@ def dockerfile(self_module, input_params):
             'script_variation_tags': variation_tags
         }
         docker_settings = metadata.get('docker', {})
-        docker_settings['dockerfile_env'] = dockerfile_environment_vars
+        docker_settings['dockerfile_env'] = dockerfile_env
         state_data['docker'] = docker_settings
         add_deps_recursive = input_params.get('add_deps_recursive', {})
 
         # Update state with metadata and variations
         update_state_result = self_module.update_state_from_meta(
-            metadata, environment_vars, state_data, constant_vars, constant_state,
+            metadata, env, state_data, constant_vars, constant_state,
             deps=[],
             post_deps=[],
             prehook_deps=[],
@@ -77,7 +77,7 @@ def dockerfile(self_module, input_params):
         update_variations_result = self_module._update_state_from_variations(
             input_params, metadata, variation_tags, metadata.get(
                 'variations', {}),
-            environment_vars, state_data, constant_vars, constant_state,
+            env, state_data, constant_vars, constant_state,
             deps=[],  # Add your dependencies if needed
             post_deps=[],  # Add post dependencies if needed
             prehook_deps=[],  # Add prehook dependencies if needed
@@ -93,24 +93,39 @@ def dockerfile(self_module, input_params):
             return update_variations_result
 
         # Set Docker-specific configurations
-        docker_settings = state_data.get('docker', {})
-        docker_settings['dockerfile_env'] = dockerfile_environment_vars
-        dockerfile_environment_vars['MLC_RUN_STATE_DOCKER'] = True
+        docker_settings = state_data['docker']
+        docker_settings['dockerfile_env'] = dockerfile_env
+        dockerfile_env['MLC_RUN_STATE_DOCKER'] = True
 
         if not docker_settings.get('run', True) and not input_params.get(
                 'docker_run_override', False):
             logging.info("Docker 'run' is set to False in meta.json")
             continue
 
         # Handle build dependencies
-        build_dependencies = docker_settings.get('build_deps', [])
-        if build_dependencies:
-            deps_result = self_module._run_deps(
-                build_dependencies, run_state=run_state, verbose=input_params.get(
-                    'v', False)
-            )
-            if deps_result['return'] > 0:
-                return deps_result
+        show_time = input_params.get('show_time', False)
+        deps = docker_settings.get('build_deps', [])
+        if deps:
+            r = self_module._run_deps(
+                deps, [], env, {}, {}, {}, {}, '', [], '', False, '', verbose,
+                show_time, ' ', run_state)
+            if r['return'] > 0:
+                return r
+
+        update_state_result = self_module.update_state_from_meta(
+            metadata, env, state_data, constant_vars, constant_state,
+            deps=[],
+            post_deps=[],
+            prehook_deps=[],
+            posthook_deps=[],
+            new_env_keys=[],
+            new_state_keys=[],
+            run_state=run_state,
+            i=input_params
+        )
+        if update_state_result['return'] > 0:
+            return update_state_result
+        docker_settings = state_data['docker']
 
         # Prune temporary environment variables
         run_command = copy.deepcopy(run_command_arc)
@@ -154,15 +169,15 @@ def dockerfile(self_module, input_params):
 
         # Push Docker image if specified
         if input_params.get('docker_push_image') in [True, 'True', 'yes']:
-            environment_vars['MLC_DOCKER_PUSH_IMAGE'] = 'yes'
+            env['MLC_DOCKER_PUSH_IMAGE'] = 'yes'
 
         # Generate Dockerfile
         mlc_docker_input = {
             'action': 'run', 'automation': 'script', 'tags': 'build,dockerfile',
             'fake_run_option': " " if docker_inputs.get('real_run') else " --fake_run",
             'comments': comments, 'run_cmd': f"{run_command_string} --quiet",
-            'script_tags': input_params.get('tags'), 'env': environment_vars,
-            'dockerfile_env': dockerfile_environment_vars,
+            'script_tags': input_params.get('tags'), 'env': env,
+            'dockerfile_env': dockerfile_env,
             'quiet': True, 'v': input_params.get('v', False), 'real_run': True
         }
         mlc_docker_input.update(docker_inputs)

automation/script/module.py

@@ -3797,10 +3797,6 @@ def _get_readme(self, cmd_parts, run_state):
 pip install mlcflow
 ```
 
-Check [this readme](https://github.com/mlcommons/ck/blob/master/docs/installation.md)
-with more details about installing MLC and dependencies across different platforms
-(Ubuntu, MacOS, Windows, RHEL, ...).
-
 """
 
         current_mlc_repo = run_state['script_repo_alias']

script/app-mlperf-inference/meta.yaml

@@ -351,7 +351,8 @@ variations:
           MLC_HOST_PLATFORM_FLAVOR:
             - x86_64
         docker:
-          base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.1-cuda12.4-pytorch24.04-ubuntu22.04-aarch64-GraceHopper-release
+          base_image_off: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.1-cuda12.4-pytorch24.04-ubuntu22.04-aarch64-GraceHopper-release
+          base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.0-cuda12.2-cudnn8.9-aarch64-ubuntu22.04-public
         env:
             MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp310-cp310-linux_aarch64.whl'
 

script/get-ml-model-resnet50/meta.yaml

@@ -135,7 +135,8 @@ variations:
   onnx,opset-8:
     env:
       MLC_DOWNLOAD_CHECKSUM: a638cf028b5870da29e09ccc2f7182e7
-      MLC_PACKAGE_URL: https://zenodo.org/record/2592612/files/resnet50_v1.onnx
+      MLC_PACKAGE_URL: https://armi.in/files/resnet50_v1.onnx
+      MLC_PACKAGE_URL1: https://zenodo.org/record/2592612/files/resnet50_v1.onnx
   onnxruntime:
     alias: onnx
   opset-11: