diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 005f41d88..85039c7d5 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -11,7 +11,6 @@ on: - VERSION - setup.py - jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index d117afa8d..a4ec7d3d0 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -63,10 +63,10 @@ jobs: - name: Run docker container from dockerhub on linux if: runner.os == 'linux' run: | - mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --image_name=cm-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=cknowledge --quiet + mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=cm-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=cknowledge --quiet - name: Run docker container locally on linux if: runner.os == 'linux' run: | - mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet + mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index 67775b14f..026db0b14 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -1,5 +1,3 @@ -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: MLPerf Automotive POC Test on: @@ -20,7 +18,7 @@ jobs: python-version: [ "3.8", "3.12" ] backend: [ "pytorch" ] implementation: [ "python" ] - docker: [ "", " --docker --docker_os=ubuntu --docker_os_version=22.04 --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt=yes" ] + docker: [ "", " --docker --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt" ] extra-args: [ "--adr.compiler.tags=gcc", "--env.MLC_MLPERF_LOADGEN_BUILD_FROM_SRC=off" ] exclude: - os: ubuntu-24.04 @@ -30,16 +28,16 @@ jobs: - os: windows-latest extra-args: "--adr.compiler.tags=gcc" - os: windows-latest - docker: " --docker --docker_it=no --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt=yes" + docker: " --docker --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt" # windows docker image is not supported in CM yet - os: macos-latest python-version: "3.8" - os: macos-13 python-version: "3.8" - os: macos-latest - docker: " --docker --docker_it=no --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt=yes" + docker: " --docker --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt" - os: macos-13 - docker: " --docker --docker_it=no --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt=yes" + docker: " --docker --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --docker_dt" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-mlperf-inference-rgat.yml b/.github/workflows/test-mlperf-inference-rgat.yml index dc2a7eb5c..f745cb4e0 100644 --- a/.github/workflows/test-mlperf-inference-rgat.yml +++ b/.github/workflows/test-mlperf-inference-rgat.yml @@ -35,6 +35,7 @@ jobs: - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }} run: | mlcr --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet -v --target_qps=1 + - name: Push Results if: github.repository_owner == 'gateoverflow' env: diff --git a/automation/script/docker.py b/automation/script/docker.py index f77462fa9..fb8dc526a 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -200,7 +200,7 @@ def docker_run(self_module, i): regenerate_docker_file = not i.get('docker_noregenerate', False) recreate_docker_image = i.get('docker_recreate', False) - if i.get('docker_skip_build', False): + if is_true(i.get('docker_skip_build', False)): regenerate_docker_file = False recreate_docker_image = False env['MLC_DOCKER_SKIP_BUILD'] = 'yes' @@ -209,7 +209,6 @@ def docker_run(self_module, i): r = prune_input({'input': i, 'extra_keys_starts_with': ['docker_']}) f_run_cmd = r['new_input'] - print(f"regenerate_docker_file = {regenerate_docker_file}") # Regenerate Dockerfile if required if regenerate_docker_file: r = dockerfile(self_module, i) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 2b5b07b27..2d70a68f4 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -72,9 +72,16 @@ def prepare_docker_inputs(input_params, docker_settings, if (value := input_params.get(f"docker_{key}", docker_settings.get(key, get_docker_default(key)))) is not None } - if docker_inputs.get('detached', docker_inputs.get('dt')): + if is_true(docker_inputs.get('detached', docker_inputs.get('dt', ''))): docker_inputs['interactive'] = False docker_inputs['detached'] = True + elif is_true(docker_inputs.get('interactive', docker_inputs.get('it', ''))): + docker_inputs['interactive'] = True + docker_inputs['detached'] = False + + for key in ["dt", "it"]: + if docker_inputs.get(key): + del (docker_inputs[key]) # Determine Dockerfile suffix and path docker_base_image = docker_inputs.get('base_image') @@ -268,9 +275,7 @@ def get_docker_default(key): "skip_run_cmd": False, "pre_run_cmds": [], "run_cmd_prefix": '', - "port_maps": [], - "detached": False, - "interactive": True + "port_maps": [] } if key in defaults: return defaults[key] diff --git a/automation/utils.py b/automation/utils.py index 61cc08e28..fd024061d 100644 --- a/automation/utils.py +++ b/automation/utils.py @@ -1,6 +1,14 @@ from mlc.utils import * +def is_true(val): + return str(val).lower() in ["1", "true", "on", "yes"] + + +def is_false(val): + return str(val).lower() in ["0", "false", "off", "no"] + + def get_host_os_info(i={}): """ Get some host platform name (currently windows or linux) and OS bits diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index be0ce6333..896008c39 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -100,7 +100,7 @@ docker: use_host_user_id: True real_run: false interactive: True - mlc_repos: 'mlc pull repo mlcommons@cm4abtf --branch=poc' + mlc_repos_off: 'mlc pull repo mlcommons@cm4abtf --branch=poc' deps: - tags: get,abtf,scratch,space mounts: diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 491c86976..fe680c3b1 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -264,7 +264,7 @@ def preprocess(i): DOCKER_GROUP = "-g $GID -o" user_shell = json.loads(shell) - f.write('RUN useradd ' + DOCKER_USER_ID + DOCKER_GROUP + ' --create-home --shell ' + user_shell[0] + ' ' + f.write(f"""RUN id -u {docker_user} > /dev/null 2>&1 || useradd """ + DOCKER_USER_ID + DOCKER_GROUP + ' --create-home --shell ' + user_shell[0] + ' ' + docker_user + EOL) f.write( 'RUN echo "' + @@ -272,7 +272,7 @@ def preprocess(i): ' ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers' + EOL) f.write('USER ' + docker_user + ":" + docker_group + EOL) - f.write('ENV HOME=/home/mlcuser' + EOL) + f.write(f"""ENV HOME=/home/{docker_user}""" + EOL) else: f.write('ENV HOME=/root' + EOL) @@ -284,7 +284,7 @@ def preprocess(i): docker_env_key + "=" + str(dockerfile_env[docker_env_key]) workdir = get_value(env, config, 'WORKDIR', 'MLC_DOCKER_WORKDIR') - if workdir and ("/home/mlcuser" not in workdir or str(env.get('MLC_DOCKER_USE_DEFAULT_USER', '')).lower() not in [ + if workdir and (f"""/home/{docker_user}""" not in workdir or str(env.get('MLC_DOCKER_USE_DEFAULT_USER', '')).lower() not in [ "yes", "1", "true"]): f.write('WORKDIR ' + workdir + EOL) diff --git a/script/build-dockerfile/dockerinfo.json b/script/build-dockerfile/dockerinfo.json index 05d712418..a6ff5056c 100644 --- a/script/build-dockerfile/dockerinfo.json +++ b/script/build-dockerfile/dockerinfo.json @@ -26,10 +26,11 @@ "WORKDIR": "/home/mlcuser", "distros": { "ubuntu": { + "USER": "ubuntu", "package-manager-update-cmd": "apt-get update -y", "package-manager-get-cmd": "apt-get install -y", "packages": [ - "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl" + "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl", "unzip" ], "versions": { "18.04": { diff --git a/script/build-dockerfile/meta.yaml b/script/build-dockerfile/meta.yaml index e6237fb6b..0c85f1606 100644 --- a/script/build-dockerfile/meta.yaml +++ b/script/build-dockerfile/meta.yaml @@ -19,7 +19,7 @@ default_env: ' MLC_DOCKER_OS: ubuntu MLC_DOCKER_NOT_PULL_UPDATE: False - MLC_MLOPS_REPO_BRANCH: mlc + MLC_MLOPS_REPO_BRANCH: dev input_mapping: build: MLC_BUILD_DOCKER_IMAGE diff --git a/script/get-sys-utils-cm/run-ubuntu.sh b/script/get-sys-utils-cm/run-ubuntu.sh index d6fb9f7ed..334515507 100644 --- a/script/get-sys-utils-cm/run-ubuntu.sh +++ b/script/get-sys-utils-cm/run-ubuntu.sh @@ -14,7 +14,7 @@ fi MLC_APT_TOOL=${MLC_APT_TOOL:-apt-get} ${MLC_SUDO} ${MLC_APT_TOOL} update && \ - ${MLC_SUDO} DEBIAN_FRONTEND=noninteractive ${MLC_APT_TOOL} install -y --no-install-recommends \ + ${MLC_SUDO} DEBIAN_FRONTEND=noninteractive ${MLC_APT_TOOL} install -y --no-install-recommends --ignore-missing \ apt-utils \ git \ wget \