From 226df0de5223ad0d334b8a42169f7700926a253f Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Jun 2025 13:22:02 +0530 Subject: [PATCH 1/2] add test for automotive submission generation --- ...lperf-automotive-submission-generation.yml | 85 +++++++++++++++++++ ...mlperf-inference-submission-generation.yml | 8 +- 2 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-mlperf-automotive-submission-generation.yml diff --git a/.github/workflows/test-mlperf-automotive-submission-generation.yml b/.github/workflows/test-mlperf-automotive-submission-generation.yml new file mode 100644 index 000000000..64e2257e8 --- /dev/null +++ b/.github/workflows/test-mlperf-automotive-submission-generation.yml @@ -0,0 +1,85 @@ +# This workflow will test the MLPerf Automotive Submission Generation using MLPerf Automation + +name: MLC based MLPerf Automotive Submission Generation + + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-automotive-submission-generation.yml' + - '**' + - '!**.md' + workflow_call: + inputs: + ref: + required: true + type: string + repo-url: + required: true + type: string + +jobs: + submission_generation: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + division: ["closed", "open", "closed-open"] + category: ["edge"] + round: ["0.5"] + case: ["closed"] + action: ["run", "docker"] + exclude: + - os: macos-latest + - os: windows-latest + - case: "closed-edge" + - division: "closed-open" + - division: "closed" + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install mlcflow + run: | + pip install mlcflow + pip install tabulate + - name: Pull MLOps repository + run: | + if [ -z "${{ inputs.repo-url }}" ]; then + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + else + mlc pull repo mlcommons@mlperf-automations --branch=dev + fi + - name: Pull repo where test cases are uploaded + run: | + git clone -b submission-generation-examples https://github.com/mlcommons/mlperf_automotive.git submission_generation_examples + - name: Run Submission Generation - round-${{ matrix.round }}${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} + run: | + if [ "${{ matrix.case }}" == "closed" ]; then + description="Test submission - contains closed edge" + elif [ "${{ matrix.case }}" == "closed-power" ]; then + description="Test submission - contains closed-power edge results" + fi + # Dynamically set the log group to simulate a dynamic step name + echo "::group::$description" + + docker_tags="" + if [ "${{ matrix.action }}" == "docker" ] && [ -z "${{ inputs.repo-url }}" ]; then + docker_tags="--docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }}" + fi + + if [ -n "${{ inputs.repo-url }}" ]; then + mlc ${{ matrix.action }} script --tags=generate,inference,submission,_automotive --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/r${{ matrix.round }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker,_automotive --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + else + mlc ${{ matrix.action }} script --tags=generate,inference,submission,_automotive --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/r${{ matrix.round }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker,_automotive --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags + fi + + echo "::endgroup::" \ No newline at end of file diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 1b25e3b8c..001e86acf 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -104,11 +104,11 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.case }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --skip_calibration_check=yes $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --skip_calibration_check=yes $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.case }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.case }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --skip_calibration_check=yes $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --skip_calibration_check=yes $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.case }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi echo "::endgroup::" \ No newline at end of file From 70a53458474faf065bf3de891e0be0460735ac89 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 9 Jun 2025 13:25:22 +0530 Subject: [PATCH 2/2] Update wg tag --- .../test-mlperf-automotive-submission-generation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-automotive-submission-generation.yml b/.github/workflows/test-mlperf-automotive-submission-generation.yml index 64e2257e8..d658ff55c 100644 --- a/.github/workflows/test-mlperf-automotive-submission-generation.yml +++ b/.github/workflows/test-mlperf-automotive-submission-generation.yml @@ -75,11 +75,11 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission,_automotive --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/r${{ matrix.round }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker,_automotive --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission,_wg-automotive --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/r${{ matrix.round }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker,_wg-automotive --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission,_automotive --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/r${{ matrix.round }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker,_automotive --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission,_wg-automotive --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/r${{ matrix.round }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker,_wg-automotive --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi echo "::endgroup::" \ No newline at end of file