add support to produce domain-based metrics/plots #79
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| on: | |
| pull_request: | |
| branches: [main, nwm-main, development, release-candidate] | |
| push: | |
| branches: [main, nwm-main, development, release-candidate] | |
| release: | |
| types: [published] | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| env: | |
| REGISTRY: ghcr.io | |
| PYTHON_VERSION: '3.10.14' | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_base: ${{ steps.vars.outputs.image_base }} | |
| pr_tag: ${{ steps.vars.outputs.pr_tag }} | |
| commit_sha: ${{ steps.vars.outputs.commit_sha }} | |
| commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} | |
| test_image_tag: ${{ steps.vars.outputs.test_image_tag }} | |
| steps: | |
| - name: Compute image vars | |
| id: vars | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" | |
| REPO="$(basename "${GITHUB_REPOSITORY}")" | |
| IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" | |
| echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" | |
| if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then | |
| PR_NUM="${{ github.event.pull_request.number }}" | |
| PR_TAG="pr-${PR_NUM}-build" | |
| echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" | |
| echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" = "push" ]; then | |
| COMMIT_SHA="${GITHUB_SHA}" | |
| SHORT_SHA="${COMMIT_SHA:0:12}" | |
| echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| fi | |
| build: | |
| name: build | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log in to registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| build-args: | | |
| NWM_EVAL_MGR_TAG=${{ env.NWM_EVAL_MGR_TAG || 'development' }} | |
| unit-test: | |
| name: unit-test | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build] | |
| container: | |
| image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| steps: | |
| - name: run unit tests (with coverage xml) | |
| run: | | |
| echo "Add unit tests here..." | |
| codeql-scan: | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build] | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v3 | |
| with: | |
| languages: python | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| # Install the package (and dev extras if defined) since there is no requirements.txt | |
| if python -c "import setuptools, pkgutil; exit(0)" 2>/dev/null; then | |
| # Try with develop extras first (provides pytest, etc.) | |
| pip install '.[develop]' || pip install . | |
| else | |
| pip install . | |
| fi | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v3 | |
| container-scanning: | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build] | |
| steps: | |
| - name: Scan container with Trivy | |
| uses: aquasecurity/trivy-action@0.20.0 | |
| with: | |
| image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} | |
| format: 'template' | |
| template: '@/contrib/sarif.tpl' | |
| output: 'trivy-results.sarif' | |
| severity: 'CRITICAL,HIGH' | |
| - name: Upload Trivy SARIF | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| deploy-latest-on-development: | |
| name: deploy-latest-on-development | |
| if: github.event_name == 'push' && github.ref_name == 'development' | |
| runs-on: ubuntu-latest | |
| needs: [setup, build, unit-test, codeql-scan, container-scanning] | |
| steps: | |
| - name: Tag image with 'latest' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" | |
| SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}" | |
| # ensure skopeo is available | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest" | |
| release: | |
| name: release | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Get commit sha for the tag | |
| id: rev | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| TAG="${{ github.event.release.tag_name }}" | |
| REPO="${{ github.repository }}" | |
| # ensure jq is available | |
| if ! command -v jq >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends jq | |
| fi | |
| # ensure gh cli is available | |
| if ! command -v gh >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends gh | |
| fi | |
| REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")" | |
| OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")" | |
| OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")" | |
| if [ "$OBJ_TYPE" = "tag" ]; then | |
| TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")" | |
| COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")" | |
| else | |
| COMMIT_SHA="$OBJ_SHA" | |
| fi | |
| SHORT_SHA="${COMMIT_SHA:0:12}" | |
| echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" | |
| - name: Tag image with release tag | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| IMAGE_BASE="${{ needs.setup.outputs.image_base }}" | |
| SHORT_SHA="${{ steps.rev.outputs.short_sha }}" | |
| RELEASE_TAG="${{ github.event.release.tag_name }}" | |
| # ensure skopeo is available | |
| if ! command -v skopeo >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y --no-install-recommends skopeo | |
| fi | |
| skopeo copy \ | |
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | |
| docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}" |