Add dev package support to build_packages.yml. (#667)

Progress on #584. ~~Depends on
#666 (the first commit).~~

This is refactors the `build_packages.yml` workflow so it can be used
via `workflow_call` as part of a "pkgci" setup, as an alternative to
creating a new `pkgci_build_packages.yml` workflow as originally
proposed in #589. This lets us
reuse the same workflow for building stable, nightly, and dev packages,
all across the same matrix of Python versions and operating systems.
Package builds take about 2 minutes (wall time) across the full matrix,
so we might as well build them all, instead of artificially constraining
ourselves to a subset like only Linux on Python 3.11.

Triggers for the workflow are now this:

Trigger | Scenario | Build type(s)
-- | -- | --
`schedule` | Nightly pre-release build | `rc`
`workflow_dispatch` | Workflow testing, manual releasing | `rc` default,
`stable` and `dev` possible
`workflow_call` | Pull request or push "pkgci" dev builds | `dev`
default, `stable` and `rc` possible

With this workflow behavior:

Build type | Version suffix | Cache enabled? | Tracing enabled? | Pushes
to release?
-- | -- | -- | -- | --
`stable` | None | No | Yes | No
`rc` | `rcYYYYMMDD` | No | Yes | Yes
`dev` | `.dev0+${{ github.sha }}` | Yes | No | No

Tested over at
https://github.com/ScottTodd/shark-ai/actions/workflows/build_packages.yml.
Example run:
https://github.com/ScottTodd/shark-ai/actions/runs/12245900071 (warm
cache)

Author: ScottTodd

.github/workflows/build_packages.yml

@@ -7,7 +7,21 @@
 name: Build packages
 
 on:
+  # Trigger from another workflow (typically to build dev packages and then test them)
+  workflow_call:
+    inputs:
+      build_type:
+        description: The type of build version to produce ("stable", "rc", or "dev")
+        type: string
+        default: "dev"
+  # Trigger manually (typically to test the workflow or manually build a release [candidate])
   workflow_dispatch:
+    inputs:
+      build_type:
+        description: The type of build version to produce ("stable", "rc", or "dev")
+        type: string
+        default: "rc"
+  # Trigger on a schedule to build nightly release candidates.
   schedule:
     # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8)
     - cron: '0 11 * * *'
@@ -16,14 +30,12 @@ permissions:
   contents: read
 
 jobs:
-  # Note: metadata generation could happen in a separate trigger/schedule
-  # workflow. For cross platform builds, it's useful to just generate the
-  # metadata on Linux and pass that to later jobs using artifacts.
+  # Generate metadata on Linux and pass to later jobs.
   setup_metadata:
     if: ${{ github.repository_owner == 'nod-ai' || github.event_name != 'schedule' }}
     runs-on: ubuntu-24.04
     outputs:
-      version_suffix: ${{ steps.version_rc.outputs.version_suffix }}
+      version_suffix: ${{ steps.version_local.outputs.version_suffix }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Setup Python
@@ -34,18 +46,35 @@ jobs:
       - name: Install Python packages
         run: pip install packaging
 
-      - name: Generate release candidate versions
-        id: version_rc
+      # Compute version suffix based on inputs (default to 'rc')
+      - name: Compute stable version suffix
+        if: ${{ inputs.build_type == 'stable' }}
+        run: |
+          version_suffix=""
+          echo "version_suffix=${version_suffix}" >> $GITHUB_ENV
+      - name: Compute rc version suffix
+        if: ${{ inputs.build_type == 'rc' || inputs.build_type == '' }}
         run: |
           version_suffix="$(printf 'rc%(%Y%m%d)T')"
+          echo "version_suffix=${version_suffix}" >> $GITHUB_ENV
+      - name: Compute dev version suffix
+        if: ${{ inputs.build_type == 'dev' }}
+        run: |
+          version_suffix=".dev0+${{ github.sha }}"
+          echo "version_suffix=${version_suffix}" >> $GITHUB_ENV
+
+      - name: Write version local files
+        id: version_local
+        run: |
           echo "version_suffix=${version_suffix}" >> $GITHUB_OUTPUT
           python3 build_tools/python_deploy/compute_local_version.py --version-suffix=${version_suffix} sharktank
           python3 build_tools/python_deploy/compute_local_version.py --version-suffix=${version_suffix} shortfin
           python3 build_tools/python_deploy/compute_common_version.py -rc --version-suffix=${version_suffix} --write-json
-      - name: Upload version_local.json
+
+      - name: Upload version_local.json files
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
-          name: version_local
+          name: version_local_files
           path: |
             sharktank/version_local.json
             shortfin/version_local.json
@@ -57,6 +86,8 @@ jobs:
     permissions:
       contents: write
     needs: [setup_metadata]
+    env:
+      OUTPUT_DIR: "${{ github.workspace }}/bindist"
     strategy:
       fail-fast: false
       matrix:
@@ -101,47 +132,59 @@ jobs:
         path: "c" # Windows can hit path length limits, so use a short path.
         submodules: false
 
-    - name: Download version_local.json
+    - name: Download version_local.json files
       uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
       with:
-        name: version_local
+        name: version_local_files
         path: ./c/
         merge-multiple: true
 
+    # Dev builds can use a cache and disable tracing to halve the build time.
+    # Other build use no cache for supply chain security and keep tracing enabled.
+    - name: Apply dev settings
+      if: ${{ inputs.build_type == 'dev' }}
+      run: |
+        echo "CACHE_DIR=${{ github.workspace }}/.shark-ai-cache" >> $GITHUB_ENV
+        echo "SHORTFIN_ENABLE_TRACING=OFF" >> $GITHUB_ENV
+    - name: Setup cache
+      if: ${{ inputs.build_type == 'dev' }}
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+      with:
+        path: ${{ env.CACHE_DIR }}
+        key: build-packages-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}-v1-${{ github.sha }}
+        restore-keys: |
+          build-packages-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}-v1-
+
+    # Build packages.
     - name: Build shark-ai (Linux x86_64)
       if: "matrix.package == 'shark-ai' && matrix.platform == 'linux-x86_64'"
-      env:
-        OUTPUT_DIR: "${{ github.workspace }}/bindist"
       run: |
         [ -e ./bindist/* ] && rm ./bindist/*
         ./c/build_tools/python_deploy/write_requirements.py --version-suffix=${{ needs.setup_metadata.outputs.version_suffix }}
         ./c/shark-ai/build_tools/build_linux_package.sh
-
     - name: Build sharktank (Linux x86_64)
       if: "matrix.package == 'sharktank' && matrix.platform == 'linux-x86_64'"
-      env:
-        OUTPUT_DIR: "${{ github.workspace }}/bindist"
       run: |
         [ -e ./bindist/* ] && rm ./bindist/*
         ./c/sharktank/build_tools/build_linux_package.sh
-
     - name: Build shortfin (Linux x86_64, ${{ matrix.python-version }})
       if: "matrix.package == 'shortfin' && matrix.platform == 'linux-x86_64'"
       env:
-        OUTPUT_DIR: "${{ github.workspace }}/bindist"
         OVERRIDE_PYTHON_VERSIONS: "${{ matrix.python-version }}"
       run: |
         [ -e ./bindist/* ] && rm ./bindist/*
         ./c/shortfin/build_tools/build_linux_package.sh
 
+    # Always upload to GitHub artifacts.
     - name: Upload python wheels
       uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       with:
         if-no-files-found: error
         name: snapshot-${{ matrix.package }}-${{ matrix.platform }}-${{ matrix.python-version }}
         path: bindist
-
-    - name: Release python wheels
+    # Upload release candidate versions to a 'dev-wheels' GitHub release.
+    - name: Release rc python wheels
+      if: ${{ inputs.build_type == 'rc' || inputs.build_type == '' }}
       uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0
       with:
         artifacts: bindist/*.whl