ci: split building, aggregating and uploading in separate jobs (#171)

* ci: split building, aggregating and uploading in separate jobs

This should also make it easier for external repositories to depend on these artifacts

* Use sha

* Use target platform for name

* Update generate-version-matrix

* Another reorder

* Rename jobs

* Update artifact name

* Update matrix script

* Aggregate per target

* Temporary simplification to test things

* Checkout repo

* Revert "Temporary simplification to test things"

This reverts commit 5dab5fc.

* Don't run on pull requests

Author: Hofer-Julian

.github/workflows/rattler-build.yml renamed to .github/workflows/build-upload.yml

@@ -8,11 +8,10 @@ on:
       - "pixi-build-rust-v[0-9]+.[0-9]+.[0-9]+"
     # Build all backends on main branch
     branches: [main]
-  pull_request:
   workflow_dispatch:
 
 
-name: "Build and publish backends"
+name: "Conda Packages"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}
@@ -23,36 +22,31 @@ permissions:
   contents: read
 
 jobs:
-  generate-version:
+  generate-matrix:
     runs-on: ubuntu-latest
     outputs:
-      version_matrix: ${{ steps.set_version.outputs.version_matrix }}
+      matrix: ${{ steps.set_version.outputs.matrix }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
-      - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1
       - uses: prefix-dev/setup-pixi@19eac09b398e3d0c747adc7921926a6d802df4da # v0.8.8
-        with:
-          environments: release
       - name: Extract versions
         id: set_version
         run: |
           # extract names and versions from cargo metadata
           # and generate a matrix entries for the build job
-          MATRIX_JSON=$(pixi run python scripts/generate_version_matrix.py)
+          MATRIX_JSON=$(pixi run generate-matrix)
 
           echo "Generated matrix: $MATRIX_JSON"
-          echo "version_matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
+          echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
 
-  rattler-build:
-    needs: generate-version
+  build:
+    needs: generate-matrix
     env:
       REPO_NAME: "prefix-dev/pixi-build-backends"
     strategy:
       matrix:
-        bins: ${{ fromJSON(needs.generate-version.outputs.version_matrix) }}
-
+        bins: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
       fail-fast: false
-
     runs-on: ${{ matrix.bins.os }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -71,24 +65,54 @@ jobs:
       - name: Build ${{ matrix.bins.bin }}
         shell: bash
         env:
-          TARGET_PLATFORM: ${{ matrix.bins.target }}
           RATTLER_BUILD_ENABLE_GITHUB_INTEGRATION: "true"
           RATTLER_BUILD_COLOR: "always"
         run: |
-          pixi run build-recipe-ci $RUNNER_TEMP recipe/${{ matrix.bins.bin }}.yaml ${{ env.TARGET_PLATFORM }}
-      - name: Upload OSX or Linux packages
+          pixi run build-recipe-ci $RUNNER_TEMP recipe/${{ matrix.bins.bin }}.yaml ${{ matrix.bins.target }}
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: conda-packages-${{ matrix.bins.bin }}-${{ matrix.bins.target }}
+          path: ${{ runner.temp }}/**/*.conda
+
+  aggregate:
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [linux-64, linux-aarch64, linux-ppc64le, win-64, osx-64, osx-arm64]
+    steps:
+      - name: Download conda package artifacts for ${{ matrix.target }}
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: conda-packages-*-${{ matrix.target }}
+          path: conda-artifacts-${{ matrix.target }}
+          merge-multiple: true
+      - name: Upload aggregated conda packages for ${{ matrix.target }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: conda-packages-${{ matrix.target }}
+          path: conda-artifacts-${{ matrix.target }}/**/*.conda
+
+  upload:
+    needs: aggregate
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags') && github.repository == 'prefix-dev/pixi-build-backends' }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - name: Download all conda packages
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: conda-packages-*
+          path: conda-packages
+          merge-multiple: true
+      - uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: build
+      - name: Upload packages
         shell: bash
-        if: ${{ startsWith(github.ref, 'refs/tags') && matrix.bins.os != 'windows-latest' && github.repository == env.REPO_NAME }}
         run: |
-          for file in "$RUNNER_TEMP"/**/*.conda; do
+          for file in conda-packages/*.conda; do
             echo "Uploading ${file}"
             pixi run -e build rattler-build upload prefix -c pixi-build-backends "$file"
           done
-      - name: Upload Windows packages
-        shell: pwsh
-        if: ${{ startsWith(github.ref, 'refs/tags') && matrix.bins.os == 'windows-latest' && github.repository == env.REPO_NAME }}
-        run: |
-          Get-ChildItem -Path $env:RUNNER_TEMP -Filter *.conda -Recurse | ForEach-Object {
-            Write-Host "Uploading $($_.FullName)"
-            pixi run -e build rattler-build upload prefix -c pixi-build-backends "$($_.FullName)"
-          }

.github/workflows/rust.yml renamed to .github/workflows/lint-test.yml

@@ -14,6 +14,7 @@ build-release = "cargo build --release"
 nextest = "cargo nextest run --workspace --all-targets"
 doctest = "cargo test --doc"
 test = [{ task = "nextest" }, { task = "doctest" }]
+generate-matrix = "python scripts/generate-matrix.py"
 
 
 install-pixi-build-python = { cmd = "cargo install --path crates/pixi-build-python --locked --force" }
@@ -27,6 +28,7 @@ install-pixi-backends = { depends-on = [
   "install-pixi-build-rust",
 ] }
 
+
 [dependencies]
 rust = ">=1.86.0,<1.87"
 python = ">=3.12.4,<4"
@@ -37,6 +39,7 @@ libzlib = ">=1.3.1,<2"
 liblzma = ">=5.8.1,<6"
 cargo-nextest = ">=0.9.96,<0.10"
 
+
 [target.linux-64.dependencies]
 clang = ">=18.1.8,<19.0"
 mold = ">=2.33.0,<3.0"
@@ -66,8 +69,6 @@ depends-on = [
 ]
 args = ["recipe", "target_platform"]
 
-[feature.release.dependencies]
-python = ">=3.12.4,<4"
 
 [feature.schema.tasks]
 generate-schema = "python schema/model.py > schema/schema.json"
@@ -98,4 +99,3 @@ default = { solve-group = "default" }
 lint = { features = ["lint"], solve-group = "default" }
 schema = { features = ["schema"], no-default-feature = true }
 build = { features = ["build"], no-default-feature = true }
-release = { features = ["release"], no-default-feature = false }

pixi.toml

@@ -85,11 +85,9 @@ def generate_matrix():
                 matrix.append(
                     {
                         "bin": package["name"],
-                        "version": package["version"],
-                        "env_name": package["name"].replace("-", "_").upper()
-                        + "_VERSION",
-                        "crate_name": package["name"],
                         "target": target["target"],
+                        "version": package["version"],
+                        "env_name": f"{package["name"].replace("-", "_").upper()}_VERSION",
                         "os": target["os"],
                     }
                 )