[DO NOT MERGE] Optimize CI pipeline with artifact sharing and enhanced caching

.github/workflows/ci-coordinator.yaml

@@ -0,0 +1,95 @@
+name: CI Coordinator
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, reopened, synchronize, labeled]
+  workflow_dispatch:
+  schedule:
+    # Three times a day for integration tests
+    - cron: '33 3,10,15 * * *'
+
+permissions:
+  contents: read
+
+env:
+  RUST_CHANNEL: '1.89.0'
+
+jobs:
+  # Matrix-based Rust builds and tests
+  rust_tests:
+    name: Rust Tests (${{ matrix.platform.name }})
+    strategy:
+      matrix:
+        platform:
+          - name: Ubuntu
+            target: x86_64
+            runner: ubuntu-latest
+            include_docker: true
+          - name: macOS ARM
+            target: aarch64
+            runner: macos-14
+            include_docker: false
+          - name: Windows
+            target: x86_64
+            runner: windows-latest
+            include_docker: false
+    uses: ./.github/workflows/rust-testing-safe.yaml
+    with:
+      target: ${{ matrix.platform.target }}
+      runner: ${{ matrix.platform.runner }}
+      include_docker_tests: ${{ matrix.platform.include_docker }}
+      rust_channel: '1.89.0'
+
+  # Matrix-based Python tests with sccache
+  python_tests:
+    name: Python Tests (${{ matrix.platform.name }})
+    strategy:
+      matrix:
+        platform:
+          - name: Ubuntu
+            target: x86_64
+            runner: ubuntu-latest
+          - name: macOS ARM
+            target: aarch64
+            runner: macos-14
+          - name: Windows
+            target: x86_64
+            runner: windows-latest
+    uses: ./.github/workflows/python-testing.yaml
+    with:
+      rust_channel: '1.89.0'
+      target: ${{ matrix.platform.target }}
+      runner: ${{ matrix.platform.runner }}
+
+  # Integration tests with cloud credentials
+  test_integration:
+    name: Test Integration
+    if: ${{
+      github.event_name == 'schedule'
+      || github.event_name == 'workflow_dispatch'
+      || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+      || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'test-with-secrets'))
+      }}
+    uses: ./.github/workflows/rust-testing-integration.yaml
+    secrets: inherit
+    with:
+      rust_channel: '1.89.0'
+      target: x86_64
+      runner: ubuntu-latest
+
+  # Code quality checks (build wheels with sccache, run in parallel)
+  linting:
+    name: Code Quality
+    uses: ./.github/workflows/linting.yaml
+
+  # Dependency checks (runs independently)
+  dependency_check:
+    name: Dependency Check
+    uses: ./.github/workflows/dependency-check.yaml
+    with:
+      rust_channel: '1.89.0'
+      target: x86_64
+      runner: ubuntu-latest

.github/workflows/dependency-check.yaml

@@ -0,0 +1,82 @@
+name: Dependency Check
+
+on:
+  workflow_call:
+    inputs:
+      rust_channel:
+        description: 'Rust channel to use'
+        required: false
+        type: string
+        default: '1.89.0'
+      target:
+        description: 'Build target architecture (x86_64, aarch64)'
+        required: false
+        type: string
+        default: 'x86_64'
+      runner:
+        description: 'GitHub runner to use'
+        required: false
+        type: string
+        default: 'ubuntu-latest'
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  CI: 1
+  RUST_BACKTRACE: short
+  RUSTFLAGS: "-D warnings -W unreachable-pub -W bare-trait-objects"
+  RUSTUP_MAX_RETRIES: 10
+  RUST_CHANNEL: ${{ inputs.rust_channel }}
+  CARGO_DENY_VERSION: '0.18.4'
+
+jobs:
+  check_dependencies:
+    name: Check Dependencies
+    runs-on: ${{ inputs.runner }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      # Rust artifacts available via Swatinem/rust-cache (no manual download needed)
+
+      - name: Install Just
+        run: sudo snap install --edge --classic just
+
+      # Cache rustup toolchain installation
+      - name: Cache rustup toolchain
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.rustup/toolchains
+            ~/.rustup/update-hashes
+            ~/.rustup/settings.toml
+          key: rustup-${{ runner.os }}-${{ inputs.rust_channel }}
+          restore-keys: |
+            rustup-${{ runner.os }}-
+
+      - name: Install Rust toolchain
+        run: |
+          rustup update --no-self-update ${{ inputs.rust_channel }}
+          rustup component add --toolchain ${{ inputs.rust_channel }} rustfmt rust-src clippy
+          rustup default ${{ inputs.rust_channel }}
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: rust-${{ inputs.rust_channel }}
+          cache-all-crates: true
+          cache-on-failure: true
+
+      - name: Cache cargo-deny
+        uses: actions/cache@v4
+        id: cargo_deny_cache
+        with:
+          path: ~/.cargo/bin/cargo-deny
+          key: cargo-deny-${{ env.CARGO_DENY_VERSION }}-${{ runner.os }}
+
+      - name: Install cargo-deny
+        if: steps.cargo_deny_cache.outputs.cache-hit != 'true'
+        run: cargo install --locked cargo-deny --version ${{ env.CARGO_DENY_VERSION }}
+
+      - name: Run dependency checks
+        run: just check-deps

.github/workflows/linting.yaml

@@ -0,0 +1,113 @@
+name: Linting & Code Quality
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  RUST_CHANNEL: '1.89.0'
+
+jobs:
+  lint:
+    name: Code Quality
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      # Install system dependencies
+      - name: Install Just
+        run: sudo snap install --edge --classic just
+
+      # Cache rustup toolchain installation
+      - name: Cache rustup toolchain
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.rustup/toolchains
+            ~/.rustup/update-hashes
+            ~/.rustup/settings.toml
+          key: rustup-${{ runner.os }}-${{ env.RUST_CHANNEL }}
+          restore-keys: |
+            rustup-${{ runner.os }}-
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install Rust toolchain
+        run: |
+          rustup update --no-self-update ${{ env.RUST_CHANNEL }}
+          rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy
+          rustup default ${{ env.RUST_CHANNEL }}
+
+      # Cache dependencies
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: rust-${{ env.RUST_CHANNEL }}
+          cache-all-crates: true
+          cache-on-failure: true
+
+      - name: Cache additional tools
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pre-commit
+            ~/.cache/codespell
+            ~/.cache/actionlint
+          key: tools-lint-${{ runner.os }}-${{ hashFiles('**/.pre-commit-config.yaml', '**/.codespellrc') }}
+          restore-keys: |
+            tools-lint-${{ runner.os }}-
+
+      # Rust linting
+      - name: Run Rust linting and formatting checks
+        working-directory: ./icechunk
+        run: just pre-commit-python
+
+      # Python setup and linting
+      - name: Build Python wheel with sccache
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: icechunk-python
+          target: x86_64
+          args: --release --out dist --find-interpreter
+          sccache: true
+
+      - name: Setup Python linting environment
+        working-directory: ./icechunk-python
+        run: |
+          uv sync --extra test --find-links dist
+          uv add pre-commit
+
+      - name: Run Python pre-commit linting
+        working-directory: ./icechunk-python
+        run: |
+          SKIP=just-rust uv run pre-commit run --all-files
+
+      - name: Run mypy type checking
+        working-directory: ./icechunk-python
+        run: |
+          uv run mypy python
+
+      # Spell checking
+      - name: Annotate locations with typos
+        uses: codespell-project/codespell-problem-matcher@v1.2.0
+
+      - name: Run codespell
+        uses: codespell-project/actions-codespell@v2
+
+      # GitHub Actions linting
+      - name: Run actionlint
+        uses: rhysd/actionlint@v1.7.7
+        with:
+          fail-on-error: true