Add upstream dev CI #5
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Check - Upstream | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| schedule: | |
| - cron: "0 0 * * *" # Daily “At 00:00” UTC | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| defaults: | |
| run: | |
| working-directory: ./icechunk-python | |
| jobs: | |
| build: | |
| name: upstream-dev | |
| runs-on: ubuntu-latest | |
| if: ${{ | |
| (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') | |
| || github.event_name == 'workflow_dispatch' | |
| || github.event_name == 'schedule' | |
| }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Just | |
| run: sudo snap install --edge --classic just | |
| - name: Stand up MinIO | |
| run: | | |
| docker compose up -d minio | |
| - name: Wait for MinIO to be ready | |
| run: | | |
| for _ in {1..10}; do | |
| if curl --silent --fail http://minio:9000/minio/health/live; then | |
| break | |
| fi | |
| sleep 3 | |
| done | |
| docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| working-directory: icechunk-python | |
| # target: ${{ matrix.platform.target }} | |
| args: --release --out dist --find-interpreter | |
| sccache: 'true' | |
| # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245 | |
| - name: setup | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| uv pip install --group upstream | |
| uv pip install icechunk --no-deps --find-links dist --force-reinstall | |
| uv pip list | |
| - name: mypy | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| mypy python | |
| - name: Restore cached hypothesis directory | |
| id: restore-hypothesis-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| restore-keys: | | |
| cache-hypothesis- | |
| - name: describe environment | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip list | |
| - name: pytest | |
| id: status | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pytest -n 4 --report-log output-pytest-log.json | |
| # explicitly save the cache so it gets updated, also do this even if it fails. | |
| - name: Save cached hypothesis directory | |
| id: save-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| - name: Generate and publish the report | |
| if: | | |
| failure() | |
| && steps.status.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| uses: xarray-contrib/issue-from-pytest-log@v1 | |
| with: | |
| log-path: output-pytest-log.jsonl | |
| xarray-backends: | |
| name: xarray-tests-upstream | |
| runs-on: ubuntu-latest | |
| if: ${{ | |
| (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') | |
| || github.event_name == 'workflow_dispatch' | |
| || github.event_name == 'schedule' | |
| }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: "icechunk" | |
| - name: Stand up MinIO | |
| working-directory: icechunk | |
| run: | | |
| docker compose up -d minio | |
| - name: Wait for MinIO to be ready | |
| working-directory: icechunk | |
| run: | | |
| for _ in {1..10}; do | |
| if curl --silent --fail http://minio:9000/minio/health/live; then | |
| break | |
| fi | |
| sleep 3 | |
| done | |
| docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: "pydata/xarray" | |
| path: "xarray" | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| working-directory: icechunk/icechunk-python | |
| args: --release --out dist --find-interpreter | |
| sccache: 'true' | |
| - name: setup | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| uv pip install icechunk['upstream'] --find-links dist --force-reinstall | |
| uv pip install pytest-mypy-plugins | |
| uv pip list | |
| - name: pytest | |
| id: status | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture | |
| pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py | |
| - name: Generate and publish the report | |
| if: | | |
| failure() | |
| && steps.status.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| uses: xarray-contrib/issue-from-pytest-log@v1 | |
| with: | |
| log-path: output-pytest-log.jsonl | |
| issue-title: "Nightly Xarray backends tests failed" |