add note about conda-forge to release guide (#1383) #1838
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Check - Upstream | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'support/**' | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| schedule: | |
| - cron: "0 0 * * *" # Daily “At 00:00” UTC | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| issues: write | |
| env: | |
| PYTHON_VERSION: "3.14" | |
| defaults: | |
| run: | |
| working-directory: ./icechunk-python | |
| jobs: | |
| build: | |
| name: upstream-dev | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| if: ${{ | |
| (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') | |
| || github.event_name == 'workflow_dispatch' | |
| || github.event_name == 'schedule' | |
| }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install Just | |
| run: sudo snap install --edge --classic just | |
| - name: Stand up MinIO | |
| run: | | |
| docker compose up -d minio | |
| - name: Wait for MinIO to be ready | |
| run: | | |
| for _ in {1..10}; do | |
| if curl --silent --fail http://localhost:9000/minio/health/live; then | |
| break | |
| fi | |
| sleep 3 | |
| done | |
| docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| working-directory: icechunk-python | |
| # target: ${{ matrix.platform.target }} | |
| args: --release --out dist --find-interpreter | |
| sccache: true | |
| # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245 | |
| - name: setup | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python --version | |
| uv pip install --group upstream --pre | |
| PY_TAG="cp${PYTHON_VERSION//./}" | |
| WHEEL=$(ls dist/*-${PY_TAG}-*.whl) | |
| uv pip install "$WHEEL" --no-deps --force-reinstall | |
| uv pip list | |
| - name: mypy | |
| id: mypy | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| mypy python 2>&1 | tee mypy-output.log | |
| - name: Create or update mypy failure issue | |
| if: | | |
| always() | |
| && steps.mypy.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| shell: bash | |
| working-directory: . | |
| run: | | |
| # Read the template and mypy output | |
| template=$(cat .github/mypy-failure-template.md) | |
| mypy_output="No mypy output captured" | |
| if [ -f "icechunk-python/mypy-output.log" ]; then | |
| mypy_output=$(cat icechunk-python/mypy-output.log) | |
| fi | |
| # Replace placeholders | |
| issue_body="${template//\{\{WORKFLOW\}\}/${{ github.workflow }}}" | |
| issue_body="${issue_body//\{\{RUN_ID\}\}/${{ github.run_id }}}" | |
| issue_body="${issue_body//\{\{RUN_URL\}\}/${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}" | |
| issue_body="${issue_body//\{\{DATE\}\}/$(date -u)}" | |
| issue_body="${issue_body//\{\{MYPY_OUTPUT\}\}/$mypy_output}" | |
| # Check for existing open issue with same title | |
| issue_title="Nightly MyPy type checking failed with upstream dependencies" | |
| existing_issue=$(gh issue list --state open --label "CI" --search "\"$issue_title\" in:title" --json number --jq '.[0].number // empty') | |
| if [ -n "$existing_issue" ]; then | |
| echo "Found existing open issue #$existing_issue, updating it..." | |
| echo "$issue_body" | gh issue edit "$existing_issue" --body-file - | |
| echo "Updated existing issue #$existing_issue with latest mypy output" | |
| else | |
| echo "No existing open issue found, creating new one..." | |
| echo "$issue_body" | gh issue create \ | |
| --title "$issue_title" \ | |
| --body-file - \ | |
| --label "CI" | |
| echo "Created new issue" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Restore cached hypothesis directory | |
| id: restore-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| restore-keys: | | |
| cache-hypothesis- | |
| - name: describe environment | |
| if: always() | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip list | |
| - name: pytest | |
| id: pytest-icechunk | |
| if: always() | |
| shell: bash | |
| working-directory: icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pytest -n 4 --report-log output-pytest-log.jsonl | |
| # explicitly save the cache so it gets updated, also do this even if it fails. | |
| - name: Save cached hypothesis directory | |
| id: save-hypothesis-cache | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: icechunk-python/.hypothesis/ | |
| key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} | |
| - name: Generate and publish the pytest report | |
| if: | | |
| always() | |
| && steps.pytest-icechunk.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| uses: xarray-contrib/issue-from-pytest-log@v1 | |
| with: | |
| log-path: icechunk-python/output-pytest-log.jsonl | |
| xarray-backends: | |
| name: xarray-tests-upstream | |
| runs-on: ubuntu-latest | |
| if: ${{ | |
| (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') | |
| || github.event_name == 'workflow_dispatch' | |
| || github.event_name == 'schedule' | |
| }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| path: "icechunk" | |
| - uses: actions/checkout@v5 | |
| with: | |
| repository: "pydata/xarray" | |
| path: "xarray" | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| working-directory: icechunk/icechunk-python | |
| args: --release --out dist --find-interpreter | |
| - name: setup | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python --version | |
| PY_TAG="cp${PYTHON_VERSION//./}" | |
| WHEEL=$(ls dist/*-${PY_TAG}-*.whl) | |
| uv pip install "$WHEEL" --group upstream | |
| uv pip install pytest-mypy-plugins | |
| uv pip list | |
| - name: Stand up MinIO | |
| working-directory: icechunk | |
| run: | | |
| docker compose up -d minio | |
| - name: Wait for MinIO to be ready | |
| working-directory: icechunk | |
| run: | | |
| for _ in {1..10}; do | |
| if curl --silent --fail http://localhost:9000/minio/health/live; then | |
| break | |
| fi | |
| sleep 3 | |
| done | |
| docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 | |
| - name: pytest | |
| id: pytest-xarray-backends | |
| shell: bash | |
| working-directory: icechunk/icechunk-python | |
| env: | |
| ICECHUNK_XARRAY_BACKENDS_TESTS: 1 | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture | |
| pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py --report-log output-pytest-log.jsonl | |
| - name: Generate and publish the xarray backends report | |
| if: | | |
| failure() | |
| && steps.pytest-xarray-backends.outcome == 'failure' | |
| && github.event_name == 'schedule' | |
| && github.repository_owner == 'earth-mover' | |
| uses: xarray-contrib/issue-from-pytest-log@v1 | |
| with: | |
| log-path: icechunk/icechunk-python/output-pytest-log.jsonl | |
| issue-title: "Nightly Xarray backends tests failed" |