Skip to content

Commit dee8dbc

Browse files
dcheriandcherian
authored
Add upstream dev CI (#862)
* Add upstream dev CI Closes #654 * Add zarr * Use uv * try again * fix xarray bit?
1 parent 3374ca4 commit dee8dbc

File tree

2 files changed

+239
-0
lines changed

2 files changed

+239
-0
lines changed

‎.github/workflows/python-upstream.yaml‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
name: Python Check - Upstream
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
types: [opened, reopened, synchronize, labeled]
9+
schedule:
10+
- cron: "0 0 * * *" # Daily “At 00:00” UTC
11+
workflow_dispatch:
12+
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
permissions:
18+
contents: read
19+
20+
defaults:
21+
run:
22+
working-directory: ./icechunk-python
23+
24+
jobs:
25+
build:
26+
name: upstream-dev
27+
runs-on: ubuntu-latest
28+
if: ${{
29+
(contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
30+
|| github.event_name == 'workflow_dispatch'
31+
|| github.event_name == 'schedule'
32+
}}
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Install Just
36+
run: sudo snap install --edge --classic just
37+
- name: Stand up MinIO
38+
run: |
39+
docker compose up -d minio
40+
41+
- name: Wait for MinIO to be ready
42+
run: |
43+
for _ in {1..10}; do
44+
if curl --silent --fail http://minio:9000/minio/health/live; then
45+
break
46+
fi
47+
sleep 3
48+
done
49+
docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123
50+
- uses: actions/setup-python@v5
51+
with:
52+
python-version: '3.13'
53+
54+
- name: Install uv
55+
uses: astral-sh/setup-uv@v5
56+
57+
- name: Build wheels
58+
uses: PyO3/maturin-action@v1
59+
with:
60+
working-directory: icechunk-python
61+
# target: ${{ matrix.platform.target }}
62+
args: --release --out dist --find-interpreter
63+
sccache: 'true'
64+
# manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245
65+
66+
- name: setup
67+
shell: bash
68+
working-directory: icechunk-python
69+
run: |
70+
set -e
71+
python3 -m venv .venv
72+
source .venv/bin/activate
73+
uv pip install --group upstream
74+
uv pip install icechunk --no-deps --find-links dist --force-reinstall
75+
uv pip list
76+
77+
- name: mypy
78+
shell: bash
79+
working-directory: icechunk-python
80+
run: |
81+
set -e
82+
python3 -m venv .venv
83+
source .venv/bin/activate
84+
mypy python
85+
86+
- name: Restore cached hypothesis directory
87+
id: restore-hypothesis-cache
88+
uses: actions/cache/restore@v4
89+
with:
90+
path: icechunk-python/.hypothesis/
91+
key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
92+
restore-keys: |
93+
cache-hypothesis-
94+
95+
- name: describe environment
96+
shell: bash
97+
working-directory: icechunk-python
98+
run: |
99+
set -e
100+
python3 -m venv .venv
101+
source .venv/bin/activate
102+
pip list
103+
104+
- name: pytest
105+
id: status
106+
shell: bash
107+
working-directory: icechunk-python
108+
run: |
109+
set -e
110+
python3 -m venv .venv
111+
source .venv/bin/activate
112+
pytest -n 4 --report-log output-pytest-log.json
113+
114+
# explicitly save the cache so it gets updated, also do this even if it fails.
115+
- name: Save cached hypothesis directory
116+
id: save-hypothesis-cache
117+
if: always()
118+
uses: actions/cache/save@v4
119+
with:
120+
path: icechunk-python/.hypothesis/
121+
key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
122+
123+
- name: Generate and publish the report
124+
if: |
125+
failure()
126+
&& steps.status.outcome == 'failure'
127+
&& github.event_name == 'schedule'
128+
&& github.repository_owner == 'earth-mover'
129+
uses: xarray-contrib/issue-from-pytest-log@v1
130+
with:
131+
log-path: output-pytest-log.jsonl
132+
133+
134+
xarray-backends:
135+
name: xarray-tests-upstream
136+
runs-on: ubuntu-latest
137+
if: ${{
138+
(contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
139+
|| github.event_name == 'workflow_dispatch'
140+
|| github.event_name == 'schedule'
141+
}}
142+
steps:
143+
- uses: actions/checkout@v4
144+
with:
145+
path: "icechunk"
146+
147+
- name: Stand up MinIO
148+
working-directory: icechunk
149+
run: |
150+
docker compose up -d minio
151+
152+
- name: Wait for MinIO to be ready
153+
working-directory: icechunk
154+
run: |
155+
for _ in {1..10}; do
156+
if curl --silent --fail http://minio:9000/minio/health/live; then
157+
break
158+
fi
159+
sleep 3
160+
done
161+
docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123
162+
163+
- uses: actions/checkout@v4
164+
with:
165+
repository: "pydata/xarray"
166+
path: "xarray"
167+
fetch-depth: 0 # Fetch all history for all branches and tags.
168+
169+
- uses: actions/setup-python@v5
170+
with:
171+
python-version: '3.11'
172+
173+
- name: Install uv
174+
uses: astral-sh/setup-uv@v5
175+
176+
- name: Build wheels
177+
uses: PyO3/maturin-action@v1
178+
with:
179+
working-directory: icechunk/icechunk-python
180+
args: --release --out dist --find-interpreter
181+
sccache: 'true'
182+
183+
- name: setup
184+
shell: bash
185+
working-directory: icechunk/icechunk-python
186+
run: |
187+
set -e
188+
python3 -m venv .venv
189+
source .venv/bin/activate
190+
uv pip install icechunk --group upstream --find-links dist
191+
uv pip install pytest-mypy-plugins
192+
uv pip list
193+
194+
- name: pytest
195+
id: status
196+
shell: bash
197+
working-directory: icechunk/icechunk-python
198+
env:
199+
ICECHUNK_XARRAY_BACKENDS_TESTS: 1
200+
run: |
201+
set -e
202+
python3 -m venv .venv
203+
source .venv/bin/activate
204+
# pass xarray's pyproject.toml so that pytest can find the `flaky` fixture
205+
pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py
206+
207+
- name: Generate and publish the report
208+
if: |
209+
failure()
210+
&& steps.status.outcome == 'failure'
211+
&& github.event_name == 'schedule'
212+
&& github.repository_owner == 'earth-mover'
213+
uses: xarray-contrib/issue-from-pytest-log@v1
214+
with:
215+
log-path: output-pytest-log.jsonl
216+
issue-title: "Nightly Xarray backends tests failed"

‎icechunk-python/pyproject.toml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,29 @@ benchmark = [
5252
"platformdirs",
5353
]
5454

55+
[dependency-groups]
56+
upstream = [
57+
"boto3",
58+
"coverage",
59+
"mypy",
60+
"pytest",
61+
"pytest-cov",
62+
"pytest-asyncio",
63+
"pytest-xdist",
64+
"pytest-reportlog",
65+
"ruff",
66+
"hypothesis",
67+
"pandas-stubs",
68+
"boto3-stubs[s3]",
69+
"termcolor",
70+
# "object-store-python @ git+https://github.com/roeap/object-store-python.git", # doesn't work with uv
71+
"object-store-python",
72+
"dask @ git+https://github.com/dask/dask.git",
73+
"distributed @ git+https://github.com/dask/distributed.git",
74+
"xarray @ git+https://github.com/pydata/xarray.git",
75+
"zarr @ git+https://github.com/zarr-developers/zarr.git",
76+
]
77+
5578
[tool.maturin]
5679
features = ["pyo3/extension-module"]
5780
module-name = "icechunk._icechunk_python"

0 commit comments

Comments
 (0)