Skip to content

Commit dd1c73b

Browse files
xgui3783xgui3783
committed
feat: allow strict download
1 parent 067a372 commit dd1c73b

File tree

2 files changed

+161
-130
lines changed

2 files changed

+161
-130
lines changed

‎api/common/data_handlers/compounds/download.py

Lines changed: 159 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,146 @@
4848

4949
LICENSE = """Please check the respective `.info.md` regarding licenses of the data."""
5050

51+
def download_region(space_id: str, parcellation_id: str, region_id: str, * , zipfile: ZipFile, **kwargs):
52+
import siibra
53+
from siibra.core import space as _space
54+
55+
try:
56+
region_filename = None
57+
space: _space.Space = siibra.spaces[space_id]
58+
region = siibra.get_region(parcellation_id, region_id)
59+
region_filename = f"{region.key}.nii.gz"
60+
regional_map = region.fetch_regional_map(space, siibra.MapType.STATISTICAL)
61+
zipfile.writestr(region_filename, gzip.compress(regional_map.to_bytes()))
62+
63+
64+
except Exception as e:
65+
zipfile.writestr(f"{region_filename or 'UNKNOWN_REGION'}.error.txt", str(e))
66+
67+
# desc
68+
try:
69+
mp = siibra.get_map(parcellation_id, space, "statistical")
70+
volidx = mp.get_index(region)
71+
vol = mp.volumes[volidx.volume]
72+
publications = "\n\n".join([
73+
f"[{url.get('citation', 'url')}]({url.get('url')})"
74+
for ds in vol.datasets
75+
for url in ds.urls
76+
])
77+
desc = "\n\n".join([ds.description for ds in vol.datasets])
78+
license_list = []
79+
for ds in vol.datasets:
80+
if isinstance(ds.LICENSE, list):
81+
license_list.extend(ds.LICENSE)
82+
if isinstance(ds.LICENSE, str):
83+
license_list.append(ds.LICENSE)
84+
85+
license = "\n\n".join(license_list)
86+
desc_str = DESC.format(name=f"Statistical map of {region.name} in {space.name}",
87+
description=desc,
88+
citations=publications,
89+
license=license)
90+
zipfile.writestr(f'{region_filename}.info.md', desc_str)
91+
except Exception as e:
92+
zipfile.writestr(f'{region_filename}.info.md.error.txt', str(e))
93+
94+
def download_parcellation(space_id: str, parcellation_id: str, * , zipfile: ZipFile, **kwargs):
95+
import siibra
96+
from siibra.core import space as _space, parcellation as _parcellation
97+
98+
try:
99+
parc_filename = None
100+
space: _space.Space = siibra.spaces[space_id]
101+
parcellation: _parcellation.Parcellation = siibra.parcellations[parcellation_id]
102+
parc_filename = f"{parcellation.key}.label.nii.gz"
103+
104+
parc_vol = parcellation.get_map(space, siibra.MapType.LABELLED).fetch()
105+
zipfile.writestr(parc_filename, gzip.compress(parc_vol.to_bytes()))
106+
107+
except Exception as e:
108+
zipfile.writestr(f"{parc_filename or 'UNKNOWN_PARCELLATION'}.error.txt", str(e))
109+
110+
info_filename = f'{parc_filename or "UNKNOWN_PARCELLATION"}.info.md'
111+
try:
112+
publications = "\n\n".join([f"[{p.get('citation', 'url')}]({p.get('url')})"
113+
if p.get('url')
114+
else p.get("citation", "CITATION")
115+
for p in parcellation.publications])
116+
desc_str = DESC.format(name=parcellation.name,
117+
description=parcellation.description,
118+
citations=publications,
119+
license=parcellation.LICENSE)
120+
zipfile.writestr(info_filename, desc_str)
121+
except Exception as e:
122+
zipfile.writestr(f"{info_filename}.error.txt", str(e))
123+
124+
def download_space(space_id: str, bbox: str, * , zipfile: ZipFile):
125+
import siibra
126+
from siibra.core import space as _space
127+
try:
128+
space_filename = None
129+
130+
space: _space.Space = siibra.spaces[space_id]
131+
space_filename = space.key
132+
133+
template = space.get_template()
134+
135+
if "neuroglancer/precomputed" in template.formats:
136+
space_filename += ".nii.gz"
137+
if bbox is None:
138+
bbox = template.boundingbox
139+
bounding_box = space.get_bounding_box(*json.loads(bbox))
140+
value = VOLUME_SIZE_LIMIT
141+
for dim in bounding_box.maxpoint - bounding_box.minpoint:
142+
value /= dim
143+
cube_rooted = math.pow(value, 1/3)
144+
space_vol = template.fetch(voi=bounding_box,
145+
format="neuroglancer/precomputed",
146+
resolution_mm=1/cube_rooted)
147+
zipfile.writestr(space_filename, gzip.compress(space_vol.to_bytes()))
148+
if "gii-mesh" in template.formats:
149+
for idx, vol in enumerate(space.volumes):
150+
variant = vol.variant or f'unknown-variant-{idx}'
151+
root = f"{space_filename}/{variant}"
152+
if "gii-mesh" in vol.providers:
153+
url = vol.providers["gii-mesh"]
154+
def write_url(base_path: str, url: str):
155+
filename = "unknownfile"
156+
try:
157+
filepath = Path(url)
158+
filename = filepath.name
159+
resp = requests.get(url)
160+
resp.raise_for_status()
161+
162+
zipfile.writestr(f"{base_path}/{filename}", resp.content)
163+
except Exception as e:
164+
zipfile.writestr(f"{base_path}/{filename}.error.txt", str(e))
165+
if isinstance(url, str):
166+
write_url(root, url)
167+
if isinstance(url, dict):
168+
for key, _url in url.items():
169+
write_url(f"{root}/{key}", _url)
170+
171+
except Exception as e:
172+
zipfile.writestr(f"{space_filename or 'UNKNOWN_SPACE'}.error.txt", str(e))
173+
174+
info_filename = f'{space_filename or "UNKNOWN_PARCELLATION"}.info.md'
175+
try:
176+
publications = "\n\n".join([f"[{p.get('citation', 'url')}]({p.get('url')})"
177+
if p.get('url')
178+
else p.get("citation", "CITATION")
179+
for p in space.publications])
180+
desc_str = DESC.format(name=space.name,
181+
description=space.description,
182+
citations=publications,
183+
license=space.LICENSE)
184+
zipfile.writestr(info_filename, desc_str)
185+
except Exception as e:
186+
zipfile.writestr(f"{info_filename}.error.txt", str(e))
187+
188+
51189
@data_decorator(ROLE)
52-
def download_all(space_id: str=None, parcellation_id: str=None, region_id: str=None, feature_id: str=None, bbox=None) -> str:
190+
def download_all(space_id: str=None, parcellation_id: str=None, region_id: str=None, feature_id: str=None, bbox: str=None, strict_mode: str=None) -> str:
53191
"""Create a download bundle (zip) for the provided specification
54192
55193
Args:
@@ -62,10 +200,9 @@ def download_all(space_id: str=None, parcellation_id: str=None, region_id: str=N
62200
63201
"""
64202
zipfilename = Path(SIIBRA_API_SHARED_DIR, f"atlas-download-{str(uuid4())}.zip")
203+
output_credit = 1 if strict_mode else 1e10
65204

66205
import siibra
67-
from api.serialization.util import instance_to_model
68-
from siibra.core import space as _space, parcellation as _parcellation, concept as _concept
69206

70207
with ZipFile(zipfilename, "w") as zipfile:
71208

@@ -78,134 +215,28 @@ def download_all(space_id: str=None, parcellation_id: str=None, region_id: str=N
78215
zipfile.write(path_to_feature_export, f"export-{feature_id}.zip")
79216
except Exception as e:
80217
zipfile.writestr(f"{feature_id}.error.txt", f"Feature exporting failed: {str(e)}")
81-
82-
def write_model(filename, obj, **kwargs):
83-
try:
84-
zipfile.writestr(filename, instance_to_model(obj, **kwargs).json(indent=2))
85-
except Exception as e:
86-
zipfile.writestr(f"{filename}.error.txt", str(e))
87218

88-
def write_desc(filename, obj, **kwargs):
89-
try:
90-
space = kwargs.get("space")
91-
if space is not None and isinstance(obj, siibra.core.parcellation.region.Region):
92-
assert isinstance(space, _space.Space)
93-
mp = siibra.get_map(obj.parcellation, space, "statistical")
94-
volidx = mp.get_index(obj)
95-
vol = mp.volumes[volidx.volume]
96-
publications = "\n\n".join([
97-
f"[{url.get('citation', 'url')}]({url.get('url')})"
98-
for ds in vol.datasets
99-
for url in ds.urls
100-
])
101-
desc = "\n\n".join([ds.description for ds in vol.datasets])
102-
license_list = []
103-
for ds in vol.datasets:
104-
if isinstance(ds.LICENSE, list):
105-
license_list.extend(ds.LICENSE)
106-
if isinstance(ds.LICENSE, str):
107-
license_list.append(ds.LICENSE)
108-
109-
license = "\n\n".join(license_list)
110-
desc_str = DESC.format(name=f"Statistical map of {obj.name} in {space.name}",
111-
description=desc,
112-
citations=publications,
113-
license=license)
114-
zipfile.writestr(filename, desc_str)
115-
return
116-
if isinstance(obj, _concept.AtlasConcept):
117-
publications = "\n\n".join([f"[{p.get('citation', 'url')}]({p.get('url')})"
118-
if p.get('url')
119-
else p.get("citation", "CITATION")
120-
for p in obj.publications])
121-
desc_str = DESC.format(name=obj.name, description=obj.description, citations=publications, license=obj.LICENSE)
122-
zipfile.writestr(filename, desc_str)
123-
return
124-
except Exception as e:
125-
zipfile.writestr(f"{filename}.error.txt", str(e))
126-
127-
if space_id is not None:
128-
injected_content=f"space_id={space_id}, parcellation_id={parcellation_id}, region_id={region_id}, bbox={bbox}"
219+
injected_content=f"space_id={space_id}, parcellation_id={parcellation_id}, region_id={region_id}, bbox={bbox}"
220+
221+
readme_txt = README.format(siibra_api_version=__version__,
222+
timestamp=str(datetime.now()),
223+
injected_content=injected_content)
224+
zipfile.writestr("README.md", readme_txt)
225+
zipfile.writestr("LICENCE.txt", LICENSE)
226+
227+
if region_id is not None:
228+
if output_credit >= 1:
229+
download_region(space_id, parcellation_id, region_id, bbox=bbox, zipfile=zipfile)
230+
output_credit -= 1
129231

130-
readme_txt = README.format(siibra_api_version=__version__,
131-
timestamp=str(datetime.now()),
132-
injected_content=injected_content)
133-
zipfile.writestr("README.md", readme_txt)
134-
zipfile.writestr("LICENCE.txt", LICENSE)
135-
try:
136-
space_filename = None
137-
138-
space: _space.Space = siibra.spaces[space_id]
139-
space_filename = space.key
140-
141-
template = space.get_template()
142-
143-
if "neuroglancer/precomputed" in template.formats:
144-
space_filename += ".nii.gz"
145-
if bbox is None:
146-
bbox = template.boundingbox
147-
bounding_box = space.get_bounding_box(*json.loads(bbox))
148-
value = VOLUME_SIZE_LIMIT
149-
for dim in bounding_box.maxpoint - bounding_box.minpoint:
150-
value /= dim
151-
cube_rooted = math.pow(value, 1/3)
152-
space_vol = template.fetch(voi=bounding_box,
153-
format="neuroglancer/precomputed",
154-
resolution_mm=1/cube_rooted)
155-
zipfile.writestr(space_filename, gzip.compress(space_vol.to_bytes()))
156-
if "gii-mesh" in template.formats:
157-
for idx, vol in enumerate(space.volumes):
158-
variant = vol.variant or f'unknown-variant-{idx}'
159-
root = f"{space_filename}/{variant}"
160-
if "gii-mesh" in vol.providers:
161-
url = vol.providers["gii-mesh"]
162-
def write_url(base_path: str, url: str):
163-
filename = "unknownfile"
164-
try:
165-
filepath = Path(url)
166-
filename = filepath.name
167-
resp = requests.get(url)
168-
resp.raise_for_status()
169-
170-
zipfile.writestr(f"{base_path}/{filename}", resp.content)
171-
except Exception as e:
172-
zipfile.writestr(f"{base_path}/{filename}.error.txt", str(e))
173-
if isinstance(url, str):
174-
write_url(root, url)
175-
if isinstance(url, dict):
176-
for key, _url in url.items():
177-
write_url(f"{root}/{key}", _url)
178-
179-
write_desc(f'{space_filename}.info.md', space)
180-
except Exception as e:
181-
zipfile.writestr(f"{space_filename or 'UNKNOWN_SPACE'}.error.txt", str(e))
182-
183232
if parcellation_id is not None:
184-
if region_id is None:
185-
try:
186-
parc_filename = None
187-
space: _space.Space = siibra.spaces[space_id]
188-
parcellation: _parcellation.Parcellation = siibra.parcellations[parcellation_id]
189-
parc_filename = f"{parcellation.key}.nii.gz"
190-
191-
parc_vol = parcellation.get_map(space, siibra.MapType.LABELLED).fetch()
192-
zipfile.writestr(parc_filename, gzip.compress(parc_vol.to_bytes()))
193-
write_desc(f'{parc_filename}.info.md', parcellation)
194-
except Exception as e:
195-
zipfile.writestr(f"{parc_filename or 'UNKNOWN_PARCELLATION'}.error.txt", str(e))
196-
197-
198-
if region_id is not None:
199-
try:
200-
region_filename = None
201-
space: _space.Space = siibra.spaces[space_id]
202-
region = siibra.get_region(parcellation_id, region_id)
203-
region_filename = f"{region.key}.nii.gz"
204-
regional_map = region.fetch_regional_map(space, siibra.MapType.STATISTICAL)
205-
zipfile.writestr(region_filename, gzip.compress(regional_map.to_bytes()))
206-
write_desc(f'{region_filename}.info.md', region, space=space)
207-
except Exception as e:
208-
zipfile.writestr(f"{region_filename or 'UNKNOWN_REGION'}.error.txt", str(e))
233+
if output_credit >= 1:
234+
download_parcellation(space_id, parcellation_id, bbox=bbox, zipfile=zipfile)
235+
output_credit -= 1
209236

237+
if space_id is not None:
238+
if output_credit >= 1:
239+
download_space(space_id, bbox=bbox, zipfile=zipfile)
240+
output_credit -= 1
210241

211242
return str(zipfilename)

‎api/server/compounds/download.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ def cleanup(filepath: Path):
2525
@router.get("")
2626
@version(*FASTAPI_VERSION)
2727
@router_decorator(ROLE, func=download_all, queue_as_async=(ROLE=="server"))
28-
def get_download_bundle(space_id: str=None, parcellation_id: str=None, bbox: str=None, region_id: str=None, feature_id: str=None, *, background: BackgroundTasks, func):
28+
def get_download_bundle(space_id: str=None, parcellation_id: str=None, bbox: str=None, region_id: str=None, feature_id: str=None, strict_mode: str=None, *, background: BackgroundTasks, func):
2929
"""Prepare the bundle. Given a specification, prepare/bundle according to the specification."""
30-
returnval = func(space_id=space_id, parcellation_id=parcellation_id, bbox=bbox, region_id=region_id, feature_id=feature_id)
30+
returnval = func(space_id=space_id, parcellation_id=parcellation_id, bbox=bbox, region_id=region_id, feature_id=feature_id, strict_mode=strict_mode)
3131
try:
3232
path_to_file = Path(returnval)
3333
except Exception as e:

0 commit comments

Comments
 (0)