Skip to content

ENH: CLI support for sharding #143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 9, 2025
Merged

ENH: CLI support for sharding #143

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 38 additions & 4 deletions ngff_zarr/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@


def _multiscales_to_ngff_zarr(
live, args, output_store, rich_dask_progress, multiscales
live, args, output_store, rich_dask_progress, multiscales, chunks_per_shard=None
):
if not args.output:
if args.quiet:
Expand All @@ -74,6 +74,7 @@ def _multiscales_to_ngff_zarr(
to_ngff_zarr(
output_store,
multiscales,
chunks_per_shard=chunks_per_shard,
progress=rich_dask_progress,
use_tensorstore=args.use_tensorstore,
version=args.ome_zarr_version,
Expand Down Expand Up @@ -223,6 +224,13 @@ def main():
help="Dask array chunking specification, either a single integer or integer per dimension, e.g. 64 or 8 16 32",
metavar="CHUNKS",
)
processing_group.add_argument(
"--chunks-per-shard",
nargs="+",
type=int,
help="Number of chunks along each axis in a shard. If not set, no sharding. Either a single integer or integer per dimension, e.g. 64 or 8 16 32",
metavar="CHUNKS_PER_SHARD",
)
processing_group.add_argument(
"-m",
"--method",
Expand Down Expand Up @@ -344,6 +352,17 @@ def shutdown_client(sig_id, frame): # noqa: ARG001
if args.quiet:
initial = None
with Live(initial, console=console) as live:
chunks_per_shard = None
if args.chunks_per_shard:
if args.ome_zarr_version == "0.4":
live.console.print(
"[red]Sharding is only supported for OME-Zarr version 0.5 and greater"
)
sys.exit(1)
if len(args.chunks_per_shard) == 1:
chunks_per_shard = args.chunks_per_shard[0]
else:
chunks_per_shard = tuple(args.chunks_per_shard)
if args.output and output_backend is ConversionBackend.ITK:
import itk

Expand All @@ -362,7 +381,12 @@ def shutdown_client(sig_id, frame): # noqa: ARG001
store = LocalStore(args.input[0])
multiscales = from_ngff_zarr(store)
_multiscales_to_ngff_zarr(
live, args, output_store, rich_dask_progress, multiscales
live,
args,
output_store,
rich_dask_progress,
multiscales,
chunks_per_shard=chunks_per_shard,
)
elif input_backend is ConversionBackend.TIFFFILE:
try:
Expand All @@ -382,7 +406,12 @@ def shutdown_client(sig_id, frame): # noqa: ARG001
method,
)
_multiscales_to_ngff_zarr(
live, args, output_store, rich_dask_progress, multiscales
live,
args,
output_store,
rich_dask_progress,
multiscales,
chunks_per_shard=chunks_per_shard,
)
except ImportError:
sys.stdout.write("[red]Please install the [i]tifffile[/i] package.\n")
Expand All @@ -394,7 +423,12 @@ def shutdown_client(sig_id, frame): # noqa: ARG001
live, ngff_image, args, progress, rich_dask_progress, subtitle, method
)
_multiscales_to_ngff_zarr(
live, args, output_store, rich_dask_progress, multiscales
live,
args,
output_store,
rich_dask_progress,
multiscales,
chunks_per_shard=chunks_per_shard,
)


Expand Down
Loading