-
-
Notifications
You must be signed in to change notification settings - Fork 348
improvements to parse_dtype
#3264
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 14 commits
ed56505
ca6196d
d04557d
6009923
082ad49
b54bbe3
d684ada
5feb937
c42edf6
d234ae2
6285005
3a82a26
b95f3be
b0edab6
385efa0
f7b5387
6cef7d7
9a7d1db
d741513
8585158
8d2c0f0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
- Expand the range of types accepted by ``parse_data_type`` to include strings and Sequences. | ||
- Move the functionality of ``parse_data_type`` to a new function called ``parse_dtype``. This change | ||
ensures that nomenclature is consistent across the codebase. ``parse_data_type`` remains, so this | ||
change is not breaking. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
from __future__ import annotations | ||
|
||
from collections.abc import Sequence | ||
from typing import TYPE_CHECKING, Final, TypeAlias | ||
|
||
from zarr.core.dtype.common import ( | ||
|
@@ -94,6 +95,7 @@ | |
"ZDType", | ||
"data_type_registry", | ||
"parse_data_type", | ||
"parse_dtype", | ||
] | ||
|
||
data_type_registry = DataTypeRegistry() | ||
|
@@ -188,43 +190,93 @@ def parse_data_type( | |
zarr_format: ZarrFormat, | ||
) -> ZDType[TBaseDType, TBaseScalar]: | ||
""" | ||
Interpret the input as a ZDType instance. | ||
Interpret the input as a ZDType. | ||
|
||
This function wraps ``parse_dtype``. The only difference is the function name. This function may | ||
be deprecated in a future version of Zarr Python in favor of ``parse_dtype``. | ||
|
||
Parameters | ||
---------- | ||
dtype_spec : ZDTypeLike | ||
The input to be interpreted as a ZDType instance. This could be a native data type | ||
(e.g., a NumPy data type), a Python object that can be converted into a native data type, | ||
a ZDType instance (in which case the input is returned unchanged), or a JSON object | ||
representation of a data type. | ||
The input to be interpreted as a ZDType. This could be a ZDType, which will be returned | ||
directly, or a JSON representation of a ZDType, or a native dtype, or a python object that | ||
can be converted into a native dtype. | ||
zarr_format : ZarrFormat | ||
The zarr format version. | ||
The Zarr format version. This parameter is required because this function will attempt to | ||
parse the JSON representation of a data type, and the JSON representation of data types | ||
varies between Zarr 2 and Zarr 3. | ||
|
||
Returns | ||
------- | ||
ZDType[TBaseDType, TBaseScalar] | ||
The ZDType instance corresponding to the input. | ||
The ZDType corresponding to the input. | ||
|
||
Examples | ||
-------- | ||
>>> from zarr.dtype import parse_data_type | ||
>>> import numpy as np | ||
>>> parse_data_type("int32", zarr_format=2) | ||
Int32(endianness='little') | ||
>>> parse_data_type(np.dtype('S10'), zarr_format=2) | ||
>>> parse_dtype(np.dtype('S10'), zarr_format=2) | ||
NullTerminatedBytes(length=10) | ||
>>> parse_data_type({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) | ||
DateTime64(endianness='little', scale_factor=10, unit='s') | ||
>>> parse_data_type("int32", zarr_format=2) | ||
Int32(endianness="little") | ||
d-v-b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
""" | ||
return parse_dtype(dtype_spec, zarr_format=zarr_format) | ||
|
||
|
||
def parse_dtype( | ||
dtype_spec: ZDTypeLike, | ||
*, | ||
zarr_format: ZarrFormat, | ||
) -> ZDType[TBaseDType, TBaseScalar]: | ||
""" | ||
Interpret the input as a ZDType. | ||
d-v-b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
Parameters | ||
---------- | ||
dtype_spec : ZDTypeLike | ||
The input to be interpreted as a ZDType. This could be a ZDType, which will be returned | ||
d-v-b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
directly, or a JSON representation of a ZDType, or a native dtype, or a python object that | ||
d-v-b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
can be converted into a native dtype. | ||
zarr_format : ZarrFormat | ||
The Zarr format version. This parameter is required because this function will attempt to | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A stretch goal (for another issue/PR) - would it be possible to make this optional for certain inputs (e.g., converting 'int32' doesn't depend on the version of the zarr spec) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would rather not have a function where one parameter is optional depending on the value of another parameter. That should probably be a different function entirely, e.g. one that specifically takes numpy dtype-like inputs and finds the corresponding zarr data type. |
||
parse the JSON representation of a data type, and the JSON representation of data types | ||
varies between Zarr 2 and Zarr 3. | ||
|
||
Returns | ||
------- | ||
ZDType[TBaseDType, TBaseScalar] | ||
The ZDType corresponding to the input. | ||
|
||
Examples | ||
-------- | ||
>>> from zarr.dtype import parse_dtype | ||
>>> import numpy as np | ||
>>> parse_dtype("int32", zarr_format=2) | ||
Int32(endianness='little') | ||
>>> parse_dtype(np.dtype('S10'), zarr_format=2) | ||
NullTerminatedBytes(length=10) | ||
>>> parse_dtype({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) | ||
DateTime64(endianness='little', scale_factor=10, unit='s') | ||
>>> parse_dtype("int32", zarr_format=2) | ||
d-v-b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
Int32(endianness="little") | ||
d-v-b marked this conversation as resolved.
Show resolved
Hide resolved
|
||
""" | ||
if isinstance(dtype_spec, ZDType): | ||
return dtype_spec | ||
# dict and zarr_format 3 means that we have a JSON object representation of the dtype | ||
if zarr_format == 3 and isinstance(dtype_spec, Mapping): | ||
return get_data_type_from_json(dtype_spec, zarr_format=3) | ||
# First attempt to interpret the input as JSON | ||
if isinstance(dtype_spec, Mapping | str | Sequence): | ||
try: | ||
return get_data_type_from_json(dtype_spec, zarr_format=zarr_format) # type: ignore[arg-type] | ||
except ValueError: | ||
# no data type matched this JSON-like input | ||
pass | ||
if dtype_spec in VLEN_UTF8_ALIAS: | ||
# If the dtype request is one of the aliases for variable-length UTF-8 strings, | ||
# return that dtype. | ||
return VariableLengthUTF8() # type: ignore[return-value] | ||
# otherwise, we have either a numpy dtype string, or a zarr v3 dtype string, and in either case | ||
# we can create a numpy dtype from it, and do the dtype inference from that | ||
# we can create a native dtype from it, and do the dtype inference from that | ||
return get_data_type_from_native_dtype(dtype_spec) # type: ignore[arg-type] |
Uh oh!
There was an error while loading. Please reload this page.