AttributeError: 'bytes' object has no attribute 'item' with open_virtual_dataset

[dschneider-wxs]

I'm am running through some example code and running into this error when trying to load my own .nc files on s3

AttributeError                            Traceback (most recent call last)
Cell In[4], line 4
      1 from virtualizarr import open_virtual_dataset
      3 virtual_datasets = [
----> 4     open_virtual_dataset(url, indexes={}, filetype="netCDF4") for url in oisst_files
      5 
      6 ]

File ~/Downloads/test-icechunk/.venv/lib/python3.13/site-packages/virtualizarr/backend.py:199, in open_virtual_dataset(filepath, filetype, group, drop_variables, loadable_variables, decode_times, cftime_variables, indexes, virtual_array_class, virtual_backend_kwargs, reader_options, backend)
    196 if backend_cls is None:
    197     raise NotImplementedError(f"Unsupported file type: {filetype.name}")
--> 199 vds = backend_cls.open_virtual_dataset(
    200     filepath,
    201     group=group,
    202     drop_variables=drop_variables,
    203     loadable_variables=loadable_variables,
    204     decode_times=decode_times,
    205     indexes=indexes,
    206     virtual_backend_kwargs=virtual_backend_kwargs,
    207     reader_options=reader_options,
    208 )
    210 return vds

File ~/Downloads/test-icechunk/.venv/lib/python3.13/site-packages/virtualizarr/readers/hdf/hdf.py:86, in HDFVirtualBackend.open_virtual_dataset(filepath, group, drop_variables, loadable_variables, decode_times, indexes, virtual_backend_kwargs, reader_options)
     77 drop_variables, loadable_variables = check_for_collisions(
     78     drop_variables,
     79     loadable_variables,
     80 )
     82 filepath = validate_and_normalize_path_to_uri(
     83     filepath, fs_root=Path.cwd().as_uri()
     84 )
---> 86 virtual_vars = HDFVirtualBackend._virtual_vars_from_hdf(
     87     path=filepath,
     88     group=group,
     89     drop_variables=drop_variables + loadable_variables,
     90     reader_options=reader_options,
     91 )
     93 loadable_vars, indexes = maybe_open_loadable_vars_and_indexes(
     94     filepath,
     95     loadable_variables=loadable_variables,
   (...)    100     decode_times=decode_times,
    101 )
    103 attrs = HDFVirtualBackend._get_group_attrs(
    104     path=filepath, reader_options=reader_options, group=group
    105 )

File ~/Downloads/test-icechunk/.venv/lib/python3.13/site-packages/virtualizarr/readers/hdf/hdf.py:419, in HDFVirtualBackend._virtual_vars_from_hdf(path, group, drop_variables, reader_options)
    417 if key not in drop_variables:
    418     if isinstance(g[key], h5py.Dataset):
--> 419         variable = HDFVirtualBackend._dataset_to_variable(
    420             path=path,
    421             dataset=g[key],
    422             group=group_name,
    423         )
    424         if variable is not None:
    425             variables[key] = variable

File ~/Downloads/test-icechunk/.venv/lib/python3.13/site-packages/virtualizarr/readers/hdf/hdf.py:342, in HDFVirtualBackend._dataset_to_variable(path, dataset, group)
    339 else:
    340     dtype = dataset.dtype
--> 342 fill_value = dataset.fillvalue.item()
    344 filters = [codec.get_config() for codec in codecs]
    345 zarray = ZArray(
    346     chunks=chunks,  # type: ignore
    347     compressor=None,
   (...)    353     zarr_format=2,
    354 )

AttributeError: 'bytes' object has no attribute 'item'

I'm using the code from the oisst example with my own bucket and objects. The .nc files were written from xarray objects with ds.to_netcdf()

import fsspec
from virtualizarr import open_virtual_dataset

fs = fsspec.filesystem("s3")
oisst_files = fs.glob("s3://wxs-dev-era5-store/hourly/era5*.nc")
oisst_files = sorted(["s3://" + f for f in oisst_files])

virtual_datasets = [
    open_virtual_dataset(url, indexes={}) for url in oisst_files
    
]

I also tried adding filetype="netCDF4" to open_virtual_dataset.

MacOS, py3.13

Package                 Version
----------------------- -----------
aiobotocore             2.21.1
aiohappyeyeballs        2.6.1
aiohttp                 3.11.16
aioitertools            0.12.0
aiosignal               1.3.2
appnope                 0.1.4
asttokens               3.0.0
attrs                   25.3.0
botocore                1.37.1
certifi                 2025.1.31
cftime                  1.6.4.post1
charset-normalizer      3.4.1
click                   8.1.8
cloudpickle             3.1.1
comm                    0.2.2
crc32c                  2.7.1
dask                    2025.3.0
debugpy                 1.8.13
decorator               5.2.1
deprecated              1.2.18
donfig                  0.8.1.post1
executing               2.2.0
frozenlist              1.5.0
fsspec                  2025.3.2
h5py                    3.13.0
icechunk                0.2.12
idna                    3.10
iniconfig               2.1.0
ipykernel               6.29.5
ipython                 9.0.2
ipython-pygments-lexers 1.1.1
jedi                    0.19.2
jmespath                1.0.1
jupyter-client          8.6.3
jupyter-core            5.7.2
locket                  1.0.0
matplotlib-inline       0.1.7
multidict               6.3.2
nest-asyncio            1.6.0
netcdf4                 1.7.2
numcodecs               0.15.1
numpy                   2.2.4
packaging               24.2
pandas                  2.2.3
parso                   0.8.4
partd                   1.4.2
pexpect                 4.9.0
platformdirs            4.3.7
pluggy                  1.5.0
pooch                   1.8.2
prompt-toolkit          3.0.50
propcache               0.3.1
psutil                  7.0.0
ptyprocess              0.7.0
pure-eval               0.2.3
pygments                2.19.1
pytest                  8.3.5
python-dateutil         2.9.0.post0
pytz                    2025.2
pyyaml                  6.0.2
pyzmq                   26.3.0
requests                2.32.3
s3fs                    2025.3.2
six                     1.17.0
stack-data              0.6.3
toolz                   1.0.0
tornado                 6.4.2
traitlets               5.14.3
typing-extensions       4.13.1
tzdata                  2025.2
ujson                   5.10.0
universal-pathlib       0.2.6
urllib3                 2.3.0
virtualizarr            1.3.2
wcwidth                 0.2.13
wrapt                   1.17.2
xarray                  2025.3.1
yarl                    1.18.3
zarr                    3.0.6

[TomNicholas]

This might be some kind of bug with the HDF reader. I'll leave it to @sharkinsspatial to take a look.

[dschneider-wxs]

Looks like it might be my fault again, but it is a misleading error message. When I tried reading the files with kerchunk I get a permissions error:

ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden

import fsspec
import kerchunk.hdf

fs = fsspec.filesystem("s3")
urls = fs.glob("s3://wxs-dev-era5-store/hourly/era5*.nc")
urls = sorted(["s3://" + f for f in urls])

so = dict(anon=True, default_fill_cache=False, default_cache_type="first")
singles = []
for u in urls:
    with fsspec.open(u, **so) as inf:
        h5chunks = kerchunk.hdf.SingleHdf5ToZarr(inf, u, inline_threshold=100)
        singles.append(h5chunks.translate())

not sure why this isn't authenticating with my user.

[dschneider-wxs]

I see, forgot to set anon=False