Cross registration / saved results opening issue

[Marekthegeck]

Hi there,
First of all many thanks for the incredible work on the pipeline! I was running the main parts successfully on our data but now I encounter problems during cross registration.
My folder structure is AnimalID/Date/Session/
minian_path = "." dpath = "D:/MiniscopeRecordings/Marek/PR_minian_analysis/69" f_pattern = r"minian$" id_dims = ["animal","session"] subdirs = ["2022_05_12", "2022_05_18"]

minian_ds = open_minian_mf( dpath, id_dims, pattern=f_pattern, sub_dirs=subdirs)

opening dataset under D:\MiniscopeRecordings\Marek\PR_minian_analysis\69\2022_05_12\FR3
opening minian

GroupNotFoundError Traceback (most recent call last)
Input In [70], in <cell line: 1>()
----> 1 minian_ds = open_minian_mf(
2 dpath, id_dims, pattern=f_pattern, sub_dirs=subdirs)

File ~.conda\envs\minian\lib\site-packages\minian\utilities.py:425, in open_minian_mf(dpath, index_dims, result_format, pattern, sub_dirs, exclude, **kwargs)
423 fname = flist[-1]
424 print("opening {}".format(fname))
--> 425 minian = open_minian(dpath=os.path.join(nextdir, fname), **kwargs)
426 key = tuple([np.array_str(minian[d].values) for d in index_dims])
427 minian_dict[key] = minian

File ~.conda\envs\minian\lib\site-packages\minian\utilities.py:333, in open_minian(dpath, post_process, return_dict)
331 arr_path = pjoin(dpath, d)
332 if isdir(arr_path):
--> 333 arr = list(xr.open_zarr(arr_path).values())[0]
334 arr.data = darr.from_zarr(
335 os.path.join(arr_path, arr.name), inline_array=True
336 )
337 dslist.append(arr)

File ~.conda\envs\minian\lib\site-packages\xarray\backends\zarr.py:675, in open_zarr(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, decode_timedelta, use_cftime, **kwargs)
664 raise TypeError(
665 "open_zarr() got unexpected keyword arguments " + ",".join(kwargs.keys())
666 )
668 backend_kwargs = {
669 "synchronizer": synchronizer,
670 "consolidated": consolidated,
671 "overwrite_encoded_chunks": overwrite_encoded_chunks,
672 "chunk_store": chunk_store,
673 }
--> 675 ds = open_dataset(
676 filename_or_obj=store,
677 group=group,
678 decode_cf=decode_cf,
679 mask_and_scale=mask_and_scale,
680 decode_times=decode_times,
681 concat_characters=concat_characters,
682 decode_coords=decode_coords,
683 engine="zarr",
684 chunks=chunks,
685 drop_variables=drop_variables,
686 backend_kwargs=backend_kwargs,
687 decode_timedelta=decode_timedelta,
688 use_cftime=use_cftime,
689 )
691 return ds

File ~.conda\envs\minian\lib\site-packages\xarray\backends\api.py:572, in open_dataset(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, autoclose, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime, decode_timedelta)
567 overwrite_encoded_chunks = backend_kwargs.pop(
568 "overwrite_encoded_chunks", None
569 )
571 opener = _get_backend_cls(engine)
--> 572 store = opener(filename_or_obj, **extra_kwargs, **backend_kwargs)
574 with close_on_error(store):
575 ds = maybe_decode_store(store, chunks)

File ~.conda\envs\minian\lib\site-packages\xarray\backends\zarr.py:296, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, append_dim, write_region)
294 zarr_group = zarr.open_consolidated(store, **open_kwargs)
295 else:
--> 296 zarr_group = zarr.open_group(store, **open_kwargs)
297 return cls(zarr_group, consolidate_on_close, append_dim, write_region)

File ~.conda\envs\minian\lib\site-packages\zarr\hierarchy.py:1347, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version)
1345 if contains_array(store, path=path):
1346 raise ContainsArrayError(path)
-> 1347 raise GroupNotFoundError(path)
1349 elif mode == 'w':
1350 init_group(store, overwrite=True, path=path, chunk_store=chunk_store)

GroupNotFoundError: group not found at path ''

I'm receiving this and similar errors whether or not defining subdirs. I've not changed folder names after running the pipeline, but I have tried to run it in different structured folder systems (like leaving out the Date and so on), and moved the data to C:, where minian is living. Did not help so far. Also, when I wanted to check how the metadata looks after running the pipeline I received basically the same error:

dpath = "D:/MiniscopeRecordings/Marek/PR_minian_analysis/69/2022_05_18/PR1"
minian_ds = open_minian(dpath)

---

GroupNotFoundError Traceback (most recent call last)
InGroupNotFoundError Traceback (most recent call last)
Input In [220], in <cell line: 1>()
----> 1 minian_ds = open_minian(dpath)
2 minian_ds

File ~.conda\envs\minian\lib\site-packages\minian\utilities.py:333, in open_minian(dpath, post_process, return_dict)
331 arr_path = pjoin(dpath, d)
332 if isdir(arr_path):
--> 333 arr = list(xr.open_zarr(arr_path).values())[0]
334 arr.data = darr.from_zarr(
335 os.path.join(arr_path, arr.name), inline_array=True
336 )
337 dslist.append(arr)

File ~.conda\envs\minian\lib\site-packages\xarray\backends\zarr.py:675, in open_zarr(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, decode_timedelta, use_cftime, **kwargs)
664 raise TypeError(
665 "open_zarr() got unexpected keyword arguments " + ",".join(kwargs.keys())
666 )
668 backend_kwargs = {
669 "synchronizer": synchronizer,
670 "consolidated": consolidated,
671 "overwrite_encoded_chunks": overwrite_encoded_chunks,
672 "chunk_store": chunk_store,
673 }
--> 675 ds = open_dataset(
676 filename_or_obj=store,
677 group=group,
678 decode_cf=decode_cf,
679 mask_and_scale=mask_and_scale,
680 decode_times=decode_times,
681 concat_characters=concat_characters,
682 decode_coords=decode_coords,
683 engine="zarr",
684 chunks=chunks,
685 drop_variables=drop_variables,
686 backend_kwargs=backend_kwargs,
687 decode_timedelta=decode_timedelta,
688 use_cftime=use_cftime,
689 )
691 return ds

File ~.conda\envs\minian\lib\site-packages\xarray\backends\api.py:572, in open_dataset(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, autoclose, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime, decode_timedelta)
567 overwrite_encoded_chunks = backend_kwargs.pop(
568 "overwrite_encoded_chunks", None
569 )
571 opener = _get_backend_cls(engine)
--> 572 store = opener(filename_or_obj, **extra_kwargs, **backend_kwargs)
574 with close_on_error(store):
575 ds = maybe_decode_store(store, chunks)

File ~.conda\envs\minian\lib\site-packages\xarray\backends\zarr.py:296, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, append_dim, write_region)
294 zarr_group = zarr.open_consolidated(store, **open_kwargs)
295 else:
--> 296 zarr_group = zarr.open_group(store, **open_kwargs)
297 return cls(zarr_group, consolidate_on_close, append_dim, write_region)

File ~.conda\envs\minian\lib\site-packages\zarr\hierarchy.py:1347, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version)
1345 if contains_array(store, path=path):
1346 raise ContainsArrayError(path)
-> 1347 raise GroupNotFoundError(path)
1349 elif mode == 'w':
1350 init_group(store, overwrite=True, path=path, chunk_store=chunk_store)

GroupNotFoundError: group not found at path ''put In [220], in <cell line: 1>()
----> 1 minian_ds = open_minian(dpath)
2 minian_ds

File ~.conda\envs\minian\lib\site-packages\minian\utilities.py:333, in open_minian(dpath, post_process, return_dict)
331 arr_path = pjoin(dpath, d)
332 if isdir(arr_path):
--> 333 arr = list(xr.open_zarr(arr_path).values())[0]
334 arr.data = darr.from_zarr(
335 os.path.join(arr_path, arr.name), inline_array=True
336 )
337 dslist.append(arr)

File ~.conda\envs\minian\lib\site-packages\xarray\backends\zarr.py:675, in open_zarr(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, decode_timedelta, use_cftime, **kwargs)
664 raise TypeError(
665 "open_zarr() got unexpected keyword arguments " + ",".join(kwargs.keys())
666 )
668 backend_kwargs = {
669 "synchronizer": synchronizer,
670 "consolidated": consolidated,
671 "overwrite_encoded_chunks": overwrite_encoded_chunks,
672 "chunk_store": chunk_store,
673 }
--> 675 ds = open_dataset(
676 filename_or_obj=store,
677 group=group,
678 decode_cf=decode_cf,
679 mask_and_scale=mask_and_scale,
680 decode_times=decode_times,
681 concat_characters=concat_characters,
682 decode_coords=decode_coords,
683 engine="zarr",
684 chunks=chunks,
685 drop_variables=drop_variables,
686 backend_kwargs=backend_kwargs,
687 decode_timedelta=decode_timedelta,
688 use_cftime=use_cftime,
689 )
691 return ds

File ~.conda\envs\minian\lib\site-packages\xarray\backends\api.py:572, in open_dataset(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, autoclose, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime, decode_timedelta)
567 overwrite_encoded_chunks = backend_kwargs.pop(
568 "overwrite_encoded_chunks", None
569 )
571 opener = _get_backend_cls(engine)
--> 572 store = opener(filename_or_obj, **extra_kwargs, **backend_kwargs)
574 with close_on_error(store):
575 ds = maybe_decode_store(store, chunks)

File ~.conda\envs\minian\lib\site-packages\xarray\backends\zarr.py:296, in ZarrStore.open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, append_dim, write_region)
294 zarr_group = zarr.open_consolidated(store, **open_kwargs)
295 else:
--> 296 zarr_group = zarr.open_group(store, **open_kwargs)
297 return cls(zarr_group, consolidate_on_close, append_dim, write_region)

File ~.conda\envs\minian\lib\site-packages\zarr\hierarchy.py:1347, in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options, zarr_version)
1345 if contains_array(store, path=path):
1346 raise ContainsArrayError(path)
-> 1347 raise GroupNotFoundError(path)
1349 elif mode == 'w':
1350 init_group(store, overwrite=True, path=path, chunk_store=chunk_store)

GroupNotFoundError: group not found at path ''

However, opening data from the intermediate results works like charme and shows that metadata is loaded in correctly (or at least distinguishable)>

I feel this might be related to #146 and #52

Hope it's just a small change in my folder structure thats needed, but at the moment I can't find a solution and help is much appreciated :)

[Marekthegeck]

I found that the minian output folders contained only little information (few hundred kB). When I save the data before running CNMFviewer, everything is fine (data is stored correctly and can be opened). Most times, I have not actively used the CNMFviewer so far.