Spatial update

[Antorhythms]

Hi,

I am trying to use the minian pipeline to analyze miniscope data.

I am starting CNMFe spatial update with this line
A_new, b_new, f_new, mask = update_spatial(
Y_hw_chk, A, b, C, f, sn_spatial, **param_first_spatial
)

but i get the error below.

These are my param_first_spatial
{'dl_wnd': 5,
'sparse_penal': 0.005,
'update_background': True,
'size_thres': (25, None)}

Thanks a lot in advance for your help.

ValueError Traceback (most recent call last)
in

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial(Y, A, b, C, f, sn, dl_wnd, sparse_penal, update_background, normalize, size_thres, in_memory)
410 },
411 )
--> 412 A_new = save_minian(
413 A_new.rename("A_new"),
414 intpath,

C:\Program Files\anaconda3\minian\minian\utilities.py in save_minian(var, dpath, meta_dict, overwrite, chunks, compute, mem_limit)
520 except FileNotFoundError:
521 pass
--> 522 arr = ds.to_zarr(fp, compute=compute, mode=md)
523 if (chunks is not None) and compute:
524 chunks = {d: var.sizes[d] if v <= 0 else v for d, v in chunks.items()}

C:\Program Files\anaconda3\envs\minian\lib\site-packages\xarray\core\dataset.py in to_zarr(self, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region)
1743 encoding = {}
1744
-> 1745 return to_zarr(
1746 self,
1747 store=store,

C:\Program Files\anaconda3\envs\minian\lib\site-packages\xarray\backends\api.py in to_zarr(dataset, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region)
1480 # TODO: figure out how to properly handle unlimited_dims
1481 dump_to_store(dataset, zstore, writer, encoding=encoding)
-> 1482 writes = writer.sync(compute=compute)
1483
1484 if compute:

C:\Program Files\anaconda3\envs\minian\lib\site-packages\xarray\backends\common.py in sync(self, compute)
153 # targets = [dask.delayed(t) for t in self.targets]
154
--> 155 delayed_store = da.store(
156 self.sources,
157 self.targets,

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\array\core.py in store(sources, targets, lock, regions, compute, return_stored, **kwargs)
1020
1021 if compute:
-> 1022 result.compute(**kwargs)
1023 return None
1024 else:

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\base.py in compute(self, **kwargs)
279 dask.base.compute
280 """
--> 281 (result,) = compute(self, traverse=False, **kwargs)
282 return result
283

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\base.py in compute(*args, **kwargs)
561 postcomputes.append(x.dask_postcompute())
562
--> 563 results = schedule(dsk, keys, **kwargs)
564 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
565

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
2653 should_rejoin = False
2654 try:
-> 2655 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
2656 finally:
2657 for f in futures.values():

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in gather(self, futures, errors, direct, asynchronous)
1962 else:
1963 local_worker = None
-> 1964 return self.sync(
1965 self._gather,
1966 futures,

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
836 return future
837 else:
--> 838 return sync(
839 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
840 )

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
338 if error[0]:
339 typ, exc, tb = error[0]
--> 340 raise exc.with_traceback(tb)
341 else:
342 return result[0]

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\utils.py in f()
322 if callback_timeout is not None:
323 future = asyncio.wait_for(future, callback_timeout)
--> 324 result[0] = yield future
325 except Exception as exc:
326 error[0] = sys.exc_info()

C:\Program Files\anaconda3\envs\minian\lib\site-packages\tornado\gen.py in run(self)
760
761 try:
--> 762 value = future.result()
763 except Exception:
764 exc_info = sys.exc_info()

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in _gather(self, futures, errors, direct, local_worker)
1827 exc = CancelledError(key)
1828 else:
-> 1829 raise exception.with_traceback(traceback)
1830 raise exc
1831 if errors == "skip":

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\optimization.py in call()
961 if not len(args) == len(self.inkeys):
962 raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 963 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
964
965 def reduce(self):

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\core.py in get()
149 for key in toposort(dsk):
150 task = dsk[key]
--> 151 result = _execute_task(task, cache)
152 cache[key] = result
153 result = _execute_task(out, cache)

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\core.py in _execute_task()
119 # temporaries by their reference count and can execute certain
120 # operations in-place.
--> 121 return func(*(_execute_task(a, cache) for a in args))
122 elif not ishashable(arg):
123 return arg

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\utils.py in apply()
33 def apply(func, args, kwargs=None):
34 if kwargs:
---> 35 return func(*args, **kwargs)
36 else:
37 return func(*args)

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial_block()
567 data_ls = []
568 for h, w in zip(*sub.any(axis=-1).nonzero()):
--> 569 res = update_spatial_perpx(y[h, w, :], alpha[h, w], sub[h, w, :], C_store, f)
570 crd = res.coords
571 crd = np.concatenate([np.full_like(crd, h), np.full_like(crd, w), crd], axis=0)

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial_perpx()
516 idx = sub.nonzero()[0]
517 try:
--> 518 C = C_store.get_orthogonal_selection((idx, slice(None))).T
519 except AttributeError:
520 C = C_store[idx, :].T

C:\Program Files\anaconda3\envs\minian\lib\site-packages\zarr\core.py in get_orthogonal_selection()
939 indexer = OrthogonalIndexer(selection, self)
940
--> 941 return self._get_selection(indexer=indexer, out=out, fields=fields)
942
943 def get_coordinate_selection(self, selection, out=None, fields=None):

C:\Program Files\anaconda3\envs\minian\lib\site-packages\zarr\core.py in _get_selection()
1122 else:
1123 # allow storage to get multiple items at once
-> 1124 lchunk_coords, lchunk_selection, lout_selection = zip(*indexer)
1125 self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
1126 drop_axes=indexer.drop_axes, fields=fields)

ValueError: not enough values to unpack (expected 3, got 0)

[Antorhythms]

Also on restarting the kernel and trying on another video, we get the following:

estimating penalty parameter
computing subsetting matrix
fitting spatial matrix
updating background

TypeError Traceback (most recent call last)
in

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial(Y, A, b, C, f, sn, dl_wnd, sparse_penal, update_background, normalize, size_thres, in_memory)
435 f_new = f.compute()
436 if size_thres:
--> 437 low, high = size_thres
438 A_bin = A_new > 0
439 mask = np.ones(A_new.sizes["unit_id"], dtype=bool)

TypeError: cannot unpack non-iterable int object

Any help would be greatly appreciated!

[phildong]

Hi,

I am trying to use the minian pipeline to analyze miniscope data.

I am starting CNMFe spatial update with this line A_new, b_new, f_new, mask = update_spatial( Y_hw_chk, A, b, C, f, sn_spatial, **param_first_spatial )

but i get the error below.

These are my param_first_spatial {'dl_wnd': 5, 'sparse_penal': 0.005, 'update_background': True, 'size_thres': (25, None)}

Thanks a lot in advance for your help.

ValueError Traceback (most recent call last) in

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial(Y, A, b, C, f, sn, dl_wnd, sparse_penal, update_background, normalize, size_thres, in_memory) 410 }, 411 ) --> 412 A_new = save_minian( 413 A_new.rename("A_new"), 414 intpath,

C:\Program Files\anaconda3\minian\minian\utilities.py in save_minian(var, dpath, meta_dict, overwrite, chunks, compute, mem_limit) 520 except FileNotFoundError: 521 pass --> 522 arr = ds.to_zarr(fp, compute=compute, mode=md) 523 if (chunks is not None) and compute: 524 chunks = {d: var.sizes[d] if v <= 0 else v for d, v in chunks.items()}

C:\Program Files\anaconda3\envs\minian\lib\site-packages\xarray\core\dataset.py in to_zarr(self, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region) 1743 encoding = {} 1744 -> 1745 return to_zarr( 1746 self, 1747 store=store,

C:\Program Files\anaconda3\envs\minian\lib\site-packages\xarray\backends\api.py in to_zarr(dataset, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region) 1480 # TODO: figure out how to properly handle unlimited_dims 1481 dump_to_store(dataset, zstore, writer, encoding=encoding) -> 1482 writes = writer.sync(compute=compute) 1483 1484 if compute:

C:\Program Files\anaconda3\envs\minian\lib\site-packages\xarray\backends\common.py in sync(self, compute) 153 # targets = [dask.delayed(t) for t in self.targets] 154 --> 155 delayed_store = da.store( 156 self.sources, 157 self.targets,

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\array\core.py in store(sources, targets, lock, regions, compute, return_stored, **kwargs) 1020 1021 if compute: -> 1022 result.compute(**kwargs) 1023 return None 1024 else:

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\base.py in compute(self, **kwargs) 279 dask.base.compute 280 """ --> 281 (result,) = compute(self, traverse=False, **kwargs) 282 return result 283

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\base.py in compute(*args, **kwargs) 561 postcomputes.append(x.dask_postcompute()) 562 --> 563 results = schedule(dsk, keys, **kwargs) 564 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)]) 565

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs) 2653 should_rejoin = False 2654 try: -> 2655 results = self.gather(packed, asynchronous=asynchronous, direct=direct) 2656 finally: 2657 for f in futures.values():

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in gather(self, futures, errors, direct, asynchronous) 1962 else: 1963 local_worker = None -> 1964 return self.sync( 1965 self._gather, 1966 futures,

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs) 836 return future 837 else: --> 838 return sync( 839 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs 840 )

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\utils.py in sync(loop, func, callback_timeout, *args, **kwargs) 338 if error[0]: 339 typ, exc, tb = error[0] --> 340 raise exc.with_traceback(tb) 341 else: 342 return result[0]

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\utils.py in f() 322 if callback_timeout is not None: 323 future = asyncio.wait_for(future, callback_timeout) --> 324 result[0] = yield future 325 except Exception as exc: 326 error[0] = sys.exc_info()

C:\Program Files\anaconda3\envs\minian\lib\site-packages\tornado\gen.py in run(self) 760 761 try: --> 762 value = future.result() 763 except Exception: 764 exc_info = sys.exc_info()

C:\Program Files\anaconda3\envs\minian\lib\site-packages\distributed\client.py in _gather(self, futures, errors, direct, local_worker) 1827 exc = CancelledError(key) 1828 else: -> 1829 raise exception.with_traceback(traceback) 1830 raise exc 1831 if errors == "skip":

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\optimization.py in call() 961 if not len(args) == len(self.inkeys): 962 raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args))) --> 963 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args))) 964 965 def reduce(self):

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\core.py in get() 149 for key in toposort(dsk): 150 task = dsk[key] --> 151 result = _execute_task(task, cache) 152 cache[key] = result 153 result = _execute_task(out, cache)

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\core.py in _execute_task() 119 # temporaries by their reference count and can execute certain 120 # operations in-place. --> 121 return func(*(_execute_task(a, cache) for a in args)) 122 elif not ishashable(arg): 123 return arg

C:\Program Files\anaconda3\envs\minian\lib\site-packages\dask\utils.py in apply() 33 def apply(func, args, kwargs=None): 34 if kwargs: ---> 35 return func(*args, **kwargs) 36 else: 37 return func(*args)

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial_block() 567 data_ls = [] 568 for h, w in zip(*sub.any(axis=-1).nonzero()): --> 569 res = update_spatial_perpx(y[h, w, :], alpha[h, w], sub[h, w, :], C_store, f) 570 crd = res.coords 571 crd = np.concatenate([np.full_like(crd, h), np.full_like(crd, w), crd], axis=0)

C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial_perpx() 516 idx = sub.nonzero()[0] 517 try: --> 518 C = C_store.get_orthogonal_selection((idx, slice(None))).T 519 except AttributeError: 520 C = C_store[idx, :].T

C:\Program Files\anaconda3\envs\minian\lib\site-packages\zarr\core.py in get_orthogonal_selection() 939 indexer = OrthogonalIndexer(selection, self) 940 --> 941 return self._get_selection(indexer=indexer, out=out, fields=fields) 942 943 def get_coordinate_selection(self, selection, out=None, fields=None):

C:\Program Files\anaconda3\envs\minian\lib\site-packages\zarr\core.py in _get_selection() 1122 else: 1123 # allow storage to get multiple items at once -> 1124 lchunk_coords, lchunk_selection, lout_selection = zip(*indexer) 1125 self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection, 1126 drop_axes=indexer.drop_axes, fields=fields)

ValueError: not enough values to unpack (expected 3, got 0)

so the spatial update function has a hard-coded requirement, it looks for the C variable stored as zarr under minian intermediate folder. The name of that folder must be C.name + ".zarr".

So my guess for this error is that either that folder doesn't exist, or it's storing data from some previous run that doesn't correspond to what you're doing when you get the error (like different recordings and/or number of cells). Maybe try clearing the intermediate folder and restart the pipeline from beginning?

[phildong]

Also on restarting the kernel and trying on another video, we get the following:

estimating penalty parameter

computing subsetting matrix
fitting spatial matrix
updating background
TypeError Traceback (most recent call last)
in
C:\Program Files\anaconda3\minian\minian\cnmf.py in update_spatial(Y, A, b, C, f, sn, dl_wnd, sparse_penal, update_background, normalize, size_thres, in_memory)
435 f_new = f.compute()
436 if size_thres:
--> 437 low, high = size_thres
438 A_bin = A_new > 0
439 mask = np.ones(A_new.sizes["unit_id"], dtype=bool)
TypeError: cannot unpack non-iterable int object

Any help would be greatly appreciated!

The size_thres parameter has to be a tuple, check here for more detail