Strange output from open_mfdataset

[pecos27]

I have a 30 different NetCDF files (time, depth, lat and lon). The netcdfs are split up in time, say from 1993-01 to 1995-01, etc.
I am trying to open all the files into one xarray data array using:

 inpath='filepath/'

infile='*nc'

ds=xr.open_mfdataset(inpath+infile,combine='nested',chunks={'depth': 1}) 

But I keep getting this error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<timed exec> in <module>

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/backends/api.py in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
    926             # Combined nested list by successive concat and merge operations
    927             # along each dimension, using structure given by "ids"
--> 928             combined = _nested_combine(
    929                 datasets,
    930                 concat_dims=concat_dim,

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/combine.py in _nested_combine(datasets, concat_dims, compat, data_vars, coords, ids, fill_value, join, combine_attrs)
    343 
    344     # Apply series of concatenate or merge operations along each dimension
--> 345     combined = _combine_nd(
    346         combined_ids,
    347         concat_dims,

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/combine.py in _combine_nd(combined_ids, concat_dims, data_vars, coords, compat, fill_value, join, combine_attrs)
    217     # element of the tuple
    218     for concat_dim in concat_dims:
--> 219         combined_ids = _combine_all_along_first_dim(
    220             combined_ids,
    221             dim=concat_dim,

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/combine.py in _combine_all_along_first_dim(combined_ids, dim, data_vars, coords, compat, fill_value, join, combine_attrs)
    253         combined_ids = dict(sorted(group))
    254         datasets = combined_ids.values()
--> 255         new_combined_ids[new_id] = _combine_1d(
    256             datasets, dim, compat, data_vars, coords, fill_value, join, combine_attrs
    257         )

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/combine.py in _combine_1d(datasets, concat_dim, compat, data_vars, coords, fill_value, join, combine_attrs)
    276     if concat_dim is not None:
    277         try:
--> 278             combined = concat(
    279                 datasets,
    280                 dim=concat_dim,

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/concat.py in concat(objs, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs)
    240             f"objects, got {type(first_obj)}"
    241         )
--> 242     return f(
    243         objs, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs
    244     )

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/concat.py in _dataset_concat(datasets, dim, data_vars, coords, compat, positions, fill_value, join, combine_attrs)
    438     datasets = [ds.copy() for ds in datasets]
    439     datasets = list(
--> 440         align(*datasets, join=join, copy=False, exclude=[dim], fill_value=fill_value)
    441     )
    442 

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/alignment.py in align(join, copy, indexes, exclude, fill_value, *objects)
    356             new_obj = obj.copy(deep=copy)
    357         else:
--> 358             new_obj = obj.reindex(
    359                 copy=copy, fill_value=fill_value, indexers=valid_indexers
    360             )

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/dataset.py in reindex(self, indexers, method, tolerance, copy, fill_value, **indexers_kwargs)
   2907 
   2908         """
-> 2909         return self._reindex(
   2910             indexers,
   2911             method,

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/dataset.py in _reindex(self, indexers, method, tolerance, copy, fill_value, sparse, **indexers_kwargs)
   2936             raise ValueError(f"invalid reindex dimensions: {bad_dims}")
   2937 
-> 2938         variables, indexes = alignment.reindex_variables(
   2939             self.variables,
   2940             self.sizes,

~/miniconda3/envs/py3_std_maps/lib/python3.9/site-packages/xarray/core/alignment.py in reindex_variables(variables, sizes, indexes, indexers, method, tolerance, copy, fill_value, sparse)
    568 
    569             if not index.is_unique:
--> 570                 raise ValueError(
    571                     f"cannot reindex or align along dimension {dim!r} because the "
    572                     "index has duplicate values"

ValueError: cannot reindex or align along dimension 'time' because the index has duplicate values 

I tried doing a few individual files and trying open_mf_dataset and there was no issues. What might be the error? Is there a way to drop_duplicates when using open_mfdataset?

[pecos27]

Resolved! Turns out that some data was empty (specific feature to the type of data)

[TomNicholas]

Glad you found a solution!

Is there a way to drop_duplicates when using open_mfdataset?

You could call .drop_duplicates in a function that you pass to the preprocess kwarg of open_mfdataset.

[pecos27]

Thank you! Nice to know for next time :)