dynamic table in extension

[HuijeongJeong]

Hi! I'm trying to have a DynamicTable w/ a ragged array column in our lab extension. I had no issue in generating an extension, and generating nwbfile using my extension. But I'm getting this error when writing nwbfile:

Traceback (most recent call last):
 File "D:\anaconda3\envs\pynwb\lib\site-packages\IPython\core\interactiveshell.py", line 3433, in run_code
   exec(code_obj, self.user_global_ns, self.user_ns)
 File "<ipython-input-32-7348167725cf>", line 1, in <module>
   with NWBHDF5IO(filename,'w') as io:
 File "D:\anaconda3\envs\pynwb\lib\site-packages\hdmf\utils.py", line 583, in func_call
   return func(args[0], **pargs)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\pynwb\__init__.py", line 246, in __init__
   super(NWBHDF5IO, self).__init__(path, manager=manager, mode=mode, file=file_obj, comm=comm, driver=driver)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\hdmf\utils.py", line 583, in func_call
   return func(args[0], **pargs)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 79, in __init__
   super().__init__(manager, source=path)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\hdmf\utils.py", line 583, in func_call
   return func(args[0], **pargs)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\hdmf\backends\io.py", line 22, in __init__
   self.open()
 File "D:\anaconda3\envs\pynwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 774, in open
   self.__file = File(self.source, open_flag, **kwargs)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\h5py\_hl\files.py", line 533, in __init__
   fid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr)
 File "D:\anaconda3\envs\pynwb\lib\site-packages\h5py\_hl\files.py", line 232, in make_fid
   fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
 File "h5py\_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
 File "h5py\_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
 File "h5py\h5f.pyx", line 126, in h5py.h5f.create
OSError: [Errno 0] Unable to create file (unable to lock file, errno = 0, error message = 'No error', Win32 GetLastError() = 33)

This only happens when I include a ragged array column. Can I get any advice?
Thank you!

[bendichter]

Hi @HuijeongJeong!

Thank for including the traceback, but I don't think I can decipher what is going wrong from that. It looks like maybe you are trying to write to an HDF5 file after closing it. I think I need more information to help you. Could you share your lab extension with me? Adding a DynamicTable to a lab extension makes sense, but you shouldn't need to extend DynamicTable in order to add a ragged column. If it's easier, I'd be happy to work through this on a call. You can schedule one with me at calendly.com/ben-dichter

[HuijeongJeong]

Thanks for the quick reply @bendichter!
Yes sure, this is part of 'create_extension_spec.py' of our extension, which I modified the ndx-events. I think the issue comes from ParameterTable.

def main():
    # these arguments were auto-generated from your cookiecutter inputs
    ns_builder = NWBNamespaceBuilder(
        doc="""extension for behavior eventlog""",
        name="""ndx-eventlog-namlab""",
        version="""0.1.0""",
        author=list(map(str.strip, """Huijeong Jeong""".split(','))),
        contact=list(map(str.strip, """huijeong.jeong@ucsf.edu""".split(',')))
    )

    # TODO: specify the neurodata_types that are used by the extension as well
    # as in which namespace they are found.
    # this is similar to specifying the Python modules that need to be imported
    # to use your new data types.
    # all types included or used by the types specified here will also be
    # included.
    ns_builder.include_type('NWBDataInterface', namespace='core')
    ns_builder.include_type('DynamicTable', namespace='core')
    ns_builder.include_type('VectorData', namespace='core')
    ns_builder.include_type('VectorIndex', namespace='core')

    # TODO: define your new data types
    # see https://pynwb.readthedocs.io/en/latest/extensions.html#extending-nwb
    # for more information
    eventtime = NWBDatasetSpec(
        name='eventtime',
        dtype='float32',
        dims=['num_events'],
        shape=[None],
        doc=('Event timestamps, in milliseconds, from arduino; does not match with raw photometry timestamp'),
        attributes=[
            NWBAttributeSpec(
                name='unit',
                dtype='text',
                value='milliseconds',
                doc="Unit of measurement for timestamps, which is fixed to 'milliseconds'.",
            )
        ]
    )

    eventindex = NWBDatasetSpec(
        name='eventindex',
        dtype='int',
        dims=['num_events'],
        shape=[None],
        doc=('Event index; see label to know what each index means'),
    )

    nonsolenoidflag = NWBDatasetSpec(
        name='nonsolenoidflag',
        dtype='int',
        dims=['num_events'],
        shape=[None],
        doc=('Indicates solenoid omission: if =1, no solenoid was actually given. Or cue identity: 0=first cue, 1=second cue'),
    )

    event_index = NWBDatasetSpec(
        name='event_index',
        neurodata_type_inc='VectorData',
        dtype='int',
        doc='Index for each event type.'
    )

    event_description = NWBDatasetSpec(
        name='event_description',
        neurodata_type_inc='VectorData',
        dtype='text',
        doc='Description for each event type.'
    )

    eventtable = NWBGroupSpec(
        neurodata_type_def='AnnotatedEventsTable',
        neurodata_type_inc='DynamicTable',
        doc=("Table to hold event label, description for each label. "
             "Each row corresponds to a different event type."),
        datasets=[event_index, event_description],
        #datasets=[index_col, description_col,eventtimes, nonsolenoidflags],
    )

    parameter_description = NWBDatasetSpec(
        name='parameter_description',
        neurodata_type_inc='VectorData',
        dtype='text',
        doc='Description for each event type.'
    )

    parameter = NWBDatasetSpec(
        name='parameter',
        neurodata_type_inc='VectorData',
        dtype='int',
        doc='values for each parameter.',
    )

    parameter_index = NWBDatasetSpec(
        name='parameter_index',
        neurodata_type_inc='VectorIndex',
        doc='values for each parameter.',
    )

    parameters = NWBGroupSpec(
        neurodata_type_def='ParameterTable',
        neurodata_type_inc='DynamicTable',
        doc=("Table to hold task parameters."
             "Each row corresponds to a different parameter."),
        datasets=[parameter_description, parameter_index, parameter],
    )

    eventlog = NWBGroupSpec(
        neurodata_type_def='Eventlog',
        neurodata_type_inc='NWBDataInterface',
        doc='A list of timestamp, index, and nonsolenoidflag of events',
        quantity = '*',
        datasets=[eventtime, eventindex, nonsolenoidflag]
    )


    # TODO: add all of your new data types to this list
    new_data_types = [eventlog, parameters, eventtable]`


Also, this is how I register classes:

`Eventlog = get_class('Eventlog','ndx-eventlog-namlab')

@register_class('AnnotatedEventsTable', 'ndx-eventlog-namlab')
class AnnotatedEventsTable(DynamicTable):

    __columns__ = (
        {'name': 'event_index', 'description': 'Index for each event type.'},
        {'name': 'event_description', 'description': 'Description for each event type.'},
        # note that the name 'description' cannot be used because it is already an attribute on VectorData
    )

    @docval({'name': 'description', 'type': str, 'doc': 'Description of what is in this table'},
            {'name': 'name', 'type': str, 'doc': 'Name of this AnnotatedEventsTable',
             'default': 'AnnotatedEventsTable'},
            *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames'))
    def __init__(self, **kwargs):
        call_docval_func(super().__init__, kwargs)

    @docval({'name': 'event_index', 'type': int, 'doc': 'Index for each event type.'},
            {'name': 'event_description', 'type': str, 'doc': 'Description for each event type.'},
            {'name': 'id', 'type': int, 'doc': 'ID for each unit', 'default': None},
            allow_extra=True)
    def add_event_type(self, **kwargs):
        """Add an event type as a row to this table."""
        super().add_row(**kwargs)

@register_class('ParameterTable', 'ndx-eventlog-namlab')
class ParameterTable(DynamicTable):

    __columns__ = (
        {'name': 'parameter_description', 'description': 'Description for each parameter.'},
        {'name': 'parameter', 'description': 'Parameters of task.','index':True},
        # note that the name 'description' cannot be used because it is already an attribute on VectorData
    )

    @docval({'name': 'description', 'type': str, 'doc': 'Description of what is in this table'},
            {'name': 'name', 'type': str, 'doc': 'Name of this ParameterTable',
             'default': 'ParameterTable'},
            *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames'))
    def __init__(self, **kwargs):
        call_docval_func(super().__init__, kwargs)

    @docval({'name': 'parameter_description', 'type': str, 'doc': 'Description for each parameter.'},
            {'name': 'parameter', 'type': 'array_data', 'doc': 'Parameters of task.','shape':(None,)},
            {'name': 'id', 'type': int, 'doc': 'ID for each unit', 'default': None},
            allow_extra=True)
    def add_parameter(self, **kwargs):
        """Add an event type as a row to this table."""
        super().add_row(**kwargs)