reading nwb file w/ extension through matnwb

[namboodirilab]

Hi! I'm trying to read nwb file w/ extension (https://dandiarchive.org/dandiset/000351?pos=1) using matnwb, and I keep getting error like below.

I had no issue in streaming this dataset through pynwb, and it looks like my extension classes are created fine. Can I get any advice?
Thank you so much for help!
Huijeong

[lawrence-mbf]

The data in /acquisition/eventidx_table/event_index does not appear to agree with the specifications provided in the file. In the specifications, VectorData should be in the hdmf-common namespace, but in the event_index fields, it provides hdmf-experimental as its namespace instead.

@bendichter is there some kind of fallback check for pynwb when it encounters something like this? I believe MatNWB assumes that the namespace attribute is correct for the neurodata_type

[namboodirilab]

Sure! this is very long and messy, but here you go

from pynwb import NWBFile, load_namespaces, get_class, NWBHDF5IO, register_class
from pynwb.file import Subject
from pynwb.core import DynamicTable
from hdmf.utils import docval, get_docval,call_docval_func
from datetime import datetime
from dateutil import tz
from scipy.io import loadmat
import functions.load as fl
import pandas as pd
import numpy as np
import os
import re

ns_path_photomtry = "C:\\Users\\Huijeong Jeong\\ndx-photometry-namlab\\spec\\ndx-photometry-namlab.namespace.yaml"
ns_path_eventlog = "C:\\Users\\Huijeong Jeong\\ndx-eventlog-namlab\\spec\\ndx-eventlog-namlab.namespace.yaml"
load_namespaces(ns_path_photomtry)
load_namespaces(ns_path_eventlog)

FiberPhotometrySeries = get_class('FiberPhotometrySeries','ndx-photometry-namlab')
dffSeries = get_class('DffSeries','ndx-photometry-namlab')
Eventlog = get_class('Eventlog','ndx-eventlog-namlab')

@register_class('AnnotatedEventsTable', 'ndx-eventlog-namlab')
class AnnotatedEventsTable(DynamicTable):

    __columns__ = (
        {'name': 'event_index', 'description': 'Index for each event type.'},
        {'name': 'event_description', 'description': 'Description for each event type.'},
        # note that the name 'description' cannot be used because it is already an attribute on VectorData
    )

    @docval({'name': 'description', 'type': str, 'doc': 'Description of what is in this table'},
            {'name': 'name', 'type': str, 'doc': 'Name of this AnnotatedEventsTable',
             'default': 'AnnotatedEventsTable'},
            *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames'))
    def __init__(self, **kwargs):
        call_docval_func(super().__init__, kwargs)

    @docval({'name': 'event_index', 'type': int, 'doc': 'Index for each event type.'},
            {'name': 'event_description', 'type': str, 'doc': 'Description for each event type.'},
            {'name': 'id', 'type': int, 'doc': 'ID for each unit', 'default': None},
            allow_extra=True)
    def add_event_type(self, **kwargs):
        """Add an event type as a row to this table."""
        super().add_row(**kwargs)


def find_files(pattern, path):
    import fnmatch
    result = []
    days = []
    for root, dirs, files in os.walk(path):
        for name in files:
            if isinstance(pattern, list):
                for ip in pattern:
                    if fnmatch.fnmatch(name, ip):
                        result.append(os.path.join(root, name))
                        if 'Day' in root:
                            days.append(int(re.split('Day|_',os.path.basename(root))[1]))
                        else:
                            days.append(np.nan)
            else:
                if fnmatch.fnmatch(name, pattern):
                    result.append(os.path.join(root, name))
                    if 'Day' in root:
                        days.append(int(re.split('Day|_', os.path.basename(root))[1]))
                    else:
                        days.append(np.nan)
    result = [x for _,x in sorted(zip(days,result))]
    days = days.sort()
    return result, days

eventidx_list = {0:'session end',1:'Lick1 onset',2:'Lick1 offset',3:'Lick2 onset',4:'Lick2 offset',5:'Lick3 onset',
                 6:'Lick3 offset',7:'Background solenoid',8:'Fixed solenoid 1',9:'Fixed solenoid 2',10:'Fixed solenoid 3',
                 11:'Fixed solenoid 4',12:'Lick retract solenoid 1',13:'Lick retract solenoid',14:'Vacuum/trial end',
                 15:'Sound 1',16:'Sound 2',17:'Sound 3',18:'Sound 4',21:'Light 1',22:'Light 2',23:'Light 3',24:'Light 4',
                 31:'Laser',35:'Lick retract solenoid 1 and 2'}
ttlidentity_list = {'RandomRewards':'rewards','Pavlovian':'cues'}

path = 'D:\OneDrive - University of California, San Francisco\Huijeong'
savedir = 'D:\\OneDrive - University of California, San Francisco\\nwbfile\\'

ppddatanames = ['analog_1','analog_2','digital_1','digital_2']
doricdatanames = ['AIn-1 - Dem (AOut-1)', 'AIn-1 - Dem (AOut-2)', 'DI--O-1','DI--O-2']
datanames_new = ['405', '470', 'ttl1', 'ttl2']
doricversion = 5

session_file,_ = find_files('sessions.csv',path)
session_data = pd.read_csv(session_file[0])

recording_area = 'NAcc'
sensor = 'dLight1.3b'
experimenter = 'Huijeong'
folderList = ['Pavlovian']

animal_name = ['HJ_FP_WT_stGtACR_M3']

for ia in animal_name:
    animal_idx = [i for i, x in enumerate(session_data['Animal ID']==ia) if x]
    date_birth = session_data['DOB'][animal_idx[0]].split('/')
    date_birth = datetime(int(date_birth[-1]),int(date_birth[0]),int(date_birth[1]))

    if not os.path.exists(os.path.join(savedir,ia)):
        os.mkdir(os.path.join(savedir,ia))

    for ifolder in folderList:
        photometry_files,_ = find_files(['*.ppd','*.doric'],os.path.join(path,ia,ifolder))

        for iF in photometry_files[4:]:
            try:
                dfffile,_ = find_files('*.p',os.path.dirname(iF))
                if np.logical_and('out' in iF, len(dfffile)==0):
                    continue
                behfile,_ = find_files(ia+'*.mat',os.path.dirname(iF))
                file_create_time = datetime.fromtimestamp(os.path.getctime(iF),tz =tz.gettz('US/Pacific'))

                # general information of nwbfile
                tasktype = os.path.split(iF)[0].split('\\')[-2]
                nwbfile = NWBFile(
                    # task description; folder name - should be the description of task
                    session_description = tasktype,
                    # recording method, recording area, sensor,animal name
                    identifier = 'photometry-'+recording_area+'-'+sensor+'-'+ia,
                    # time when behavior (mat) file was saved
                    session_start_time = file_create_time,
                    # session number ('Day')
                    session_id = os.path.split(iF)[0].split('\\')[-1]+'_'+tasktype,
                    lab = 'Namboodiri lab',
                    institution = 'University of California, San Francisco'
                )

                # information of subject
                pob = file_create_time.date()-date_birth.date()
                nwbfile.subject = Subject(
                    subject_id = ia, # animal name
                    age = 'P'+str(pob.days)+'D', # POB at surgery date
                    description = 'serial number '+str(animal_idx[0]), # serial number on sheet
                    species = 'Mus musculus', # what they recommended... we can do just mouse though
                    sex = session_data['Sex'][animal_idx[0]],
                    genotype = session_data['Genotype'][animal_idx[0]]
                )

                # photometry raw data
                if '.ppd' in iF:
                    photometry_data = fl.load_ppd(iF, ppddatanames, datanames_new)
                elif '.doric' in iF:
                    photometry_data = fl.load_doric(iF, doricversion, doricdatanames, datanames_new)

                time_raw = [x/1000 for x in photometry_data['time'].tolist()] # raw timestamps from photometry system (s)
                resolution = np.nanmean(np.diff(time_raw))
                data = np.vstack((photometry_data['405'],photometry_data['470'])).transpose().tolist()
                ttl = np.vstack((photometry_data['ttl1'],photometry_data['ttl2'])).transpose().tolist()

                virus = session_data['Volume of injection (including dilution and stock number) at placement INCLUDE COORDINATES'][animal_idx[0]]
                coordinate = session_data['Placement of lens or fiber INCLUDE COORDINATES'][animal_idx[0]]
                fp = FiberPhotometrySeries('photometry',data,'mV',timestamps=time_raw,
                                           ttl=ttl,resolution=resolution,data_identity='raw405,raw470',
                                           ttl_identity='session start/stop,'+ttlidentity_list[tasktype],virus=virus,coordinate=coordinate,
                                           optic_fiber_diameter='400um',laser_intensity='40uw')
                nwbfile.add_acquisition(fp)

                # photometry dff data & timestamps aligned to behavior timestamps
                dff_data = fl.load_pickle(dfffile[0])
                dff_data = dff_data[0]
                time_sync = [x/1000 for x in dff_data['time'].tolist()] # timestamps corrected to the arduino timestamps (s)
                dffseries = dffSeries('dff',dff_data['dff'].flatten(),unit='%',
                                      description='calculated dff and timestamps aligned to behavior timestamps',
                                      timestamps=time_sync,resolution=np.nanmean(np.diff(time_sync)))
                dff_module = nwbfile.create_processing_module(name='photometry',description='photometry')
                dff_module.add(dffseries)

                # behavior data
                behdata = loadmat(behfile[0])
                # table of event indices and their descriptions
                eventidx_table = AnnotatedEventsTable(name='eventidx_table',description='event index table')
                [eventidx_table.add_event_type(event_index=x,event_description=eventidx_list[x]) for x in np.unique(behdata['eventlog'][:,0])]
                eventlog = Eventlog('eventlog',eventtime=behdata['eventlog'][:,1]/1000,eventtime__unit='seconds',
                                    eventindex=behdata['eventlog'][:,0],nonsolenoidflag=behdata['eventlog'][:,2])
                nwbfile.add_acquisition(eventlog)
                nwbfile.add_acquisition(eventidx_table)

                filename = os.path.join(savedir,ia+'\\'+ia+'_'+os.path.basename(os.path.dirname(iF))+'_'+tasktype+'.nwb')

                with NWBHDF5IO(filename,'w') as io:
                    io.write(nwbfile)
                    io.close()
                print(filename + ' saved!\n')
            except:
                print(iF + ' failed!\n')

[namboodirilab]

The part I save eventtable, which makes an error in matnwb, comes after comment '# behavior data'. Also, again I didn't have any issue in saving and importing these files using pynwb, but I only get an error when I use matnwb to open them

[HuijeongJeong]

Hi @rly ! Did you happen to find any issue in the code above? We're in process of publishing data and codes for a paper, so we are hoping to fix things before publication... Please let me know if you need anything else from me!! Thanks again for your help!

[bendichter]

I think this is a version issue. Could you try uninstalling pynwb and hdmf and then reinstalling them?

pip uninstall pynwb hdmf
pip install pynwb

then run your conversion code again?

[HuijeongJeong]

Hi! thanks for the reply! hmm I tried to run my conversion code again after uninstalling and reinstalling everything, but sill have the same issue -- no issue in creating/loading in python using pynwb, but I can't load it in matlab w/ the same error that I had in original posting.
This is what my environment looks like