Merge remote-tracking branch 'origin/convert_to_nwb' into convert_to_nwb

Author: bendichter

element_array_ephys/__init__.py

@@ -1,16 +1,2 @@
-import datajoint as dj
-import logging
-import os
-
-
-dj.config['enable_python_native_blobs'] = True
-
-
-def get_logger(name):
-    log = logging.getLogger(name)
-    log.setLevel(os.getenv('LOGLEVEL', 'INFO'))
-    return log
-
-
 # ephys_acute as default
-import element_array_ephys.ephys_acute as ephys
+import element_array_ephys.ephys_acute as ephys

element_array_ephys/ephys_acute.py

@@ -57,9 +57,9 @@ def activate(ephys_schema_name, probe_schema_name=None, *, create_schema=True,
 
 def get_ephys_root_data_dir() -> list:
     """
-    All data paths, directories in DataJoint Elements are recommended to be
-    stored as relative paths, with respect to some user-configured "root"
-    directory, which varies from machine to machine (e.g. different mounted
+    All data paths, directories in DataJoint Elements are recommended to be 
+    stored as relative paths, with respect to some user-configured "root" 
+    directory, which varies from machine to machine (e.g. different mounted 
     drive locations)
 
     get_ephys_root_data_dir() -> list
@@ -130,7 +130,9 @@ class EphysRecording(dj.Imported):
     ---
     -> probe.ElectrodeConfig
     -> AcquisitionSoftware
-    sampling_rate: float # (Hz) 
+    sampling_rate: float # (Hz)
+    recording_datetime: datetime # datetime of the recording from this probe
+    recording_duration: float # (seconds) duration of the recording from this probe
     """
 
     class EphysFile(dj.Part):
@@ -142,7 +144,7 @@ class EphysFile(dj.Part):
 
     def make(self, key):
 
-        session_dir = find_full_path(get_ephys_root_data_dir(),
+        session_dir = find_full_path(get_ephys_root_data_dir(), 
                                      get_session_directory(key))
 
         inserted_probe_serial_number = (ProbeInsertion * probe.Probe & key).fetch1('probe')
@@ -189,9 +191,12 @@ def make(self, key):
             self.insert1({**key,
                           **generate_electrode_config(probe_type, electrode_group_members),
                           'acq_software': acq_software,
-                          'sampling_rate': spikeglx_meta.meta['imSampRate']})
+                          'sampling_rate': spikeglx_meta.meta['imSampRate'],
+                          'recording_datetime': spikeglx_meta.recording_time,
+                          'recording_duration': (spikeglx_meta.recording_duration
+                                        or spikeglx.retrieve_recording_duration(meta_filepath))})
 
-            root_dir = find_root_directory(get_ephys_root_data_dir(),
+            root_dir = find_root_directory(get_ephys_root_data_dir(), 
                                            meta_filepath)
             self.EphysFile.insert1({
                 **key,
@@ -223,7 +228,9 @@ def make(self, key):
             self.insert1({**key,
                           **generate_electrode_config(probe_type, electrode_group_members),
                           'acq_software': acq_software,
-                          'sampling_rate': probe_data.ap_meta['sample_rate']})
+                          'sampling_rate': probe_data.ap_meta['sample_rate'],
+                          'recording_datetime': probe_data.recording_info['recording_datetimes'][0],
+                          'recording_duration': np.sum(probe_data.recording_info['recording_durations'])})
 
             root_dir = find_root_directory(get_ephys_root_data_dir(),
                 probe_data.recording_info['recording_files'][0])
@@ -294,8 +301,7 @@ def make(self, key):
                 shank, shank_col, shank_row, _ = spikeglx_recording.apmeta.shankmap['data'][recorded_site]
                 electrode_keys.append(probe_electrodes[(shank, shank_col, shank_row)])
         elif acq_software == 'Open Ephys':
-
-            session_dir = find_full_path(get_ephys_root_data_dir(),
+            session_dir = find_full_path(get_ephys_root_data_dir(), 
                                          get_session_directory(key))
 
             loaded_oe = openephys.OpenEphys(session_dir)
@@ -457,9 +463,8 @@ class Curation(dj.Manual):
 
     def create1_from_clustering_task(self, key, curation_note=''):
         """
-        A function to create a new corresponding "Curation" for a particular
-        "ClusteringTask", which assumes that no curation was performed on the
-        dataset
+        A function to create a new corresponding "Curation" for a particular 
+        "ClusteringTask"
         """
         if key not in Clustering():
             raise ValueError(f'No corresponding entry in Clustering available'
@@ -473,9 +478,9 @@ def create1_from_clustering_task(self, key, curation_note=''):
         # Synthesize curation_id
         curation_id = dj.U().aggr(self & key, n='ifnull(max(curation_id)+1,1)').fetch1('n')
         self.insert1({**key, 'curation_id': curation_id,
-                      'curation_time': creation_time,
+                      'curation_time': creation_time, 
                       'curation_output_dir': output_dir,
-                      'quality_control': is_qc,
+                      'quality_control': is_qc, 
                       'manual_curation': is_curated,
                       'curation_note': curation_note})
 
@@ -623,7 +628,7 @@ def yield_unit_waveforms():
                 spikeglx_meta_filepath = get_spikeglx_meta_filepath(key)
                 neuropixels_recording = spikeglx.SpikeGLX(spikeglx_meta_filepath.parent)
             elif acq_software == 'Open Ephys':
-                session_dir = find_full_path(get_ephys_root_data_dir(),
+                session_dir = find_full_path(get_ephys_root_data_dir(), 
                                              get_session_directory(key))
                 openephys_dataset = openephys.OpenEphys(session_dir)
                 neuropixels_recording = openephys_dataset.probes[probe_serial_number]
@@ -670,7 +675,7 @@ def get_spikeglx_meta_filepath(ephys_recording_key):
     except FileNotFoundError:
         # if not found, search in session_dir again
         if not spikeglx_meta_filepath.exists():
-            session_dir = find_full_path(get_ephys_root_data_dir(),
+            session_dir = find_full_path(get_ephys_root_data_dir(), 
                                          get_session_directory(
                                              ephys_recording_key))
             inserted_probe_serial_number = (ProbeInsertion * probe.Probe
@@ -709,7 +714,7 @@ def get_neuropixels_channel2electrode_map(ephys_recording_key, acq_software):
             for recorded_site, (shank, shank_col, shank_row, _) in enumerate(
                 spikeglx_meta.shankmap['data'])}
     elif acq_software == 'Open Ephys':
-        session_dir = find_full_path(get_ephys_root_data_dir(),
+        session_dir = find_full_path(get_ephys_root_data_dir(), 
                                      get_session_directory(ephys_recording_key))
         openephys_dataset = openephys.OpenEphys(session_dir)
         probe_serial_number = (ProbeInsertion & ephys_recording_key).fetch1('probe')

element_array_ephys/ephys_chronic.py

@@ -132,7 +132,9 @@ class EphysRecording(dj.Imported):
     ---
     -> probe.ElectrodeConfig
     -> AcquisitionSoftware
-    sampling_rate: float # (Hz) 
+    sampling_rate: float # (Hz)
+    recording_datetime: datetime # datetime of the recording from this probe
+    recording_duration: float # (seconds) duration of the recording from this probe
     """
 
     class EphysFile(dj.Part):
@@ -168,7 +170,8 @@ def make(self, key):
                     break
             else:
                 raise FileNotFoundError(
-                    'No SpikeGLX data found for probe insertion: {}'.format(key))
+                    f'No SpikeGLX data found for probe insertion: {key}' + 
+                    ' The probe serial number does not match.')
 
             if re.search('(1.0|2.0)', spikeglx_meta.probe_model):
                 probe_type = spikeglx_meta.probe_model
@@ -190,7 +193,10 @@ def make(self, key):
             self.insert1({**key,
                           **generate_electrode_config(probe_type, electrode_group_members),
                           'acq_software': acq_software,
-                          'sampling_rate': spikeglx_meta.meta['imSampRate']})
+                          'sampling_rate': spikeglx_meta.meta['imSampRate'],
+                          'recording_datetime': spikeglx_meta.recording_time,
+                          'recording_duration': (spikeglx_meta.recording_duration
+                                        or spikeglx.retrieve_recording_duration(meta_filepath))})
 
             root_dir = find_root_directory(get_ephys_root_data_dir(), 
                                            meta_filepath)
@@ -224,7 +230,9 @@ def make(self, key):
             self.insert1({**key,
                           **generate_electrode_config(probe_type, electrode_group_members),
                           'acq_software': acq_software,
-                          'sampling_rate': probe_data.ap_meta['sample_rate']})
+                          'sampling_rate': probe_data.ap_meta['sample_rate'],
+                          'recording_datetime': probe_data.recording_info['recording_datetimes'][0],
+                          'recording_duration': np.sum(probe_data.recording_info['recording_durations'])})
 
             root_dir = find_root_directory(get_ephys_root_data_dir(),
                                 probe_data.recording_info['recording_files'][0])

element_array_ephys/export/nwb/nwb.py

@@ -14,11 +14,11 @@
 from spikeinterface import extractors
 from tqdm import tqdm
 
-from ... import probe, ephys_acute, ephys_chronic, ephys_no_curation
+from ... import probe, ephys_acute
 
 assert probe.schema.is_activated(), 'probe not yet activated'
 
-for ephys in (ephys_acute):
+for ephys in (ephys_acute,):
     if ephys.schema.is_activated():
         break
 else:
@@ -206,7 +206,7 @@ def create_units_table(
     )
 
     for unit in tqdm(
-        (clustering_query @ ephys.CuratedClustering.Unit).fetch(as_dict=True),
+        (ephys.CuratedClustering.Unit & clustering_query).fetch(as_dict=True),
         desc=f"creating units table for paramset {paramset_record['paramset_idx']}",
     ):
 

element_array_ephys/probe.py

@@ -153,7 +153,7 @@ class ElectrodeConfig(dj.Lookup):
     electrode_config_hash: uuid  
     ---
     -> ProbeType
-    electrode_config_name: varchar(64)  # user friendly name
+    electrode_config_name: varchar(4000)  # user friendly name
     """
 
     class Electrode(dj.Part):