Merge branch 'run_kilosort' of https://github.com/datajoint/element-array-ephys into rk

Author: CBroz1

element_array_ephys/ephys_acute.py

@@ -407,7 +407,7 @@ def make(self, key):
                                 for key in electrode_query.fetch('KEY')}
 
             electrode_keys.extend(probe_electrodes[channel_idx]
-                                  for channel_idx in oe_probe.lfp_meta['channels_indices'])
+                                  for channel_idx in lfp_channel_ind)
         else:
             raise NotImplementedError(f'LFP extraction from acquisition software'
                                       f' of type {acq_software} is not yet implemented')
@@ -428,7 +428,8 @@ class ClusteringMethod(dj.Lookup):
     clustering_method_desc: varchar(1000)
     """
 
-    contents = [('kilosort2.5', 'kilosort2.5 clustering method'),
+    contents = [('kilosort2', 'kilosort2 clustering method'),
+                ('kilosort2.5', 'kilosort2.5 clustering method'),
                 ('kilosort3', 'kilosort3 clustering method')]
 
 

element_array_ephys/ephys_chronic.py

@@ -4,11 +4,15 @@
 import numpy as np
 import inspect
 import importlib
+from decimal import Decimal
 
 from element_interface.utils import find_root_directory, find_full_path, dict_to_uuid
 
 from .readers import spikeglx, kilosort, openephys
-from . import probe
+from . import probe, get_logger
+
+
+log = get_logger(__name__)
 
 schema = dj.schema()
 
@@ -350,7 +354,7 @@ def make(self, key):
                                 for key in electrode_query.fetch('KEY')}
 
             electrode_keys.extend(probe_electrodes[channel_idx]
-                                  for channel_idx in oe_probe.lfp_meta['channels_indices'])
+                                  for channel_idx in lfp_channel_ind)
         else:
             raise NotImplementedError(f'LFP extraction from acquisition software'
                                       f' of type {acq_software} is not yet implemented')
@@ -371,7 +375,8 @@ class ClusteringMethod(dj.Lookup):
     clustering_method_desc: varchar(1000)
     """
 
-    contents = [('kilosort2.5', 'kilosort2.5 clustering method'),
+    contents = [('kilosort2', 'kilosort2 clustering method'),
+                ('kilosort2.5', 'kilosort2.5 clustering method'),
                 ('kilosort3', 'kilosort3 clustering method')]
 
 

element_array_ephys/ephys_no_curation.py

@@ -406,7 +406,7 @@ def make(self, key):
                                 for key in electrode_query.fetch('KEY')}
 
             electrode_keys.extend(probe_electrodes[channel_idx]
-                                  for channel_idx in oe_probe.lfp_meta['channels_indices'])
+                                  for channel_idx in lfp_channel_ind)
         else:
             raise NotImplementedError(f'LFP extraction from acquisition software'
                                       f' of type {acq_software} is not yet implemented')
@@ -427,7 +427,8 @@ class ClusteringMethod(dj.Lookup):
     clustering_method_desc: varchar(1000)
     """
 
-    contents = [('kilosort2.5', 'kilosort2.5 clustering method'),
+    contents = [('kilosort2', 'kilosort2 clustering method'),
+                ('kilosort2.5', 'kilosort2.5 clustering method'),
                 ('kilosort3', 'kilosort3 clustering method')]
 
 

element_array_ephys/readers/openephys.py

@@ -35,11 +35,18 @@ class OpenEphys:
     def __init__(self, experiment_dir):
         self.session_dir = pathlib.Path(experiment_dir)
 
-        openephys_file = pyopenephys.File(self.session_dir.parent)  # this is on the Record Node level
+        if self.session_dir.name.startswith('recording'):
+            openephys_file = pyopenephys.File(self.session_dir.parent.parent)  # this is on the Record Node level
+            self._is_recording_folder = True
+        else:
+            openephys_file = pyopenephys.File(self.session_dir.parent)  # this is on the Record Node level
+            self._is_recording_folder = False
 
         # extract the "recordings" for this session
         self.experiment = next(experiment for experiment in openephys_file.experiments
-                               if pathlib.Path(experiment.absolute_foldername) == self.session_dir)
+                               if pathlib.Path(experiment.absolute_foldername) == (
+                                   self.session_dir.parent if self._is_recording_folder else self.session_dir)
+                               )
 
         # extract probe data
         self.probes = self.load_probe_data()
@@ -76,26 +83,39 @@ def load_probe_data(self):
                               if isinstance(sigchain['PROCESSOR'], list)
                               else [sigchain['PROCESSOR']])
             for processor in processor_iter:
-                if processor['@pluginName'] in ('Neuropix-PXI', 'Neuropix-3a'):
+                if processor['@pluginName'] in ('Neuropix-3a', 'Neuropix-PXI'):
+                    if 'STREAM' in processor:  # only on version >= 0.6.0
+                        ap_streams = [stream for stream in processor['STREAM'] if not stream['@name'].endswith('LFP')]
+                    else:
+                        ap_streams = None
+
                     if (processor['@pluginName'] == 'Neuropix-3a'
                             or 'NP_PROBE' not in processor['EDITOR']):
                         if isinstance(processor['EDITOR']['PROBE'], dict):
-                            probe = Probe(processor, 0)
-                            probes[probe.probe_SN] = probe
+                            probe_indices = (0,)
                         else:
-                            for probe_index in range(len(processor['EDITOR']['PROBE'])):
-                                probe = Probe(processor, probe_index)
-                                probes[probe.probe_SN] = probe
-                    else:  # Neuropix-PXI
-                        for probe_index in range(len(processor['EDITOR']['NP_PROBE'])):
-                            probe = Probe(processor, probe_index)
-                            probes[probe.probe_SN] = probe
-                        
+                            probe_indices = range(len(processor['EDITOR']['PROBE']))
+                    elif processor['@pluginName'] == 'Neuropix-PXI':
+                        probe_indices = range(len(processor['EDITOR']['NP_PROBE']))
+                    else:
+                        raise NotImplementedError
+                else:  # not a processor for Neuropixels probe
+                    continue
+
+                for probe_index in probe_indices:
+                    probe = Probe(processor, probe_index)
+                    if ap_streams:
+                        probe.probe_info['ap_stream'] = ap_streams[probe_index]
+                    probes[probe.probe_SN] = probe
+
         for probe_index, probe_SN in enumerate(probes):
 
             probe = probes[probe_SN]
 
             for rec in self.experiment.recordings:
+                if self._is_recording_folder and rec.absolute_foldername != self.session_dir:
+                    continue
+
                 assert len(rec._oebin['continuous']) == len(rec.analog_signals), \
                     f'Mismatch in the number of continuous data' \
                     f' - expecting {len(rec._oebin["continuous"])} (from structure.oebin file),' \
@@ -106,17 +126,26 @@ def load_probe_data(self):
                     if continuous_info['source_processor_id'] != probe.processor_id:
                         continue
 
-                    # determine if this is continuous data for AP or LFP
-                    if 'source_processor_sub_idx' in continuous_info:
+                    # determine if this is continuous data for AP or LFP for the current probe
+                    if 'ap_stream' in probe.probe_info:
+                        if probe.probe_info['ap_stream']['@name'].split('-')[0] != continuous_info['stream_name'].split('-')[0]:
+                            continue  # not continuous data for the current probe
+                        match = re.search('-(AP|LFP)$', continuous_info['stream_name'])
+                        if match:
+                            continuous_type = match.groups()[0].lower()
+                        else:
+                            continuous_type = 'ap'
+                    elif 'source_processor_sub_idx' in continuous_info:
                         if continuous_info['source_processor_sub_idx'] == probe_index * 2:  # ap data
                             assert continuous_info['sample_rate'] == analog_signal.sample_rate == 30000
                             continuous_type = 'ap'
                         elif continuous_info['source_processor_sub_idx'] == probe_index * 2 + 1:  # lfp data
                             assert continuous_info['sample_rate'] == analog_signal.sample_rate == 2500
                             continuous_type = 'lfp'
+                        else:
+                            continue  # not continuous data for the current probe
                     else:
-                        match = re.search('-(AP|LFP)$', continuous_info['folder_name'].strip('/'))
-                        continuous_type = match.groups()[0].lower()
+                        raise ValueError(f'Unable to infer type (AP or LFP) for the continuous data from:\n\t{continuous_info}')
 
                     if continuous_type == 'ap':
                         probe.recording_info['recording_count'] += 1
@@ -179,6 +208,15 @@ def __init__(self, processor, probe_index=0):
             self.probe_info = processor['EDITOR']['NP_PROBE'][probe_index]
             self.probe_SN = self.probe_info['@probe_serial_number']
             self.probe_model = _probe_model_name_mapping[self.probe_info['@probe_name']]
+
+            if 'ELECTRODE_XPOS' in self.probe_info:
+                self.probe_info['ELECTRODE_XPOS'] = {int(re.search(r'\d+$', k).group()): int(v)
+                                                     for k, v in self.probe_info.pop('ELECTRODE_XPOS').items()}
+                self.probe_info['ELECTRODE_YPOS'] = {int(re.search(r'\d+$', k).group()): int(v)
+                                                     for k, v in self.probe_info.pop('ELECTRODE_YPOS').items()}
+                self.probe_info['ELECTRODE_SHANK'] = {int(re.search(r'\d+$', k).group()): int(v)
+                                                      for k, v in self.probe_info['CHANNELS'].items()}
+
             self._channels_connected = {int(re.search(r'\d+$', k).group()): 1
                                         for k in self.probe_info.pop('CHANNELS')}