Merge branch 'datajoint:main' into main

Author: ttngu207

.github/workflows/u24_element_release_call.yml

@@ -17,7 +17,6 @@ jobs:
     secrets:
       TWINE_USERNAME: ${{secrets.TWINE_TEST_USERNAME}}
       TWINE_PASSWORD: ${{secrets.TWINE_TEST_PASSWORD}}
-      GOOGLE_ANALYTICS_KEY: ${{secrets.GOOGLE_ANALYTICS_KEY}}
   call_u24_elements_release_alpine:
     if: >-
       github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' && !contains(github.event.workflow_run.head_branch, 'test')
@@ -27,5 +26,4 @@ jobs:
     secrets:
       TWINE_USERNAME: ${{secrets.TWINE_USERNAME}}
       TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}}
-      GOOGLE_ANALYTICS_KEY: ${{secrets.GOOGLE_ANALYTICS_KEY}}
 

CHANGELOG.md

@@ -3,6 +3,35 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
  [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.2.10] - 2023-05-26
+
++ Add - Kilosort, NWB, and DANDI citations
++ Fix - CSS to improve readability of tables in dark mode
++ Update - mkdocs.yaml
+
+## [0.2.9] - 2023-05-11
+
++ Fix - `.ipynb` dark mode output for all notebooks.
+
+## [0.2.8] - 2023-04-28
+
++ Fix - `.ipynb` output in tutorials is not visible in dark mode.
+
+## [0.2.7] - 2023-04-19
+
++ Bugfix - A name remapping dictionary was added to ensure consistency between the column names of the `metrics.csv` file and the attribute names of the `QualityMetrics` table
+
+## [0.2.6] - 2023-04-17
+
++ Fix - Update Pandas DataFrame column name to insert `pt_ratio` in `QualityMetrics.Waveform` table
+
+## [0.2.5] - 2023-04-12
+
++ Add - docstrings for quality metric tables
++ Fix - docstring errors
++ Update - `concepts.md`
++ Update - schema diagrams with quality metrics tables
+
 ## [0.2.4] - 2023-03-10
 
 + Update - Requirements with `ipywidgets` and `scikit-image` for plotting widget
@@ -90,6 +119,12 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS,
   2.0 - MS
 
+[0.2.10]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.10
+[0.2.9]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.9
+[0.2.8]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.8
+[0.2.7]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.7
+[0.2.6]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.6
+[0.2.5]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.5
 [0.2.4]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.4
 [0.2.3]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.3
 [0.2.2]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.2

docs/.docker/pip_requirements.txt

@@ -8,4 +8,5 @@ mkdocs-gen-files
 mkdocs-literate-nav
 mkdocs-exclude-search
 mkdocs-markdownextradata-plugin
-mkdocs-jupyter
+mkdocs-jupyter
+mkdocs-section-index

docs/docker-compose.yaml

@@ -14,7 +14,6 @@ services:
       - PACKAGE
       - UPSTREAM_REPO
       - MODE
-      - GOOGLE_ANALYTICS_KEY
       - PATCH_VERSION
     volumes:
       - ../docs:/main/docs

docs/mkdocs.yaml

@@ -1,4 +1,4 @@
-# ---------------------- PROJECT SPECIFIC ---------------------------
+--- # ---------------------- PROJECT SPECIFIC ---------------------------
 
 site_name: DataJoint Documentation
 site_url: http://localhost/docs/elements/element-array-ephys
@@ -7,19 +7,20 @@ repo_name: datajoint/element-array-ephys
 nav:
   - Element Array Ephys: index.md
   - Concepts: concepts.md
-  - Tutorials: 
-    - Overview: tutorials/index.md
-    - Data Download: tutorials/00-data-download-optional.ipynb
-    - Configure: tutorials/01-configure.ipynb
-    - Workflow Structure: tutorials/02-workflow-structure-optional.ipynb
-    - Process: tutorials/03-process.ipynb
-    - Automate: tutorials/04-automate-optional.ipynb
-    - Explore: tutorials/05-explore.ipynb
-    - Drop: tutorials/06-drop-optional.ipynb
-    - Downstream Analysis: tutorials/07-downstream-analysis.ipynb
-    - Visualizations: tutorials/10-data_visualization.ipynb
-    - Electrode Localization: tutorials/08-electrode-localization.ipynb
-    - NWB Export: tutorials/09-NWB-export.ipynb
+  - Tutorials:
+      - Overview: tutorials/index.md
+      - Data Download: tutorials/00-data-download-optional.ipynb
+      - Configure: tutorials/01-configure.ipynb
+      - Workflow Structure: tutorials/02-workflow-structure-optional.ipynb
+      - Process: tutorials/03-process.ipynb
+      - Automate: tutorials/04-automate-optional.ipynb
+      - Explore: tutorials/05-explore.ipynb
+      - Drop: tutorials/06-drop-optional.ipynb
+      - Downstream Analysis: tutorials/07-downstream-analysis.ipynb
+      - Visualizations: tutorials/10-data_visualization.ipynb
+      - Electrode Localization: tutorials/08-electrode-localization.ipynb
+      - NWB Export: tutorials/09-NWB-export.ipynb
+      - Quality Metrics: tutorials/quality_metrics.ipynb
   - Citation: citation.md
   - API: api/ # defer to gen-files + literate-nav
   - Changelog: changelog.md
@@ -35,7 +36,7 @@ nav:
 # 02. Instead of designating codeblocks with bash, use console. For example..
 #     ```console
 #     cd ../my_dir
-#     ``` 
+#     ```
 # 03. Links across docs should ...
 #     A. Not involve line breaks.
 #     B. Use relative paths to docs in the same repo
@@ -54,19 +55,16 @@ nav:
 #     UPSTREAM_REPO=https://github.com/datajoint/element-{ELEMENT}.git \
 #     HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
 #     ```
-# 02. Site analytics depend on a local environment variable GOOGLE_ANALYTICS_KEY
-#     You can find this in LastPass or declare with any string to suppress errors
-# 03. The API section will pull docstrings.
+# 02. The API section will pull docstrings.
 #     A. Follow google style guide e.g., 
 #        https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
 #        With typing suggestions: https://docs.python.org/3/library/typing.html
 #     B. To pull a specific workflow fork, change ./docs/src/api/make_pages.py#L19
-# 04. To see your fork of the workflow-{element} in this render, change the
+# 03. To see your fork of the workflow-{element} in this render, change the
 #     URL in ./docs/src/api/make_pages.py#L19 to your fork.
-# 05. For redirecting options For redirect options, see 'redirects' below.
-# 06. To deploy this site on your fork, 
+# 04. To deploy this site on your fork, 
 #     A. declare a branch called gh-pages
-#     B. go to the your fork > settings > pages 
+#     B. go to the your fork > settings > pages
 #     C. direct pages to render from the gh-pages branch at root
 #     D. push a tag to your fork with the format test*.*.*
 #
@@ -99,9 +97,6 @@ theme:
 plugins:
   - markdownextradata: {}
   - search
-  # - redirects: # OPTIONAL REDIRECTS
-  #     redirect_maps:
-  #       "index.md": "getting_started.md"
   - mkdocstrings:
       default_handler: python
       handlers:
@@ -112,14 +107,15 @@ plugins:
             line_length: 88
   - gen-files:
       scripts:
-      - ./src/api/make_pages.py
+        - ./src/api/make_pages.py
   - literate-nav:
       nav_file: navigation.md
   - exclude-search:
       exclude:
         - "*/navigation.md"
   - mkdocs-jupyter:
       ignore_h1_titles: True
+  - section-index
 markdown_extensions:
   - attr_list
   - toc:
@@ -141,13 +137,12 @@ markdown_extensions:
   - pymdownx.inlinehilite
   - pymdownx.snippets
   - footnotes
-  
+  - pymdownx.magiclink # Displays bare URLs as links
+  - pymdownx.tasklist: # Renders check boxes in tasks lists
+      custom_checkbox: true
 extra:
   PATCH_VERSION: !ENV PATCH_VERSION
   generator: false # Disable watermark
-  analytics:
-    provider: google
-    property: !ENV GOOGLE_ANALYTICS_KEY
   version:
     provider: mike
   social:
@@ -182,4 +177,4 @@ extra_css:
   - assets/stylesheets/extra.css
 
 extra_javascript:
-  - https://js-na1.hs-scripts.com/23133402.js  # HubSpot chatbot
+  - https://js-na1.hs-scripts.com/23133402.js # HubSpot chatbot

docs/src/.overrides/assets/stylesheets/extra.css

@@ -91,3 +91,22 @@ html a[title="YouTube"].md-social__link svg {
     /* previous/next text */
     /* --md-footer-fg-color: var(--dj-white); */
 }
+
+table {
+    border-collapse: collapse;
+}
+
+tr {
+    border-left: 1px solid var(--dj-black);
+    border-right: 1px solid var(--dj-black);
+}
+
+td, th {
+    border-top: 1px solid var(--dj-black);
+    border-bottom: 1px solid var(--dj-black);
+}
+
+[data-md-color-scheme="slate"] td, th {
+    background-color: var(--dj-white);
+    color: var(--dj-black);
+}

docs/src/citation.md

@@ -1,11 +1,19 @@
 # Citation
 
-If your work uses this Element, please cite the following manuscript and Research
-Resource Identifier (RRID):
+If your work uses the following resources, please cite the respective manuscript and/or Research Resource Identifier (RRID):
 
-+ Yatsenko D, Nguyen T, Shen S, Gunalan K, Turner CA, Guzman R, Sasaki M, Sitonic D,
-  Reimer J, Walker EY, Tolias AS. DataJoint Elements: Data Workflows for
-  Neurophysiology. bioRxiv. 2021 Jan 1. doi: https://doi.org/10.1101/2021.03.30.437358
++ DataJoint Element Array Electrophysiology - Version {{ PATCH_VERSION }}
+  + Yatsenko D, Nguyen T, Shen S, Gunalan K, Turner CA, Guzman R, Sasaki M, Sitonic D,
+    Reimer J, Walker EY, Tolias AS. DataJoint Elements: Data Workflows for
+    Neurophysiology. bioRxiv. 2021 Jan 1. doi: https://doi.org/10.1101/2021.03.30.437358
 
-+ DataJoint Elements ([RRID:SCR_021894](https://scicrunch.org/resolver/SCR_021894)) -
-  Element Array Electrophysiology (version {{ PATCH_VERSION }})
+  + [RRID:SCR_021894](https://scicrunch.org/resolver/SCR_021894)
+
++ Kilosort
+  + [Manuscripts](https://github.com/MouseLand/Kilosort#citation-requirement)
+
++ NWB
+  + [Manuscript](https://www.nwb.org/publications/)
+
++ DANDI
+  + [Citation options](https://www.dandiarchive.org/handbook/10_using_dandi/#citing-dandi)

docs/src/concepts.md

@@ -107,6 +107,10 @@ is a table within the Element or a table connected to the Element.
 
 ![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/attached_array_ephys_element_precluster.svg)
 
+### `ephys_no_curation` module
+
+![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/attached_array_ephys_element_no_curation.svg)
+
 ### `subject` schema ([API docs](https://datajoint.com/docs/elements/element-animal/api/element_animal/subject))
 
 Although not required, most choose to connect the `Session` table to a `Subject` table.
@@ -156,12 +160,15 @@ Tables for storing probe or unit-level visualization results.
 | --- | --- |
 | ProbeLevelReport | A table to store drift map figures generated from each recording probe. |
 | UnitLevelReport | A table to store figures (waveforms, autocorrelogram, peak waveform + neighbors) generated for each unit. |
+| QualityMetricCutoffs | A table to store cut-off values for cluster quality metrics. |
+| QualityMetricSet | A manual table to match a set of cluster quality metric values with desired cut-offs. |
+| QualityMetricReport | A table to store quality metric figures. |
 
 ## Element Development
 
 Through our interviews and direct collaboration on the precursor projects, we identified
 the common motifs to create the 
-[Array ElectrophysiologyElement](https://github.com/datajoint/element-array-ephys).
+[Array Electrophysiology Element](https://github.com/datajoint/element-array-ephys).
 
 Major features of the Array Electrophysiology Element include:
 

element_array_ephys/ephys_acute.py

@@ -1353,14 +1353,20 @@ def make(self, key):
         kilosort_dir = find_full_path(get_ephys_root_data_dir(), output_dir)
 
         metric_fp = kilosort_dir / "metrics.csv"
+        rename_dict = {
+            "isi_viol": "isi_violation",
+            "num_viol": "number_violation",
+            "contam_rate": "contamination_rate",
+        }
 
         if not metric_fp.exists():
             raise FileNotFoundError(f"QC metrics file not found: {metric_fp}")
 
         metrics_df = pd.read_csv(metric_fp)
         metrics_df.set_index("cluster_id", inplace=True)
         metrics_df.replace([np.inf, -np.inf], np.nan, inplace=True)
-
+        metrics_df.columns = metrics_df.columns.str.lower()
+        metrics_df.rename(columns=rename_dict, inplace=True)
         metrics_list = [
             dict(metrics_df.loc[unit_key["unit"]], **unit_key)
             for unit_key in (CuratedClustering.Unit & key).fetch("KEY")

element_array_ephys/ephys_chronic.py

@@ -1282,14 +1282,20 @@ def make(self, key):
         kilosort_dir = find_full_path(get_ephys_root_data_dir(), output_dir)
 
         metric_fp = kilosort_dir / "metrics.csv"
+        rename_dict = {
+            "isi_viol": "isi_violation",
+            "num_viol": "number_violation",
+            "contam_rate": "contamination_rate",
+        }
 
         if not metric_fp.exists():
             raise FileNotFoundError(f"QC metrics file not found: {metric_fp}")
 
         metrics_df = pd.read_csv(metric_fp)
         metrics_df.set_index("cluster_id", inplace=True)
         metrics_df.replace([np.inf, -np.inf], np.nan, inplace=True)
-
+        metrics_df.columns = metrics_df.columns.str.lower()
+        metrics_df.rename(columns=rename_dict, inplace=True)
         metrics_list = [
             dict(metrics_df.loc[unit_key["unit"]], **unit_key)
             for unit_key in (CuratedClustering.Unit & key).fetch("KEY")

element_array_ephys/ephys_no_curation.py

@@ -937,9 +937,9 @@ class Unit(dj.Part):
 
         Attributes:
             CuratedClustering (foreign key): CuratedClustering primary key.
-            unit (foreign key, int): Unique integer identifying a single unit.
-            probe.ElectrodeConfig.Electrode (dict): probe.ElectrodeConfig.Electrode primary key.
-            ClusteringQualityLabel (dict): CLusteringQualityLabel primary key.
+            unit (int): Unique integer identifying a single unit.
+            probe.ElectrodeConfig.Electrode (foreign key): probe.ElectrodeConfig.Electrode primary key.
+            ClusteringQualityLabel (foreign key): CLusteringQualityLabel primary key.
             spike_count (int): Number of spikes in this recording for this unit.
             spike_times (longblob): Spike times of this unit, relative to start time of EphysRecording.
             spike_sites (longblob): Array of electrode associated with each spike.
@@ -1293,14 +1293,20 @@ def make(self, key):
         kilosort_dir = find_full_path(get_ephys_root_data_dir(), output_dir)
 
         metric_fp = kilosort_dir / "metrics.csv"
+        rename_dict = {
+            "isi_viol": "isi_violation",
+            "num_viol": "number_violation",
+            "contam_rate": "contamination_rate",
+        }
 
         if not metric_fp.exists():
             raise FileNotFoundError(f"QC metrics file not found: {metric_fp}")
 
         metrics_df = pd.read_csv(metric_fp)
         metrics_df.set_index("cluster_id", inplace=True)
         metrics_df.replace([np.inf, -np.inf], np.nan, inplace=True)
-
+        metrics_df.columns = metrics_df.columns.str.lower()
+        metrics_df.rename(columns=rename_dict, inplace=True)
         metrics_list = [
             dict(metrics_df.loc[unit_key["unit"]], **unit_key)
             for unit_key in (CuratedClustering.Unit & key).fetch("KEY")

element_array_ephys/ephys_precluster.py

@@ -616,7 +616,6 @@ def make(self, key):
                 ][recorded_site]
                 electrode_keys.append(probe_electrodes[(shank, shank_col, shank_row)])
         elif acq_software == "Open Ephys":
-
             session_dir = find_full_path(
                 get_ephys_root_data_dir(), get_session_directory(key)
             )
@@ -1270,13 +1269,20 @@ def make(self, key):
         kilosort_dir = find_full_path(get_ephys_root_data_dir(), output_dir)
 
         metric_fp = kilosort_dir / "metrics.csv"
+        rename_dict = {
+            "isi_viol": "isi_violation",
+            "num_viol": "number_violation",
+            "contam_rate": "contamination_rate",
+        }
 
         if not metric_fp.exists():
             raise FileNotFoundError(f"QC metrics file not found: {metric_fp}")
 
         metrics_df = pd.read_csv(metric_fp)
         metrics_df.set_index("cluster_id", inplace=True)
-
+        metrics_df.replace([np.inf, -np.inf], np.nan, inplace=True)
+        metrics_df.columns = metrics_df.columns.str.lower()
+        metrics_df.rename(columns=rename_dict, inplace=True)
         metrics_list = [
             dict(metrics_df.loc[unit_key["unit"]], **unit_key)
             for unit_key in (CuratedClustering.Unit & key).fetch("KEY")