Merge branch 'main' of https://github.com/sidhulyalkar/element-array-ephys

Author: sidhulyalkar

.gitignore

@@ -102,6 +102,7 @@ ENV/
 
 # mkdocs documentation
 docs/site
+docs/src/tutorials/*ipynb
 
 # mypy
 .mypy_cache/
@@ -123,3 +124,6 @@ docker-compose.y*ml
 # vscode settings
 .vscode
 *.code-workspace
+
+# exports/notes
+temp*

.markdownlint.yaml

@@ -0,0 +1,16 @@
+# Markdown Linter configuration for docs
+# https://github.com/DavidAnson/markdownlint
+# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md
+MD009: false # permit trailing spaces
+MD007: false # List indenting - permit 4 spaces
+MD013:
+    line_length: "88" # Line length limits
+    tables: false # disable for tables
+    headings: false # disable for headings
+MD030: false # Number of spaces after a list
+MD033: # HTML elements allowed
+    allowed_elements:
+        - "br"
+MD034: false # Permit bare URLs
+MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling
+MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling

.pre-commit-config.yaml

@@ -0,0 +1,58 @@
+default_stages: [commit, push]
+exclude: (^.github/|^docs/|^images/)
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files # prevent giant files from being committed
+      - id: requirements-txt-fixer
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+        description: Forces to replace line ending by the UNIX 'lf' character.
+
+  # black
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+      - id: black-jupyter
+        args:
+          - --line-length=88
+
+  # isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+        description: Sorts imports in an alphabetical order
+
+  # flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: # arguments to configure flake8
+          # making isort line length compatible with black
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+
+          # these are errors that will be ignored by flake8
+          # https://www.flake8rules.com/rules/{code}.html
+          - "--ignore=E203,E501,W503,W605,E402"
+          # E203 - Colons should not have any space before them.
+          #        Needed for list indexing
+          # E501 - Line lengths are recommended to be no greater than 79 characters.
+          #        Needed as we conform to 88
+          # W503 - Line breaks should occur after the binary operator.
+          #        Needed because not compatible with black
+          # W605 - a backslash-character pair that is not a valid escape sequence now
+          #        generates a DeprecationWarning. This will eventually become a SyntaxError.
+          #        Needed because we use \d as an escape sequence
+          # E402 - Place module level import at the top.
+          #        Needed to prevent circular import error

CHANGELOG.md

@@ -3,11 +3,35 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
  [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
-## [0.2.1] - Unreleased
+## [0.2.3] - Unreleased
 
++ Add - extras_require install options for nwb and development requirement sets
++ Add - mkdocs notebook rendering
++ Add - markdown linting and spellcheck config files, with implementation edits
++ Update - license for 2023
++ Update - blackify previous updates
+
+## [0.2.2] - 2022-01-11
+
++ Bugfix - Revert import order in `__init__.py` to avoid circular import error.
++ Update - `.pre-commit-config.yaml` to disable automatic positioning of import
+  statement at the top.
++ Bugfix - Update docstrings to render API for documentation website.
+
+## [0.2.1] - 2022-01-06
+
++ Add - `build_electrode_layouts` function in `probe.py` to compute the electrode layout
+  for all types of probes.
++ Update - parameterize run_CatGT step from parameters retrieved from
+  `ClusteringParamSet` table
 + Update - clustering step, update duration for "median_subtraction" step
 + Bugfix - handles single probe recording in "Neuropix-PXI" format
 + Update - safeguard in creating/inserting probe types upon probe activation
++ Add - quality control metric dashboard
++ Update & fix docstrings
++ Update - `ephys_report.UnitLevelReport` to add `ephys.ClusterQualityLabel` as a
+  foreign key reference
++ Add - `.pre-commit-config.yaml`
 
 ## [0.2.0] - 2022-10-28
 
@@ -16,15 +40,17 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Add `ephys_no_curation` and routines to trigger spike-sorting analysis
   using Kilosort (2.0, 2.5)
 + Add - mkdocs for Element Documentation
-+ Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the spike sorting analysis.
++ Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the
+  spike sorting analysis.
 
 ## [0.1.4] - 2022-07-11
 
 + Bugfix - Handle case where `spike_depths` data is present.
 
 ## [0.1.3] - 2022-06-16
 
-+ Update - Allow for the `precluster_output_dir` attribute to be nullable when no pre-clustering is performed.
++ Update - Allow for the `precluster_output_dir` attribute to be nullable when no
+  pre-clustering is performed.
 
 ## [0.1.2] - 2022-06-09
 
@@ -37,8 +63,9 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 ## [0.1.0] - 2022-05-26
 
 + Update - Rename module for acute probe insertions from `ephys.py` to `ephys_acute.py`.
-+ Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of `ephys_acute.py`.
-+ Add - Module for chronic probe insertions (`ephys_chronic.py`).  
++ Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of
+  `ephys_acute.py`.
++ Add - Module for chronic probe insertions (`ephys_chronic.py`).
 + Bugfix - Missing `fileTimeSecs` key in SpikeGLX meta file.
 + Update - Move common functions to `element-interface` package.
 + Add - NWB export function
@@ -47,7 +74,6 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - Processing with Kilosort and pyKilosort for Open Ephys and SpikeGLX
 
-
 ## [0.1.0b0] - 2021-05-07
 
 + Update - First beta release
@@ -60,6 +86,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS,
   2.0 - MS
 
+[0.2.3]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.3
+[0.2.2]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.2
 [0.2.1]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.1
 [0.2.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.0
 [0.1.4]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.4

CODE_OF_CONDUCT.md

@@ -1,4 +1,3 @@
-
 # Contributor Covenant Code of Conduct
 
 ## Our Pledge

CONTRIBUTING.md

@@ -1,3 +1,5 @@
 # Contribution Guidelines
 
-This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details.
+This project follows the
+[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/).
+Please reference the link for more full details.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 DataJoint NEURO
+Copyright (c) 2023 DataJoint
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -8,5 +8,5 @@ fully functional pipeline.
 
 ![diagram](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/diagram_flowchart.svg)
 
-Installation and usage instructions can be found at the 
+Installation and usage instructions can be found at the
 [Element documentation](https://datajoint.com/docs/elements/element-array-ephys).

cspell.json

@@ -0,0 +1,156 @@
+// cSpell Settings
+//https://github.com/streetsidesoftware/vscode-spell-checker
+{
+    "version": "0.2", // Version of the setting file.  Always 0.2
+    "language": "en", // language - current active spelling language
+    "enabledLanguageIds": [
+        "markdown", "yaml", "python"
+    ],
+    // flagWords - list of words to be always considered incorrect
+    // This is useful for offensive words and common spelling errors.
+    // For example "hte" should be "the"
+    "flagWords": [],
+    "allowCompoundWords": true,
+    "ignorePaths": [
+        "./element_array_ephys.egg-info/*",
+        "./images/*"
+    ],
+    "words": [
+        "acorr",
+        "aggr",
+        "Alessio",
+        "Andreas",
+        "apmeta",
+        "arange",
+        "asarray",
+        "astype",
+        "autocorrelogram",
+        "Axona",
+        "bbins",
+        "bdist",
+        "Binarize",
+        "Brody",
+        "Buccino",
+        "catgt",
+        "cbar",
+        "cbin",
+        "cdat",
+        "Chans",
+        "chans",
+        "chns",
+        "Clust",
+        "clusterings",
+        "cmap",
+        "correlogram",
+        "correlograms",
+        "curations",
+        "DANDI",
+        "decomp",
+        "DISTRO",
+        "djbase",
+        "dtype",
+        "ecephys",
+        "electrophysiogical",
+        "elif",
+        "Ephys",
+        "gblcar",
+        "gfix",
+        "hdmf",
+        "HHMI",
+        "hstack",
+        "ibllib",
+        "ifnull",
+        "Imax",
+        "IMAX",
+        "imax",
+        "imec",
+        "imread",
+        "imro",
+        "imrotbl",
+        "imshow",
+        "inlinehilite",
+        "ipynb",
+        "ipywidgets",
+        "kcoords",
+        "Klusta",
+        "Kwik",
+        "lfmeta",
+        "linenums",
+        "mdict",
+        "Mesoscale",
+        "mkdocs",
+        "mkdocstrings",
+        "Moser",
+        "mtscomp",
+        "Nchan",
+        "nchan",
+        "ndarray",
+        "ndim",
+        "ndimage",
+        "Neuralynx",
+        "NEURO",
+        "neuroconv",
+        "Neurodata",
+        "Neuropix",
+        "neuropixel",
+        "NeuroPixels",
+        "nwbfile",
+        "NWBHDF",
+        "oebin",
+        "openephys",
+        "openephys",
+        "openpyxl",
+        "phylog",
+        "plotly",
+        "PSTH",
+        "pykilosort",
+        "pymdownx",
+        "pynwb",
+        "pyopenephys",
+        "pyplot",
+        "pytest",
+        "Reimer",
+        "repolarization",
+        "Roboto",
+        "RRID",
+        "Rxiv",
+        "Sasaki",
+        "scipy",
+        "sdist",
+        "Siegle",
+        "sess",
+        "SGLX",
+        "Shen",
+        "Sitonic",
+        "spikeglx",
+        "spkcount",
+        "Stereotaxic",
+        "tcat",
+        "tickvals",
+        "tofile",
+        "Tolias",
+        "tqdm",
+        "vline",
+        "Vmax",
+        "vmax",
+        "xanchor",
+        "xaxes",
+        "xaxis",
+        "xcoords",
+        "xcorr",
+        "xlabel",
+        "xlim",
+        "XPOS",
+        "xtick",
+        "Yatsenko",
+        "yaxes",
+        "yaxis",
+        "ycoord",
+        "ycoords",
+        "ylabel",
+        "ylim",
+        "YPOS",
+        "yref",
+        "yticks"
+    ]
+}

docs/.docker/pip_requirements.txt

@@ -7,4 +7,5 @@ mdx-truly-sane-lists
 mkdocs-gen-files
 mkdocs-literate-nav
 mkdocs-exclude-search
-mkdocs-markdownextradata-plugin
+mkdocs-markdownextradata-plugin
+mkdocs-jupyter

docs/docker-compose.yaml

@@ -28,9 +28,17 @@ services:
       - |
         git config --global --add safe.directory /main
         set -e
+        export ELEMENT_UNDERSCORE=$$(echo $${PACKAGE} | sed 's/element_//g')
+        export ELEMENT_HYPHEN=$$(echo $${ELEMENT_UNDERSCORE} | sed 's/_/-/g')
         export PATCH_VERSION=$$(cat /main/$${PACKAGE}/version.py | grep -oE '\d+\.\d+\.[a-z0-9]+')
+        git clone https://github.com/datajoint/workflow-$${ELEMENT_HYPHEN}.git /main/delete || true
+        if [ -d /main/delete/ ]; then
+          mv /main/delete/workflow_$${ELEMENT_UNDERSCORE} /main/
+          mv /main/delete/notebooks/*ipynb /main/docs/src/tutorials/
+          rm -fR /main/delete
+        fi
         if echo "$${MODE}" | grep -i live &>/dev/null; then
-            mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
+            mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 2>&1 | tee docs/temp_mkdocs.log
         elif echo "$${MODE}" | grep -iE "qa|push" &>/dev/null; then
             echo "INFO::Delete gh-pages branch"
             git branch -D gh-pages || true