Merge branch 'main' of https://github.com/sidhulyalkar/element-array-ephys

Author: sidhulyalkar

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,39 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'bug'
+assignees: ''
+
+---
+
+## Bug Report
+
+### Description
+
+A clear and concise description of what is the overall operation that is intended to be
+performed that resulted in an error.
+
+### Reproducibility
+Include:
+- OS (WIN | MACOS | Linux)
+- DataJoint Element Version
+- MySQL Version
+- MySQL Deployment Strategy (local-native | local-docker | remote)
+- Minimum number of steps to reliably reproduce the issue
+- Complete error stack as a result of evaluating the above steps
+
+### Expected Behavior
+A clear and concise description of what you expected to happen.
+
+### Screenshots
+If applicable, add screenshots to help explain your problem.
+
+### Additional Research and Context
+Add any additional research or context that was conducted in creating this report.
+
+For example:
+- Related GitHub issues and PR's either within this repository or in other relevant
+  repositories.
+- Specific links to specific lines or a focus within source code.
+- Relevant summary of Maintainers development meetings, milestones, projects, etc.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: DataJoint Contribution Guideline
+    url: https://docs.datajoint.org/python/community/02-Contribute.html
+    about: Please make sure to review the DataJoint Contribution Guidelines

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,57 @@
+---
+name: Feature request
+about: Suggest an idea for a new feature
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+
+## Feature Request
+
+### Problem
+
+A clear and concise description how this idea has manifested and the context. Elaborate
+on the need for this feature and/or what could be improved. Ex. I'm always frustrated
+when [...]
+
+### Requirements
+
+A clear and concise description of the requirements to satisfy the new feature. Detail
+what you expect from a successful implementation of the feature. Ex. When using this
+feature, it should [...]
+
+### Justification
+
+Provide the key benefits in making this a supported feature. Ex. Adding support for this
+feature would ensure [...]
+
+### Alternative Considerations
+
+Do you currently have a work-around for this? Provide any alternative solutions or
+features you've considered.
+
+### Related Errors
+Add any errors as a direct result of not exposing this feature.
+
+Please include steps to reproduce provided errors as follows:
+- OS (WIN | MACOS | Linux)
+- DataJoint Element Version
+- MySQL Version
+- MySQL Deployment Strategy (local-native | local-docker | remote)
+- Minimum number of steps to reliably reproduce the issue
+- Complete error stack as a result of evaluating the above steps
+
+### Screenshots
+If applicable, add screenshots to help explain your feature.
+
+### Additional Research and Context
+Add any additional research or context that was conducted in creating this feature request.
+
+For example:
+- Related GitHub issues and PR's either within this repository or in other relevant
+  repositories.
+- Specific links to specific lines or a focus within source code.
+- Relevant summary of Maintainers development meetings, milestones, projects, etc.
+- Any additional supplemental web references or links that would further justify this
+  feature request.

.github/workflows/development.yaml

@@ -1,9 +1,17 @@
 name: Development
 on:
-  pull_request:
   push:
+    branches:
+      - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
+      - '!stage*' # exclude branches beginning with stage
     tags:
-      - "*.*.*"
+      - '\d+\.\d+\.\d+' # only semver tags
+  pull_request:
+    branches:
+      - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
+      - '!stage*' # exclude branches beginning with stage
 jobs:
   test-changelog:
     runs-on: ubuntu-latest
@@ -73,7 +81,9 @@ jobs:
           path: dist
           retention-days: 1
   publish-release:
-    if: github.event_name == 'push'
+    if: |
+      github.event_name == 'push' &&
+      startsWith(github.ref, 'refs/tags')
     needs: build
     runs-on: ubuntu-latest
     env:
@@ -146,15 +156,22 @@ jobs:
           asset_path: ${{env.PKG_SDIST_PATH}}
           asset_name: pip-${{env.SDIST_PKG_NAME}}-${{env.PKG_VERSION}}.tar.gz
           asset_content_type: application/gzip
-  # Open Source documentation deploy workflow
-  deploy-docs:
-    if: github.event_name == 'push'
+  publish-docs:
+    if: |
+      github.event_name == 'push' &&
+      startsWith(github.ref, 'refs/tags')
     needs: build
     runs-on: ubuntu-latest
+    env:
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.x
-      - run: pip install mkdocs-material mkdocs-redirects mkdocstrings mkdocstrings-python
-      - run: mkdocs gh-deploy --force --config-file ./docs/mkdocs.yml
+      - name: Deploy docs
+        run: |
+          export MODE=BUILD
+          export PACKAGE=element_array_ephys
+          export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
+          export HOST_UID=$(id -u)
+          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
+          git push origin gh-pages

.gitignore

@@ -101,7 +101,7 @@ ENV/
 .ropeproject
 
 # mkdocs documentation
-/site
+docs/site
 
 # mypy
 .mypy_cache/
@@ -115,4 +115,7 @@ dj_local_conf_old.json
 **/#*#
 **/.#*
 
-docker-compose.y*ml
+docker-compose.y*ml
+
+# include
+!docs/docker-compose.yaml

CHANGELOG.md

@@ -3,6 +3,10 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
  [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.1.5] - 2022-07-11
+
++ Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the spike sorting analysis.
+
 ## [0.1.4] - 2022-07-11
 
 + Bugfix - Handle case where `spike_depths` data is present.
@@ -44,10 +48,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Readers for: `SpikeGLX`, `Open Ephys`, `Kilosort`
 + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS, 2.0 - MS
 
-[0.1.4]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.4
-[0.1.3]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.3
-[0.1.2]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.2
-[0.1.1]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.1
+[0.1.4]: https://github.com/datajoint/element-array-ephys/compare/0.1.3...0.1.4
+[0.1.3]: https://github.com/datajoint/element-array-ephys/compare/0.1.2...0.1.3
+[0.1.2]: https://github.com/datajoint/element-array-ephys/compare/0.1.1...0.1.2
+[0.1.1]: https://github.com/datajoint/element-array-ephys/compare/0.1.0...0.1.1
 [0.1.0]: https://github.com/datajoint/element-array-ephys/compare/0.1.0b0...0.1.0
-[0.1.0b0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.0b0
+[0.1.0b0]: https://github.com/datajoint/element-array-ephys/compare/0.1.0a5...0.1.0b0
 [0.1.0a5]: https://github.com/datajoint/element-array-ephys/releases/tag/0.1.0a5

README.md

@@ -3,13 +3,19 @@
 + This repository features DataJoint pipeline design for extracellular array electrophysiology, 
 with Neuropixels probe and Kilosort spike sorting method. 
 
-+ The pipeline presented here is not a complete pipeline by itself, but rather a modular 
-design of tables and dependencies specific to the extracellular electrophysiology workflow. 
++ This module is designed to be integrated into a complete data pipeline and provides
+data tables and computational dependencies specific to extracellular electrophysiology
+processing and analysis.
 
 + This modular pipeline element can be flexibly attached downstream 
 to any particular design of experiment session, thus assembling a fully functional 
 ephys pipeline.
 
++ This Element provides three schema options, selected via the DataJoint config file, with `dj.config['custom']['ephys_mode']`
+  + `acute` probe insertion, with curated clustering
+  + `chronic` probe insertion, with curated clustering
+  + `no-curation`, with kilosort triggered clustering
+
 + See the [Element Array Electrophysiology documentation](https://elements.datajoint.org/description/array_ephys/) for the background information and development timeline.
 
 + For more information on the DataJoint Elements project, please visit https://elements.datajoint.org.  This work is supported by the National Institutes of Health.

docs/.docker/Dockerfile

@@ -0,0 +1,15 @@
+FROM datajoint/miniconda3:4.10.3-py3.9-alpine
+ARG PACKAGE
+WORKDIR /main
+COPY --chown=anaconda:anaconda ./docs/.docker/apk_requirements.txt ${APK_REQUIREMENTS}
+COPY --chown=anaconda:anaconda ./docs/.docker/pip_requirements.txt ${PIP_REQUIREMENTS}
+RUN \
+    /entrypoint.sh echo "Dependencies installed" && \
+    git config --global user.name "GitHub Action" && \
+    git config --global user.email "action@github.com"&& \
+    git config --global pull.rebase false && \
+    git init
+COPY --chown=anaconda:anaconda ./${PACKAGE} /main/${PACKAGE}
+COPY --chown=anaconda:anaconda ./docs/mkdocs.yaml /main/docs/mkdocs.yaml
+COPY --chown=anaconda:anaconda ./docs/src /main/docs/src
+COPY --chown=anaconda:anaconda ./CHANGELOG.md /main/docs/src/about/changelog.md

docs/.docker/apk_requirements.txt

@@ -0,0 +1 @@
+git

docs/.docker/pip_requirements.txt

@@ -0,0 +1,9 @@
+mkdocs-material
+mkdocs-redirects
+mkdocstrings
+mkdocstrings-python
+mike
+mdx-truly-sane-lists
+mkdocs-gen-files
+mkdocs-literate-nav
+mkdocs-exclude-search

docs/docker-compose-docs-serve.yaml

@@ -0,0 +1,40 @@
+# MODE="LIVE|QA|BUILD" PACKAGE=element_array_ephys UPSTREAM_REPO=https://github.com/datajoint/element-array-ephys.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
+#
+# navigate to http://localhost/
+version: "2.4"
+services:
+  docs:
+    build:
+      dockerfile: docs/.docker/Dockerfile
+      context: ../
+      args:
+        - PACKAGE
+    image: ${PACKAGE}-docs
+    environment:
+      - PACKAGE
+      - UPSTREAM_REPO
+      - MODE
+    volumes:
+      - ..:/main
+    user: ${HOST_UID}:anaconda
+    ports:
+      - 80:80
+    command:
+      - sh
+      - -c
+      - |
+        set -e
+        if echo "$${MODE}" | grep -i live &>/dev/null; then
+            mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
+        elif echo "$${MODE}" | grep -iE "qa|build" &>/dev/null; then
+            git branch -D gh-pages || true
+            git fetch $${UPSTREAM_REPO} gh-pages:gh-pages || true
+            mike deploy --config-file ./docs/mkdocs.yaml -u $$(grep -oE '\d+\.\d+' /main/$${PACKAGE}/version.py) latest
+            mike set-default --config-file ./docs/mkdocs.yaml latest
+            if echo "$${MODE}" | grep -i qa &>/dev/null; then
+                mike serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
+            fi
+        else
+            echo "Unexpected mode..."
+            exit 1
+        fi