Merge pull request #173 from kushalbakshi/codespace

Add DevContainers + Codespaces tutorial

Author: MilagrosMarin

.devcontainer/Dockerfile

@@ -0,0 +1,48 @@
+FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f
+
+ENV PATH /usr/local/bin:$PATH
+ENV PYTHON_VERSION 3.9.17
+
+RUN \
+    adduser --system --disabled-password --shell /bin/bash vscode && \
+    # install docker
+    apt-get update && \
+    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    mkdir -m 0755 -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
+    usermod -aG docker vscode && \
+    apt-get clean
+
+RUN \
+    # dev setup
+    apt update && \
+    apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
+    usermod -aG sudo vscode && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    pip install --no-cache-dir --upgrade black pip nbconvert && \
+    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
+    apt-get clean
+
+COPY ./ /tmp/element-array-ephys/
+
+RUN \
+    # pipeline dependencies
+    apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \
+    pip install --no-cache-dir -e /tmp/element-array-ephys[elements,tests] && \
+    # clean up
+    rm -rf /tmp/element-array-ephys && \
+    apt-get clean
+
+ENV DJ_HOST fakeservices.datajoint.io
+ENV DJ_USER root
+ENV DJ_PASS simple
+
+ENV EPHYS_ROOT_DATA_DIR /workspaces/element-array-ephys/example_data
+ENV DATABASE_PREFIX neuro_
+
+USER vscode
+CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd"

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+{
+	"name": "Environment + Data",
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+	},
+	"onCreateCommand": "mkdir -p ${EPHYS_ROOT_DATA_DIR} && pip install -e .",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${EPHYS_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
+	"hostRequirements": {
+		"cpus": 4,
+		"memory": "8gb",
+		"storage": "32gb"
+	},
+	"forwardPorts": [
+		3306
+	],
+	"customizations": {
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+		},
+		"vscode": {
+			"extensions": [
+				"ms-python.python@2023.8.0",
+				"ms-toolsai.jupyter@2023.3.1201040234"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+  app:
+    cpus: 4
+    mem_limit: 8g
+    build:
+      context: ..
+      dockerfile: ./.devcontainer/Dockerfile
+    # image: datajoint/element_array_ephys:latest
+    extra_hosts:
+      - fakeservices.datajoint.io:127.0.0.1
+    environment:
+      - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-array-ephys-benchmark/v2
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - ..:/workspaces/element-array-ephys:cached
+      - docker_data:/var/lib/docker # persist docker images
+    privileged: true # only because of dind
+volumes:
+  docker_data:

.github/workflows/release.yaml

@@ -16,12 +16,12 @@ jobs:
     uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
     permissions: 
       contents: write
-  # devcontainer-build:
-  #   uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
-  # devcontainer-publish:
-  #   needs: 
-  #     - devcontainer-build
-  #   uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
-  #   secrets:
-  #     DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
-  #     DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-publish:
+    needs: 
+      - devcontainer-build
+    uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
+    secrets:
+      DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
+      DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}

.github/workflows/test.yaml

@@ -4,8 +4,8 @@ on:
   pull_request:
   workflow_dispatch:
 jobs:
-  # devcontainer-build:
-  #   uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
   tests:
     runs-on: ubuntu-latest
     strategy:

.gitignore

@@ -116,14 +116,15 @@ dj_local_conf_old.json
 **/#*#
 **/.#*
 
-docker-compose.y*ml
 
 # include
 !docs/docker-compose.yaml
 
 # vscode settings
-.vscode
 *.code-workspace
 
 # exports/notes
 temp*
+
+# Codespaces
+example_data/

.markdownlint.yaml

@@ -11,6 +11,8 @@ MD030: false # Number of spaces after a list
 MD033: # HTML elements allowed
     allowed_elements:
         - "br"
+        - "figure"
+        - "figcaption"
 MD034: false # Permit bare URLs
 MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling
 MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "editor.formatOnPaste": false,
+    "editor.formatOnSave": true,
+    "editor.rulers": [
+        88
+    ],
+    "python.formatting.provider": "black",
+    "[python]": {
+        "editor.defaultFormatter": null
+    },
+    "[markdown]": {
+        "editor.defaultFormatter": "disable"
+    },
+    "files.autoSave": "off"
+}

CHANGELOG.md

@@ -3,6 +3,13 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
  [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.3.0] - 2023-10-25
+
++ Add - DevContainer for codespaces
++ Add - `tutorial_pipeline.py`
++ Add - 60 min tutorial using Jupyter Notebooks, short demo notebook
++ Update - General improvements to increase consistency with other DataJoint Elements
+
 ## [0.2.11] - 2023-06-29
 
 + Update - Improve kilosort triggering routine - better logging, remove temporary files, robust resumable processing
@@ -126,6 +133,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS,
   2.0 - MS
 
+[0.3.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.3.0
+[0.2.11]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.11
 [0.2.10]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.10
 [0.2.9]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.9
 [0.2.8]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.8