Update tutorial and codespace

.devcontainer/Dockerfile

@@ -0,0 +1,53 @@
+FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f
+
+ENV PATH /usr/local/bin:$PATH
+ENV PYTHON_VERSION 3.9.17
+
+RUN \
+    adduser --system --disabled-password --shell /bin/bash vscode && \
+    # install docker
+    apt-get update && \
+    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    mkdir -m 0755 -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
+    usermod -aG docker vscode && \
+    apt-get clean
+
+RUN \
+    # dev setup
+    apt update && \
+    apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
+    usermod -aG sudo vscode && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    pip install --no-cache-dir --upgrade black pip nbconvert && \
+    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
+    apt-get clean
+
+COPY ./ /tmp/element-deeplabcut/
+
+RUN \
+    # pipeline dependencies
+    apt-get update && \
+    apt-get install -y gcc ffmpeg graphviz && \
+    pip install --no-cache-dir -e /tmp/element-deeplabcut[elements,dlc_default] && \
+    # clean up
+    rm -rf /tmp/element-deeplabcut/ && \
+    apt-get clean
+
+ENV DJ_HOST fakeservices.datajoint.io
+ENV DJ_USER root
+ENV DJ_PASS simple
+
+ENV DATA_MOUNTPOINT /workspaces/element-deeplabcut/example_data
+ENV DLC_ROOT_DATA_DIR $DATA_MOUNTPOINT/inbox
+ENV DLC_PROCESSED_DATA_DIR $DATA_MOUNTPOINT/outbox
+ENV DATABASE_PREFIX neuro_
+
+USER vscode
+CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd"
+
+ENV LD_LIBRARY_PATH="/lib:/opt/conda/lib"

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+{
+	"name": "Environment + Data",
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+	},
+	"onCreateCommand": "mkdir -p ${DATA_MOUNTPOINT} && pip install -e .",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${DATA_MOUNTPOINT} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
+	"hostRequirements": {
+		"cpus": 4,
+		"memory": "8gb",
+		"storage": "32gb"
+	},
+	"forwardPorts": [
+		3306
+	],
+	"customizations": {
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+		},
+		"vscode": {
+			"extensions": [
+				"[email protected]",
+				"[email protected]"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+  app:
+    cpus: 4
+    mem_limit: 8g
+    build:
+      context: ..
+      dockerfile: ./.devcontainer/Dockerfile
+    #image: datajoint/element_deeplabcut:latest
+    extra_hosts:
+      - fakeservices.datajoint.io:127.0.0.1
+    environment:
+      - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-dlc-data/v2
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - ..:/workspaces/element-deeplabcut:cached
+      - docker_data:/var/lib/docker # persist docker images
+    privileged: true # only because of dind
+volumes:
+  docker_data:

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: DataJoint Contribution Guideline
-    url: https://docs.datajoint.org/python/community/02-Contribute.html
+    url: https://datajoint.com/docs/about/contribute/
     about: Please make sure to review the DataJoint Contribution Guidelines

.github/workflows/release.yaml

@@ -16,3 +16,12 @@ jobs:
     uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
     permissions: 
       contents: write
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-publish:
+    needs: 
+      - devcontainer-build
+    uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
+    secrets:
+      DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
+      DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}

.github/workflows/test.yaml

@@ -6,6 +6,8 @@ on:
   schedule:
     - cron: "0 8 * * 1"
 jobs:
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
   tests:
     runs-on: ubuntu-latest
     strategy:
@@ -31,3 +33,4 @@ jobs:
         run: |
           python_version=${{matrix.py_ver}}
           black element_deeplabcut --check --verbose --target-version py${python_version//.}
+          black notebooks --check --verbose --target-version py${python_version//.}

.gitignore

@@ -1,9 +1,15 @@
+# User data
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 
-# Distribution, packaging, PyInstaller
+# C extensions
+*.so
+
+# Distribution, packaging
 .Python
 env/
 build/
@@ -21,11 +27,17 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+.idea/
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+
+# Installer logs
 pip-log.txt
 pip-delete*.txt
-.idea/
 
 # Unit test / coverage reports
 htmlcov/
@@ -39,54 +51,82 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 
-# C extension, Translations
-*.so
+# Translations
 *.mo
 *.pot
 
-# editors: vscode, emacs, Mac
-.vscode
-**/*~
-**/#*#
-**/.#*
-.DS_Store
-
-# Django, Flask, Scrapy, Sphinx, mkdocs:
-# PyBuilder, Jupyter, SageMath, celery beat
+# Django stuff:
 *.log
 local_settings.py
+
+# Flask stuff:
 instance/
 .webassets-cache
+
+# Scrapy stuff:
 .scrapy
 scratchpaper.*
+
+# Sphinx documentation
 docs/_build/
-/site
+
+# PyBuilder
 target/
+
+# Jupyter Notebook
 .ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
 celerybeat-schedule
+
+# SageMath parsed files
 *.sage.py
 
-# dotenv, virtualenv, pyenv, mypy
-./.env
+# dotenv
+.env
+
+# virtualenv
 .venv
 venv/
 ENV/
-.python-version
-.mypy_cache/
+./.env
 
-# Spyder/Rope project settings
+# Spyder project settings
 .spyderproject
 .spyproject
+
+# Rope project settings
 .ropeproject
 
-# datajoint, notes, nwb export
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# datajoint
 dj_local_c*.json
+dj_*.y*ml
 temp*
 temp/*
-*nwb
-workflow_deeplabcut/
 
 # docs
-/docs/site/
+/docs/site
 /docs/src/tutorials/*ipynb
-/docs/mike-mkdocs*
+
+# emacs
+**/*~
+**/#*#
+**/.#*
+
+example_data
+
+#nwb export
+*nwb
+
+# vscode
+*.code-workspace
+.vscode

.markdownlint.yaml

@@ -3,14 +3,15 @@
 # https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md
 MD009: false # permit trailing spaces
 MD007: false # List indenting - permit 4 spaces
-MD013: 
+MD013:
     line_length: "88" # Line length limits
     tables: false # disable for tables
     headings: false # disable for headings
 MD030: false # Number of spaces after a list
 MD033: # HTML elements allowed
-    allowed_elements: 
-        - "br"
+    allowed_elements:
+        - "figure"
+        - "figcaption"
 MD034: false # Permit bare URLs
 MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling
 MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling

.pre-commit-config.yaml

@@ -0,0 +1,59 @@
+default_stages: [commit, push]
+exclude: (^.github/|^docs/|^images/|^notebooks/|^tests/)
+# Current tests/__init__ violates many flake8. Excluding pending change to conftest
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files # prevent giant files from being committed
+      - id: requirements-txt-fixer
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+        description: Forces to replace line ending by the UNIX 'lf' character.
+
+  # black
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+      - id: black-jupyter
+        args:
+          - --line-length=88
+
+  # isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+        description: Sorts imports in an alphabetical order
+
+  # flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: # arguments to configure flake8
+          # making isort line length compatible with black
+          - "--max-line-length=88"
+          - "--max-complexity=18"
+          - "--select=B,C,E,F,W,T4,B9"
+
+          # these are errors that will be ignored by flake8
+          # https://www.flake8rules.com/rules/{code}.html
+          - "--ignore=E203,E501,W503,W605,E402"
+          # E203 - Colons should not have any space before them.
+          #        Needed for list indexing
+          # E501 - Line lengths are recommended to be no greater than 79 characters.
+          #        Needed as we conform to 88
+          # W503 - Line breaks should occur after the binary operator.
+          #        Needed because not compatible with black
+          # W605 - a backslash-character pair that is not a valid escape sequence now
+          #        generates a DeprecationWarning. This will eventually become a SyntaxError.
+          #        Needed because we use \d as an escape sequence
+          # E402 - Place module level import at the top.
+          #        Needed to prevent circular import error