Merge pull request #1 from ttngu207/main

initial clean up, conforming to Works deployment

.github/workflows/codebook-profile-build-caller.yaml

@@ -0,0 +1,56 @@
+name: codebook-profile-build
+on:
+  workflow_dispatch:
+    inputs:
+      jhub_ver:
+        description: "JupyterHub version"
+        required: true
+        default: "4.0.2"
+      py_ver:
+        description: "Python version"
+        required: true
+        default: "3.10"
+      worker_type:
+        description: "Use worker image as codebook profile image"
+        required: true
+        type: choice
+        default: "standard"
+        options:
+          - standard
+
+jobs:
+  get_meta:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: get_version
+        name: Get build metadata
+        run: |
+          ## Get workflow version as codebook env image version
+          echo "workflow_version=$(cat $(find . -name "version.py" | head -n 1) | grep -oP '\d+\.\d+\.[a-z0-9]+')" >> $GITHUB_OUTPUT
+    outputs:
+      workflow_version: ${{steps.get_version.outputs.workflow_version}}
+  call_sciops_docker_image_debian:
+    needs: [get_meta]
+    uses: dj-sciops/.github/.github/workflows/codebook-profile-build.yaml@main
+    # uses: yambottle/dj-sciops.github/.github/workflows/codebook-profile-build.yaml@main
+    with:
+      jhub_ver: ${{ inputs.jhub_ver }}
+      py_ver: ${{ inputs.py_ver }}
+      worker_type: ${{ inputs.worker_type }}
+      workflow_version: ${{needs.get_meta.outputs.workflow_version}}
+      runner_timeout: 180 # default, optional
+    secrets:
+      RUNNER_PAT: ${{secrets.RUNNER_PAT}}
+      RUNNER_REGION: ${{secrets.RUNNER_REGION}}
+      RUNNER_TYPE: ${{secrets.RUNNER_TYPE}}
+      RUNNER_AMI_ID: ${{secrets.RUNNER_AMI_ID}}
+      RUNNER_SUBNET_ID: ${{secrets.RUNNER_SUBNET_ID}}
+      RUNNER_SG_ID: ${{secrets.RUNNER_SG_ID}}
+      RUNNER_AWS_ACCESS_KEY_ID: ${{secrets.RUNNER_AWS_ACCESS_KEY_ID}}
+      RUNNER_AWS_SECRET_ACCESS_KEY: ${{secrets.RUNNER_AWS_SECRET_ACCESS_KEY}}
+      DEPLOY_SSH_KEY_BASE64: ${{secrets.DEPLOY_SSH_KEY_BASE64}}
+      DOCKER_REGISTRY_HOST: ${{secrets.DOCKER_REGISTRY_HOST}}
+      DOCKER_REGISTRY_REPO: ${{secrets.DOCKER_REGISTRY_REPO}}
+      DOCKER_REGISTRY_USERNAME: ${{secrets.DOCKER_REGISTRY_USERNAME}}
+      DOCKER_REGISTRY_PASSWORD: ${{secrets.DOCKER_REGISTRY_PASSWORD}}

.github/workflows/worker-build-caller.yaml

@@ -0,0 +1,64 @@
+name: worker-build
+on:
+  workflow_dispatch:
+    inputs:
+      worker_type:
+        description: "Worker Type"
+        required: true
+        type: choice
+        options:
+          - standard
+          - spike_processing
+          - spike_sorting
+          - dlc
+      staging:
+        description: "Staging"
+        required: true
+        default: stable
+        type: choice
+        options:
+          - unstable
+          - stable
+run-name: ${{ inputs.worker_type}}-${{ inputs.staging }}-worker-build
+
+jobs:
+  get_meta:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: get_version
+        name: Get build metadata
+        run: |
+          ## Get workflow version as codebook env image version
+          echo "workflow_version=$(cat $(find . -name "version.py" | head -n 1) | grep -oP '\d+\.\d+\.[a-z0-9]+')" >> $GITHUB_OUTPUT
+    outputs:
+      workflow_version: ${{steps.get_version.outputs.workflow_version}}
+  call-worker-ami-build:
+    needs: [get_meta]
+    uses: dj-sciops/.github/.github/workflows/worker-ami-build-1.0.0.yaml@main
+    # uses: yambottle/dj-sciops.github/.github/workflows/worker-ami-build.yaml@main
+    with:
+      worker_type: ${{github.event.inputs.worker_type}}
+      workflow_version: ${{needs.get_meta.outputs.workflow_version}}
+      staging: ${{github.event.inputs.staging}}
+      runner_timeout: 180 # default, optional
+    secrets:
+      RUNNER_PAT: ${{secrets.RUNNER_PAT}}
+      RUNNER_REGION: ${{secrets.RUNNER_REGION}}
+      RUNNER_TYPE: ${{secrets.RUNNER_TYPE}}
+      RUNNER_AMI_ID: ${{secrets.RUNNER_AMI_ID}}
+      RUNNER_SUBNET_ID: ${{secrets.RUNNER_SUBNET_ID}}
+      RUNNER_SG_ID: ${{secrets.RUNNER_SG_ID}}
+      RUNNER_AWS_ACCESS_KEY_ID: ${{secrets.RUNNER_AWS_ACCESS_KEY_ID}}
+      RUNNER_AWS_SECRET_ACCESS_KEY: ${{secrets.RUNNER_AWS_SECRET_ACCESS_KEY}}
+      DJ_HOST: ${{secrets.DJ_HOST}}
+      DJ_USER: ${{secrets.DJ_USER}}
+      DJ_PASS: ${{secrets.DJ_PASS}}
+      BUILD_PAT: ${{secrets.BUILD_PAT}}
+      BUILD_AWS_ACCESS_KEY_ID: ${{secrets.BUILD_AWS_ACCESS_KEY_ID}}
+      BUILD_AWS_SECRET_ACCESS_KEY: ${{secrets.BUILD_AWS_SECRET_ACCESS_KEY}}
+      DEPLOY_SSH_KEY_BASE64: ${{secrets.DEPLOY_SSH_KEY_BASE64}}
+      DOCKER_REGISTRY_HOST: ${{secrets.DOCKER_REGISTRY_HOST}}
+      DOCKER_REGISTRY_REPO: ${{secrets.DOCKER_REGISTRY_REPO}}
+      DOCKER_REGISTRY_USERNAME: ${{secrets.DOCKER_REGISTRY_USERNAME}}
+      DOCKER_REGISTRY_PASSWORD: ${{secrets.DOCKER_REGISTRY_PASSWORD}}

.gitignore

@@ -28,6 +28,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+.idea/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -129,3 +130,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# datajoint
+dj_local_c*.json
+

docker/standard_worker/dist/debian/docker-compose-standard_worker.yaml

@@ -0,0 +1,39 @@
+# docker compose --env-file=../../.env  -f docker-compose-standard_worker.yaml -p microns_trippy_standard build
+services:
+  standard_worker:
+    build:
+      # only necessary if rebuilding image
+      context: .
+      dockerfile: standard_worker.Dockerfile
+      args:
+        - PY_VER=${PY_VER:-3.9}
+        - DEPLOY_KEY
+        - REPO_OWNER
+        - REPO_NAME
+        - REPO_BRANCH=${REPO_BRANCH:-main}
+    image: ${DOCKER_IMAGE_TAG}
+    environment:
+      - DJ_HOST
+      - DJ_USER
+      - DJ_PASS
+      - DATABASE_PREFIX
+      - AWS_ACCESS_KEY
+      - AWS_ACCESS_SECRET
+      - RAW_ROOT_DATA_DIR=${S3_ROOT_MOUNTPOINT:-./s3_root}/inbox
+      - PROCESSED_ROOT_DATA_DIR=${OUTBOX_MOUNTPOINT:-./outbox}
+      - WORKER_MAX_IDLED_CYCLE
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - /dev/fuse
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - ${OUTBOX_MOUNTPOINT:-./outbox}:${OUTBOX_MOUNTPOINT:-./outbox}
+      - ${S3_ROOT_MOUNTPOINT:-./s3_root}:${S3_ROOT_MOUNTPOINT:-./s3_root}
+    scale: ${WORKER_COUNT:-1}
+    command:
+      - /bin/bash
+      - -c
+      - |
+        run_workflow standard_worker

docker/standard_worker/dist/debian/standard_worker.Dockerfile

@@ -0,0 +1,31 @@
+ARG PY_VER
+FROM jupyter/docker-stacks-foundation:python-${PY_VER:-3.9}
+# FROM mambaorg/micromamba:debian12-slim
+
+USER root
+RUN apt update && \
+   apt install -y ssh git graphviz && \
+   apt-get update && apt-get install ffmpeg libsm6 libxext6 -y && \
+   pip install --upgrade pip && \
+   pip install gateway_provisioners && \
+   jupyter image-bootstrap install --languages python && \
+   chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \
+   chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \
+   chown -R jovyan:users /usr/local/bin/kernel-launchers && \
+   mamba install -n base -y -c conda-forge graphviz && \
+   mamba clean --all -y
+CMD /usr/local/bin/bootstrap-kernel.sh
+WORKDIR $HOME
+USER jovyan
+
+ARG DEPLOY_KEY
+COPY --chown=jovyan $DEPLOY_KEY $HOME/.ssh/id_ed25519
+RUN chmod u=r,g-rwx,o-rwx $HOME/.ssh/id_ed25519 && \
+    ssh-keyscan github.com >> $HOME/.ssh/known_hosts
+
+ARG REPO_OWNER
+ARG REPO_NAME
+ARG REPO_BRANCH
+
+RUN git clone -b ${REPO_BRANCH} [email protected]:${REPO_OWNER}/${REPO_NAME}.git --recurse-submodules
+RUN pip install -e ./${REPO_NAME}

microns_phase3/__init__.py

@@ -1,3 +1,5 @@
+import os
+
 from .version import __version__
 
 import datajoint as dj
@@ -9,4 +11,17 @@
 # try:
 #     import coregister.solve as cs
 # except ModuleNotFoundError:
-#     raise ModuleNotFoundError('Coregistration package missing. Run "pip3 install git+https://github.com/AllenInstitute/em_coregistration.git@phase3"') 
+#     raise ModuleNotFoundError('Coregistration package missing. Run "pip3 install git+https://github.com/AllenInstitute/em_coregistration.git@phase3"')
+
+
+for key in ("custom", "stores"):
+    if key not in dj.config:
+        dj.config[key] = {}
+
+# overwrite dj.config['custom'] values with environment variables if available
+
+dj.config["custom"]["database.prefix"] = os.getenv(
+    "DATABASE_PREFIX", dj.config["custom"].get("database.prefix", "")
+)
+
+DB_PREFIX: str = dj.config["custom"].get("database.prefix", "")

microns_phase3/nda.py

@@ -8,14 +8,15 @@
 from scipy.interpolate import interp1d
 
 import datajoint as dj
+from microns_phase3 import DB_PREFIX
 
 # # BCM specific export schemas
 # from stimulus import stimulus
 # from stimline import tune
 # from pipeline import meso, experiment, stack
 # m65p3 = dj.create_virtual_module('microns_minnie65_02','microns_minnie65_02')
 
-schema = dj.schema('microns_phase3_nda', create_tables=True)
+schema = dj.schema(DB_PREFIX + 'phase3_nda', create_tables=True)
 # schema.spawn_missing_classes()
 
 params = {'ignore_extra_fields':True,'skip_duplicates':True}

microns_phase3/utils.py

@@ -7,7 +7,8 @@
 from coregister.transform.transform import Transform
 
 import datajoint as dj
-nda = dj.create_virtual_module('microns_phase3_nda','microns_phase3_nda')
+from microns_phase3 import DB_PREFIX
+nda = dj.create_virtual_module('phase3_nda', DB_PREFIX + 'phase3_nda')
 
 def get_grid(field_key, desired_res=1):
     """ Get registered grid for this registration. """

requirements.txt

@@ -7,4 +7,7 @@ matplotlib==3.7.1
 nglui==2.14.2
 caveclient==5.7.0
 em_coregistration@git+https://github.com/AllenInstitute/em_coregistration@phase3#egg=em_coregistration
-torch==1.9.0
+torch==1.9.0
+cryptography==44.0.0
+seaborn
+ipykernel