ready for aws cloudformation testing

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: '^docs/|/migrations/|devcontainer.json'
+exclude: '^docs/|/migrations/|devcontainer.json|/aws_cloudform.json|^rds_redis_ec2_config.yml'
 default_stages: [commit]
 
 default_language_version:

check_spot_termination.py

@@ -0,0 +1,31 @@
+import time
+from subprocess import call
+
+import requests
+
+
+def check_spot_termination():
+    """Check for Spot Instance termination notice."""
+    termination_url = "http://169.254.169.254/latest/meta-data/spot/instance-action"
+    try:
+        response = requests.get(termination_url, timeout=2)
+        if response.status_code == 200:
+            return True
+    except requests.exceptions.RequestException:
+        pass
+    return False
+
+
+def graceful_shutdown():
+    """Initiate a graceful shutdown of the Celery worker."""
+    # Replace `celery_worker_name` with the actual name of your worker
+    call(["pkill", "-9", "celery"])
+
+
+if __name__ == "__main__":
+    while True:
+        if check_spot_termination():
+            print("Spot Instance termination notice detected. Initiating graceful shutdown.")
+            graceful_shutdown()
+            break
+        time.sleep(30)  # Check every 30 seconds

compose/aws_services/celery_worker/Dockerfile

@@ -27,4 +27,4 @@ COPY . /usr/src/app/
 COPY ./compose/production/django/celery/worker/start /start-celeryworker
 RUN chmod +x /start-celeryworker
 
-CMD ["/start-celeryworker"]
+CMD ["/start-celeryworker"]

compose/aws_services/django/entrypoint

@@ -7,7 +7,7 @@ set -o nounset
 
 
 # N.B. If only .env files supported variable expansion...
-export CELERY_BROKER_URL="${REDIS_URL}"
+export CELERY_BROKER_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
 
 
 if [ -z "${POSTGRES_USER}" ]; then

compose/local/django/celery/worker/start

@@ -4,4 +4,4 @@ set -o errexit
 set -o nounset
 
 
-exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker -l INFO'
+exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker -l INFO --concurrency=3'

compose/production/django/entrypoint

@@ -7,7 +7,7 @@ set -o nounset
 
 
 # N.B. If only .env files supported variable expansion...
-export CELERY_BROKER_URL="${REDIS_URL}"
+export CELERY_BROKER_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
 
 
 if [ -z "${POSTGRES_USER}" ]; then

compose/production/django/entrypoint_modified

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# N.B. If only .env files supported variable expansion...
+export CELERY_BROKER_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0"
+
+if [ -z "${POSTGRES_USER}" ]; then
+    base_postgres_image_default_user='postgres'
+    export POSTGRES_USER="${base_postgres_image_default_user}"
+fi
+export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
+
+python << END
+import sys
+import time
+
+import psycopg
+
+def database_exists(conn_params, dbname):
+    with psycopg.connect(**conn_params) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (dbname,))
+            return cur.fetchone() is not None
+
+def create_database(conn_params, dbname):
+    with psycopg.connect(**conn_params) as conn:
+        conn.autocommit = True
+        with conn.cursor() as cur:
+            cur.execute(f"CREATE DATABASE \"{dbname}\"")
+
+conn_params = {
+    "dbname": "postgres",  # connect to the default database to check/create
+    "user": "${POSTGRES_USER}",
+    "password": "${POSTGRES_PASSWORD}",
+    "host": "${POSTGRES_HOST}",
+    "port": "${POSTGRES_PORT}"
+}
+
+dbname = "${POSTGRES_DB}"
+
+if not database_exists(conn_params, dbname):
+    print("Database does not exist. Creating database: {}".format(dbname))
+    create_database(conn_params, dbname)
+else:
+    print("Database {} already exists.".format(dbname))
+
+# Now connect to the target database
+conn_params["dbname"] = dbname
+with psycopg.connect(**conn_params) as conn:
+    print('Connected to the database successfully')
+
+END
+
+>&2 echo 'PostgreSQL is available'
+
+exec "$@"

config/aws_cloudform.json

@@ -553,4 +553,4 @@
 			}
 		}
 	}
-}
+}

config/settings/production.py

@@ -1,13 +1,15 @@
+import os
+
 from .base import *  # noqa
 from .base import env
-import os
 
 # GENERAL
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
 SECRET_KEY = env("DJANGO_SECRET_KEY")
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["example.com"])
+REDIS_URL = "redis://" + env("REDIS_HOST", "redis") + ":" + env("REDIS_PORT", "6379") + "/0"
 
 # DATABASES
 # ------------------------------------------------------------------------------
@@ -18,7 +20,7 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": env("REDIS_URL"),
+        "LOCATION": REDIS_URL,
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             # Mimicing memcache behavior.

docs/conf.py

@@ -12,6 +12,7 @@
 
 import os
 import sys
+
 import django
 
 if os.getenv("READTHEDOCS", default=False) == "True":
@@ -20,8 +21,10 @@
     os.environ["USE_DOCKER"] = "no"
 else:
     sys.path.insert(0, os.path.abspath("/app"))
+
+REDIS_URL = "redis://" + os.getenv("REDIS_HOST", "redis") + ":" + os.getenv("REDIS_PORT", "6379") + "/0"
 os.environ["DATABASE_URL"] = "sqlite:///readthedocs.db"
-os.environ["CELERY_BROKER_URL"] = os.getenv("REDIS_URL", "redis://redis:6379")
+os.environ["CELERY_BROKER_URL"] = REDIS_URL
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
 django.setup()
 

docs/model_diagrams/Binding.json

@@ -0,0 +1,89 @@
+{
+  "model_name": "Binding",
+  "fields": [
+    {
+      "name": "id",
+      "type": "BigAutoField"
+    },
+    {
+      "name": "upload_date",
+      "type": "DateField"
+    },
+    {
+      "name": "modified_date",
+      "type": "DateTimeField"
+    },
+    {
+      "name": "batch",
+      "type": "CharField"
+    },
+    {
+      "name": "replicate",
+      "type": "PositiveIntegerField"
+    },
+    {
+      "name": "source_orig_id",
+      "type": "CharField"
+    },
+    {
+      "name": "strain",
+      "type": "CharField"
+    },
+    {
+      "name": "condition",
+      "type": "CharField"
+    },
+    {
+      "name": "file",
+      "type": "FileField"
+    },
+    {
+      "name": "genomic_inserts",
+      "type": "PositiveIntegerField"
+    },
+    {
+      "name": "mito_inserts",
+      "type": "PositiveIntegerField"
+    },
+    {
+      "name": "plasmid_inserts",
+      "type": "PositiveIntegerField"
+    },
+    {
+      "name": "notes",
+      "type": "CharField"
+    }
+  ],
+  "relationships": [
+    {
+      "name": "bindingmanualqc",
+      "related_model": "BindingManualQC",
+      "type": "UnknownRelation"
+    },
+    {
+      "name": "promotersetsig",
+      "related_model": "PromoterSetSig",
+      "type": "UnknownRelation"
+    },
+    {
+      "name": "uploader",
+      "related_model": "User",
+      "type": "ForeignKey"
+    },
+    {
+      "name": "modifier",
+      "related_model": "User",
+      "type": "ForeignKey"
+    },
+    {
+      "name": "regulator",
+      "related_model": "Regulator",
+      "type": "ForeignKey"
+    },
+    {
+      "name": "source",
+      "related_model": "DataSource",
+      "type": "ForeignKey"
+    }
+  ]
+}

docs/model_diagrams/BindingManualQC.json

@@ -0,0 +1,54 @@
+{
+  "model_name": "BindingManualQC",
+  "fields": [
+    {
+      "name": "id",
+      "type": "BigAutoField"
+    },
+    {
+      "name": "upload_date",
+      "type": "DateField"
+    },
+    {
+      "name": "modified_date",
+      "type": "DateTimeField"
+    },
+    {
+      "name": "best_datatype",
+      "type": "CharField"
+    },
+    {
+      "name": "data_usable",
+      "type": "CharField"
+    },
+    {
+      "name": "passing_replicate",
+      "type": "CharField"
+    },
+    {
+      "name": "rank_recall",
+      "type": "CharField"
+    },
+    {
+      "name": "notes",
+      "type": "CharField"
+    }
+  ],
+  "relationships": [
+    {
+      "name": "uploader",
+      "related_model": "User",
+      "type": "ForeignKey"
+    },
+    {
+      "name": "modifier",
+      "related_model": "User",
+      "type": "ForeignKey"
+    },
+    {
+      "name": "binding",
+      "related_model": "Binding",
+      "type": "ForeignKey"
+    }
+  ]
+}