diff --git a/compose/aws_services/celery_worker/Dockerfile b/compose/aws_services/celery_worker/Dockerfile deleted file mode 100644 index 3603f61..0000000 --- a/compose/aws_services/celery_worker/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ - -# Set environment variables -ENV BUILD_ENVIRONMENT=production -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 - -# Set work directory -WORKDIR /usr/src/app - -# Install system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - libpq-dev \ - && apt-get purge -y --auto-remove \ - && rm -rf /var/lib/apt/lists/* - -# Requirements are installed here to ensure they will be cached. -COPY ./requirements . - -# Create Python Dependency and Sub-Dependency Wheels. -RUN pip wheel --wheel-dir /usr/src/app/wheels \ - -r ${BUILD_ENVIRONMENT}.txt - -# Copy the Django project -COPY . /usr/src/app/ - -# Start script for Celery Worker -COPY ./compose/production/django/celery/worker/start /start-celeryworker -RUN chmod +x /start-celeryworker - -CMD ["/start-celeryworker"] diff --git a/compose/aws_services/django/Dockerfile b/compose/aws_services/django/Dockerfile deleted file mode 100644 index 7155110..0000000 --- a/compose/aws_services/django/Dockerfile +++ /dev/null @@ -1,80 +0,0 @@ - -# define an alias for the specific python version used in this file. -FROM python:3.11.7-slim-bullseye as python - -# Python build stage -FROM python as python-build-stage - -ARG BUILD_ENVIRONMENT=production - -# Install apt packages -RUN apt-get update && apt-get install --no-install-recommends -y \ - # dependencies for building Python packages - build-essential \ - # psycopg2 dependencies - libpq-dev \ - # HTSlib dependencies - zlib1g-dev \ - libbz2-dev \ - liblzma-dev - -# Requirements are installed here to ensure they will be cached. -COPY ./requirements . - -# Create Python Dependency and Sub-Dependency Wheels. -RUN pip wheel --wheel-dir /usr/src/app/wheels \ - -r ${BUILD_ENVIRONMENT}.txt - - -# Python 'run' stage -FROM python as python-run-stage - -ARG BUILD_ENVIRONMENT=production -ARG APP_HOME=/app - -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 -ENV BUILD_ENV ${BUILD_ENVIRONMENT} - -WORKDIR ${APP_HOME} - -RUN addgroup --system django \ - && adduser --system --ingroup django django - - -# Install required system dependencies -RUN apt-get update && apt-get install --no-install-recommends -y \ - # psycopg2 dependencies - libpq-dev \ - # Translations dependencies - gettext \ - # cleaning up unused files - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - && rm -rf /var/lib/apt/lists/* - -# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction -# copy python dependency wheels from python-build-stage -COPY --from=python-build-stage /usr/src/app/wheels /wheels/ - -# use wheels to install python dependencies -RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \ - && rm -rf /wheels/ - -COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint -RUN sed -i 's/\r$//g' /entrypoint -RUN chmod +x /entrypoint - -# copy application code to WORKDIR -COPY --chown=django:django . ${APP_HOME} - -# make django owner of the WORKDIR directory as well. -RUN chown django:django ${APP_HOME} - -USER django - -RUN DATABASE_URL="" \ - CELERY_BROKER_URL="" \ - DJANGO_SETTINGS_MODULE="config.settings.test" \ - python manage.py compilemessages - -ENTRYPOINT ["/entrypoint"] diff --git a/compose/aws_services/django/celery/beat/start b/compose/aws_services/django/celery/beat/start deleted file mode 100644 index 42ddca9..0000000 --- a/compose/aws_services/django/celery/beat/start +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - - -exec celery -A config.celery_app beat -l INFO diff --git a/compose/aws_services/django/celery/flower/start b/compose/aws_services/django/celery/flower/start deleted file mode 100644 index 4180d67..0000000 --- a/compose/aws_services/django/celery/flower/start +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset - - -exec celery \ - -A config.celery_app \ - -b "${CELERY_BROKER_URL}" \ - flower \ - --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" diff --git a/compose/aws_services/django/entrypoint b/compose/aws_services/django/entrypoint deleted file mode 100644 index f15d0b5..0000000 --- a/compose/aws_services/django/entrypoint +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - - - -# N.B. If only .env files supported variable expansion... -export CELERY_BROKER_URL="redis://${REDIS_HOST}:${REDIS_PORT}/0" - - -if [ -z "${POSTGRES_USER}" ]; then - base_postgres_image_default_user='postgres' - export POSTGRES_USER="${base_postgres_image_default_user}" -fi -export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" - -python << END -import sys -import time - -import psycopg - -suggest_unrecoverable_after = 30 -start = time.time() - -while True: - try: - psycopg.connect( - dbname="${POSTGRES_DB}", - user="${POSTGRES_USER}", - password="${POSTGRES_PASSWORD}", - host="${POSTGRES_HOST}", - port="${POSTGRES_PORT}", - ) - break - except psycopg.OperationalError as error: - sys.stderr.write("Waiting for PostgreSQL to become available...\n") - - if time.time() - start > suggest_unrecoverable_after: - sys.stderr.write(" This is taking longer than expected. The following exception may be indicative of an unrecoverable error: '{}'\n".format(error)) - - time.sleep(1) -END - ->&2 echo 'PostgreSQL is available' - -exec "$@" diff --git a/compose/aws_services/django/start b/compose/aws_services/django/start deleted file mode 100644 index 83ffc8f..0000000 --- a/compose/aws_services/django/start +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - - -python /app/manage.py collectstatic --noinput - -exec /usr/local/bin/gunicorn config.asgi --bind 0.0.0.0:5000 --chdir=/app -k uvicorn.workers.UvicornWorker diff --git a/singularity_local.sh b/singularity_local.sh new file mode 100755 index 0000000..fb291fb --- /dev/null +++ b/singularity_local.sh @@ -0,0 +1,311 @@ +#!/bin/bash + +# Function to check service readiness +check_service_ready() { + local service_name=$1 + local check_command=$2 + local counter=0 + + echo "Waiting for $service_name to start..." + while ! eval $check_command; do + if [ "$counter" -lt "$TIMEOUT_SECONDS" ]; then + sleep 1 + ((counter++)) + echo "Checking $service_name... ($counter seconds)" + else + echo "Timeout reached. $service_name did not start within $TIMEOUT_MINUTES minutes." + exit 1 + fi + done + echo "$service_name is up and running." +} + +# Function to start each service +start_service() { + case $1 in + postgres) + singularity exec --bind $yeastregulatorydb_local_postgres_data:/var/lib/postgresql/data \ + --bind $yeastregulatorydb_local_postgres_data_backups:/backups \ + --env-file ./.envs/.local/.postgres $postgres_sif \ + &>./postgres_log.txt & + check_service_ready "PostgreSQL" "pg_isready -h localhost -p 5432" + ;; + redis) + singularity exec $redis_sif redis-server &> redis_log.txt & + check_service_ready "Redis" "redis-cli ping > /dev/null 2>&1" + ;; + django) + singularity exec --bind .:/app \ + --env-file ./.envs/.local/.django \ + --env-file ./.envs/.local/.postgres \ + --env-file ./.envs/.local/.regulatory_data \ + $django_sif /start &> django_log.txt & + check_service_ready "Django app" "curl -s http://localhost:8000 > /dev/null" + ;; + docs) + singularity exec --bind .:/app \ + --env-file ./.envs/.local/.django \ + $docs_sif /start-docs &>docs_log.txt & + check_service_ready "Docs" "echo 'Docs is ready'" + ;; + celeryworker) + singularity exec --bind .:/app \ + --env-file ./.envs/.local/.django \ + --env-file ./.envs/.local/.postgres \ + --env-file ./.envs/.local/.regulatory_data \ + $django_sif /start-celeryworker &> celeryworker_log.txt & + check_service_ready "Celery worker" "echo 'Celery worker is ready'" + ;; + celerybeat) + singularity exec --bind .:/app \ + --env-file ./.envs/.local/.django \ + --env-file ./.envs/.local/.postgres \ + --env-file ./.envs/.local/.regulatory_data \ + $django_sif /start-celerybeat &> celerybeat_log.txt & + check_service_ready "Celery beat" "echo 'Celery beat is ready'" + ;; + celeryflower) + singularity exec --bind .:/app \ + --env-file ./.envs/.local/.django \ + --env-file ./.envs/.local/.postgres \ + --env-file ./.envs/.local/.regulatory_data \ + $django_sif /start-flower &> celeryflower_log.txt & + check_service_ready "Celery flower" "echo 'Celery flower is ready'" + ;; + *) + echo "Unknown service: $1" + ;; + esac +} + +# Start specified services +for service in "${services_to_start[@]}"; do + start_service $service +done + +show_help() { +cat << EOF +Usage: ${0##*/} [OPTIONS]... +Launch Singularity containers for a Django application with optional services. + + -h, --help display this help and exit + -p, --postgres_sif PATH path to the PostgreSQL Singularity image file + -r, --redis_sif PATH path to the Redis Singularity image file + -d, --django_sif PATH path to the Django Singularity image file + -o, --docs_sif PATH path to the Docs Singularity image file + -g, --postgres_data PATH path to the PostgreSQL data directory + -b, --postgres_backup PATH path to the PostgreSQL data backups directory + -t, --timeout MINUTES optional timeout in minutes for service readiness checks (default: 3) + -s, --services "SERVICE1 SERVICE2" optional comma separated, no spaces, list of services to start (e.g., postgres,redis,django) + +Examples: + ${0##*/} -p /path/to/postgres.sif -r /path/to/redis.sif -d /path/to/django.sif -o /path/to/docs.sif \\ + -g /path/to/postgres_data -b /path/to/postgres_backup -t 3 -s "postgres redis django" +EOF +} + +# Initialize variables +postgres_sif="" +redis_sif="" +django_sif="" +docs_sif="" +postgres_data="" +postgres_backup="" +timeout_minutes=3 +services_to_start=() + +# Parse options +OPTS=$(getopt -o hp:r:d:o:g:b:t:s: --long help,postgres_sif:,redis_sif:,django_sif:,docs_sif:,postgres_data:,postgres_backup:,timeout:,services: -n 'parse-options' -- "$@") +if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi + +eval set -- "$OPTS" + +while true; do + case "$1" in + -h | --help ) show_help; exit 0 ;; + -p | --postgres_sif ) postgres_sif="$2"; shift 2 ;; + -r | --redis_sif ) redis_sif="$2"; shift 2 ;; + -d | --django_sif ) django_sif="$2"; shift 2 ;; + -o | --docs_sif ) docs_sif="$2"; shift 2 ;; + -g | --postgres_data ) postgres_data="$2"; shift 2 ;; + -b | --postgres_backup ) postgres_backup="$2"; shift 2 ;; + -t | --timeout ) timeout_minutes="$2"; shift 2 ;; + -s | --services ) + IFS=',' read -r -a services_to_start <<< "$2" + shift 2 ;; + -- ) shift; break ;; + * ) break ;; + esac +done + +main() { + # Loop over the array of packages to load them + spack_packages=("postgresql" "singularityce" "redis") + for pkg in "${spack_packages[@]}"; do + if ! eval $(spack load --sh $pkg); then + echo "Error loading $pkg package with spack. Ensure the package exists and is available." + exit 1 + fi + echo "$pkg loaded successfully." + done + + let "timeout_seconds=timeout_minutes * 60" + + # Start specified services + for service in "${services_to_start[@]}"; do + start_service $service + done +} + +# Call main function +main "$@" + + + +# let "timeout_seconds=timeout_minutes * 60" + +# # Implement the rest of your script here, including loading spack packages, +# # starting the services as specified in `services_to_start`, and checking for their readiness. +# # You'll replace or modify the previous loop and checks according to the new variables used. + + +# # Define an array of Spack packages to load +# spack_packages=("postgresql" "singularityce" "redis") + +# # Loop over the array of packages +# for pkg in "${spack_packages[@]}"; do +# if ! eval $(spack load --sh $pkg); then +# echo "Error loading $pkg package with spack. Ensure the package exists and is available." +# exit 1 +# fi +# echo "$pkg loaded successfully." +# done + +# # pass in the path to where you want postgres data to be +# # stored on your local machine +# yeastregulatorydb_local_postgres_data=$1 +# yeastregulatorydb_local_postgres_data_backups=$2 + +# # paths to sif files +# postgres_sif=$3 +# redis_sif=$4 +# django_sif=$5 +# docs_sif=$6 + +# # optional arguments +# TIMEOUT_MINUTES="${7:-3}" + +# # Convert minutes to seconds for the timeout +# let "TIMEOUT_SECONDS=TIMEOUT_MINUTES * 60" + +# # start the postgres container +# singularity exec $postgres_sif \ +# --bind $yeastregulatorydb_local_postgres_data:/var/lib/postgresql/data \ +# --bind $yeastregulatorydb_local_postgres_data_backups:/backups \ +# --env-file ./.envs/.local/.postgres & + +# # Counter to keep track of the time elapsed +# counter=0 +# # check that the postgres container is up and running +# # before starting the next container. Timeout and issue exit +# # error if it takes too long +# while ! pg_isready -h localhost -p 5432; do +# if [ "$counter" -lt "$TIMEOUT_SECONDS" ]; then +# echo "Waiting for PostgreSQL to start... $((counter++))s" +# sleep 1 +# else +# echo "Timeout reached. PostgreSQL did not start within $TIMEOUT_MINUTES minutes." +# exit 1 +# fi +# done + +# echo "PostgreSQL is up and running." + +# # Start the redis container +# singularity exec $redis_sif redis-server & + +# counter=0 +# # Check if Redis is ready +# echo "Waiting for Redis to start..." +# while ! redis-cli ping > /dev/null 2>&1; do +# if [ "$counter" -lt "$TIMEOUT_SECONDS" ]; then +# sleep 1 +# ((counter++)) +# echo "Checking Redis... ($counter seconds)" +# else +# echo "Timeout reached. Redis did not start within $(($TIMEOUT_SECONDS / 60)) minutes." +# exit 1 +# fi +# done + +# echo "Redis is up and running." + +# singularity exec $django_sif \ +# --bind .:/app \ +# --env-file ./.envs/.local/.django \ +# --env-file ./.envs/.local/.postgres \ +# --env-file ./.envs/.local/.regulatory_data \ +# --port 8000:8000 \ +# /start & + +# # Check if the Django app is ready +# counter=0 +# echo "Waiting for Django app to start..." +# while ! curl -s http://localhost:8000 > /dev/null; do +# if [ "$counter" -lt "$TIMEOUT_SECONDS" ]; then +# sleep 1 +# ((counter++)) +# echo "Checking Django app... ($counter seconds)" +# else +# echo "Timeout reached. Django did not start within $(($TIMEOUT_SECONDS / 60)) minutes." +# exit 1 +# fi +# done + +# echo "Django app is up and running." + + + +# # docs: +# # image: yeastregulatorydb_local_docs +# # container_name: yeastregulatorydb_local_docs +# # build: +# # context: . +# # dockerfile: ./compose/local/docs/Dockerfile +# # env_file: +# # - ./.envs/.local/.django +# # volumes: +# # - ./docs:/docs:z +# # - ./config:/app/config:z +# # - ./yeastregulatorydb:/app/yeastregulatorydb:z +# # ports: +# # - '9000:9000' +# # command: /start-docs + +# # celeryworker: +# # <<: *django +# # image: yeastregulatorydb_local_celeryworker +# # container_name: yeastregulatorydb_local_celeryworker +# # depends_on: +# # - redis +# # - postgres +# # ports: [] +# # command: /start-celeryworker + +# # celerybeat: +# # <<: *django +# # image: yeastregulatorydb_local_celerybeat +# # container_name: yeastregulatorydb_local_celerybeat +# # depends_on: +# # - redis +# # - postgres +# # ports: [] +# # command: /start-celerybeat + +# # flower: +# <<: *django +# image: yeastregulatorydb_local_flower +# container_name: yeastregulatorydb_local_flower +# ports: +# - '5555:5555' +# command: /start-flower