From 26a10b434ae9417969830d0da697f235733b2879 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 1 Nov 2021 16:16:28 -0500 Subject: [PATCH] task/FP-1120 & task/FP-1136: Allow standalone services (e.g. CMS/Docs/Portal) (#19) * ignore .idea IDE project directoy * Breaking up services into independent composable files to facilitate fully customizable suite of services. Including a default configuration for commonly deployed services * updating Makefile with full path to overrides compose overrides * Including env file in override * use relative paths * Use home environment variable to help find configs * use home var to find configs * moving base compose files * gitignoring custom overrides--not tracked here * Updating readme and changing the default docs command to build * Update Makefile with default CAMINO_HOME Co-authored-by: Sal Tijerina * task/fp 1120 1135: Update samples, gitignore, and fix CAMINO_HOME (#20) * update samples; organize .gitgnore * formatting * move CAMINO_HOME declaration to top * fix CAMINO_HOME var * ignore all but example in conf/portal * add sample "CAMINO_HOME" setting Co-authored-by: Sal Tijerina --- .gitignore | 21 +- Makefile | 20 +- README.md | 50 ++--- conf/camino/sample.env | 2 + conf/cms/secrets.sample.py | 65 ++++--- conf/compose/docker-compose.cms.yml | 16 ++ conf/compose/docker-compose.core.cms.yml | 63 ++++++ conf/compose/docker-compose.core.docs.cms.yml | 78 ++++++++ conf/compose/docker-compose.core.yml | 19 ++ conf/compose/docker-compose.docs.yml | 6 + conf/compose/docker-compose.memcached.yml | 7 + conf/compose/docker-compose.nginx.yml | 8 + conf/compose/docker-compose.rabbitmq.yml | 16 ++ conf/compose/docker-compose.redis.yml | 15 ++ conf/compose/docker-compose.websockets.yml | 14 ++ conf/compose/docker-compose.workers.yml | 16 ++ conf/portal/settings_secret.example.py | 181 +++--------------- conf/rabbitmq/rabbitmq.env.sample | 6 +- docker-compose.yml | 114 ----------- 19 files changed, 386 insertions(+), 331 deletions(-) create mode 100644 conf/compose/docker-compose.cms.yml create mode 100644 conf/compose/docker-compose.core.cms.yml create mode 100644 conf/compose/docker-compose.core.docs.cms.yml create mode 100644 conf/compose/docker-compose.core.yml create mode 100644 conf/compose/docker-compose.docs.yml create mode 100644 conf/compose/docker-compose.memcached.yml create mode 100644 conf/compose/docker-compose.nginx.yml create mode 100644 conf/compose/docker-compose.rabbitmq.yml create mode 100644 conf/compose/docker-compose.redis.yml create mode 100644 conf/compose/docker-compose.websockets.yml create mode 100644 conf/compose/docker-compose.workers.yml delete mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index e4a7d08..16d786e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__/ *.py[cod] *$py.class .idea/ + # C extensions *.so @@ -31,17 +32,17 @@ MANIFEST local_settings.py db.sqlite3 db.sqlite3-journal -rabbitmq.env -elasticsearch.yml -*secrets.py -*secrets.env -*.secret -*settings_secret.py -*settings_secret.env + +# Camino configuraitons from Core-Portal-Deployments .env conf/camino/* !conf/camino/*sample.* -camino_env -compose_file -make_env +# Secrets and Configurations +*settings_secret.py +*settings_custom.py +*settings_local.py +*secrets.py +rabbitmq.env +conf/portal/* +!conf/portal/*example.* diff --git a/Makefile b/Makefile index ef0bdc7..c649bc3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,25 @@ #!make ENV_FILE ?= ./.env include $(ENV_FILE) -DOCKER_COMPOSE := docker-compose -f docker-compose.yml -f ./conf/camino/$(COMPOSE_FILE) --env-file=$(ENV_FILE) + +# CAMINO_HOME defaults to the path to this Makefile +ifndef CAMINO_HOME +override CAMINO_HOME := $(dir $(lastword $(abspath $(MAKEFILE_LIST)))) +endif + +ifndef BASE_COMPOSE_FILE +override BASE_COMPOSE = -f ${CAMINO_HOME}/conf/compose/docker-compose.core.cms.yml +else +override BASE_COMPOSE = -f ${CAMINO_HOME}/conf/compose/${BASE_COMPOSE_FILE} +endif + +ifndef COMPOSE_FILE +override COMPOSE_OVERRIDE = +else +override COMPOSE_OVERRIDE = -f ${CAMINO_HOME}/conf/camino/${COMPOSE_FILE} +endif + +DOCKER_COMPOSE := docker-compose ${BASE_COMPOSE} ${COMPOSE_OVERRIDE} --env-file=$(ENV_FILE) .PHONY: deploy-core deploy-core: diff --git a/README.md b/README.md index 988ec47..c2f279f 100644 --- a/README.md +++ b/README.md @@ -13,40 +13,44 @@ A compose file exists for each environment to orchestrate deployments. Each appl Setup: Clone the Camino repo to the path `/opt/portal` -At the root of the cloned project create the env file `.env` and add the name of the environment file to be used to manage deployments. The env files are named according the deployment environment they manage, like `dev.env, pprd.env, prod.env` and live in the directory `conf/camino`. With the env file set, make will load the environment variables and pass them to the compose file. +At the root of the cloned project create the env file `.env` and add the environment variables +that indicate which docker-compose files should be used. With the env file set, Make will +load the environment variables and pass them to the compose file. -Ex. To manage and deploy dev, use the filename `dev.env`, this file will be passed to docker-compose with `--env-file=./conf/camino/dev.env` +The .env file can be used to [populate values in the compose file](https://docs.docker.com/compose/environment-variables/) +and in conjunction with the Makefile helps create custom service sets. -This file can be used to [populate values in the compose file](https://docs.docker.com/compose/environment-variables/) +### Quick-Start Example -To deploy changes to a service, the image name and tag (or digest) for the service should be set in the appropriate compose file (or, if using a var, the matching env file in `conf/camino`). This means that the image must be pre-built and its tag or digest known before deployment. - -For example, to deploy a new pre-built cms image to production, the image in docker-compose.yml would be updated like below: - -``` -services: - cms: - image: taccwma/frontera-cms:{tag} - volumes: +1. Clone this repo +2. Create a `.env` file and populate with: ``` -to -``` -services: - cms: - image: taccwma/frontera-cms:{tag} - volumes: +BASE_COMPOSE_FILE=docker-compose.nginx.yml +COMPOSE_FILE= +CAMINO_HOME=${PWD} ``` +3. Run `make up` +4. Navigate to `localhost` -To update a dev environment, a similar change would happen in the docker-compose-dev.yml file. +Services can be customized without modifying base +compose files by creating overrides files in `conf/camino/` -Additionally, to facilitate continuous integration without constant updates to the Camino repo, a service in a compose file can be tagged with `:latest`. New images built and published with the tag `:latest` can then be deployed without requiring updates to Camino. +##### To override the version of nginx image used in the previous example +1. Create file `docker-compose.nginx.overrides.yml` In `conf/camino` and paste in: ``` +version: "3.8" services: - cms: - image: taccwma/frontera-cms:latest - volumes: + nginx: + image: nginx:stable ``` +2. In the .env file add entry, `COMPOSE_FILE=docker-compose.nginx.overrides.yml` +3. run make up +4. Note that the nginx image with tag `stable` is being pulled down +5. Navigating to `localhost` now displays the default page for the new image + +To facilitate continuous integration without updating the Camino repo, +a service in a compose file can be tagged with `:latest`. New images built and published with the tag `:latest` can then be deployed without requiring updates to Camino. Deploying with Jenkins: diff --git a/conf/camino/sample.env b/conf/camino/sample.env index c0462b4..64bcd57 100644 --- a/conf/camino/sample.env +++ b/conf/camino/sample.env @@ -5,3 +5,5 @@ COMPOSE_FILE= CMS_TAG= PORTAL_TAG= SSH_HOST= +CUSTOM_SETTINGS_PORTAL= +CAMINO_HOME=/opt/portal/Camino diff --git a/conf/cms/secrets.sample.py b/conf/cms/secrets.sample.py index b7d31e3..b88b2b5 100644 --- a/conf/cms/secrets.sample.py +++ b/conf/cms/secrets.sample.py @@ -1,28 +1,43 @@ -_SECRET_KEY = 'change_me' -_DEBUG = True -_ALLOWED_HOSTS = ['*'] -_SITE_ID = 1 -_CMS_TEMPLATES = ( - ## Customize this - ('base.html', 'Base'), - ('cms_menu.html', 'CMS Menu'), - ('fullwidth.html', 'Fullwidth'), - ('sidebar_left.html', 'Sidebar Left'), - ('sidebar_right.html', 'Sidebar Right') -) +# SECRET SETTINGS VALUES. +# SAMPLE.TACC.UTEXAS.EDU -# As needed: -#_CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', -# 'LOCATION': 'frontera_portal_memcached:11211', -# } -#} +######################## +# DJANGO SETTINGS +######################## +SECRET_KEY = 'change_me' +ALLOWED_HOSTS = ['sample.tacc.utexas.edu'] +CEP_AUTH_VERIFICATION_ENDPOINT = 'https://sample.tacc.utexas.edu' -_DATABASE_ENGINE = 'django.db.backends.postgresql' -_DATABASE_NAME = 'postgres' -_DATABASE_USERNAME = 'postgres' -_DATABASE_PASSWORD = 'secretpw' -_DATABASE_HOST = 'postgres_host' -_DATABASE_PORT = 5432 \ No newline at end of file +######################## +# DATABASE SETTINGS +######################## + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'PORT': '5432', + 'NAME': 'sample_cms', + 'USER': 'sample_cms', + 'PASSWORD': 'change_me', + 'HOST': 'portal_postgres' + } +} + +######################## +# ELASTICSEARCH +######################## + +ES_HOSTS = 'https://esclient:9200' +ES_AUTH = 'sample:change_me' +ES_INDEX_PREFIX = 'sample-{}' +ES_DOMAIN = 'https://sample.tacc.utexas.edu' + +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', + 'URL': ES_HOSTS, + 'INDEX_NAME': ES_INDEX_PREFIX.format('cms'), + 'KWARGS': {'http_auth': ES_AUTH } + } +} diff --git a/conf/compose/docker-compose.cms.yml b/conf/compose/docker-compose.cms.yml new file mode 100644 index 0000000..04bf0f2 --- /dev/null +++ b/conf/compose/docker-compose.cms.yml @@ -0,0 +1,16 @@ +version: "3.8" +services: + cms: + image: ${CMS_IMAGE}:${CMS_TAG} + volumes: + - ${CAMINO_HOME}/conf/cms/secrets.py:/code/taccsite_cms/secrets.py + - ${CAMINO_HOME}/conf/uwsgi/uwsgi_cms.ini:/code/uwsgi.ini + - /var/www/portal/cms/static:/code/static + - /var/www/portal/cms/media:/code/media + command: ["uwsgi", "--ini", "uwsgi.ini"] + container_name: portal_cms + logging: + driver: syslog + options: + tag: portal_cms + restart: always diff --git a/conf/compose/docker-compose.core.cms.yml b/conf/compose/docker-compose.core.cms.yml new file mode 100644 index 0000000..ac97496 --- /dev/null +++ b/conf/compose/docker-compose.core.cms.yml @@ -0,0 +1,63 @@ +version: "3.8" +services: + nginx: + extends: + file: ${CAMINO_HOME}/conf/compose/docker-compose.nginx.yml + service: nginx + volumes: + - ${CAMINO_HOME}/conf/nginx/nginx.conf:/etc/nginx/nginx.conf + - ${CAMINO_HOME}/conf/uwsgi/uwsgi_params:/etc/nginx/uwsgi_params + - /etc/ssl/dhparam.pem:/etc/ssl/dhparam.pem + - /var/www/portal/cms/static:/var/www/portal/cms/static + - /var/www/portal/cms/media:/var/www/portal/cms/media + - /var/www/portal/portal/media:/var/www/portal/portal/media + - /var/www/portal/portal/static:/var/www/portal/portal/static + ports: + - 80:80 + - 443:443 + container_name: portal_nginx + logging: + driver: syslog + options: + tag: portal_nginx + restart: always + + memcached: + extends: + file: docker-compose.memcached.yml + service: memcached + + redis: + extends: + file: docker-compose.redis.yml + service: redis + + rabbitmq: + extends: + file: docker-compose.rabbitmq.yml + service: rabbitmq + + websockets: + extends: + file: docker-compose.websockets.yml + service: websockets + + core: + extends: + file: docker-compose.core.yml + service: core + + workers: + extends: + file: docker-compose.workers.yml + service: workers + + cms: + extends: + file: docker-compose.cms.yml + service: cms + +volumes: + portal_es_data: + portal_redis_data: + portal_rabbitmq_data: diff --git a/conf/compose/docker-compose.core.docs.cms.yml b/conf/compose/docker-compose.core.docs.cms.yml new file mode 100644 index 0000000..af0606a --- /dev/null +++ b/conf/compose/docker-compose.core.docs.cms.yml @@ -0,0 +1,78 @@ +version: "3.8" +services: + nginx: + extends: + file: ${CAMINO_HOME}/conf/compose/docker-compose.nginx.yml + service: nginx + volumes: + - ${CAMINO_HOME}/conf/nginx/nginx.conf:/etc/nginx/nginx.conf + - ${CAMINO_HOME}/conf/uwsgi/uwsgi_params:/etc/nginx/uwsgi_params + - /etc/ssl/dhparam.pem:/etc/ssl/dhparam.pem + - /var/www/portal/cms/static:/var/www/portal/cms/static + - /var/www/portal/cms/media:/var/www/portal/cms/media + - /var/www/portal/portal/media:/var/www/portal/portal/media + - /var/www/portal/portal/static:/var/www/portal/portal/static + ports: + - 80:80 + - 443:443 + container_name: portal_nginx + logging: + driver: syslog + options: + tag: portal_nginx + restart: always + + docs: + extends: + file: docker-compose.docs.yml + service: docs + image: taccwma/frontera-docs:latest + volumes: + - /var/www/portal/docs:/docs/site + command: ["mkdocs", "build"] + container_name: portal_user_guide + logging: + driver: syslog + options: + tag: portal_user_guide + + memcached: + extends: + file: docker-compose.memcached.yml + service: memcached + + redis: + extends: + file: docker-compose.redis.yml + service: redis + + rabbitmq: + extends: + file: docker-compose.rabbitmq.yml + service: rabbitmq + + websockets: + extends: + file: docker-compose.websockets.yml + service: websockets + + core: + extends: + file: docker-compose.core.yml + service: core + + workers: + extends: + file: docker-compose.workers.yml + service: workers + + cms: + extends: + file: docker-compose.cms.yml + service: cms + +volumes: + portal_es_data: + portal_redis_data: + portal_rabbitmq_data: + diff --git a/conf/compose/docker-compose.core.yml b/conf/compose/docker-compose.core.yml new file mode 100644 index 0000000..1955696 --- /dev/null +++ b/conf/compose/docker-compose.core.yml @@ -0,0 +1,19 @@ +version: "3.8" +services: + core: + image: ${PORTAL_IMAGE}:${PORTAL_TAG} + volumes: + - ${CAMINO_HOME}/conf/portal/settings_secret.py:/srv/www/portal/server/portal/settings/settings_secret.py + - ${CAMINO_HOME}/conf/portal/settings_custom.py:/srv/www/portal/server/portal/settings/settings_custom.py + - /var/www/portal/portal/media:/srv/www/portal/server/media + - /var/www/portal/portal/static:/srv/www/portal/server/static + - ${CAMINO_HOME}/conf/uwsgi/uwsgi_core.ini:/srv/www/portal/server/conf/uwsgi/uwsgi_core.ini + stdin_open: true + tty: true + command: uwsgi --ini /srv/www/portal/server/conf/uwsgi/uwsgi_core.ini + container_name: portal_django + logging: + driver: syslog + options: + tag: portal_django + restart: always diff --git a/conf/compose/docker-compose.docs.yml b/conf/compose/docker-compose.docs.yml new file mode 100644 index 0000000..c04c49b --- /dev/null +++ b/conf/compose/docker-compose.docs.yml @@ -0,0 +1,6 @@ +version: "3.8" +services: + docs: + image: taccwma/frontera-docs:latest + command: ["mkdocs", "build"] + container_name: portal_user_guide diff --git a/conf/compose/docker-compose.memcached.yml b/conf/compose/docker-compose.memcached.yml new file mode 100644 index 0000000..c5432c3 --- /dev/null +++ b/conf/compose/docker-compose.memcached.yml @@ -0,0 +1,7 @@ +version: "3.8" +services: + memcached: + image: memcached:latest + command: ["-m", "1024m"] + container_name: portal_memcached + restart: always diff --git a/conf/compose/docker-compose.nginx.yml b/conf/compose/docker-compose.nginx.yml new file mode 100644 index 0000000..1f9b7ce --- /dev/null +++ b/conf/compose/docker-compose.nginx.yml @@ -0,0 +1,8 @@ +version: "3.8" +services: + nginx: + image: nginx + ports: + - 80:80 + container_name: portal_nginx + restart: always diff --git a/conf/compose/docker-compose.rabbitmq.yml b/conf/compose/docker-compose.rabbitmq.yml new file mode 100644 index 0000000..c7766c8 --- /dev/null +++ b/conf/compose/docker-compose.rabbitmq.yml @@ -0,0 +1,16 @@ +version: "3.8" +services: + rabbitmq: + image: rabbitmq:3.7.17-management + volumes: + - portal_rabbitmq_data:/var/lib/rabbitmq/mnesia/rabbit@portal_rabbitmq + env_file: ${CAMINO_HOME}/conf/rabbitmq/rabbitmq.env + container_name: portal_rabbitmq + logging: + driver: syslog + options: + tag: portal_rabbitmq + restart: always + +volumes: + portal_rabbitmq_data: diff --git a/conf/compose/docker-compose.redis.yml b/conf/compose/docker-compose.redis.yml new file mode 100644 index 0000000..dec4425 --- /dev/null +++ b/conf/compose/docker-compose.redis.yml @@ -0,0 +1,15 @@ +version: "3.8" +services: + redis: + image: redis:5.0 + volumes: + - portal_redis_data:/data + container_name: portal_redis + logging: + driver: syslog + options: + tag: portal_redis + restart: always + +volumes: + portal_redis_data: diff --git a/conf/compose/docker-compose.websockets.yml b/conf/compose/docker-compose.websockets.yml new file mode 100644 index 0000000..0bc8f84 --- /dev/null +++ b/conf/compose/docker-compose.websockets.yml @@ -0,0 +1,14 @@ +version: "3.8" +services: + websockets: + image: ${PORTAL_IMAGE}:${PORTAL_TAG} + command: 'daphne -b 0.0.0.0 -p 9000 -e ssl:443:privateKey=/etc/ssl/private/portal.key:certKey=/etc/ssl/certs/portal.cer --root-path=/srv/www/portal/server --access-log - --proxy-headers portal.asgi:application' + volumes: + - ${CAMINO_HOME}/conf/portal/settings_secret.py:/srv/www/portal/server/portal/settings/settings_secret.py + - ${CAMINO_HOME}/conf/portal/settings_custom.py:/srv/www/portal/server/portal/settings/settings_custom.py + container_name: portal_websockets + logging: + driver: syslog + options: + tag: portal_websockets + restart: always diff --git a/conf/compose/docker-compose.workers.yml b/conf/compose/docker-compose.workers.yml new file mode 100644 index 0000000..e486b48 --- /dev/null +++ b/conf/compose/docker-compose.workers.yml @@ -0,0 +1,16 @@ +version: "3.8" +services: + workers: + extends: + file: docker-compose.core.yml + service: core + volumes: + - ${CAMINO_HOME}/conf/portal/settings_secret.py:/srv/www/portal/server/portal/settings/settings_secret.py + - ${CAMINO_HOME}/conf/portal/settings_custom.py:/srv/www/portal/server/portal/settings/settings_custom.py + command: "celery -A portal worker -Q default,indexing,files,api,onboard --concurrency=10" + container_name: portal_workers + logging: + driver: syslog + options: + tag: portal_workers + restart: always diff --git a/conf/portal/settings_secret.example.py b/conf/portal/settings_secret.example.py index ec546e4..11ba9b1 100644 --- a/conf/portal/settings_secret.example.py +++ b/conf/portal/settings_secret.example.py @@ -7,56 +7,36 @@ ######################## _SECRET_KEY = 'CHANGE ME !' -_DEBUG = True - -#_WSGI_APPLICATION = 'ws4redis.django_runserver.application' # DEV -_WSGI_APPLICATION = 'portal.wsgi.application' # PROD - -# Namespace for portal -_PORTAL_NAMESPACE = 'FRONTERA' -_PORTAL_DOMAIN = 'frontera.dev' - -# Admin account -_PORTAL_ADMIN_USERNAME = 'portal_admin' - -_SYSTEM_MONITOR_DISPLAY_LIST = ['frontera.tacc.utexas.edu', 'stampede2.tacc.utexas.edu', 'lonestar5.tacc.utexas.edu'] ######################## # DJANGO SETTINGS LOCAL ######################## # Database. -_DJANGO_DB_ENGINE= 'django.db.backends.postgresql' -_DJANGO_DB_HOST= 'frontera_prtl_postgres' -_DJANGO_DB_PORT= '5432' -_DJANGO_DB_NAME= 'dev' -_DJANGO_DB_USER= 'dev' -_DJANGO_DB_PASSWORD= 'dev' +_DJANGO_DB_ENGINE = 'django.db.backends.postgresql' +_DJANGO_DB_HOST = 'portal_postgres' +_DJANGO_DB_PORT = '5432' +_DJANGO_DB_NAME = 'dev' +_DJANGO_DB_USER = 'dev' +_DJANGO_DB_PASSWORD = 'dev' # TAS Authentication. -_TAS_URL='https://tas-dev.tacc.utexas.edu/api' -_TAS_CLIENT_KEY='key' -_TAS_CLIENT_SECRET='secret' +_TAS_URL = 'https://tas-dev.tacc.utexas.edu/api' +_TAS_CLIENT_KEY = 'key' +_TAS_CLIENT_SECRET = 'secret' # Redmine Tracker Authentication. -_RT_HOST='https://consult.tacc.utexas.edu/REST/1.0' -_RT_UN='username' -_RT_PW='password' -_RT_QUEUE = 'QUEUE' -_RT_TAG = 'CEP_portal' - -# Recaptcha Authentication. -_RECAPTCHA_PUBLIC_KEY='public_key' -_RECAPTCHA_PRIVATE_KEY='private_key' -_RECAPTCHA_USE_SSL='True' -_NOCAPTCHA='True' - -_REQUEST_ACCESS=False +_RT_HOST = 'https://consult.tacc.utexas.edu/REST/1.0' +_RT_UN = 'username' +_RT_PW = 'password' ######################## # AGAVE SETTINGS ######################## +# Admin account +_PORTAL_ADMIN_USERNAME = 'portal_admin' + # Agave Tenant. _AGAVE_TENANT_ID = 'tenant_name' _AGAVE_TENANT_BASEURL = 'https://agave.mytenant.org' @@ -65,98 +45,39 @@ _AGAVE_CLIENT_KEY = 'TH1$_!$-MY=K3Y!~' _AGAVE_CLIENT_SECRET = 'TH1$_!$-My=S3cr3t!~' _AGAVE_SUPER_TOKEN = 'S0m3T0k3n_tHaT-N3v3r=3xp1R35' -_AGAVE_STORAGE_SYSTEM = 'my.storage.default' -_AGAVE_COMMUNITY_DATA_SYSTEM = 'storage_system' -_AGAVE_PUBLIC_DATA_SYSTEM = 'storage_system' -_AGAVE_DEFAULT_TRASH_NAME = 'Trash' - -_AGAVE_JWT_HEADER = 'HTTP_X_JWT_ASSERTION_PORTALS' ######################## # RABBITMQ SETTINGS ######################## -_BROKER_URL_USERNAME = 'dev' +_BROKER_URL_USERNAME = 'portal_rabbitmq' _BROKER_URL_PWD = 'dev' -_BROKER_URL_HOST = 'frontera_prtl_rabbitmq' +_BROKER_URL_HOST = 'portal_rabbitmq' _BROKER_URL_PORT = '5672' -_BROKER_URL_VHOST = 'dev' - -_RESULT_BACKEND_USERNAME = 'dev' -_RESULT_BACKEND_PWD = 'dev' -_RESULT_BACKEND_HOST = 'frontera_prtl_redis' -_RESULT_BACKEND_PORT = '6379' -_RESULT_BACKEND_DB = '0' +_BROKER_URL_VHOST = 'portal_rabbitmq' ######################## # ELASTICSEARCH SETTINGS ######################## -_ES_HOSTS = 'frontera_prtl_elasticsearch' +_ES_HOSTS = 'portal_elasticsearch:9200' +_ES_AUTH = 'username:password' +_ES_INDEX_PREFIX = 'cep-dev-{}' ######################## # CELERY SETTINGS ######################## -# TBD. - -######################## -# LOGGING SETTINGS -######################## - -# TBD. - -######################## -# DJANGO APP: WORKSPACE -######################## - -_PORTAL_APPS_METADATA_NAMES = ['portal_apps'] -_PORTAL_ALLOCATION = 'TACC-ACI' - -# NOTE: set _WH_BASE_URL to ngrok redirect for local dev testing (i.e. _WH_BASE_URL = 'https://12345.ngrock.io', see https://ngrok.com/) -_WH_BASE_URL = '' - -######################## -# DJANGO APP: DATA DEPOT -######################## - -# Absolute path where home directories should be created. -# Absolute with respect to the host -# Use only if all home directories are under one parent directory. -_PORTAL_DATA_DEPOT_DEFAULT_HOME_DIR_ABS_PATH = '/corral-repl/tacc/aci/CEP/home_dirs/' -# Relative path from the default sotrage system where home directories -# should be created. -# Use only if all home directories are under one parent directory. -# NOTE: Replace PORTAL_NAME with name of project (e.g. - cep). -_PORTAL_DATA_DEPOT_DEFAULT_HOME_DIR_REL_PATH = 'home_dirs' -_PORTAL_DATA_DEPOT_USER_SYSTEM_PREFIX = 'cep.dev.home.{}' -_PORTAL_DATA_DEPOT_STORAGE_HOST = 'data.tacc.utexas.edu' -_PORTAL_USER_HOME_MANAGER = 'portal.apps.accounts.managers.user_home.UserHomeManager' -_PORTAL_KEYS_MANAGER = 'portal.apps.accounts.managers.ssh_keys.KeysManager' -_PORTAL_USER_ACCOUNT_SETUP_STEPS = [ ] -_PORTAL_DATA_DEPOT_WORK_HOME_DIR_FS = '/work' -_PORTAL_DATA_DEPOT_WORK_HOME_DIR_EXEC_SYSTEM = 'EXECUTION_SYSTEM' -_PORTAL_JUPYTER_URL = "https://jupyter.tacc.cloud" -_PORTAL_JUPYTER_SYSTEM_MAP = { - "cep.home.{username}" : "/tacc-work", -} +_RESULT_BACKEND_HOST = 'portal_redis' +_RESULT_BACKEND_PORT = '6379' +_RESULT_BACKEND_DB = '0' ####################### -# PROJECTS SETTING +# PROJECTS SETTINGS ####################### -_PORTAL_DATA_DEPOT_PROJECTS_SYSTEM_PREFIX = 'cep.project' -_PORTAL_PROJECTS_NAME_PREFIX = _PORTAL_DATA_DEPOT_PROJECTS_SYSTEM_PREFIX -_PORTAL_PROJECTS_ID_PREFIX = _PORTAL_NAMESPACE.upper() -_PORTAL_PROJECTS_ROOT_DIR = '/corral-repl/tacc/aci/CEP/projecs' -_PORTAL_PROJECTS_ROOT_SYSTEM_NAME = '{}.root'.format( - _PORTAL_DATA_DEPOT_PROJECTS_SYSTEM_PREFIX -) -_PORTAL_PROJECTS_ROOT_HOST = _PORTAL_DATA_DEPOT_STORAGE_HOST + _PORTAL_PROJECTS_PRIVATE_KEY = '' _PORTAL_PROJECTS_PUBLIC_KEY = '' -_PORTAL_PROJECTS_FS_EXEC_SYSTEM_ID = '' -_PORTAL_PROJECTS_PEMS_APP_ID = '' -_PORTAL_USER_ACCOUNT_SETUP_WEBHOOK_PWD = '123' ######################## # EXTERNAL DATA RESOURCES SETTINGS @@ -179,53 +100,3 @@ "directory": "external-resources" } } - - -######################## -# DJANGO CMS SETTINGS -######################## - -# CMS Site (allows for multiple sites on a single CMS) -_SITE_ID=1 - -_HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', - 'URL': 'frontera_prtl_elasticsearch:9200/', - 'INDEX_NAME': 'cms', - } -} - -_COMMUNITY_INDEX_SCHEDULE = None - -######################## -# Custom Portal Template Assets -# Asset path root is static files output dir. -# {% static %} won't work in conjunction with {{ VARIABLE }} so use full paths. -######################## - -# No Art. -#_PORTAL_ICON_FILENAME='' # Empty string yields NO icon. -_PORTAL_LOGO_FILENAME='' # Empty string yields text 'CEP'. -#_PORTAL_NAVBAR_BACKGROUND_FILENAME='' # Empty string yields NO bg art. - -# Defaul Art. -_PORTAL_ICON_FILENAME='/static/img/favicon.ico' -#_PORTAL_LOGO_FILENAME='/static/img/TACC.A.png' -_PORTAL_NAVBAR_BACKGROUND_FILENAME='' - -# Custom Art (example using old CEP art). -#_PORTAL_ICON_FILENAME='/static/img/favicon.cep.png' -#_PORTAL_LOGO_FILENAME='/static/img/logo.cep.png' -#_PORTAL_NAVBAR_BACKGROUND_FILENAME='/static/img/network-Header.jpg' - -######################## -# GOOGLE ANALYTICS -######################## - -# Using test account under personal email. -# To use during dev, Tracking Protection in browser needs to be turned OFF. -# Need to setup an admin account to aggregate tracking properties for portals. -# NOTE: Use the _AGAVE_TENANT_ID URL value when setting up the tracking property. -_GOOGLE_ANALYTICS_PROPERTY_ID = 'UA-XXXXX-Y' -_GOOGLE_ANALYTICS_PRELOAD = True diff --git a/conf/rabbitmq/rabbitmq.env.sample b/conf/rabbitmq/rabbitmq.env.sample index 1eeb5cd..b8fdf37 100644 --- a/conf/rabbitmq/rabbitmq.env.sample +++ b/conf/rabbitmq/rabbitmq.env.sample @@ -2,6 +2,6 @@ #More info: https://hub.docker.com/_/rabbitmq/ #Use this to configure rabbitmq container #RABBIT_ERLANG_COOKIE= -#RABBITMQ_DEFAULT_USER=dev -#RABBITMQ_DEFAULT_PASS=dev -#RABBITMQ_DEFAULT_VHOST=dev +RABBITMQ_DEFAULT_USER=portal_rabbitmq +RABBITMQ_DEFAULT_PASS=dev +RABBITMQ_DEFAULT_VHOST=portal_rabbitmq diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 793e52d..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,114 +0,0 @@ -version: "3.8" -services: - cms: - image: ${CMS_IMAGE}:${CMS_TAG} - volumes: - - ./conf/cms/secrets.py:/code/taccsite_cms/secrets.py - - ./conf/uwsgi/uwsgi_cms.ini:/code/uwsgi.ini - - /var/www/portal/cms/static:/code/static - - /var/www/portal/cms/media:/code/media - command: ["uwsgi", "--ini", "uwsgi.ini"] - container_name: portal_cms - logging: - driver: syslog - options: - tag: portal_cms - restart: always - - nginx: - image: nginx - volumes: - - ./conf/nginx/nginx.conf:/etc/nginx/nginx.conf - - ./conf/uwsgi/uwsgi_params:/etc/nginx/uwsgi_params - - /etc/ssl/dhparam.pem:/etc/ssl/dhparam.pem - - /var/www/portal/cms/static:/var/www/portal/cms/static - - /var/www/portal/cms/media:/var/www/portal/cms/media - - /var/www/portal/portal/media:/var/www/portal/portal/media - - /var/www/portal/portal/static:/var/www/portal/portal/static - ports: - - 80:80 - - 443:443 - container_name: portal_nginx - logging: - driver: syslog - options: - tag: portal_nginx - restart: always - - memcached: - image: memcached:latest - command: ["-m", "1024m"] - container_name: portal_memcached - restart: always - - redis: - image: redis:5.0 - volumes: - - portal_redis_data:/data - container_name: portal_redis - logging: - driver: syslog - options: - tag: portal_redis - restart: always - - rabbitmq: - image: rabbitmq:3.7.17-management - volumes: - - portal_rabbitmq_data:/var/lib/rabbitmq/mnesia/rabbit@portal_rabbitmq - env_file: ./conf/rabbitmq/rabbitmq.env - container_name: portal_rabbitmq - logging: - driver: syslog - options: - tag: portal_rabbitmq - restart: always - - websockets: - image: ${PORTAL_IMAGE}:${PORTAL_TAG} - command: 'daphne -b 0.0.0.0 -p 9000 -e ssl:443:privateKey=/etc/ssl/private/portal.key:certKey=/etc/ssl/certs/portal.cer --root-path=/srv/www/portal/server --access-log - --proxy-headers portal.asgi:application' - volumes: - - ./conf/portal/settings_secret.py:/srv/www/portal/server/portal/settings/settings_secret.py - - ./conf/portal/settings_custom.py:/srv/www/portal/server/portal/settings/settings_custom.py - container_name: portal_websockets - logging: - driver: syslog - options: - tag: portal_websockets - restart: always - - core: - image: ${PORTAL_IMAGE}:${PORTAL_TAG} - volumes: - - ./conf/portal/settings_secret.py:/srv/www/portal/server/portal/settings/settings_secret.py - - ./conf/portal/settings_custom.py:/srv/www/portal/server/portal/settings/settings_custom.py - - /var/www/portal/portal/media:/srv/www/portal/server/media - - /var/www/portal/portal/static:/srv/www/portal/server/static - - ./conf/uwsgi/uwsgi_core.ini:/srv/www/portal/server/conf/uwsgi/uwsgi_core.ini - stdin_open: true - tty: true - command: uwsgi --ini /srv/www/portal/server/conf/uwsgi/uwsgi_core.ini - container_name: portal_django - logging: - driver: syslog - options: - tag: portal_django - restart: always - - workers: - image: ${PORTAL_IMAGE}:${PORTAL_TAG} - volumes: - - ./conf/portal/settings_secret.py:/srv/www/portal/server/portal/settings/settings_secret.py - - ./conf/portal/settings_custom.py:/srv/www/portal/server/portal/settings/settings_custom.py - command: "celery -A portal worker -Q default,indexing,files,api,onboard --concurrency=10" - container_name: portal_workers - logging: - driver: syslog - options: - tag: portal_workers - restart: always - -volumes: - portal_redis_data: - portal_es_data: - portal_rabbitmq_data: