node details overrides + logging utils

CHANGES.md

@@ -23,7 +23,8 @@
     respective messages to report logging messages in a standard approach.
     Colors can be disabled with `BIRDHOUSE_COLOR=0` and logging level can be set with `BIRDHOUSE_LOG_LEVEL={LEVEL}`
     where all levels above or equal to the configured one will be displayed (default logging level is `INFO`).
-  * Move `read-configs.include.sh` into `birdhouse/scripts` along other include scripts.
+  * Unify all `birdhouse/scripts` utilities to employ the same `COMPOSE_DIR` variable (auto-resolved or explicitly set)
+    in order to include or source any relevant dependencies they might have within the `birdhouse-deploy` repository.
   * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call.
     This can be used to perform a "dry-run" of the command and validate that was is loaded is as expected, by inspecting
     provided log messages.

birdhouse/components/geoserver/pre-docker-compose-up

@@ -1,17 +1,17 @@
 #!/bin/sh
 
 THIS_FILE="$(readlink -f "$0" || realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
-COMPOSE_DIR="$THIS_DIR/../.."
+THIS_DIR="$(dirname "${THIS_FILE}")"
+COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}/..")}"
 
-if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then
-  . "${COMPOSE_DIR}/scripts/read-configs.include.sh"
+if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then
+  . "${COMPOSE_DIR}/read-configs.include.sh"
 
   # resolve GEOSERVER_DATA_DIR
   read_configs
 fi
 
 if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then
   log INFO "fix GeoServer data dir permission on first run only, when data dir do not exist yet."
-  FIRST_RUN_ONLY=1 "${COMPOSE_DIR}"/deployment/fix-geoserver-data-dir-perm
+  FIRST_RUN_ONLY=1 "${COMPOSE_DIR}/deployment/fix-geoserver-data-dir-perm"
 fi

birdhouse/components/proxy/pre-docker-compose-up

@@ -4,7 +4,7 @@
 # enabled components. These files will be served by the nginx proxy as static files.
 
 THIS_FILE="$(readlink -f "$0" || realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
 
 mkdir -p "${THIS_DIR}/static"
 

birdhouse/default.env

@@ -29,6 +29,9 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}'
 export DELAYED_EVAL="
   $DELAYED_EVAL
   PAVICS_FQDN_PUBLIC
+  DOC_URL
+  SUPPORT_EMAIL
+  SSL_CERTIFICATE
   DATA_PERSIST_SHARED_ROOT
   WPS_OUTPUTS_DIR
   SERVER_NAME
@@ -47,29 +50,35 @@ export DELAYED_EVAL="
 #   Previous defaults are defined for backward-compatibility.
 #   If not overridden explicitly by their non '__' prefixed variant,
 #   a WARN message will be displayed by pavics-compose.
-export __DEFAULT__SERVER_NAME="PAVICS"
-export __DEFAULT__SERVER_DESCRIPTION="
+__DEFAULT__SERVER_NAME="PAVICS"
+__DEFAULT__SERVER_DESCRIPTION="
 The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of
 climate analysis services served through Open Geospatial Consortium (OGC) protocols.
 These services include data access, processing and visualization. Both data and algorithms
 can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS,
 or a custom web interface.
 "
-export __DEFAULT__SERVER_INSTITUTION="Ouranos"
-export __DEFAULT__SERVER_SUBJECT="Climatology"
+__DEFAULT__SERVER_INSTITUTION="Ouranos"
+__DEFAULT__SERVER_SUBJECT="Climatology"
 # below can be a CSV list of tags
-export __DEFAULT__SERVER_TAGS="Climatology"
-export __DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html"
-export __DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md"
-export __DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues"
+__DEFAULT__SERVER_TAGS="Climatology"
+__DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html"
+__DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md"
+__DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues"
 # NOTE:
 #   This value does not use the previously hard coded default.
 #   Previous default pointed at the wrong repository with a mismatching LICENSE file.
-export __DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE"
+__DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE"
+__DEFAULT__SUPPORT_EMAIL="[email protected]"
+__DEFAULT__DOC_URL="https://www.example.com/"
+__DEFAULT__PAVICS_FQDN="hostname.domainname"
+__DEFAULT__SSL_CERTIFICATE="/path/to/ssl/cert.pem"
 
 # apply overrides or fallback above defaults with delayed evaluation
-# see also: 'SUPPORT_EMAIL'
-# This parameter is relevant to server details, but uses the old name for backward compatibility.
+# exceptions for 'SUPPORT_EMAIL' and 'DOC_URL' using the old name for backward compatibility.
+export SUPPORT_EMAIL='${__DEFAULT__SUPPORT_EMAIL}'
+export DOC_URL='${__DEFAULT__DOC_URL}'
+export SSL_CERTIFICATE='${__DEFAULT__SSL_CERTIFICATE}'
 export SERVER_NAME='${__DEFAULT__SERVER_NAME}'
 export SERVER_DESCRIPTION='${__DEFAULT__SERVER_DESCRIPTION}'
 export SERVER_INSTITUTION='${__DEFAULT__SERVER_INSTITUTION}'
@@ -83,24 +92,24 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}'
 # Defaults for required variables recommended for override for security reasons.
 # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance.
 # These values would be detected only if the instance was configured using a copy of 'env.local.example'.
-# Any default value change here should be synchronized with the corresponding value in 'env.local.example'.
-export __DEFAULT__MAGPIE_SECRET="itzaseekrit"
-export __DEFAULT__MAGPIE_ADMIN_USERNAME="admin"
-export __DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!"
-export __DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics"
-export __DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty"
-export __DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie"
-export __DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty"
-export __DEFAULT__GEOSERVER_ADMIN_USER="admingeo"
-export __DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass"
+# Any default value should use the corresponding '${__DEFAULT__{var}}' reference in 'env.local.example'.
+__DEFAULT__MAGPIE_SECRET="itzaseekrit"
+__DEFAULT__MAGPIE_ADMIN_USERNAME="admin"
+__DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!"
+__DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics"
+__DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty"
+__DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie"
+__DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty"
+__DEFAULT__GEOSERVER_ADMIN_USER="admingeo"
+__DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass"
 #############################################################################
 # Deprecated vars (for components in the ./deprecated-components directory)
 #############################################################################
-export __DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass"
-export __DEFAULT__CATALOG_USERNAME="admin-catalog"
-export __DEFAULT__CATALOG_PASSWORD="qwerty"
-export __DEFAULT__PHOENIX_PASSWORD="phoenix_pass"
-export __DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234"
+__DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass"
+__DEFAULT__CATALOG_USERNAME="admin-catalog"
+__DEFAULT__CATALOG_PASSWORD="qwerty"
+__DEFAULT__PHOENIX_PASSWORD="phoenix_pass"
+__DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234"
 
 export DEFAULT_CONF_DIRS='
   ./components/proxy

birdhouse/deployment/certbotwrapper

@@ -47,7 +47,7 @@ certbotwrapper START_TIME=$START_TIME"
 set -x
 
 THIS_FILE="$(readlink -f "$0" || realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
 SAVED_PWD="`pwd`"
 
 . "$THIS_DIR/../read-configs.include.sh"

birdhouse/deployment/deploy.sh

@@ -53,9 +53,9 @@
 #   are re-read.  docker-compose is not aware of any changes outside of the
 #   docker-compose.yml file.
 
-if [ ! -z "$AUTODEPLOY_SILENT" ]; then
+if [ ! -z "${AUTODEPLOY_SILENT}" ]; then
     LOG_FILE="/var/log/PAVICS/autodeploy.log"
-    exec >>$LOG_FILE 2>&1
+    exec >> "${LOG_FILE}" 2>&1
 fi
 
 usage() {
@@ -65,31 +65,30 @@ usage() {
 COMPOSE_DIR="$1"
 ENV_LOCAL_FILE="$2"
 
-if [ -z "$COMPOSE_DIR" ]; then
+if [ -z "${COMPOSE_DIR}" ]; then
     echo "ERROR: please provide path to PAVICS docker-compose dir." 1>&2
     usage
     exit 2
 else
     shift
 fi
 
-if [ -z "$ENV_LOCAL_FILE" ]; then
-    ENV_LOCAL_FILE="$COMPOSE_DIR/env.local"
+if [ -z "${ENV_LOCAL_FILE}" ]; then
+    ENV_LOCAL_FILE="${COMPOSE_DIR}/env.local"
 else
     shift
 fi
 
-COMPOSE_DIR="$(realpath "$COMPOSE_DIR")"
-REPO_ROOT="$(realpath "$COMPOSE_DIR/..")"
+COMPOSE_DIR="$(realpath "${COMPOSE_DIR}")"
 
-if [ ! -f "$COMPOSE_DIR/docker-compose.yml" -o \
-     ! -f "$COMPOSE_DIR/pavics-compose.sh" ]; then
-    echo "ERROR: missing docker-compose.yml or pavics-compose.sh file in '$COMPOSE_DIR'" 1>&2
+if [ ! -f "${COMPOSE_DIR}/docker-compose.yml" ] || \
+   [ ! -f "${COMPOSE_DIR}/pavics-compose.sh" ]; then
+    echo "ERROR: missing docker-compose.yml or pavics-compose.sh file in '${COMPOSE_DIR}'" 1>&2
     exit 2
 fi
 
-if [ ! -f "$ENV_LOCAL_FILE" ]; then
-    echo "ERROR: env.local '$ENV_LOCAL_FILE' not found, please instantiate from '$COMPOSE_DIR/env.local.example'" 1>&2
+if [ ! -f "${ENV_LOCAL_FILE}" ]; then
+    echo "ERROR: env.local '${ENV_LOCAL_FILE}' not found, please instantiate from '${COMPOSE_DIR}/env.local.example'" 1>&2
     exit 2
 fi
 
@@ -100,56 +99,56 @@ fi
 # Setup COMPOSE_DIR and PWD for sourcing env.local.
 # Prevent un-expected difference when this script is run inside autodeploy
 # container and manually from the host.
-cd $COMPOSE_DIR
+cd "${COMPOSE_DIR}" || exit
 
-START_TIME="`date -Isecond`"
-echo "deploy START_TIME=$START_TIME"
+START_TIME="$(date -Isecond)"
+echo "deploy START_TIME=${START_TIME}"
 
-. "${COMPOSE_DIR}/scripts/read-configs.include.sh"
+. "${COMPOSE_DIR}/read-configs.include.sh"
 
 # Read AUTODEPLOY_EXTRA_REPOS
 read_basic_configs_only
 
 set -x
 
-for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do
-    if [ -d "$adir" ]; then
-        cd $adir
+for adir in "${COMPOSE_DIR}" ${AUTODEPLOY_EXTRA_REPOS}; do
+    if [ -d "${adir}" ]; then
+        cd "${adir}" || exit
 
         # fail fast if unclean checkout
-        if [ ! -z "`git status -u --porcelain`" ]; then
-            echo "ERROR: unclean repo '$adir'" 1>&2
+        if [ ! -z "$(git status -u --porcelain)" ]; then
+            echo "ERROR: unclean repo '${adir}'" 1>&2
             exit 1
         fi
     else
-        echo "WARNING: extra repo '$adir' do not exist"
+        echo "WARNING: extra repo '${adir}' do not exist"
     fi
 done
 
-cd $COMPOSE_DIR
+cd "${COMPOSE_DIR}" || exit
 
 read_basic_configs_only
 
 # stop all to force reload any changed config that are volume-mount into the containers
 ./pavics-compose.sh stop
 
-for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do
-    if [ -d "$adir" ]; then
-        cd $adir
+for adir in "${COMPOSE_DIR}" ${AUTODEPLOY_EXTRA_REPOS}; do
+    if [ -d "${adir}" ]; then
+        cd "${adir}" || exit
 
-        EXTRA_REPO="`git rev-parse --show-toplevel`"
-        DEPLOY_KEY="$AUTODEPLOY_DEPLOY_KEY_ROOT_DIR/`basename "$EXTRA_REPO"`_deploy_key"
-        DEFAULT_DEPLOY_KEY="$AUTODEPLOY_DEPLOY_KEY_ROOT_DIR/id_rsa_git_ssh_read_only"
-        if [ ! -e "$DEPLOY_KEY" -a -e "$DEFAULT_DEPLOY_KEY" ]; then
-            DEPLOY_KEY="$DEFAULT_DEPLOY_KEY"
+        EXTRA_REPO="$(git rev-parse --show-toplevel)"
+        DEPLOY_KEY="${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}/$(basename "${EXTRA_REPO}")_deploy_key"
+        DEFAULT_DEPLOY_KEY="${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}/id_rsa_git_ssh_read_only"
+        if [ ! -e "${DEPLOY_KEY}" ] && [ -e "${DEFAULT_DEPLOY_KEY}" ]; then
+            DEPLOY_KEY="${DEFAULT_DEPLOY_KEY}"
         fi
 
         export GIT_SSH_COMMAND=""  # git ver 2.3+
-        if [ -e "$DEPLOY_KEY" ]; then
+        if [ -e "${DEPLOY_KEY}" ]; then
             # override git ssh command for private repos only
             #
             # https://git-scm.com/docs/git-config#Documentation/git-config.txt-sshvariant
-            export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=$DEPLOY_KEY"
+            export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=${DEPLOY_KEY}"
         else
             unset GIT_SSH_COMMAND
         fi
@@ -160,18 +159,18 @@ for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do
         # This runs as the root user so new/updated files will be owned by root after the git pull, this sets the
         # owner of the code to CODE_OWNERSHIP if set. CODE_OWNERSHIP should contain uids instead of usernames since
         # usernames within a docker container will not necessarily line up with those on the host system.
-        if [ -n "$CODE_OWNERSHIP" ]; then
-          chown -R "$CODE_OWNERSHIP" "$(git rev-parse --show-toplevel)"
+        if [ -n "${CODE_OWNERSHIP}" ]; then
+          chown -R "${CODE_OWNERSHIP}" "$(git rev-parse --show-toplevel)"
         fi
     else
-        echo "WARNING: extra repo '$adir' do not exist"
+        echo "WARNING: extra repo '${adir}' do not exist"
     fi
 done
 
-cd $COMPOSE_DIR
+cd "${COMPOSE_DIR}" || exit
 
 # reload again after git pull because this file could be changed by the pull
-. "${COMPOSE_DIR}/scripts/read-configs.include.sh"
+. "${COMPOSE_DIR}/read-configs.include.sh"
 
 # reload again after default.env since env.local can override default.env
 # (ex: JUPYTERHUB_USER_DATA_DIR)
@@ -183,8 +182,8 @@ read_basic_configs_only
 set +x
 
 echo "
-deploy finished START_TIME=$START_TIME
-deploy finished   END_TIME=`date -Isecond`"
+deploy finished START_TIME=${START_TIME}
+deploy finished   END_TIME=$(date -Isecond)"
 
 
 # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4

birdhouse/deployment/fix-geoserver-data-dir-perm

@@ -9,28 +9,26 @@
 # FIRST_RUN_ONLY, this script will always execute.
 
 THIS_FILE="$(readlink -f "$0" || realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
+COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}")}"
 
-# Go to repo root.
-cd $THIS_DIR/../..
-
-. birdhouse/scripts/read-configs.include.sh
+. "${COMPOSE_DIR}/read-configs.include.sh"
 
 # Get BASH_IMAGE
 # Get GEOSERVER_DATA_DIR
 read_configs
 
 set -x
 
-DATA_DIR="$GEOSERVER_DATA_DIR"
+DATA_DIR="${GEOSERVER_DATA_DIR}"
 
 if [ -n "$1" ]; then
     DATA_DIR="$1"; shift
 fi
 
 docker run --rm --name fix-geoserver-data-dir-perm \
-    --volume ${DATA_DIR}:/datadir \
+    --volume "${DATA_DIR}":/datadir \
     --env FIRST_RUN_ONLY \
-    $BASH_IMAGE \
+    "${BASH_IMAGE}" \
     bash -xc 'if [ -z "$FIRST_RUN_ONLY" -o ! -f /datadir/global.xml ]; \
     then chown -R 1000:10001 /datadir; else echo "No execute."; fi'

birdhouse/deployment/fix-write-perm

@@ -38,25 +38,26 @@
 # localized (only the directories we need) solution.
 
 THIS_FILE="$(readlink -f "$0" || realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
+COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}")}"
 
-# Go to repo root.
-cd $THIS_DIR/../..
-
-. birdhouse/scripts/read-configs.include.sh
+. "${COMPOSE_DIR}/read-configs.include.sh"
 
 # Get GEOSERVER_DATA_DIR, JUPYTERHUB_USER_DATA_DIR, MAGPIE_PERSIST_DIR
 read_configs
 
-DEFAULT_EXTRA_DATA_DIR="$GEOSERVER_DATA_DIR/ $JUPYTERHUB_USER_DATA_DIR/ \
-    $MAGPIE_PERSIST_DIR/"
+DEFAULT_EXTRA_DATA_DIR="
+    ${GEOSERVER_DATA_DIR}/
+    ${JUPYTERHUB_USER_DATA_DIR}/ \
+    ${MAGPIE_PERSIST_DIR}/ \
+"
 
 EXTRA_DATA_DIR=""
 if [ -n "$FIX_WRITE_PERM_EXTRA" ]; then
-    EXTRA_DATA_DIR="$DEFAULT_EXTRA_DATA_DIR"
+    EXTRA_DATA_DIR="${DEFAULT_EXTRA_DATA_DIR}"
 fi
 
 set -x
 
-sudo setfacl -Rdm u:$USER:rwX $PWD $AUTODEPLOY_EXTRA_REPOS $EXTRA_DATA_DIR "$@"  # for future files
-sudo setfacl -Rm u:$USER:rwX $PWD $AUTODEPLOY_EXTRA_REPOS $EXTRA_DATA_DIR "$@"  # for existing files
+sudo setfacl -Rdm "u:${USER}:rwX" "${PWD}" ${AUTODEPLOY_EXTRA_REPOS} ${EXTRA_DATA_DIR} "$@"  # for future files
+sudo setfacl -Rm "u:${USER}:rwX" "${PWD}" ${AUTODEPLOY_EXTRA_REPOS} ${EXTRA_DATA_DIR} "$@"  # for existing files