From 9c67f685dd5602a2435bb15c8c575d036ca8fe33 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 14 Dec 2023 20:41:18 -0500 Subject: [PATCH 01/34] node details overrides + logging utils --- CHANGES.md | 21 +- .../docker_configuration.py.template | 14 +- .../geoserver/pre-docker-compose-up | 6 +- .../conf.extra-service.d/canarie-api.conf | 6 + .../conf.extra-service.d/canarie-api.conf | 6 + .../canarie-api/docker_configuration.py | 229 ++++++++++++++ birdhouse/config/catalog/catalog.cfg | 34 ++ .../canarie-api/canarie_api_monitoring.py | 34 ++ .../catalog/config/magpie/permissions.cfg | 5 + .../catalog/config/magpie/providers.cfg | 43 +++ .../config/proxy/canarie_api_monitoring.py | 5 + .../canarie-api/canarie_api_monitoring.py | 112 +++++++ .../config/finch/config/magpie/providers.cfg | 9 + .../config/proxy/canarie_api_monitoring.py | 71 +++++ birdhouse/config/finch/service-config.json | 25 ++ birdhouse/config/finch/wps.cfg | 15 + .../config/magpie/permissions.cfg | 5 + .../flyingpigeon/config/magpie/providers.cfg | 8 + .../config/flyingpigeon/service-config.json | 25 ++ birdhouse/config/flyingpigeon/wps.cfg | 10 + .../canarie-api/canarie_api_monitoring.py | 5 + .../proxy/conf.extra-service.d/frontend.conf | 6 + birdhouse/config/frontend/frontend.env | 8 + .../canarie-api/canarie_api_monitoring.py | 40 +++ .../config/proxy/canarie_api_monitoring.py | 6 + .../proxy/conf.extra-service.d/geoserver.conf | 30 ++ .../config/geoserver/service-config.json | 23 ++ .../canarie-api/canarie_api_monitoring.py | 38 +++ .../hummingbird/config/magpie/providers.cfg | 8 + birdhouse/config/hummingbird/custom.cfg | 8 + .../config/hummingbird/service-config.json | 25 ++ .../canarie-api/canarie_api_monitoring.py | 35 +++ .../jupyterhub/config/magpie/providers.cfg | 10 + .../config/proxy/canarie_api_monitoring.py | 7 + .../conf.extra-service.d/jupyterhub.conf | 11 + .../jupyterhub/custom_templates/login.html | 45 +++ .../config/jupyterhub/jupyterhub_config.py | 296 ++++++++++++++++++ .../config/jupyterhub/service-config.json | 20 ++ .../canarie-api/canarie_api_monitoring.py | 42 +++ .../config/proxy/canarie_api_monitoring.py | 9 + .../proxy/conf.extra-service.d/magpie.conf | 6 + .../docker-entrypoint-initdb.d/create-db.sql | 1 + birdhouse/config/magpie/magpie.ini | 183 +++++++++++ birdhouse/config/magpie/permissions.cfg | 22 ++ .../config/magpie/postgres-credentials.env | 7 + birdhouse/config/magpie/providers.cfg | 1 + .../canarie-api/canarie_api_monitoring.py | 34 ++ .../malleefowl/config/magpie/providers.cfg | 8 + .../config/proxy/canarie_api_monitoring.py | 5 + birdhouse/config/malleefowl/custom.cfg | 30 ++ birdhouse/config/ncops/ncops.cfg | 2 + .../config/ncwms2/config/magpie/providers.cfg | 8 + .../proxy/conf.extra-service.d/ncwms2.conf | 7 + birdhouse/config/ncwms2/custom.cfg | 11 + .../canarie-api/canarie_api_monitoring.py | 34 ++ .../config/proxy/canarie_api_monitoring.py | 5 + birdhouse/config/phoenix/custom.cfg | 10 + .../proxy/conf.extra-service.d/portainer.conf | 5 + birdhouse/config/postgres/credentials.env | 3 + .../canarie-api/canarie_api_monitoring.py | 6 + .../conf.extra-service.d/project-api.conf | 6 + .../config/proxy/conf.d/all-services.include | 28 ++ birdhouse/config/proxy/conf.d/frontend.conf | 48 +++ birdhouse/config/proxy/nginx.conf | 41 +++ birdhouse/config/proxy/static/components.json | 26 ++ birdhouse/config/proxy/static/services.json | 221 +++++++++++++ birdhouse/config/proxy/static/version.json | 1 + .../canarie-api/canarie_api_monitoring.py | 39 +++ .../config/raven/config/magpie/providers.cfg | 9 + .../config/proxy/canarie_api_monitoring.py | 39 +++ birdhouse/config/raven/service-config.json | 25 ++ birdhouse/config/raven/wps.cfg | 15 + .../canarie-api/canarie_api_monitoring.py | 39 +++ .../config/proxy/canarie_api_monitoring.py | 8 + birdhouse/config/thredds/catalog.xml | 53 ++++ .../canarie-api/canarie_api_monitoring.py | 72 +++++ .../thredds/config/magpie/providers.cfg | 35 +++ .../config/proxy/canarie_api_monitoring.py | 41 +++ .../proxy/conf.extra-service.d/thredds.conf | 10 + birdhouse/config/thredds/service-config.json | 26 ++ birdhouse/config/thredds/threddsConfig.xml | 137 ++++++++ .../canarie-api/canarie_api_monitoring.py | 56 ++++ .../config/proxy/canarie_api_monitoring.py | 9 + .../proxy/conf.extra-service.d/twitcher.conf | 8 + birdhouse/config/twitcher/twitcher.ini | 141 +++++++++ .../canarie-api/canarie_api_monitoring.py | 1 + .../config/proxy/canarie_api_monitoring.py | 1 + .../wps_outputs-volume.conf | 5 + birdhouse/default.env | 43 ++- birdhouse/deployment/deploy.sh | 4 +- .../deployment/fix-geoserver-data-dir-perm | 2 +- birdhouse/deployment/fix-write-perm | 2 +- birdhouse/deployment/install-deploy-notebook | 2 +- birdhouse/deployment/trigger-deploy-notebook | 4 +- birdhouse/deployment/triggerdeploy.sh | 2 +- birdhouse/pavics-compose.sh | 44 +-- birdhouse/scripts/check-autodeploy-repos | 4 +- birdhouse/scripts/check-instance-ready | 4 +- birdhouse/scripts/check-wps-database.sh | 8 +- .../scripts/clear-running-wps-jobs-in-db.sh | 12 +- birdhouse/scripts/create-magpie-users | 9 +- .../scripts/deprecated/trigger-pavicscrawler | 4 +- .../detect-user-install-in-jupyter-env | 9 +- .../scripts/get-components-json.include.sh | 11 +- .../scripts/get-services-json.include.sh | 9 +- birdhouse/scripts/logging.include.sh | 18 ++ .../migrate-jupyterhub-user-persistence | 2 +- .../{ => scripts}/read-configs.include.sh | 59 +++- birdhouse/scripts/sync-data | 4 +- 109 files changed, 3066 insertions(+), 73 deletions(-) create mode 100644 birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf create mode 100644 birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf create mode 100644 birdhouse/config/canarie-api/docker_configuration.py create mode 100644 birdhouse/config/catalog/catalog.cfg create mode 100644 birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/catalog/config/magpie/permissions.cfg create mode 100644 birdhouse/config/catalog/config/magpie/providers.cfg create mode 100644 birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/finch/config/magpie/providers.cfg create mode 100644 birdhouse/config/finch/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/finch/service-config.json create mode 100644 birdhouse/config/finch/wps.cfg create mode 100644 birdhouse/config/flyingpigeon/config/magpie/permissions.cfg create mode 100644 birdhouse/config/flyingpigeon/config/magpie/providers.cfg create mode 100644 birdhouse/config/flyingpigeon/service-config.json create mode 100644 birdhouse/config/flyingpigeon/wps.cfg create mode 100644 birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf create mode 100644 birdhouse/config/frontend/frontend.env create mode 100644 birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf create mode 100644 birdhouse/config/geoserver/service-config.json create mode 100644 birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/hummingbird/config/magpie/providers.cfg create mode 100644 birdhouse/config/hummingbird/custom.cfg create mode 100644 birdhouse/config/hummingbird/service-config.json create mode 100644 birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/jupyterhub/config/magpie/providers.cfg create mode 100644 birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf create mode 100644 birdhouse/config/jupyterhub/custom_templates/login.html create mode 100644 birdhouse/config/jupyterhub/jupyterhub_config.py create mode 100644 birdhouse/config/jupyterhub/service-config.json create mode 100644 birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf create mode 100644 birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql create mode 100644 birdhouse/config/magpie/magpie.ini create mode 100644 birdhouse/config/magpie/permissions.cfg create mode 100644 birdhouse/config/magpie/postgres-credentials.env create mode 100644 birdhouse/config/magpie/providers.cfg create mode 100644 birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/malleefowl/config/magpie/providers.cfg create mode 100644 birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/malleefowl/custom.cfg create mode 100644 birdhouse/config/ncops/ncops.cfg create mode 100644 birdhouse/config/ncwms2/config/magpie/providers.cfg create mode 100644 birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf create mode 100644 birdhouse/config/ncwms2/custom.cfg create mode 100644 birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/phoenix/custom.cfg create mode 100644 birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf create mode 100644 birdhouse/config/postgres/credentials.env create mode 100644 birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf create mode 100644 birdhouse/config/proxy/conf.d/all-services.include create mode 100644 birdhouse/config/proxy/conf.d/frontend.conf create mode 100644 birdhouse/config/proxy/nginx.conf create mode 100644 birdhouse/config/proxy/static/components.json create mode 100644 birdhouse/config/proxy/static/services.json create mode 100644 birdhouse/config/proxy/static/version.json create mode 100644 birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/raven/config/magpie/providers.cfg create mode 100644 birdhouse/config/raven/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/raven/service-config.json create mode 100644 birdhouse/config/raven/wps.cfg create mode 100644 birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/solr/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/thredds/catalog.xml create mode 100644 birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/thredds/config/magpie/providers.cfg create mode 100644 birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf create mode 100644 birdhouse/config/thredds/service-config.json create mode 100644 birdhouse/config/thredds/threddsConfig.xml create mode 100644 birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf create mode 100644 birdhouse/config/twitcher/twitcher.ini create mode 100644 birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py create mode 100644 birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py create mode 100644 birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf create mode 100644 birdhouse/scripts/logging.include.sh rename birdhouse/{ => scripts}/read-configs.include.sh (86%) diff --git a/CHANGES.md b/CHANGES.md index ee8768271..a64bd3439 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,26 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes +- Compose script utilities: + * Add `BIRDHOUSE_COLOR` option and various logging/messaging definitions in `birdhouse/scripts/logging.include.sh`. + * Replace all explicit color "logging" related `echo` in scripts by utility variables + `MSG_DEBUG`, `MSG_INFO`, `MSG_WARN` and `MSG_ERROR` as applicable per respective messages. + * Move `read-configs.include.sh` into `birdhouse/scripts` along other include scripts. + +- Defaults: + * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS. + These variables use a special combination of `DELAYED_EVAL` and `OPTIONAL_VARS` definitions that can make use + of a variable formatted as `='${__DEFAULT__}'` that will print a warning messages indicating + that the default is employed, although *STRONGLY* recommended to be overridden. This allows a middle ground between + backward-compatible `env.local` while flagging potentially misused configurations. + +## Fixes +- Canarie-API: updated references + * Use the new `SERVER_[...]` variables. + * Replace the LICENSE URL of the server node pointing + at [Ouranosinc/pavics-sdi](https://github.com/Ouranosinc/pavics-sdi) instead + of intended [bird-house/birdhouse-deploy](https://github.com/bird-house/birdhouse-deploy). [2.0.1](https://github.com/bird-house/birdhouse-deploy/tree/2.0.1) (2023-12-11) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index 3e70d2735..c4478dd27 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -111,23 +111,23 @@ SERVICES = { # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. 'version': '2.0.1', 'releaseTime': '2023-12-11T22:30:20Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', + 'institution': '${SERVER_INSTITUTION}', + 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', 'category': 'Resource/Cloud Management', - 'tags': ['Climatology'] + 'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()], }, 'stats': { 'method': '.*', 'route': '(?!)' # this will be set by CANARIE_STATS_ROUTES (see below) }, 'redirect': { - 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', - 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md', - 'support': 'https://github.com/bird-house/birdhouse-deploy/issues', + 'doc': '${SERVER_DOCUMENTATION_URL}', + 'releasenotes': '${SERVER_RELEASE_NOTES_URL}', + 'support': '${SERVER_SUPPORT_URL}', 'source': 'https://github.com/bird-house/birdhouse-deploy', 'tryme': 'https://${PAVICS_FQDN_PUBLIC}', - 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', + 'licence': '${SERVER_LICENSE_URL}', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' }, 'monitoring': {} # filled in after processing everything, see end of script diff --git a/birdhouse/components/geoserver/pre-docker-compose-up b/birdhouse/components/geoserver/pre-docker-compose-up index 78cc13880..f3ac3f686 100755 --- a/birdhouse/components/geoserver/pre-docker-compose-up +++ b/birdhouse/components/geoserver/pre-docker-compose-up @@ -4,7 +4,11 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="$THIS_DIR/../.." +if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/logging.include.sh" +fi + if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then - echo "fix GeoServer data dir permission on first run only, when data dir do not exist yet." + echo "${MSG_INFO}fix GeoServer data dir permission on first run only, when data dir do not exist yet." FIRST_RUN_ONLY=1 "$COMPOSE_DIR"/deployment/fix-geoserver-data-dir-perm fi diff --git a/birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf b/birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf new file mode 100644 index 000000000..e843f54f8 --- /dev/null +++ b/birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf @@ -0,0 +1,6 @@ + location /canarie/ { + proxy_pass http://127.0.0.1:2000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header X-Script-Name /canarie; + } diff --git a/birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf b/birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf new file mode 100644 index 000000000..e843f54f8 --- /dev/null +++ b/birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf @@ -0,0 +1,6 @@ + location /canarie/ { + proxy_pass http://127.0.0.1:2000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header X-Script-Name /canarie; + } diff --git a/birdhouse/config/canarie-api/docker_configuration.py b/birdhouse/config/canarie-api/docker_configuration.py new file mode 100644 index 000000000..4f03e1528 --- /dev/null +++ b/birdhouse/config/canarie-api/docker_configuration.py @@ -0,0 +1,229 @@ +# coding: utf-8 +import datetime +import os +import logging + +# note: +# Can only be dependencies also available in CanarieAPI docker image. +# Otherwise, they must be installed after the fact with the entrypoint script. +import requests_cache # see entrypoint script + +logger = logging.getLogger("canarie-api-config") + +MY_SERVER_NAME = 'https://10.0.2.15/canarie' + +DATABASE = { + 'filename': '/opt/local/src/CanarieAPI/stats.db', + 'access_log': '/var/log/nginx/access_file.log', + 'log_pid': '/var/run/nginx.pid' +} + + +# Below is to avoid rate-limiting as much as possible on multiple runs. +# Because the logparser and monitoring must load the application context and configuration, +# the requests performed to retrieve version numbers and release times occurs on every cron interval. +REQUEST_CACHE = "/tmp/canarieapi-config-cache" +os.makedirs(REQUEST_CACHE, exist_ok=True) +SESSION = requests_cache.CachedSession( + "canarieapi-config-cache", + use_temp=True, + backend="filesystem", + expire_after=datetime.timedelta(days=1), + cache_control=False, # prefer our request cache options, not ones provided by responses + stale_if_error=True, + allowable_codes=[200], + allowable_methods=["GET"], +) + + +def _get_release_time_from_github_tag(repository_reference: str, tagged_version: str) -> str: + repo = repository_reference.split("github.com/")[-1] + url = f"https://api.github.com/repos/{repo}/tags" + resp = SESSION.get(url, headers={"Accept": "application/json"}) + if resp.status_code != 200: + raise ValueError( + f"HTTP status from tags list: {resp.status_code} from [{url}] with detail:\n{resp.text!s}" + ) + + for tag_info in resp.json(): + tag_ver = tag_info.get("name") + tag_commit_url = tag_info.get("commit", {}).get("url") + if tag_ver == tagged_version or tag_ver == "v" + tagged_version and tag_commit_url: + break + else: + raise ValueError(f"No matching version in tagged releases under [{url}].") + + resp = SESSION.get(tag_commit_url, headers={"Accept": "application/json"}) + if resp.status_code != 200: + raise ValueError( + f"HTTP status from tag info: {resp.status_code} from [{tag_commit_url}] with detail:\n{resp.text!s}" + ) + + commit = resp.json()["commit"] + commit_date = commit["committer"]["date"] + return commit_date + + +def _get_release_time_from_docker_tag(repository_reference: str, tagged_version: str) -> str: + ns, repo = repository_reference.split("/", 1) + url = f"https://hub.docker.com/v2/namespaces/{ns}/repositories/{repo}/tags/{tagged_version}" + resp = SESSION.get(url, headers={"Accept": "application/json"}) + if resp.status_code != 200: + raise ValueError( + f"HTTP status from tag info: {resp.status_code} from [{url}] with detail:\n{resp.text!s}" + ) + + updated = resp.json()["last_updated"] + return updated + + +def get_release_time_from_repo_tag(repository_type: str, repository_reference: str, tagged_version: str) -> str: + try: + if repository_type == "github": + release = _get_release_time_from_github_tag(repository_reference, tagged_version) + elif repository_type == "docker": + release = _get_release_time_from_docker_tag(repository_reference, tagged_version) + else: + raise ValueError(f"Unknown repository parsing type: [{repository_type}]") + if release: + logger.error("canarie-api: found version [%s] release time [%s] from [%s: %s]", + tagged_version, release, repository_type, repository_reference) + return release + raise ValueError("no release time value found") + except Exception as exc: + logger.error("canarie-api: failed retrieving version [%s] release time from [%s: %s]", + tagged_version, repository_type, repository_reference, exc_info=exc) + return "unknown" + + +SERVICES = { + 'node': { # partial duplicate of first item from 'PLATFORM' for backward compatibility + 'info': { + 'name': 'Node', + 'synopsis': ( + 'Nodes are data, compute and index endpoints accessed through the platform or external clients. ' + 'The Node service is the backend that allows: data storage, harvesting, indexation and discovery of ' + 'local and federated data; authentication and authorization; server registration and management. ' + 'Node service is therefore composed of several other services.' + ), + # NOTE: + # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. + # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. + 'version': '1.40.0', + 'releaseTime': '2023-11-30T18:27:41Z', + 'institution': 'Ouranos', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Resource/Cloud Management', + 'tags': ['Climatology'] + }, + 'stats': { + 'method': '.*', + 'route': '(?!)' # this will be set by CANARIE_STATS_ROUTES (see below) + }, + 'redirect': { + 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', + 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md', + 'support': 'https://github.com/bird-house/birdhouse-deploy/issues', + 'source': 'https://github.com/bird-house/birdhouse-deploy', + 'tryme': 'https://10.0.2.15', + 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', + 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' + }, + 'monitoring': {} # filled in after processing everything, see end of script + } +} + +PLATFORMS = { + 'server': { + 'info': { + 'name': 'PAVICS', + 'synopsis': """ +The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of +climate analysis services served through Open Geospatial Consortium (OGC) protocols. +These services include data access, processing and visualization. Both data and algorithms +can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, +or a custom web interface. +""".replace("\n", " ").strip(), + # NOTE: + # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. + # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. + 'version': '1.40.0', + 'releaseTime': '2023-11-30T18:27:41Z', + 'institution': 'Ouranos', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '(?!)' # this can be set by individual components (eg: the legacy project-api component) + }, + 'redirect': { + 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', + 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/releases', + 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', + 'source': 'https://github.com/Ouranosinc/pavics-sdi', + 'tryme': 'https://pavics.ouranos.ca', + 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', + 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html', + 'factsheet': 'http://www.canarie.ca/software/pavics' + }, + 'monitoring': {} + } +} + +CANARIE_MONITORING_EXTRA_CONF_DIR = os.environ.get( + 'CANARIE_MONITORING_EXTRA_CONF_DIR', '/bogus-notexist' +) + + +def exec_file(filepath, **kwargs): + """ + Backward compatible ``execfile`` equivalent for Python 3.x that allows keyword arguments. + + See https://stackoverflow.com/a/41658338/5936364. + """ + _globals = kwargs.get("globals") or globals() + _locals = kwargs.get("locals") or _globals # as per exec/execfile documentation + _globals.update({ + "__file__": filepath, + "__name__": "__main__", + }) + with open(filepath, 'rb') as file: + exec(compile(file.read(), filepath, "exec"), _globals, _locals) + + +CANARIE_STATS_ROUTES = [] + +if os.path.exists(CANARIE_MONITORING_EXTRA_CONF_DIR): + # alphabetically sorted for reproducible override precedence + for extra_conf in sorted(os.listdir(CANARIE_MONITORING_EXTRA_CONF_DIR)): + extra_conf_full_path = f"{CANARIE_MONITORING_EXTRA_CONF_DIR}/{extra_conf}" + # only handle files ending with .py + if os.path.isfile(extra_conf_full_path) and extra_conf_full_path.endswith(".py"): + logger.info("canarie-api: loading extra config '%s'", extra_conf_full_path) + exec_file(extra_conf_full_path, locals=locals()) + else: + logger.info("canarie-api: ignoring extra config '%s'", extra_conf_full_path) + + +# For historical reasons, the "node" service was used to contain all "services" of the current "platform". +# However, those services should instead be listed directly at the root as individual services of the current platform, +# to directly obtain their status instead of going through the "node" service sub-page. This also allows us to provide +# the individual links (docs, release, etc.) for the services themselves. The services can also be represented nested +# under the "Platform" section as well. This "Platform" section could be extended with other known platforms part of a +# DACCS network. For backward-compatibility, repopulate the "node" service using moved definitions. +_NODE = SERVICES["node"]["monitoring"] +_PLATFORM = PLATFORMS["server"]["monitoring"] +for svc_name, svc_config in SERVICES.items(): + if svc_name == "node": + continue + for mon_name, mon_config in svc_config["monitoring"].items(): + _NODE.setdefault(mon_name, mon_config) + _PLATFORM.setdefault(mon_name, mon_config) # type: ignore + +if CANARIE_STATS_ROUTES: + SERVICES['node']['stats']['route'] = '/(' + '|'.join(r.strip('/') for r in CANARIE_STATS_ROUTES) + ')/.*' + +# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/config/catalog/catalog.cfg b/birdhouse/config/catalog/catalog.cfg new file mode 100644 index 000000000..b16b4a3ca --- /dev/null +++ b/birdhouse/config/catalog/catalog.cfg @@ -0,0 +1,34 @@ +[catalog] +solr_host=http://10.0.2.15:8983/solr/birdhouse/ + +# Multiple thredds hosts can be given, comma separated +# note: this URL is also used as prefix when comparing authorizations from magpie +thredds_host=https://10.0.2.15/twitcher/ows/proxy/thredds + +# Multiple esgf nodes can be given, comma separated +esgf_nodes=https://esgf-node.llnl.gov/esg-search + +# Provide a magpie host to filter results based on access permissions. +# Must also provide credentials with read access so that the crawler can parse the thredds host(s) +# Leave as a comment for a public catalog. +magpie_host=https://10.0.2.15/magpie +magpie_user=admin-catalog +magpie_pw=qwerty +# SSL verification (true or false) +verify=true +# Comma separated list of magpie service name for the thredds hosts listed above +thredds_host_magpie_svc_name=thredds + +# WMS service url with replaced by each instance of the thredds_host, +# without the port number and replaced by the base url in thredds. +# Leave as comment to use the default WMS service +wms_alternate_server=https://10.0.2.15/twitcher/ows/proxy/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ + +[pywps] +outputurl=https://10.0.2.15/wpsoutputs/catalog +parallelprocesses=30 + +[logging] +#level=DEBUG +#file=/tmp/wps.log +database=postgresql://postgres-pavics:postgres-qwerty@postgres/catalog diff --git a/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..0ca41f89c --- /dev/null +++ b/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,34 @@ +SERVICES['Catalog'] = { + 'info': { + 'name': 'Catalog', + 'synopsis': 'Catalog', + 'version': "0.6.11", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/pavics-datacatalog", "0.6.11"), + 'researchSubject': 'Climate', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Catalog', + 'tags': ['Catalog', 'Data'] + }, + 'stats': { + 'method': '.*', + 'route': 'http://10.0.2.15:8086/.*' + }, + 'redirect': { + 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#pavics-datacatalog', + 'releasenotes': 'https://github.com/Ouranosinc/PAVICS-DataCatalog/blob/master/CHANGES.md', + 'support': 'https://github.com/ouranosinc/PAVICS-DataCatalog/issues', + 'source': 'https://github.com/ouranosinc/PAVICS-DataCatalog', + 'tryme': 'http://10.0.2.15:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities', + 'licence': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#credits', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#pavics-datacatalog' + }, + "monitoring": { + "Catalog": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://10.0.2.15:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities' + } + } + } +} diff --git a/birdhouse/config/catalog/config/magpie/permissions.cfg b/birdhouse/config/catalog/config/magpie/permissions.cfg new file mode 100644 index 000000000..79f098e2e --- /dev/null +++ b/birdhouse/config/catalog/config/magpie/permissions.cfg @@ -0,0 +1,5 @@ +permissions: + - service: thredds + permission: read + user: admin-catalog + action: create diff --git a/birdhouse/config/catalog/config/magpie/providers.cfg b/birdhouse/config/catalog/config/magpie/providers.cfg new file mode 100644 index 000000000..c094415b1 --- /dev/null +++ b/birdhouse/config/catalog/config/magpie/providers.cfg @@ -0,0 +1,43 @@ +providers: + catalog: + url: http://10.0.2.15:8086/pywps + title: Catalog + public: true + c4i: false + type: wps + sync_type: wps + + thredds: + url: http://10.0.2.15:8083/twitcher/ows/proxy/thredds + title: Thredds + public: true + c4i: false + type: thredds + sync_type: thredds + # below is a custom config to indicate how magpie should convert thredds path elements into resources/permissions + # see: https://pavics-magpie.readthedocs.io/en/latest/services.html#servicethredds + configuration: + skip_prefix: "thredds" # prefix to ignore, below prefixes will be matched against whatever comes after in path + file_patterns: + # note: make sure to employ quotes and double escapes to avoid parsing YAML error + - ".+\\.ncml" # match longest extension first to avoid tuncating it by match of sorter '.nc' + - ".+\\.nc" + metadata_type: + prefixes: + - null # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed + - "\\w+\\.gif" # threddsIcon, folder icon, etc. + - "\\w+\\.ico" # favicon + - "\\w+\\.txt" # licence + - "\\w+\\.css" # tds.css + - "catalog\\.\\w+" # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE` + - catalog + - ncml + - uddc + - iso + data_type: + prefixes: + - fileServer + - dodsC + - wcs + - wms + - ncss diff --git a/birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py b/birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..3fc1bbc26 --- /dev/null +++ b/birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,5 @@ +SERVICES['node']['monitoring']['Catalog'] = { + 'request': { + 'url': 'http://10.0.2.15:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities' + } +} diff --git a/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..9f59a0e74 --- /dev/null +++ b/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,112 @@ +# used more than once, avoid unnecessary requests +FINCH_VERSION = "0.9.2" +FINCH_RELEASE_TIME = get_release_time_from_repo_tag("github", "bird-house/finch", FINCH_VERSION) + +SERVICES['indices'] = { + 'info': { + 'name': 'Climate indices', + 'synopsis': ( + 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, ' + 'the number of degree-days of cooling, the duration of heatwaves, etc. ' + 'This returns annual values of the most popular climate indices.' + ), + 'version': FINCH_VERSION, + 'institution': 'Ouranos', + 'releaseTime': FINCH_RELEASE_TIME, + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/finch.*' + }, + 'redirect': { + 'doc': 'https://finch.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/bird-house/finch/releases', + 'support': 'https://github.com/bird-house/finch/issues', + 'source': 'https://github.com/bird-house/finch', + 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Finch': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} +SERVICES['slicer'] = { + 'info': { + 'name': 'Spatial and temporal data slicer', + 'synopsis': 'These services allow the subsetting and download of NetCDF over user-defined regions and periods.', + 'version': '0.7.1', + 'institution': 'Ouranos', + 'releaseTime': '2021-03-25T00:00:00Z', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/finch.*' + }, + 'redirect': { + 'doc': 'https://finch.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/bird-house/finch/releases', + 'support': 'https://github.com/bird-house/finch/issues', + 'source': 'https://github.com/bird-house/finch', + 'tryme': 'https://pavics-sdi.readthedocs.io/en/latest/notebooks/regridding.html', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Finch': { + 'request': { + 'url': 'http://finch:5000/?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} + +SERVICES['Finch'] = { + 'info': { + 'name': 'Finch', + 'synopsis': ( + 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, ' + 'the number of degree-days of cooling, the duration of heatwaves, etc. This returns annual values of ' + 'the most popular climate indices.' + ), + 'version': FINCH_VERSION, + 'institution': 'Ouranos', + 'releaseTime': FINCH_RELEASE_TIME, + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/finch.*' + }, + 'redirect': { + 'doc': 'https://finch.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/bird-house/finch/releases', + 'support': 'https://github.com/bird-house/finch/issues', + 'source': 'https://github.com/bird-house/finch', + 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Finch': { + 'request': { + 'url': 'http://finch:5000/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + }, +} diff --git a/birdhouse/config/finch/config/magpie/providers.cfg b/birdhouse/config/finch/config/magpie/providers.cfg new file mode 100644 index 000000000..e9eebd1a8 --- /dev/null +++ b/birdhouse/config/finch/config/magpie/providers.cfg @@ -0,0 +1,9 @@ +providers: + finch: + url: http://finch:5000/wps + title: Finch + public: true + c4i: false + type: wps + sync_type: wps + diff --git a/birdhouse/config/finch/config/proxy/canarie_api_monitoring.py b/birdhouse/config/finch/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..19f4d2b26 --- /dev/null +++ b/birdhouse/config/finch/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,71 @@ +SERVICES['node']['monitoring']['Finch'] = { + 'request': { + 'url': 'http://10.0.2.15:8095/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } +} +SERVICES['indices'] = { + 'info': { + 'name': 'Climate indices', + 'synopsis': 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, the number of degree-days of cooling, the duration of heatwaves, etc. This returns annual values of the most popular climate indices.', + 'version': '0.7.1', + 'institution': 'Ouranos', + 'releaseTime': '2021-03-25T00:00:00Z', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/finch.*' + }, + 'redirect': { + 'doc': 'https://finch.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/bird-house/finch/releases', + 'support': 'https://github.com/bird-house/finch/issues', + 'source': 'https://github.com/bird-house/finch', + 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Finch': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} +SERVICES['slicer'] = { + 'info': { + 'name': 'Spatial and temporal data slicer', + 'synopsis': 'These services allows the subsetting and download of NetCDF over user-defined regions and periods.', + 'version': '0.7.1', + 'institution': 'Ouranos', + 'releaseTime': '2021-03-25T00:00:00Z', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/finch.*' + }, + 'redirect': { + 'doc': 'https://finch.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/bird-house/finch/releases', + 'support': 'https://github.com/bird-house/finch/issues', + 'source': 'https://github.com/bird-house/finch', + 'tryme': 'https://pavics-sdi.readthedocs.io/en/latest/notebooks/regridding.html', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Finch': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} diff --git a/birdhouse/config/finch/service-config.json b/birdhouse/config/finch/service-config.json new file mode 100644 index 000000000..7b0fde606 --- /dev/null +++ b/birdhouse/config/finch/service-config.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "finch", + "keywords": [ + "service-wps" + ], + "description": "A Web Processing Service for Climate Indicators.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://finch.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" + } + ] +} diff --git a/birdhouse/config/finch/wps.cfg b/birdhouse/config/finch/wps.cfg new file mode 100644 index 000000000..baf222d42 --- /dev/null +++ b/birdhouse/config/finch/wps.cfg @@ -0,0 +1,15 @@ +[server] +outputurl = https://10.0.2.15/wpsoutputs/finch +outputpath = /data/wpsoutputs/finch + +# default 3mb, fix "Broken pipe" between the proxy and the wps service +maxrequestsize = 400mb + +# default 2, too low for a production server with capable CPUs +parallelprocesses = 10 + +[logging] +level = INFO +database=postgresql://postgres-pavics:postgres-qwerty@postgres/finch + + diff --git a/birdhouse/config/flyingpigeon/config/magpie/permissions.cfg b/birdhouse/config/flyingpigeon/config/magpie/permissions.cfg new file mode 100644 index 000000000..9db2a983a --- /dev/null +++ b/birdhouse/config/flyingpigeon/config/magpie/permissions.cfg @@ -0,0 +1,5 @@ +permissions: + - service: flyingpigeon + permission: getcapabilities + group: administrators + action: create diff --git a/birdhouse/config/flyingpigeon/config/magpie/providers.cfg b/birdhouse/config/flyingpigeon/config/magpie/providers.cfg new file mode 100644 index 000000000..08f74cebd --- /dev/null +++ b/birdhouse/config/flyingpigeon/config/magpie/providers.cfg @@ -0,0 +1,8 @@ +providers: + flyingpigeon: + url: http://flyingpigeon:8093/wps + title: Flyingpigeon + public: true + c4i: false + type: wps + sync_type: wps diff --git a/birdhouse/config/flyingpigeon/service-config.json b/birdhouse/config/flyingpigeon/service-config.json new file mode 100644 index 000000000..c303d439e --- /dev/null +++ b/birdhouse/config/flyingpigeon/service-config.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", + "name": "flyingpigeon", + "keywords": [ + "service-wps" + ], + "description": "WPS for climate model data, indices and extreme events.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/flyingpigeon?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://flyingpigeon.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/flyingpigeon?service=WPS&request=GetCapabilities" + } + ] +} diff --git a/birdhouse/config/flyingpigeon/wps.cfg b/birdhouse/config/flyingpigeon/wps.cfg new file mode 100644 index 000000000..b2587b600 --- /dev/null +++ b/birdhouse/config/flyingpigeon/wps.cfg @@ -0,0 +1,10 @@ +[server] +outputurl = https://10.0.2.15/wpsoutputs/flyingpigeon +outputpath = /data/wpsoutputs/flyingpigeon +maxsingleinputsize = 2097152000.0 + +[logging] +level = INFO +database=postgresql://postgres-pavics:postgres-qwerty@postgres/flyingpigeon + + diff --git a/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..e70b9c487 --- /dev/null +++ b/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,5 @@ +PLATFORMS['server']['monitoring']['Frontend'] = { + 'request': { + 'url': 'https://10.0.2.15' + } +} diff --git a/birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf b/birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf new file mode 100644 index 000000000..e15a9e280 --- /dev/null +++ b/birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf @@ -0,0 +1,6 @@ + + location /frontend/ { + proxy_pass http://10.0.2.15:3000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + } diff --git a/birdhouse/config/frontend/frontend.env b/birdhouse/config/frontend/frontend.env new file mode 100644 index 000000000..feab043c9 --- /dev/null +++ b/birdhouse/config/frontend/frontend.env @@ -0,0 +1,8 @@ +PAVICS_FRONTEND_IP=10.0.2.15 +PAVICS_FRONTEND_PORT=443 +PAVICS_FRONTEND_PROTO=https +BIRDHOUSE_HOST=10.0.2.15 +NODE_TLS_REJECT_UNAUTHORIZED=0 +NCWMS_HOST=https://10.0.2.15/twitcher/ows/proxy/ncWMS2/wms +CATALOG_HOST=https://10.0.2.15/twitcher/ows/proxy/catalog/pywps +MALLEEFOWL_HOST=https://10.0.2.15/twitcher/ows/proxy/malleefowl/wps diff --git a/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..2a098004c --- /dev/null +++ b/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,40 @@ +SERVICES['GeoServer'] = { + 'info': { + 'name': 'GeoServer', + 'synopsis': ( + 'GeoServer is the reference implementation of the Open Geospatial Consortium (OGC) ' + 'Web Feature Service (WFS) and Web Coverage Service (WCS) standards, as well as a high performance ' + 'certified compliant Web Map Service (WMS), compliant Catalog Service for the Web (CSW) and ' + 'implementing Web Processing Service (WPS). GeoServer forms a core component of the Geospatial Web.' + ), + 'version': "2.22.2", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/geoserver", "2.22.2-kartoza-build20230226-r7-allow-change-context-root-and-fix-missing-stable-plugins-and-avoid-chown-datadir"), + 'researchSubject': 'Geospatial', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Catalog', + 'tags': ['Data', 'Geospatial', 'Catalog', 'OGC', 'WFS', 'WMS', 'WPS'] + }, + 'stats': { + 'method': '.*', + 'route': "/geoserver/.*" + }, + 'redirect': { + 'doc': 'https://docs.geoserver.org/', + 'releasenotes': 'https://geoserver.org/release/2.22.2/', + 'support': 'https://github.com/kartoza/docker-geoserver/issues', + 'source': 'https://github.com/kartoza/docker-geoserver', + 'tryme': 'https://10.0.2.15/geoserver/', + 'licence': 'https://github.com/geoserver/geoserver/blob/2.22.2/LICENSE.txt', + 'provenance': 'https://github.com/kartoza/docker-geoserver' + }, + "monitoring": { + "GeoServer": { + 'request': { + 'url': 'https://10.0.2.15/geoserver/web/' + } + } + } +} + +CANARIE_STATS_ROUTES.append('geoserver') diff --git a/birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py b/birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..c5ebc7483 --- /dev/null +++ b/birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,6 @@ +SERVICES['node']['monitoring'].update({'Geoserver': { + 'request': { + 'url': 'https://10.0.2.15/geoserver/web/' + } +}}) +CANARIE_STATS_ROUTES.append('geoserver') diff --git a/birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf b/birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf new file mode 100644 index 000000000..2d8abd4b7 --- /dev/null +++ b/birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf @@ -0,0 +1,30 @@ + location /geoserver/ { + auth_request /secure-geoserver-auth; + auth_request_set $auth_status $upstream_status; + + proxy_pass http://geoserver:8080/geoserver/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header Accept-Encoding ""; + gzip_proxied any; + gzip on; + gzip_comp_level 1; + gzip_types application/json text/plain application/xml text/html; + } + + location = /secure-geoserver-auth { + internal; + + + # If GEOSERVER_SKIP_AUTH is "True" then the following section is skipped and this + # location block will always return 200 (which means that the /geoserver/ location, above, + # will be publicly available. + proxy_pass https://10.0.2.15/twitcher/ows/verify/geoserver$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host:$server_port; + } diff --git a/birdhouse/config/geoserver/service-config.json b/birdhouse/config/geoserver/service-config.json new file mode 100644 index 000000000..71cdc4455 --- /dev/null +++ b/birdhouse/config/geoserver/service-config.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "geoserver", + "keywords": [ + "data", + "service-wms", + "service-wfs", + "service-wps" + ], + "description": "GeoServer is a server that allows users to view and edit geospatial data.", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://10.0.2.15/geoserver/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://docs.geoserver.org/" + } + ] +} diff --git a/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..903c217ff --- /dev/null +++ b/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,38 @@ +HUMMINGBIRD_VERSION = "0.5_dev" +HUMMINGBIRD_RELEASE = get_release_time_from_repo_tag("docker", "pavics/hummingbird", HUMMINGBIRD_VERSION) + +SERVICES['hummingbird'] = { + 'info': { + 'name': 'Climatology compliance checker.', + 'synopsis': ( + 'A Web Processing Service for compliance checks used in the climate science community.' + ), + 'version': HUMMINGBIRD_VERSION, + 'institution': 'bird-house', + 'releaseTime': HUMMINGBIRD_RELEASE, + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Processing', + 'tags': ['Climatology', 'Checker', 'Compliance', 'CF-conventions', 'WPS', 'OGC'], + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/hummingbird.*' + }, + 'redirect': { + 'doc': 'https://hummingbird.readthedocs.io/', + 'releasenotes': 'https://github.com/bird-house/hummingbird/blob/master/CHANGES.rst', + 'support': 'https://github.com/bird-house/hummingbird/issues', + 'source': 'https://github.com/bird-house/hummingbird', + 'tryme': 'https://10.0.2.15/twitcher/ows/proxy/hummingbird/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'licence': 'https://github.com/bird-house/hummingbird/blob/master/LICENSE.txt', + 'provenance': 'https://github.com/bird-house/hummingbird' + }, + 'monitoring': { + 'Hummingbird': { + 'request': { + 'url': 'http://hummingbird:8080/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} diff --git a/birdhouse/config/hummingbird/config/magpie/providers.cfg b/birdhouse/config/hummingbird/config/magpie/providers.cfg new file mode 100644 index 000000000..f2b6855ac --- /dev/null +++ b/birdhouse/config/hummingbird/config/magpie/providers.cfg @@ -0,0 +1,8 @@ +providers: + hummingbird: + url: http://hummingbird:8080/wps + title: Hummingbird + public: true + c4i: false + type: wps + sync_type: wps diff --git a/birdhouse/config/hummingbird/custom.cfg b/birdhouse/config/hummingbird/custom.cfg new file mode 100644 index 000000000..11ee63b92 --- /dev/null +++ b/birdhouse/config/hummingbird/custom.cfg @@ -0,0 +1,8 @@ +[buildout] +extends = profiles/docker.cfg + +[pywps] +enable-https = true +https-port = 443 +https-output-port = 443 +database=postgresql://postgres-pavics:postgres-qwerty@postgres/hummingbird diff --git a/birdhouse/config/hummingbird/service-config.json b/birdhouse/config/hummingbird/service-config.json new file mode 100644 index 000000000..4d17f2cc9 --- /dev/null +++ b/birdhouse/config/hummingbird/service-config.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "hummingbird", + "keywords": [ + "service-wps" + ], + "description": "A WPS for metadata compliance checks used in the climate science community.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://birdhouse-hummingbird.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" + } + ] +} diff --git a/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..5c8e285ba --- /dev/null +++ b/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,35 @@ +SERVICES['Jupyter'] = { + 'info': { + 'name': 'Jupyter', + 'synopsis': 'Jupyter notebooks portal.', + 'version': "${JUPYTER_VERSION}", + 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/jupyterhub", "latest"), + 'institution': 'Ouranos', + 'researchSubject': 'Any', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Research', + 'tags': ['Development', 'Research', 'Notebooks'] + }, + 'stats': { + 'method': '.*', + 'route': '/jupyter/.*' + }, + 'redirect': { + 'doc': 'https://jupyter.org/hub', + 'releasenotes': 'https://github.com/Ouranosinc/jupyterhub/tags', # no CHANGES file available + 'support': 'https://github.com/Ouranosinc/jupyterhub/issues', + 'source': 'https://github.com/Ouranosinc/jupyterhub', + 'tryme': 'https://10.0.2.15/jupyter/', + 'licence': 'https://github.com/Ouranosinc/jupyterhub/blob/latest/LICENSE', + 'provenance': '' + }, + "monitoring": { + "Jupyter": { + 'request': { + 'url': 'https://10.0.2.15/jupyter/hub/login' + }, + } + } +} + +CANARIE_STATS_ROUTES.append('jupyter') diff --git a/birdhouse/config/jupyterhub/config/magpie/providers.cfg b/birdhouse/config/jupyterhub/config/magpie/providers.cfg new file mode 100644 index 000000000..f768d7837 --- /dev/null +++ b/birdhouse/config/jupyterhub/config/magpie/providers.cfg @@ -0,0 +1,10 @@ +providers: + jupyterhub: + # below URL is only used to fill in the required location in Magpie + # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL + url: http://proxy:80 + title: Jupyter + public: true + c4i: false + type: api + sync_type: api diff --git a/birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py b/birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..65ffb99de --- /dev/null +++ b/birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,7 @@ +SERVICES['node']['monitoring']['Jupyter'] = { + 'request': { + 'url': 'https://10.0.2.15/jupyter/hub/login' + } +} +CANARIE_STATS_ROUTES.append('jupyter') + diff --git a/birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf b/birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf new file mode 100644 index 000000000..7da931a62 --- /dev/null +++ b/birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf @@ -0,0 +1,11 @@ + location /jupyter/ { + proxy_pass http://jupyterhub:8000/jupyter/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # websocket headers + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } diff --git a/birdhouse/config/jupyterhub/custom_templates/login.html b/birdhouse/config/jupyterhub/custom_templates/login.html new file mode 100644 index 000000000..384107ceb --- /dev/null +++ b/birdhouse/config/jupyterhub/custom_templates/login.html @@ -0,0 +1,45 @@ +{% extends "templates/login.html" %} {% set announcement_login = ' + + + +

Public demo login: demo

+

+ Given this public nature, anyone can tamper with your notebooks so please + export your valuable notebooks elsewhere if you want to + preverve them. This public demo account also have limitted computing + resources. +

+

+ Contact helpdesk@example.com for information on how to + get an account and a private workspace or the password of the + public demo account. +

+

+ The only writable folder is writable-workspace + (/notebook_dir/writable-workspace in the terminal) and it is persisted + between sessions. +

+

+ Please be considerate with the amount of + disk space usage on this Jupyter instance. +

+

+ Please shutdown the kernel and close un-used + notebooks to avoid wasting computing resources. +

+

+ This Jupyter instance can restart every day. + Long running processes will be killed without notice. +

+ + +' %} + +{% set login_term_url = '' %} + +{# Example overriding the actual login terms and condition text #} +{# +{% block login_terms_text %} +Your own text here terms and conditions. +{% endblock login_terms_text %} +#} diff --git a/birdhouse/config/jupyterhub/jupyterhub_config.py b/birdhouse/config/jupyterhub/jupyterhub_config.py new file mode 100644 index 000000000..575a3b394 --- /dev/null +++ b/birdhouse/config/jupyterhub/jupyterhub_config.py @@ -0,0 +1,296 @@ +import os +from os.path import join +import logging +import subprocess + +from dockerspawner import DockerSpawner + +c = get_config() # noqa # can be called directy without import because injected by IPython + +c.JupyterHub.bind_url = 'http://:8000/jupyter' + +## Whether to shutdown single-user servers when the Hub shuts down. +c.JupyterHub.cleanup_servers = False + +c.JupyterHub.hub_ip = 'jupyterhub' + +c.JupyterHub.authenticator_class = 'jupyterhub_magpie_authenticator.MagpieAuthenticator' +c.MagpieAuthenticator.magpie_url = "http://magpie:2001" +c.MagpieAuthenticator.public_fqdn = "10.0.2.15" +c.MagpieAuthenticator.authorization_url = "http://twitcher:8000/ows/verify/jupyterhub" + +if os.getenv("JUPYTERHUB_CRYPT_KEY"): + c.MagpieAuthenticator.enable_auth_state = True + c.MagpieAuthenticator.refresh_pre_spawn = True + c.MagpieAuthenticator.auth_refresh_age = int("60") + +c.JupyterHub.cookie_secret_file = '/persist/jupyterhub_cookie_secret' +c.JupyterHub.db_url = '/persist/jupyterhub.sqlite' + +c.JupyterHub.template_paths = ['/custom_templates'] + +class CustomDockerSpawner(DockerSpawner): + @property + def escaped_name(self): + """ + Return the username without escaping. This ensures that mounted directories on the + host machine are discovered properly since we expect the username to match the username + set by Magpie. + """ + return self.user.name + + async def start(self): + if(os.environ['MOUNT_IMAGE_SPECIFIC_NOTEBOOKS'] == 'true'): + host_dir = join(os.environ['JUPYTERHUB_USER_DATA_DIR'], 'tutorial-notebooks-specific-images') + + # Mount a volume with a tutorial-notebook subfolder corresponding to the image name, if it exists + # The names are defined in the JUPYTERHUB_IMAGE_SELECTION_NAMES variable. + image_name = self.user_options.get('image') + if(os.path.isdir(join(host_dir, image_name))): + self.volumes[join(host_dir, image_name)] = { + "bind": '/notebook_dir/tutorial-notebooks', + "mode": "ro" + } + else: + # Try again, removing any colons and any following text. Useful if the image name contains + # the version number, which should not be used in the directory name. + image_name = image_name.split(':')[0] + if(os.path.isdir(join(host_dir, image_name))): + self.volumes[join(host_dir, image_name)] = { + "bind": '/notebook_dir/tutorial-notebooks', + "mode": "ro" + } + else: + # Mount the entire tutorial-notebooks directory + self.volumes[join(os.environ['JUPYTERHUB_USER_DATA_DIR'], "tutorial-notebooks")] = { + "bind": "/notebook_dir/tutorial-notebooks", + "mode": "ro" + } + return await super().start() + +c.JupyterHub.spawner_class = CustomDockerSpawner + +# Selects the first image from the list by default +c.DockerSpawner.image = os.environ['DOCKER_NOTEBOOK_IMAGES'].split()[0] +c.DockerSpawner.use_internal_ip = True +c.DockerSpawner.network_name = os.environ['DOCKER_NETWORK_NAME'] + +notebook_dir = '/notebook_dir' +jupyterhub_data_dir = os.environ['JUPYTERHUB_USER_DATA_DIR'] +container_workspace_dir = join(notebook_dir, "writable-workspace") +container_home_dir = join(container_workspace_dir, ".home") + +c.DockerSpawner.notebook_dir = notebook_dir +c.DockerSpawner.environment = { + "HOME": container_home_dir, + # https://docs.bokeh.org/en/latest/docs/user_guide/jupyter.html#jupyterhub + # Issue https://github.com/bokeh/bokeh/issues/12090 + # Post on Panel forum: + # https://discourse.holoviz.org/t/how-to-customize-the-display-url-from-panel-serve-for-use-behind-jupyterhub-with-jupyter-server-proxy/3571 + # Issue about Panel Preview: https://github.com/holoviz/panel/issues/3440 + "PAVICS_HOST_URL": "https://10.0.2.15", + # https://docs.dask.org/en/stable/configuration.html + # https://jupyterhub-on-hadoop.readthedocs.io/en/latest/dask.html + "DASK_DISTRIBUTED__DASHBOARD__LINK": "https://10.0.2.15{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" +} + +host_user_data_dir = join(os.environ['WORKSPACE_DIR'], "{username}") +c.DockerSpawner.volumes = {host_user_data_dir: container_workspace_dir} + +# Case for the cowbird setup, where the workspace_dir contains a symlink to the jupyterhub dir. +# The jupyterhub dir must also be mounted in this case. +if os.environ['WORKSPACE_DIR'] != jupyterhub_data_dir: + c.DockerSpawner.volumes[join(jupyterhub_data_dir, "{username}")] = { + "bind": join(jupyterhub_data_dir, "{username}"), + "mode": "rw" + } + c.DockerSpawner.volumes[join(os.environ['WORKSPACE_DIR'], os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR'])] = { + "bind": join(notebook_dir, os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR']), + "mode": "ro" + } + +container_gdrive_settings_path = join(container_home_dir, ".jupyter/lab/user-settings/@jupyterlab/google-drive/drive.jupyterlab-settings") +host_gdrive_settings_path = os.environ['JUPYTER_GOOGLE_DRIVE_SETTINGS'] + +if len(host_gdrive_settings_path) > 0: + c.DockerSpawner.volumes[host_gdrive_settings_path] = { + "bind": container_gdrive_settings_path, + "mode": "ro" + } + +readme = os.environ.get('JUPYTERHUB_README', default="") +if readme != "": + c.DockerSpawner.volumes[readme] = { + "bind": join(notebook_dir, "README.ipynb"), + "mode": "ro" + } + +def create_dir_hook(spawner): + username = spawner.user.name + jupyterhub_user_dir = join(jupyterhub_data_dir, username) + + if not os.path.exists(jupyterhub_user_dir): + os.mkdir(jupyterhub_user_dir, 0o755) + + subprocess.call(["chown", "-R", f"{os.environ['USER_WORKSPACE_UID']}:{os.environ['USER_WORKSPACE_GID']}", + jupyterhub_user_dir]) + + if os.environ['WORKSPACE_DIR'] != jupyterhub_data_dir: + # Case for cowbird setup. The workspace directory should also have the user's ownership, + # to have working volume mounts with the DockerSpawner. + workspace_user_dir = join(os.environ['WORKSPACE_DIR'], username) + if not os.path.exists(workspace_user_dir): + raise FileNotFoundError(f"The user {username}'s workspace doesn't exist in the workspace directory, " + "but should have been created by Cowbird already.") + subprocess.call(["chown", f"{os.environ['USER_WORKSPACE_UID']}:{os.environ['USER_WORKSPACE_GID']}", + workspace_user_dir]) + + if username == os.environ['JUPYTER_DEMO_USER']: + # Restrict resources for the public demo user + # CPU limit, seems not honored by DockerSpawner + spawner.cpu_limit = float(os.environ['JUPYTER_DEMO_USER_CPU_LIMIT']) + spawner.mem_limit = os.environ['JUPYTER_DEMO_USER_MEM_LIMIT'] + +c.Spawner.pre_spawn_hook = create_dir_hook + +## Disable per-user configuration of single-user servers. +c.Spawner.disable_user_config = True + +c.DockerSpawner.default_url = '/lab' +c.DockerSpawner.remove = True # delete containers when servers are stopped + +c.DockerSpawner.image_whitelist = { + 'jupyter/scipy-notebook': 'jupyter/scipy-notebook', + 'jupyter/r-notebook': 'jupyter/r-notebook', + 'jupyter/tensorflow-notebook': 'jupyter/tensorflow-notebook', + 'jupyter/datascience-notebook': 'jupyter/datascience-notebook', + 'jupyter/pyspark-notebook': 'jupyter/pyspark-notebook', + 'jupyter/all-spark-notebook': 'jupyter/all-spark-notebook', +} + # noqa +c.DockerSpawner.pull_policy = "always" # for images not using pinned version +c.DockerSpawner.debug = True +c.JupyterHub.log_level = logging.DEBUG + +c.Spawner.debug = True + +## Timeout (in seconds) to wait for spawners to initialize +c.JupyterHub.init_spawners_timeout = 20 # default 10 + +## Timeout (in seconds) before giving up on a spawned HTTP server +c.Spawner.http_timeout = 60 # default 30 + +## Timeout (in seconds) before giving up on starting of single-user server. +c.Spawner.start_timeout = 120 # default 60 + +## Extra arguments to be passed to the single-user server. +c.Spawner.args = [ + # Allow non-empty directory deletion which enable recursive dir deletion. + # https://jupyter-server.readthedocs.io/en/latest/other/full-config.html + "--FileContentsManager.always_delete_dir=True", + ] + +c.DockerSpawner.extra_host_config = { + # start init pid 1 process to reap defunct processes + 'init': True, + } + +c.Authenticator.admin_users = {'admin'} # noqa + +## Force refresh of auth prior to spawn. +# Do nothing right now, pending implementation of +# MagpieAuthenticator.refresh_user() (see +# https://github.com/Ouranosinc/jupyterhub/issues/2) +c.Authenticator.refresh_pre_spawn = True + +## Blacklist of usernames that are not allowed to log in. +# https://jupyterhub.readthedocs.io/en/stable/api/auth.html +# +# For security reasons, block user with known hardcoded public password or +# non real Jupyter users. +blocked_users = {'authtest', '${CATALOG_USERNAME}', 'anonymous'} +c.Authenticator.blacklist = blocked_users # v0.9+ +c.Authenticator.blocked_users = blocked_users # v1.2+ + + +# ------------------------------------------------------------------------------ +# Shutdown idle user server based on configured timeouts. +# ------------------------------------------------------------------------------ +# Timeout (in seconds, default: 3 days) to shut down the user server when no kernels or terminals +# are running and there is no activity. If undefined or set to zero, the feature will not be enabled. +jupyter_idle_server_cull_timeout = int("600" or 0) +if jupyter_idle_server_cull_timeout: + c.Spawner.args.append('--NotebookApp.shutdown_no_activity_timeout={}'.format(jupyter_idle_server_cull_timeout)) +# Timeout (in seconds, default: 1 day) after which individual +# user kernels/terminals are considered idle and ready to be culled. +jupyter_idle_kernel_cull_timeout = int("10" or 0) +# Interval (in seconds, default: half of timeout) on which to check for idle kernels exceeding the cull timeout value. +jupyter_idle_kernel_cull_interval = int("" or 0) +if jupyter_idle_kernel_cull_timeout: + if not jupyter_idle_kernel_cull_interval or jupyter_idle_kernel_cull_interval > jupyter_idle_kernel_cull_timeout: + jupyter_idle_kernel_cull_interval = max(1, int(jupyter_idle_kernel_cull_timeout / 2)) + c.Spawner.args.extend([ + '--MappingKernelManager.cull_idle_timeout={}'.format(jupyter_idle_kernel_cull_timeout), + '--MappingKernelManager.cull_interval={}'.format(jupyter_idle_kernel_cull_interval), + '--TerminalManager.cull_inactive_timeout={}'.format(jupyter_idle_kernel_cull_timeout), + '--TerminalManager.cull_interval={}'.format(jupyter_idle_kernel_cull_interval), + ]) +# Culling kernels which have one or more connections for idle but open notebooks and/or terminals. +# Otherwise, browser tabs, notebooks and terminals all have to be closed for culling to work. +if jupyter_idle_server_cull_timeout or jupyter_idle_kernel_cull_timeout: + c.Spawner.args.extend([ + '--MappingKernelManager.cull_connected=True', + '--TerminalManager.cull_connected=True', + ]) + +# ------------------------------------------------------------------------------ +# Configuration overrides +# ------------------------------------------------------------------------------ + + +# do not pull docker iamge updates each time +c.DockerSpawner.pull_policy = "ifnotpresent" + +# allow HTTP requests to /jupyter/hub/api using the following token +# {Authorization: Token } +c.JupyterHub.services = [ + { + "name": "service-admin", + "api_token": "admin-token", + }, +] +c.JupyterHub.load_roles = [ + { + "name": "service-role", + "scopes": [ + # specify the permissions the token should have + "admin:users", + "admin:servers", + "access:servers", + "proxy" + ], + "services": [ + # assign the service the above permissions + "service-admin", + ], + } +] + +# mount additional local notebook locations to imitate the results from auto-deploy script: +# - birdhouse/pavics-jupyter-base/scheduler-jobs/deploy_data_pavics_jupyter.env +# - birdhouse/pavics-jupyter-base/scheduler-jobs/deploy_data_specific_image +class OverrideDockerSpawner(CustomDockerSpawner): + async def start(self): + self.volumes["/home/francis/dev/daccs/pavics-sdi/"] = { + "bind": "/home/francis/dev/daccs/pavics-sdi/", + "mode": "ro", + } + self.volumes["/home/francis/dev/daccs/pavics-jupyter-images/"] = { + "bind": "/home/francis/dev/daccs/pavics-jupyter-images/", + "mode": "ro", + } + return await super().start() + +c.JupyterHub.spawner_class = OverrideDockerSpawner + + # noqa diff --git a/birdhouse/config/jupyterhub/service-config.json b/birdhouse/config/jupyterhub/service-config.json new file mode 100644 index 000000000..0ac71a4f6 --- /dev/null +++ b/birdhouse/config/jupyterhub/service-config.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "jupyterhub", + "keywords": [ + "jupyterhub" + ], + "description": "An interactive development environment use to create and share documents that contain live code.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://10.0.2.15/jupyter" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://jupyter.org/hub" + } + ] +} diff --git a/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..56f7cf4a0 --- /dev/null +++ b/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,42 @@ +SERVICES['Magpie'] = { + 'info': { + 'name': 'Magpie', + 'synopsis': ( + 'Magpie is service for AuthN/AuthZ accessible via a REST API. ' + 'It allows you to manage User/Group/Service/Resource/Permission management ' + 'and integrates with Twitcher.' + ), + 'version': "3.38.0", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/Magpie", "3.38.0"), + 'researchSubject': 'Security', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Security', + 'tags': ['Security', 'Management', 'Access', 'Policy Decision Point'] + }, + 'stats': { + 'method': '.*', + 'route': "/magpie/.*" + }, + 'redirect': { + 'doc': 'https://pavics-magpie.readthedocs.io/', + 'releasenotes': 'https://github.com/Ouranosinc/Magpie/blob/master/CHANGES.rst', + 'support': 'https://github.com/Ouranosinc/Magpie/issues', + 'source': 'https://github.com/Ouranosinc/Magpie', + 'tryme': 'https://10.0.2.15/magpie/', + 'licence': 'https://github.com/Ouranosinc/Magpie/blob/3.38.0/LICENSE', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + "monitoring": { + "Magpie": { + 'request': { + 'url': 'https://10.0.2.15/magpie/version' + }, + 'response': { + 'text': r'\{.*"code": 200.*"type": "application/json".*\}' + } + } + } +} + +CANARIE_STATS_ROUTES.append('magpie') diff --git a/birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py b/birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..36157bfd3 --- /dev/null +++ b/birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,9 @@ +SERVICES['node']['monitoring'].update({'Magpie': { + 'request': { + 'url': 'https://10.0.2.15/magpie/version' + }, + 'response': { + 'text': '\{.*"code": 200.*"type": "application/json".*\}' + } +}}) +CANARIE_STATS_ROUTES.append('magpie') diff --git a/birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf b/birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf new file mode 100644 index 000000000..cd043e908 --- /dev/null +++ b/birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf @@ -0,0 +1,6 @@ + location /magpie/ { + proxy_pass http://magpie:2001/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + include /etc/nginx/conf.d/cors.include; + } diff --git a/birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql b/birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql new file mode 100644 index 000000000..4d7a6c031 --- /dev/null +++ b/birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql @@ -0,0 +1 @@ +CREATE DATABASE magpiedb; diff --git a/birdhouse/config/magpie/magpie.ini b/birdhouse/config/magpie/magpie.ini new file mode 100644 index 000000000..2aa29a947 --- /dev/null +++ b/birdhouse/config/magpie/magpie.ini @@ -0,0 +1,183 @@ +### +# app configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### +# From sample +# https://github.com/Ouranosinc/Magpie/blob/master/config/magpie.ini +### + +[composite:main] +use = egg:Paste#urlmap +/ = magpie_app +#/api = api_app +#/magpie/api = api_app + +[app:magpie_app] +use = egg:magpie +filter-with = urlprefix + +pyramid.reload_templates = false +pyramid.debug_authorization = false +pyramid.debug_notfound = false +pyramid.debug_routematch = false +pyramid.default_locale_name = en +pyramid.includes = pyramid_tm ziggurat_foundations.ext.pyramid.sign_in ziggurat_foundations.ext.pyramid.get_user + +# magpie +# +# other overridable variables available in magpie/constants.py +# +magpie.port = 2001 +magpie.url = https://10.0.2.15/magpie +magpie.max_restart = 5 +magpie.push_phoenix = true +# This secret should be the same in Twitcher ! +magpie.secret = magpie +magpie.admin_user = admin +magpie.admin_password = qwertyqwerty! + +# ziggurat +ziggurat_foundations.model_locations.User = magpie.models:User +ziggurat_foundations.sign_in.username_key = user_name +ziggurat_foundations.sign_in.password_key = password +ziggurat_foundations.sign_in.came_from_key = came_from +ziggurat_foundations.sign_in.sign_in_pattern = /signin_internal +ziggurat_foundations.sign_in.sign_out_pattern = /signout +ziggurat_foundations.session_provider_callable = magpie.models:get_session_callable + +# github (login provider) +github.client_id=Iv1.afcc0662862307a4 +github.client_secret=840a968b219f10b8bc3591abd8a2d37ed98a0642 + +# temporary token definition (defaults below if omitted) +# note: token here refers to uuids employed in temporary URL endpoints, not security auth tokens +# Lifetime (in seconds) of the temporary token : +magpie.token_expire = 86400 + +# user registration and approvals + +# below values are the defaults, adjust accordingly for desired behavior +# email template overrides defaults in: magpie/api/templates +magpie.user_registration_enabled = false +magpie.user_registration_submission_email_template = +magpie.user_registration_approval_enabled = false +magpie.user_registration_approval_email_recipient = +magpie.user_registration_approval_email_template = +magpie.user_registration_approved_email_template = +magpie.user_registration_declined_email_template = +magpie.user_registration_notify_enabled = false +magpie.user_registration_notify_email_recipient = +magpie.user_registration_notify_email_template = + +# user assignment to groups with terms & conditions +magpie.group_terms_submission_email_template = +magpie.group_terms_approved_email_template = + +# smtp server configuration +magpie.smtp_user = Magpie +magpie.smtp_from = +magpie.smtp_host = +magpie.smtp_port = +magpie.smtp_ssl = +magpie.smtp_password = + +# See https://pavics-magpie.readthedocs.io/en/latest/configuration.html#envvar-MAGPIE_USER_NAME_EXTRA_REGEX for details +# User names must match the regular expression defined here as well (whether this variable is set or not): +# https://github.com/Ouranosinc/Magpie/blob/595602/magpie/api/management/user/user_utils.py#L66 +#magpie.user_name_extra_regex = + +# Caching settings for specific sections/functions - improves performance response times of recurring requests +# See Twitcher INI configuration for caching that takes effect when resolving access enforcement to actual services. +# Following cache settings apply only during Magpie API requests. +# +# NOTE: Caching will only take effect with Magpie >= 3.7 +# Detail: +# Region 'acl' takes effect whenever computing "effective permissions" of user/group onto a service/resource. +# Because Magpie is employed to manage those permissions and that computing the effective resolution of the +# complete resource hierarchy and full user-group membership inheritance is fairly rare, caching is not specifically +# needed here. It is actually more often then not useful to always refresh the latest states to be sure of which +# permissions are actually applied when the administrator manages them. Therefore, all caches are disabled here, but +# this is not the case on Twitcher side. +cache.regions = acl, service +cache.type = memory +cache.enabled = false +cache.acl.enabled = false +cache.service.enabled = false + +[app:api_app] +use = egg:Paste#static +document_root = %(here)s/ui/swagger + +[filter:urlprefix] +use = egg:PasteDeploy#prefix +# prefix must be adjusted according to 'MAGPIE_URL' / 'magpie.url' +prefix = /magpie + +### +# wsgi server configuration +### + +[server:main] +use = egg:gunicorn#main +host = 0.0.0.0 +port=2001 +timeout=10 +workers=3 +threads=4 + +# used by magpie/alembic with symlink, %(here)s corresponds to this directory +[alembic] +script_location = %(here)s/../magpie/alembic +#sqlalchemy.url = postgresql://postgres:postgres@localhost/magpie + +### +# logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, magpie, sqlalchemy, alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console +formatter = generic + +[logger_magpie] +# "level = DEBUG" logs detailed information about operations/settings (not for production, will leak sensitive data) +# "level = INFO" reports useful information, not leaking details about settings +# "level = WARN" only potential problems are reported +level = DEBUG +handlers = +qualname = magpie +formatter = generic + +[logger_sqlalchemy] +# "level = DEBUG" logs SQL queries, transactions and results +# "level = INFO" logs SQL queries (data can be identified from query field values) +# "level = WARN" logs neither (recommended for production systems, avoid anything below unless for dev/debug system) +level = WARN +handlers = +qualname = sqlalchemy.engine +formatter = generic + +[logger_alembic] +level = INFO +handlers = +qualname = alembic +formatter = generic + +[handler_console] +class = StreamHandler +args = (sys.stdout,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = [%(asctime)s] %(levelname)-10.10s [%(threadName)s][%(name)s] %(message)s diff --git a/birdhouse/config/magpie/permissions.cfg b/birdhouse/config/magpie/permissions.cfg new file mode 100644 index 000000000..dc5155ffc --- /dev/null +++ b/birdhouse/config/magpie/permissions.cfg @@ -0,0 +1,22 @@ +# +# Parameters: +# service: service name to receive the permission (directly on it if no 'resource' mentioned, must exist) +# resource (optional): tree path of the service's resource (ex: /res1/sub-res2/sub-sub-res3) +# user and/or group: user/group to apply the permission on (user/group must exist) +# permission: name of the permission to be applied (see 'magpie/permissions.py' for supported values) +# action: one of [create, remove] (default: create) +# +# Default behaviour: +# - create missing resources if supported by the service (and tree automatically resolvable), then apply permissions. +# - create missing user/group if required (default user created: (group: anonymous, password: 12345). +# - applicable service, user or group is missing, corresponding permissions are ignored and not updated. +# - unknown actions are ignored and corresponding permission are not updated, unspecified action resolves to 'create'. +# - already satisfied permission configurations are left as is. +# + +permissions: + - service: api + resource: /api + permission: read + group: anonymous + action: create diff --git a/birdhouse/config/magpie/postgres-credentials.env b/birdhouse/config/magpie/postgres-credentials.env new file mode 100644 index 000000000..4f1914a5f --- /dev/null +++ b/birdhouse/config/magpie/postgres-credentials.env @@ -0,0 +1,7 @@ +POSTGRES_USER=postgres-magpie +POSTGRES_PASSWORD=postgres-qwerty +POSTGRES_DB=magpiedb +MAGPIE_POSTGRES_USER=postgres-magpie +MAGPIE_POSTGRES_PASSWORD=postgres-qwerty +MAGPIE_POSTGRES_DB=magpiedb +MAGPIE_POSTGRES_HOST=postgres-magpie diff --git a/birdhouse/config/magpie/providers.cfg b/birdhouse/config/magpie/providers.cfg new file mode 100644 index 000000000..768fdd8b1 --- /dev/null +++ b/birdhouse/config/magpie/providers.cfg @@ -0,0 +1 @@ +providers: {} diff --git a/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..dfeb38790 --- /dev/null +++ b/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,34 @@ +SERVICES['Malleefowl'] = { + 'info': { + 'name': 'Malleefowl', + 'synopsis': 'A Web Processing Service for Climate Data Access and Workflows.', + 'version': "pavics-0.3.5", + 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/malleefowl", "pavics-0.3.5"), + 'institution': 'Ouranos', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Resource/Cloud Management', + 'tags': ['Climatology'] + }, + 'stats': { + 'method': '.*', + 'route': '/malleefowl/.*' + }, + 'redirect': { + 'doc': 'https://malleefowl.readthedocs.io/en/latest/', + 'releasenotes': 'https://github.com/Ouranosinc/malleefowl/blob/master/CHANGES.rst', + 'support': 'https://github.com/Ouranosinc/malleefowl/issues', + 'source': 'https://github.com/Ouranosinc/malleefowl', + 'tryme': 'https://10.0.2.15/malleefowl/', + 'licence': '', + 'provenance': '' + }, + 'monitoring': { + "Malleefowl": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://10.0.2.15:8091/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + } + } +} \ No newline at end of file diff --git a/birdhouse/config/malleefowl/config/magpie/providers.cfg b/birdhouse/config/malleefowl/config/magpie/providers.cfg new file mode 100644 index 000000000..e4214fbb4 --- /dev/null +++ b/birdhouse/config/malleefowl/config/magpie/providers.cfg @@ -0,0 +1,8 @@ +providers: + malleefowl: + url: http://10.0.2.15:8091/wps + title: Malleefowl + public: true + c4i: false + type: wps + sync_type: wps diff --git a/birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py b/birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..2d12e767f --- /dev/null +++ b/birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,5 @@ +SERVICES['node']['monitoring']['Malleefowl'] = { + 'request': { + 'url': 'http://10.0.2.15:8091/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } +} diff --git a/birdhouse/config/malleefowl/custom.cfg b/birdhouse/config/malleefowl/custom.cfg new file mode 100644 index 000000000..e8f1a1b38 --- /dev/null +++ b/birdhouse/config/malleefowl/custom.cfg @@ -0,0 +1,30 @@ +[buildout] +extends=profiles/docker.cfg + +[settings] +persist-path = /pavics-data +archive-root = / +thredds_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/fileServer/birdhouse +wps_url = https://${environment:PAVICS_FQDN_PUBLIC}/wpsoutputs +wps_url2 = https://${environment:PAVICS_FQDN_PUBLIC}:443/wpsoutputs +ncwms_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs +opendap_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/dodsC/birdhouse +magpie_hostname = https://${environment:HOSTNAME}/magpie + +[pywps] +enable-https = true +https-port = 443 +https-output-port = 443 +workers = 10 +parallelprocesses = 10 +maxprocesses = 100 +database=postgresql://postgres-pavics:postgres-qwerty@postgres/malleefowl +extra-options = + archive_root=${settings:archive-root} + persist_path=${settings:persist-path} + thredds_url=${settings:thredds_url} + authz_url=${settings:magpie_hostname} + authz_thredds_service_name=thredds + known_extensions=nc,nc4 + wms_mapping=${settings:wps_url},${settings:ncwms_url}/wps_outputs,${settings:wps_url2},${settings:ncwms_url}/wps_outputs,${settings:thredds_url},${settings:ncwms_url} + opendap_mapping=${settings:wps_url},${settings:opendap_url}/wps_outputs,${settings:wps_url2},${settings:opendap_url}/wps_outputs,${settings:thredds_url},${settings:opendap_url} diff --git a/birdhouse/config/ncops/ncops.cfg b/birdhouse/config/ncops/ncops.cfg new file mode 100644 index 000000000..6dedf3e03 --- /dev/null +++ b/birdhouse/config/ncops/ncops.cfg @@ -0,0 +1,2 @@ +GEOSERVER_HOST=10.0.2.15:8087 +WPS_HOST=10.0.2.15:8079 diff --git a/birdhouse/config/ncwms2/config/magpie/providers.cfg b/birdhouse/config/ncwms2/config/magpie/providers.cfg new file mode 100644 index 000000000..f78c0d350 --- /dev/null +++ b/birdhouse/config/ncwms2/config/magpie/providers.cfg @@ -0,0 +1,8 @@ +providers: + ncWMS2: + url: http://10.0.2.15:8080/ncWMS2 + title: ncWMS2 + public: true + c4i: false + type: ncwms + sync_type: ncwms diff --git a/birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf b/birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf new file mode 100644 index 000000000..6281babd1 --- /dev/null +++ b/birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf @@ -0,0 +1,7 @@ + location /ncWMS2/ { + # proxy_pass http://10.0.2.15:8080; + # proxy_set_header Host $host; + # proxy_set_header X-Forwarded-Proto $real_scheme; + # include /etc/nginx/conf.d/cors.include; + return 302 /twitcher/ows/proxy$request_uri; + } diff --git a/birdhouse/config/ncwms2/custom.cfg b/birdhouse/config/ncwms2/custom.cfg new file mode 100644 index 000000000..8e6ce602e --- /dev/null +++ b/birdhouse/config/ncwms2/custom.cfg @@ -0,0 +1,11 @@ +[buildout] +extends = buildout.cfg + +supervisor-host = * +supervisor-port = 9001 + +[settings] +tomcat-ncwms-password = ncwmspass + +[ncwms] +data_dir = /pavics-data diff --git a/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..ad08683e9 --- /dev/null +++ b/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,34 @@ +SERVICES['Phoenix'] = { + 'info': { + 'name': 'Phoenix', + 'synopsis': 'Legacy authentication. See Magpie/Twitcher instead.', + 'version': "pavics-0.2.3", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/pyramid-phoenix", "pavics-0.2.3"), + 'researchSubject': 'Authentication', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Authentication', + 'tags': ['Authentication', 'Legacy'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/geoserver/web/.*' # FIXME: original value doesn't make sense + }, + 'redirect': { + 'doc': 'http://pyramid-phoenix.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/ouranosinc/pyramid-phoenix/CHANGES.rst', + 'support': 'https://github.com/ouranosinc/pyramid-phoenix/issues', + 'source': 'https://github.com/ouranosinc/pyramid-phoenix', + 'tryme': 'https://10.0.2.15:8443/', + 'licence': 'https://github.com/ouranosinc/pyramid-phoenix/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + "monitoring": { + "Phoenix": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'https://10.0.2.15:8443/' + } + } + } +} \ No newline at end of file diff --git a/birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py b/birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..508d9175b --- /dev/null +++ b/birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,5 @@ +SERVICES['node']['monitoring']['Phoenix'] = { + 'request': { + 'url': 'https://10.0.2.15:8443/' + } +} diff --git a/birdhouse/config/phoenix/custom.cfg b/birdhouse/config/phoenix/custom.cfg new file mode 100644 index 000000000..7d91ddd76 --- /dev/null +++ b/birdhouse/config/phoenix/custom.cfg @@ -0,0 +1,10 @@ +[buildout] +extends=profiles/docker.cfg +parts += pytest + +[settings] +esgf-search-url = http://136.172.30.96/esg-search +phoenix-password = sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 +phoenix-solr = true +phoenix-wms = true +phoenix-require-csrf = false diff --git a/birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf b/birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf new file mode 100644 index 000000000..d0fa2db86 --- /dev/null +++ b/birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf @@ -0,0 +1,5 @@ + location /portainer/ { + proxy_pass http://portainer:9000/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + } diff --git a/birdhouse/config/postgres/credentials.env b/birdhouse/config/postgres/credentials.env new file mode 100644 index 000000000..672bc4ae0 --- /dev/null +++ b/birdhouse/config/postgres/credentials.env @@ -0,0 +1,3 @@ +POSTGRES_USER=postgres-pavics +POSTGRES_PASSWORD=postgres-qwerty +POSTGRES_DB=pavics diff --git a/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..f4fab156a --- /dev/null +++ b/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,6 @@ +PLATFORMS['server']['monitoring']['Project'] = { + 'request': { + 'url': 'https://10.0.2.15/project-api/explorer/' + } +} +PLATFORMS['server']['stats']['route'] = '/project-api/.*' diff --git a/birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf b/birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf new file mode 100644 index 000000000..e35658447 --- /dev/null +++ b/birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf @@ -0,0 +1,6 @@ + + location /project-api/ { + proxy_pass http://10.0.2.15:3005/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + } diff --git a/birdhouse/config/proxy/conf.d/all-services.include b/birdhouse/config/proxy/conf.d/all-services.include new file mode 100644 index 000000000..e6cff734d --- /dev/null +++ b/birdhouse/config/proxy/conf.d/all-services.include @@ -0,0 +1,28 @@ + location / { + return 302 https://10.0.2.15/stac/; + } + + location /components { + default_type application/json; + root /static; + try_files /components.json =404; + } + + location /services { + default_type application/json; + root /static; + try_files /services.json =404; + } + + location /version { + default_type application/json; + root /static; + try_files /version.json =404; + } + + location /doc { + return 302 https://www.example.com/; + } + + # for other extra components to extend Nginx + include /etc/nginx/conf.extra-service.d/*/*.conf; diff --git a/birdhouse/config/proxy/conf.d/frontend.conf b/birdhouse/config/proxy/conf.d/frontend.conf new file mode 100644 index 000000000..0d18f014b --- /dev/null +++ b/birdhouse/config/proxy/conf.d/frontend.conf @@ -0,0 +1,48 @@ +# top-level http config for websocket headers +# If Upgrade is defined, Connection = upgrade +# If Upgrade is empty, Connection = close +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +# Sets a $real_scheme variable whose value is the scheme passed by the load +# balancer in X-Forwarded-Proto (if any), defaulting to $scheme. +# Similar to how the HttpRealIp module treats X-Forwarded-For. +map $http_x_forwarded_proto $real_scheme { + default $http_x_forwarded_proto; + '' $scheme; +} + +server { + listen 80; + server_name localhost; + + include /etc/nginx/conf.d/all-services.include; + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} + +server { + listen 443; + server_name localhost; + proxy_buffering off; + + resolver 127.0.0.11; + + ssl on; + ssl_certificate /etc/nginx/cert.pem; + ssl_certificate_key /etc/nginx/cert.pem; + + include /etc/nginx/conf.d/all-services.include; + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/birdhouse/config/proxy/nginx.conf b/birdhouse/config/proxy/nginx.conf new file mode 100644 index 000000000..aad412e7f --- /dev/null +++ b/birdhouse/config/proxy/nginx.conf @@ -0,0 +1,41 @@ + +user root; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_iso8601] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access_file.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + client_max_body_size 1000m; + client_body_timeout 600s; + + # timeout for reading a response from the proxied server + proxy_read_timeout 240s; # default 60s + + include /etc/nginx/conf.d/*.conf; + + # for other extra components to extend Nginx + include /etc/nginx/conf.extra.d/*/*.conf; + +} diff --git a/birdhouse/config/proxy/static/components.json b/birdhouse/config/proxy/static/components.json new file mode 100644 index 000000000..e370c72d0 --- /dev/null +++ b/birdhouse/config/proxy/static/components.json @@ -0,0 +1,26 @@ +{"components": ["bird-house/birdhouse-deploy:config/proxy", +"bird-house/birdhouse-deploy:config/canarie-api", +"bird-house/birdhouse-deploy:config/geoserver", +"bird-house/birdhouse-deploy:config/wps_outputs-volume", +"bird-house/birdhouse-deploy:config/postgres", +"bird-house/birdhouse-deploy:config/finch", +"bird-house/birdhouse-deploy:config/raven", +"bird-house/birdhouse-deploy:config/data-volume", +"bird-house/birdhouse-deploy:config/hummingbird", +"bird-house/birdhouse-deploy:config/thredds", +"bird-house/birdhouse-deploy:config/portainer", +"bird-house/birdhouse-deploy:config/magpie", +"bird-house/birdhouse-deploy:config/twitcher", +"bird-house/birdhouse-deploy:config/jupyterhub", +"bird-house/birdhouse-deploy:optional-components/canarie-api-full-monitoring", +"bird-house/birdhouse-deploy:optional-components/wps-healthchecks", +"bird-house/birdhouse-deploy:optional-components/secure-thredds", +"bird-house/birdhouse-deploy:optional-components/testthredds", +"bird-house/birdhouse-deploy:components/weaver", +"bird-house/birdhouse-deploy:optional-components/test-weaver", +"bird-house/birdhouse-deploy:optional-components/secure-data-proxy", +"bird-house/birdhouse-deploy:optional-components/x-robots-tag-header", +"bird-house/birdhouse-deploy:components/stac", +"bird-house/birdhouse-deploy:optional-components/stac-public-access", +"bird-house/birdhouse-deploy:components/cowbird", +"bird-house/birdhouse-deploy:optional-components/stac-data-proxy"]} diff --git a/birdhouse/config/proxy/static/services.json b/birdhouse/config/proxy/static/services.json new file mode 100644 index 000000000..5cc667101 --- /dev/null +++ b/birdhouse/config/proxy/static/services.json @@ -0,0 +1,221 @@ +{"services": [ { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "geoserver", + "keywords": [ + "data", + "service-wms", + "service-wfs", + "service-wps" + ], + "description": "GeoServer is a server that allows users to view and edit geospatial data.", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://10.0.2.15/geoserver/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://docs.geoserver.org/" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "finch", + "keywords": [ + "service-wps" + ], + "description": "A Web Processing Service for Climate Indicators.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://finch.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "raven", + "keywords": [ + "service-wps" + ], + "description": "A WPS providing hydrological modeling and analysis services.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-raven.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "hummingbird", + "keywords": [ + "service-wps" + ], + "description": "A WPS for metadata compliance checks used in the climate science community.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://birdhouse-hummingbird.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "thredds", + "keywords": [ + "data", + "catalog" + ], + "description": "A server that provides catalog, metadata, and data access services for scientific data.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://10.0.2.15/thredds/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://www.unidata.ucar.edu/software/tds/" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/thredds/catalog.xml" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "jupyterhub", + "keywords": [ + "jupyterhub" + ], + "description": "An interactive development environment use to create and share documents that contain live code.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://10.0.2.15/jupyter" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://jupyter.org/hub" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "weaver", + "keywords": [ + "service-ogcapi_processes" + ], + "description": "An OGC-API flavored Execution Management Service", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://10.0.2.15/weaver/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-weaver.readthedocs.io/" + }, + { + "rel": "service-desc", + "type": "application/json", + "href": "https://10.0.2.15/weaver/" + }, + { + "rel": "conformance", + "type": "application/json", + "href": "https://10.0.2.15/weaver/conformance/" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "stac", + "keywords": [ + "catalog" + ], + "description": "Searchable spatiotemporal metadata describing climate and Earth observation datasets.", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://10.0.2.15/stac/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://github.com/radiantearth/stac-api-spec/tree/v1.0.0" + }, + { + "rel": "service-desc", + "type": "application/yaml", + "href": "https://raw.githubusercontent.com/radiantearth/stac-api-spec/main/core/openapi.yaml" + }, + { + "rel": "alternate", + "type": "text/html", + "href": "https://10.0.2.15/stac-browser/" + } + ] +}, { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "stac-browser", + "keywords": [ + "catalog" + ], + "description": "A web interface to browse the STAC catalogs.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://10.0.2.15/stac-browser/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://github.com/radiantearth/stac-browser" + }, + { + "rel": "alternate", + "type": "application/json", + "href": "https://10.0.2.15/stac/" + } + ] +}]} diff --git a/birdhouse/config/proxy/static/version.json b/birdhouse/config/proxy/static/version.json new file mode 100644 index 000000000..f870a933b --- /dev/null +++ b/birdhouse/config/proxy/static/version.json @@ -0,0 +1 @@ +{"version": "1.40.0", "release_time": "2023-11-30T18:27:41Z"} diff --git a/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..50aae7063 --- /dev/null +++ b/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,39 @@ +RAVEN_VERSION = "0.18.1" +RAVEN_RELEASE = get_release_time_from_repo_tag("github", "Ouranosinc/raven", RAVEN_VERSION) + +SERVICES['raven'] = { + 'info': { + 'name': 'Hydrological modeling and analysis', + 'synopsis': ( + 'A suite of WPS processes to calibrate and run hydrological models, ' + 'including geographical information retrieval and processing as well as time series analysis.' + ), + 'version': RAVEN_VERSION, + 'institution': 'Ouranos', + 'releaseTime': RAVEN_RELEASE, + 'researchSubject': 'Hydrology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Hydrology'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/raven.*' + }, + 'redirect': { + 'doc': 'https://pavics-raven.readthedocs.io/en/latest/', + 'releasenotes': 'https://github.com/Ouranosinc/raven/releases', + 'support': 'https://github.com/Ouranosinc/raven/issues', + 'source': 'https://github.com/Ouranosinc/raven', + 'tryme': 'https://pavics-raven.readthedocs.io/en/latest/notebooks/index.html', + 'licence': 'https://github.com/Ouranosinc/raven/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Raven': { + 'request': { + 'url': 'http://raven:9099/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} diff --git a/birdhouse/config/raven/config/magpie/providers.cfg b/birdhouse/config/raven/config/magpie/providers.cfg new file mode 100644 index 000000000..f72469649 --- /dev/null +++ b/birdhouse/config/raven/config/magpie/providers.cfg @@ -0,0 +1,9 @@ +providers: + raven: + url: http://raven:9099/wps + title: Raven + public: true + c4i: false + type: wps + sync_type: wps + diff --git a/birdhouse/config/raven/config/proxy/canarie_api_monitoring.py b/birdhouse/config/raven/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..76c0775c8 --- /dev/null +++ b/birdhouse/config/raven/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,39 @@ +SERVICES['node']['monitoring']['Raven'] = { + 'request': { + 'url': 'http://10.0.2.15:8096/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } +} +SERVICES['raven'] = { + 'info': { + 'name': 'Hydrological modeling and analysis', + 'synopsis': ('A suite of WPS processes to calibrate and run hydrological models, including geographical' + 'information retrieval and processing as well as time series analysis.'), + 'version': '0.11.1', + 'institution': 'Ouranos', + 'releaseTime': '2021-02-01T00:00:00Z', + 'researchSubject': 'Hydrology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Hydrology'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/raven.*' + }, + 'redirect': { + 'doc': 'https://pavics-raven.readthedocs.io/en/latest/', + 'releasenotes': 'https://github.com/Ouranosinc/raven/releases', + 'support': 'https://github.com/Ouranosinc/raven/issues', + 'source': 'https://github.com/Ouranosinc/raven', + 'tryme': 'https://pavics-raven.readthedocs.io/en/latest/notebooks/index.html', + 'licence': 'https://github.com/Ouranosinc/raven/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Raven': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} diff --git a/birdhouse/config/raven/service-config.json b/birdhouse/config/raven/service-config.json new file mode 100644 index 000000000..43c3f775c --- /dev/null +++ b/birdhouse/config/raven/service-config.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "raven", + "keywords": [ + "service-wps" + ], + "description": "A WPS providing hydrological modeling and analysis services.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-raven.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" + } + ] +} diff --git a/birdhouse/config/raven/wps.cfg b/birdhouse/config/raven/wps.cfg new file mode 100644 index 000000000..ef7382950 --- /dev/null +++ b/birdhouse/config/raven/wps.cfg @@ -0,0 +1,15 @@ +[server] +outputurl = https://10.0.2.15/wpsoutputs/raven +outputpath = /data/wpsoutputs/raven + +# default 3mb, fix "Broken pipe" between the proxy and the wps service +maxrequestsize = 100mb + +# default 2, too low for a production server with capable CPUs +parallelprocesses = 10 + +[logging] +level = INFO +database=postgresql://postgres-pavics:postgres-qwerty@postgres/raven + + diff --git a/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..6371110f5 --- /dev/null +++ b/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,39 @@ +SERVICES['Solr'] = { + 'info': { + 'name': 'Solr', + 'synopsis': ( + 'SOLR is a search platform part of the Apache Lucene project. ' + 'It is used in this project for its faceted search capability. ' + 'Search queries are relayed from the UI or WPS processes to the SOLR database, ' + 'which returns a json file with the links to matching files.' + ), + 'version': "5.2.1", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/solr", "5.2.1"), + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Indexation', 'Search'] + }, + 'stats': { + 'method': '.*', + 'route': '/solr/.*' + }, + 'redirect': { + 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/backend.html#indexation', + 'releasenotes': 'https://github.com/Ouranosinc/PAVICS/tags', + 'support': 'https://github.com/Ouranosinc/PAVICS/issues', + 'source': 'https://github.com/Ouranosinc/PAVICS/tree/master/birdhouse/docker/solr', + 'tryme': 'http://10.0.2.15:8983/solr/', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/arch/backend.html#indexation' + }, + "monitoring": { + 'Solr': { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://10.0.2.15:8983/solr/birdhouse/select' + } + }, + } +} diff --git a/birdhouse/config/solr/config/proxy/canarie_api_monitoring.py b/birdhouse/config/solr/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..8f61b06f5 --- /dev/null +++ b/birdhouse/config/solr/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,8 @@ +SERVICES['node']['monitoring']['Solr'] = { + 'request': { + 'url': 'http://10.0.2.15:8983/solr/birdhouse/select?q=CMIP5&fq=model:MPI-ESM-MR&fq=experiment:rcp45&fq=variable:tasmax&fq=institute:MPI-M&fq=frequency:mon&wt=json' + }, + 'response': { + 'text': '.*catalog_url\":\".+/testdata/flyingpigeon/cmip5.*/catalog.xml.*' + } +} diff --git a/birdhouse/config/thredds/catalog.xml b/birdhouse/config/thredds/catalog.xml new file mode 100644 index 000000000..aba2d4fe8 --- /dev/null +++ b/birdhouse/config/thredds/catalog.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + all + + + + + + + + + + + + + + + + + all + + + + + + + + + + + + + + + + diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..dcc606354 --- /dev/null +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,72 @@ +SERVICES['renderer'] = { + 'info': { + 'name': 'High-resolution spatial gridded data renderer', + 'synopsis': ( + 'This service renders gridded data on the server and sends images to the client for display within ' + 'mapping applications using Open Geospatial Consortium (OGC) Web Mappping Service (WMS) standard.' + ), + 'version': '4.6.15', + 'institution': 'Unidata', + 'releaseTime': '2020-06-16T00:00:00Z', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology'] + }, + 'stats': { + 'method': '.*', + 'route': '/thredds/.*' + }, + 'redirect': { + 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/frontend.html#gridded-data-rendering', + 'releasenotes': 'https://github.com/Unidata/tds/releases', + 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', + 'source': 'https://github.com/Unidata/tds', + 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/rendering.html', + 'licence': 'https://github.com/Unidata/tds/blob/master/LICENSE', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'ncWMS': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + } + }, + } +} + +SERVICES['Thredds'] = { + 'info': { + 'name': 'Thredds', + 'synopsis': 'Climate Data Catalog and Format Renderers', + 'version': "4.6.18-unidata-2022-01", + 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/thredds-docker", "4.6.18-unidata-2022-01"), + 'institution': 'Ouranos', + 'researchSubject': 'Catalog', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Catalog', + 'tags': ['Catalog', 'Climate Data'] + }, + 'stats': { + 'method': '.*', + 'route': '/twitcher/ows/proxy/thredds/.*' + }, + 'redirect': { + 'doc': 'https://www.unidata.ucar.edu/software/tds/', + 'releasenotes': 'https://docs.unidata.ucar.edu/tds/current/userguide/upgrade.html', + 'support': 'https://www.unidata.ucar.edu/software/tds/#help', + 'source': 'https://github.com/Unidata/tds', + 'tryme': 'https://10.0.2.15/twitcher/ows/proxy/thredds/', + 'licence': 'https://github.com/Unidata/tds/blob/main/LICENSE', + 'provenance': 'https://downloads.unidata.ucar.edu/tds/' + }, + "monitoring": { + "Thredds": { + 'request': { + 'url': 'http://thredds:8080//twitcher/ows/proxy/thredds/catalog.html' + } + } + } +} + +CANARIE_STATS_ROUTES.append('thredds') diff --git a/birdhouse/config/thredds/config/magpie/providers.cfg b/birdhouse/config/thredds/config/magpie/providers.cfg new file mode 100644 index 000000000..3ca5f9d84 --- /dev/null +++ b/birdhouse/config/thredds/config/magpie/providers.cfg @@ -0,0 +1,35 @@ +providers: + thredds: + url: http://thredds:8080/twitcher/ows/proxy/thredds + title: Thredds + public: true + c4i: false + type: thredds + sync_type: thredds + # below is a custom config to indicate how magpie should convert thredds path elements into resources/permissions + # see: https://pavics-magpie.readthedocs.io/en/latest/services.html#servicethredds + configuration: + skip_prefix: "thredds" # prefix to ignore, below prefixes will be matched against whatever comes after in path + file_patterns: + # note: make sure to employ quotes and double escapes to avoid parsing YAML error + - ".+\\.ncml" # match longest extension first to avoid tuncating it by match of sorter '.nc' + - ".+\\.nc" + metadata_type: + prefixes: + - null # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed + - "\\w+\\.gif" # threddsIcon, folder icon, etc. + - "\\w+\\.ico" # favicon + - "\\w+\\.txt" # licence + - "\\w+\\.css" # tds.css + - "catalog\\.\\w+" # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE` + - catalog + - ncml + - uddc + - iso + data_type: + prefixes: + - fileServer + - dodsC + - wcs + - wms + - ncss diff --git a/birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py b/birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..34d4ed188 --- /dev/null +++ b/birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,41 @@ +SERVICES['node']['monitoring'].update({'Thredds': { + 'request': { + 'url': 'http://10.0.2.15:8083/twitcher/ows/proxy/thredds/catalog.html' + } + }}) + +SERVICES['renderer'] = { + 'info': { + 'name': 'High-resolution spatial gridded data renderer', + 'synopsis': 'This service renders gridded data on the server and sends images to the client for display within mapping applications using Open Geospatial Consortium (OGC) Web Mappping Service (WMS) standard.', + 'version': '4.6.15', + 'institution': 'Unidata', + 'releaseTime': '2020-06-16T00:00:00Z', + 'researchSubject': 'Climatology', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Data Manipulation', + 'tags': ['Climatology'] + }, + 'stats': { + 'method': '.*', + 'route': '/thredds/.*' + }, + 'redirect': { + 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/frontend.html#gridded-data-rendering', + 'releasenotes': 'https://github.com/Unidata/tds/releases', + 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', + 'source': 'https://github.com/Unidata/tds', + 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/rendering.html', + 'licence': 'https://github.com/Unidata/tds/blob/master/LICENSE', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'ncWMS': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + } + }, + } +} + +CANARIE_STATS_ROUTES.append('thredds') diff --git a/birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf b/birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf new file mode 100644 index 000000000..1e5cb1911 --- /dev/null +++ b/birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf @@ -0,0 +1,10 @@ + + location /thredds/ { + #return 302 /twitcher/ows/proxy$request_uri; + proxy_pass https://10.0.2.15/twitcher/ows/proxy/thredds/; + # direct hit Thredds, bypassing twitcher, for debugging only + # proxy_pass http://thredds:8080/twitcher/ows/proxy/thredds/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } diff --git a/birdhouse/config/thredds/service-config.json b/birdhouse/config/thredds/service-config.json new file mode 100644 index 000000000..5f30820fc --- /dev/null +++ b/birdhouse/config/thredds/service-config.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", + "name": "thredds", + "keywords": [ + "data", + "catalog" + ], + "description": "A server that provides catalog, metadata, and data access services for scientific data.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://10.0.2.15/thredds/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://www.unidata.ucar.edu/software/tds/" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://10.0.2.15/thredds/catalog.xml" + } + ] +} diff --git a/birdhouse/config/thredds/threddsConfig.xml b/birdhouse/config/thredds/threddsConfig.xml new file mode 100644 index 000000000..11f037b37 --- /dev/null +++ b/birdhouse/config/thredds/threddsConfig.xml @@ -0,0 +1,137 @@ + + + + + Birdhouse Thredds Data Server + /twitcher/ows/proxy/thredds/threddsIcon.gif + TDS + + Scientific Climate Data + meteorology, atmosphere, climate, ocean, earth science, birdhouse + + + Support + Birdhouse + + + + + + Birdhouse + + + Birdhouse + + + + + true + + + + + true + false + true + 5000 + 5000 + + + + + true + false + 15 min + 30 min + + + + + true + true + true + + + + + true + 1 hour + 10 Gb + + + + + true + + + diff --git a/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..fe0835e19 --- /dev/null +++ b/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1,56 @@ + +def get_twitcher_magpie_adapter_version() -> str: + dockerfile_url = "https://raw.githubusercontent.com/Ouranosinc/Magpie/3.38.0/Dockerfile.adapter" + resp = SESSION.get(dockerfile_url) + if resp.status_code != 200: + raise ValueError( + f"HTTP status: {resp.status_code} from [{dockerfile_url}] with detail:\n{resp.text!s}" + ) + tag_line = [line for line in resp.text.splitlines() if line.startswith("FROM birdhouse/twitcher")] + if not tag_line: + raise ValueError(f"Could not locate Twitcher version from [{dockerfile_url}]") + tag_version = tag_line[0].split(":v", 1)[-1].strip() + if not tag_version: + raise ValueError(f"Could not locate Twitcher version from [{dockerfile_url}]") + return tag_version + + +TWITCHER_VERSION = get_twitcher_magpie_adapter_version() +TWITCHER_RELEASE = get_release_time_from_repo_tag("github", "bird-house/twitcher", TWITCHER_VERSION) +SERVICES['Twitcher'] = { + 'info': { + 'name': 'Twitcher', + 'synopsis': ( + 'Twitcher is a security proxy for OWS services like Web Processing Services (WPS). ' + 'The proxy service uses OAuth2 access tokens to protect the OWS service access using ' + 'Magpie permissions.' + ), + 'version': TWITCHER_VERSION, + 'institution': 'Ouranos', + 'releaseTime': TWITCHER_RELEASE, + 'researchSubject': 'Security', + 'supportEmail': 'helpdesk@example.com', + 'category': 'Security', + 'tags': ['Security', 'Proxy', 'Access', 'Policy Enforcement Point'] + }, + 'stats': { + 'method': '.*', + 'route': "/twitcher/(?!ows).*" + }, + 'redirect': { + 'doc': 'https://twitcher.readthedocs.io/', + 'releasenotes': 'https://github.com/bird-house/twitcher/blob/master/CHANGES.rst', + 'support': 'https://github.com/bird-house/twitcher/issues', + 'source': 'https://github.com/bird-house/twitcher', + 'tryme': 'https://10.0.2.15/twitcher/', + 'licence': 'https://github.com/bird-house/twitcher/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + "monitoring": { + "Twitcher": { + 'request': { + 'url': 'https://10.0.2.15/twitcher/' + } + } + } +} diff --git a/birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py b/birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..bca45380d --- /dev/null +++ b/birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py @@ -0,0 +1,9 @@ +# We can only monitor twitcher if there is an endpoint that it is protecting that we can try to access +# If there is at least one other service that provides a route protected by twitcher, monitor that route; +# otherwise do nothing. +if 'flyingpigeon': + SERVICES['node']['monitoring'].update({'Twitcher': { + 'request': { + 'url': 'https://10.0.2.15/twitcher/ows/proxy/flyingpigeon?service=WPS&version=1.0.0&request=GetCapabilities' + } + }}) diff --git a/birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf b/birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf new file mode 100644 index 000000000..becccfd9d --- /dev/null +++ b/birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf @@ -0,0 +1,8 @@ + location /twitcher/ { + proxy_pass http://twitcher:8000/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $real_scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host:$server_port; + proxy_set_header Forwarded "proto=https;host=10.0.2.15"; # Helps the STAC component to craft URLs containing the full PAVICS_FQDN_PUBLIC + } diff --git a/birdhouse/config/twitcher/twitcher.ini b/birdhouse/config/twitcher/twitcher.ini new file mode 100644 index 000000000..64c58329e --- /dev/null +++ b/birdhouse/config/twitcher/twitcher.ini @@ -0,0 +1,141 @@ +### +# app configuration +# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### +# From sample +# https://github.com/bird-house/twitcher/blob/master/development.ini +### + +[app:main] +use = egg:pyramid_twitcher + +pyramid.reload_templates = true +pyramid.debug_authorization = false +pyramid.debug_notfound = false +pyramid.debug_routematch = false +pyramid.default_locale_name = en +pyramid.includes = +# twitcher not using db, instead employ magpie adapter +sqlalchemy.url = sqlite:///%(here)s/twitcher.sqlite +#sqlalchemy.url = postgresql://${POSTGIS_USER}:${POSTGIS_PASSWORD}@postgis:${POSTGIS_PORT}/twitcher + +retry.attempts = 3 + +# Caching settings for specific sections/functions - improves performance response times of recurring requests +# For caching related to Magpie API endpoints themselves, instead refer to Magpie INI configuration file. +# +# Although related to Magpie code, cache regions settings executed by 'MagpieAdapter' are running under Twitcher +# to resolve Access Control Lists (ACL) to services/resources, and must therefore be placed in Twitcher configuration. +# Caching that takes effect in Twitcher via 'MagpieAdapter' is when requests use the URL endpoint: +# +# //proxy/[/...] +# +# NOTE: Caching will only take effect with Magpie >= 3.7 +# Detail: +# Both 'acl' and 'service' scopes occur on every permission resolution for a given user requesting any access. +# The difference is that 'acl' applies for every combination of (user/group, service/resource, permissions), +# while 'service' limits itself to the service name/type resolution from the request path. +# Since services are not expected to change often, 'service' cache can be safely increased at a much higher refresh +# interval than 'acl' which should re-validate any permission changes more frequently. +# Caching can be forced reset/ignored by using the 'Cache-Control: no-cache' header during any corresponding request. +cache.regions = acl, service +cache.type = memory +cache.enabled = true +cache.acl.enabled = true +cache.acl.expire = 20 +cache.service.enabled = true +cache.service.expire = 60 + +# By default, the toolbar only appears for clients from IP addresses +# '127.0.0.1' and '::1'. +# debugtoolbar.hosts = 127.0.0.1 ::1 + +# twitcher +twitcher.url = https://10.0.2.15/twitcher +twitcher.adapter = magpie.adapter.MagpieAdapter +twitcher.rpcinterface = false +twitcher.username = +twitcher.password = +twitcher.ows_security = true +twitcher.ows_proxy = true +twitcher.ows_proxy_delegate = false +twitcher.ows_proxy_protected_path = /ows +twitcher.ows_proxy_ssl_verify = false +twitcher.workdir = +twitcher.prefix = + +# magpie (for twitcher.adapter) +magpie.url = https://10.0.2.15/magpie +magpie.secret = magpie +magpie.admin_user = admin +magpie.admin_password = qwertyqwerty! + +### +# wsgi server configuration +### + +[alembic] +# path to migration scripts +script_location = twitcher/alembic +file_template = %%(year)d%%(month).2d%%(day).2d_%%(rev)s +# file_template = %%(rev)s_%%(slug)s + +[server:main] +use = egg:gunicorn#main +host = 0.0.0.0 +port = 8000 +timeout = 10 +workers = 10 +threads = 4 + +### +# logging configuration +# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, twitcher, magpie, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console + +[logger_twitcher] +# "level = DEBUG" logs detailed information about operations/settings (not for production, will leak sensitive data) +# "level = INFO" reports useful information, not leaking details about settings +# "level = WARN" only potential problems/unexpected results reported, such as when caching is employed +level = DEBUG +handlers = +qualname = twitcher + +# MagpieAdapter or any other Magpie utilities it employs through Twitcher proxy-adapter security check +[logger_magpie] +# "level = DEBUG" logs detailed information about operations/settings (not for production, will leak sensitive data) +# "level = INFO" reports useful information about operations, not leaking details about settings +# "level = WARN" only potential problems are reported such as missing settings in configuration +level = DEBUG +handlers = +qualname = magpie + +[logger_sqlalchemy] +# "level = DEBUG" logs SQL queries, transactions and results +# "level = INFO" logs SQL queries (data can be identified from query field values) +# "level = WARN" logs neither (recommended for production systems, avoid anything below unless for dev/debug system) +level = WARN +handlers = +qualname = sqlalchemy.engine + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py new file mode 100644 index 000000000..949946a09 --- /dev/null +++ b/birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py @@ -0,0 +1 @@ +CANARIE_STATS_ROUTES.append('wpsoutputs') diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py b/birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py new file mode 100644 index 000000000..949946a09 --- /dev/null +++ b/birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py @@ -0,0 +1 @@ +CANARIE_STATS_ROUTES.append('wpsoutputs') diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf new file mode 100644 index 000000000..990bdfbb8 --- /dev/null +++ b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf @@ -0,0 +1,5 @@ + location /wpsoutputs/ { + include /etc/nginx/conf.extra-service.d/secure-data-proxy/secure-data-auth.include; + + alias /data/wps_outputs/; + } diff --git a/birdhouse/default.env b/birdhouse/default.env index 1db0e7f32..1cbdb15f7 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -31,17 +31,54 @@ export DELAYED_EVAL=" PAVICS_FQDN_PUBLIC DATA_PERSIST_SHARED_ROOT WPS_OUTPUTS_DIR + SERVER_NAME + SERVER_DESCRIPTION + SERVER_INSTITUTION + SERVER_SUBJECT + SERVER_TAGS + SERVER_DOCUMENTATION_URL + SERVER_RELEASE_NOTES_URL + SERVER_SUPPORT_URL + SERVER_LICENSE_URL " - -export SERVER_NAME=PAVICS -export SERVER_DESCRIPTION=" +# Server Identification Details +# Following definitions should definitenly be updated. +# Previous defaults are defined for backward-compatibility. +# If not overridden explicitly by their non '__' prefixed variant, +# a WARN message will be displayed by pavics-compose. +export __DEFAULT__SERVER_NAME=PAVICS +export __DEFAULT__SERVER_DESCRIPTION=" The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of climate analysis services served through Open Geospatial Consortium (OGC) protocols. These services include data access, processing and visualization. Both data and algorithms can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, or a custom web interface. " +export __DEFAULT__SERVER_INSTITUTION=Ouranos +export __DEFAULT__SERVER_SUBJECT=Climatology +# below can be a CSV list of tags +export __DEFAULT__SERVER_TAGS=Climatology +export __DEFAULT__SERVER_DOCUMENTATION_URL=https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html +export __DEFAULT__SERVER_RELEASE_NOTES_URL=https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md +export __DEFAULT__SERVER_SUPPORT_URL=https://github.com/bird-house/birdhouse-deploy/issues +# NOTE: +# This value does not use the previously hard coded default. +# Previous default pointed at the wrong repository with a mismatching LICENSE file. +export __DEFAULT__SERVER_LICENSE_URL=https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE + +# apply overrides or fallback above defaults with delayed evaluation +# see also: 'SUPPORT_EMAIL' +# This parameter is relevant to server details, but uses the old name for backward compatibility. +export SERVER_NAME='${__DEFAULT__SERVER_NAME}' +export SERVER_DESCRIPTION='${__DEFAULT__SERVER_DESCRIPTION}' +export SERVER_INSTITUTION='${__DEFAULT__SERVER_INSTITUTION}' +export SERVER_SUBJECT='${__DEFAULT__SERVER_SUBJECT}' +export SERVER_TAGS='${__DEFAULT__SERVER_TAGS}' +export SERVER_DOCUMENTATION_URL='${__DEFAULT__SERVER_DOCUMENTATION_URL}' +export SERVER_RELEASE_NOTES_URL='${__DEFAULT__SERVER_RELEASE_NOTES_URL}' +export SERVER_SUPPORT_URL='${__DEFAULT__SERVER_SUPPORT_URL}' +export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' export DEFAULT_CONF_DIRS=' ./components/proxy diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh index 7159ae5e3..9917efece 100755 --- a/birdhouse/deployment/deploy.sh +++ b/birdhouse/deployment/deploy.sh @@ -105,7 +105,7 @@ cd $COMPOSE_DIR START_TIME="`date -Isecond`" echo "deploy START_TIME=$START_TIME" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Read AUTODEPLOY_EXTRA_REPOS read_basic_configs_only @@ -171,7 +171,7 @@ done cd $COMPOSE_DIR # reload again after git pull because this file could be changed by the pull -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/scripts/read-configs.include.sh" # reload again after default.env since env.local can override default.env # (ex: JUPYTERHUB_USER_DATA_DIR) diff --git a/birdhouse/deployment/fix-geoserver-data-dir-perm b/birdhouse/deployment/fix-geoserver-data-dir-perm index 644922674..d5616961c 100755 --- a/birdhouse/deployment/fix-geoserver-data-dir-perm +++ b/birdhouse/deployment/fix-geoserver-data-dir-perm @@ -14,7 +14,7 @@ THIS_DIR="`dirname "$THIS_FILE"`" # Go to repo root. cd $THIS_DIR/../.. -. birdhouse/read-configs.include.sh +. birdhouse/scripts/read-configs.include.sh # Get BASH_IMAGE # Get GEOSERVER_DATA_DIR diff --git a/birdhouse/deployment/fix-write-perm b/birdhouse/deployment/fix-write-perm index 49d2cd73d..a68a7b444 100755 --- a/birdhouse/deployment/fix-write-perm +++ b/birdhouse/deployment/fix-write-perm @@ -43,7 +43,7 @@ THIS_DIR="`dirname "$THIS_FILE"`" # Go to repo root. cd $THIS_DIR/../.. -. birdhouse/read-configs.include.sh +. birdhouse/scripts/read-configs.include.sh # Get GEOSERVER_DATA_DIR, JUPYTERHUB_USER_DATA_DIR, MAGPIE_PERSIST_DIR read_configs diff --git a/birdhouse/deployment/install-deploy-notebook b/birdhouse/deployment/install-deploy-notebook index baf148e57..9927a1f3a 100755 --- a/birdhouse/deployment/install-deploy-notebook +++ b/birdhouse/deployment/install-deploy-notebook @@ -29,7 +29,7 @@ if [ ! -e "$REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook" ]; then exit 2 fi -. "$REPO_ROOT/birdhouse/read-configs.include.sh" +. "$REPO_ROOT/birdhouse/scripts/read-configs.include.sh" # Get JUPYTERHUB_USER_DATA_DIR read_configs diff --git a/birdhouse/deployment/trigger-deploy-notebook b/birdhouse/deployment/trigger-deploy-notebook index bb052d3c0..e793ef466 100755 --- a/birdhouse/deployment/trigger-deploy-notebook +++ b/birdhouse/deployment/trigger-deploy-notebook @@ -34,8 +34,8 @@ if [ -z "$COMPOSE_DIR" ]; then COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")" fi -if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then - . "$COMPOSE_DIR/read-configs.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Get JUPYTERHUB_USER_DATA_DIR read_configs diff --git a/birdhouse/deployment/triggerdeploy.sh b/birdhouse/deployment/triggerdeploy.sh index 4ddce8122..bdb4f6347 100755 --- a/birdhouse/deployment/triggerdeploy.sh +++ b/birdhouse/deployment/triggerdeploy.sh @@ -173,7 +173,7 @@ START_TIME="`date -Isecond`" echo "========== triggerdeploy START_TIME=$START_TIME" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Read AUTODEPLOY_EXTRA_REPOS read_basic_configs_only diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 44636ea51..e9ff36f0c 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -10,10 +10,6 @@ # * Try to keep the same behavior/code, inside and outside of the # autodeploy container to catch error early with the autodeploy. -YELLOW=$(tput setaf 3) -RED=$(tput setaf 1) -NORMAL=$(tput sgr0) - # list of all variables to be substituted in templates # some of these variables *could* employ provided values in 'default.env', # but they must ultimately be defined one way or another for the server to work @@ -34,31 +30,38 @@ OPTIONAL_VARS=' $EXTRA_PYWPS_CONFIG $SERVER_NAME $SERVER_DESCRIPTION + $SERVER_INSTITUTION + $SERVER_SUBJECT + $SERVER_TAGS + $SERVER_DOCUMENTATION_URL + $SERVER_RELEASE_NOTES_URL + $SERVER_SUPPORT_URL + $SERVER_LICENSE_URL ' # we switch to the real directory of the script, so it still works when used from $PATH # tip: ln -s /path/to/pavics-compose.sh ~/bin/ # Setup PWD for sourcing env.local. -cd $(dirname $(readlink -f $0 || realpath $0)) +cd "$(dirname "$(readlink -f "$0" || realpath "$0")")" || (echo "Unresolved path [$0]" && exit 1) # Setup COMPOSE_DIR for sourcing env.local. # Prevent un-expected difference when this script is run inside autodeploy # container and manually from the host. COMPOSE_DIR="`pwd`" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/scripts/read-configs.include.sh" read_configs # this sets ALL_CONF_DIRS -. ./scripts/get-components-json.include.sh -. ./scripts/get-services-json.include.sh -. ./scripts/get-version-json.include.sh +. "${COMPOSE_DIR}/scripts/get-components-json.include.sh" +. "${COMPOSE_DIR}/scripts/get-services-json.include.sh" +. "${COMPOSE_DIR}/scripts/get-version-json.include.sh" for i in ${VARS} do v="${i}" if [ -z "`eval "echo ${v}"`" ] then - echo "${RED}Error${NORMAL}: Required variable $v is not set. Check env.local file." + echo "${MSG_ERROR}Required variable $v is not set. Check env.local file." exit 1 fi done @@ -67,15 +70,15 @@ done # will add delay # if [ ! -f $SSL_CERTIFICATE ] # then -# echo "Error, SSL certificate file $SSL_CERTIFICATE is missing" +# echo "${MSG_ERROR}SSL certificate file $SSL_CERTIFICATE is missing" # exit 1 # fi TIMEWAIT_REUSE=$(/sbin/sysctl -n net.ipv4.tcp_tw_reuse) -if [ ${TIMEWAIT_REUSE} -eq 0 ] +if [ "${TIMEWAIT_REUSE}" -eq 0 ] then - echo "${YELLOW}Warning:${NORMAL} the sysctl net.ipv4.tcp_tw_reuse is not enabled" - echo " It it suggested to set it to 1, otherwise the pavicscrawler may fail" + echo "${MSG_WARN}the sysctl net.ipv4.tcp_tw_reuse is not enabled. " \ + "It it suggested to set it to 1, otherwise the pavicscrawler may fail." fi export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES="" @@ -91,14 +94,14 @@ find $ALL_CONF_DIRS -name '*.template' | while read FILE do DEST=${FILE%.template} - cat ${FILE} | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > ${DEST} + cat "${FILE}" | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > "${DEST}" done if [ x"$1" = x"up" ]; then for adir in $ALL_CONF_DIRS; do COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then - echo "executing '$COMPONENT_PRE_COMPOSE_UP'" + echo "${MSG_INFO}executing '$COMPONENT_PRE_COMPOSE_UP'" sh -x "$COMPONENT_PRE_COMPOSE_UP" fi done @@ -108,6 +111,11 @@ create_compose_conf_list # this sets COMPOSE_CONF_LIST echo "COMPOSE_CONF_LIST=" echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' +if [ x"$1" = x"info" ]; then + echo "${MSG_INFO}Stopping before execution of docker-compose command." + exit 0 +fi + # the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script PROXY_SECURE_PORT=443 HOSTNAME=${PAVICS_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ERR=$? @@ -116,7 +124,7 @@ ERR=$? type post-compose 2>&1 | grep 'post-compose is a function' > /dev/null if [ $? -eq 0 ] then - [ ${ERR} -gt 0 ] && { echo "Error occurred with docker-compose, not running post-compose"; exit $?; } + [ ${ERR} -gt 0 ] && { echo "${MSG_ERROR}Error occurred with docker-compose, not running post-compose"; exit $?; } post-compose $* fi @@ -137,7 +145,7 @@ do for adir in $ALL_CONF_DIRS; do COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up" if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then - echo "executing '$COMPONENT_POST_COMPOSE_UP'" + echo "${MSG_INFO}executing '$COMPONENT_POST_COMPOSE_UP'" sh -x "$COMPONENT_POST_COMPOSE_UP" fi done diff --git a/birdhouse/scripts/check-autodeploy-repos b/birdhouse/scripts/check-autodeploy-repos index 3df3fbc9d..865193502 100755 --- a/birdhouse/scripts/check-autodeploy-repos +++ b/birdhouse/scripts/check-autodeploy-repos @@ -6,8 +6,8 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" -if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then - . "$COMPOSE_DIR/read-configs.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Get AUTODEPLOY_EXTRA_REPOS read_configs diff --git a/birdhouse/scripts/check-instance-ready b/birdhouse/scripts/check-instance-ready index 287a66940..e024bf84f 100755 --- a/birdhouse/scripts/check-instance-ready +++ b/birdhouse/scripts/check-instance-ready @@ -11,8 +11,8 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" -if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then - . "$COMPOSE_DIR/read-configs.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Get PAVICS_FQDN read_configs diff --git a/birdhouse/scripts/check-wps-database.sh b/birdhouse/scripts/check-wps-database.sh index a04869d6d..dedcc9e58 100755 --- a/birdhouse/scripts/check-wps-database.sh +++ b/birdhouse/scripts/check-wps-database.sh @@ -1,5 +1,11 @@ #!/bin/bash +THIS_FILE="`realpath "$0"`" +THIS_DIR="`dirname "$THIS_FILE"`" + +if [ -f "${THIS_DIR}/logging.include.sh" ]; then + . "${THIS_DIR}/logging.include.sh" +fi function usage(){ cat <&2 + echo "${MSG_ERROR}please provide a database name, ex: finch" 1>&2 exit 2 fi shift @@ -24,7 +32,7 @@ docker exec $POSTGRES_CONTAINER_NAME psql -U $POSTGRES_USER $DB_NAME -c "select set +x echo " -WARNING: this will crash all the above requests if currently still processing +${MSG_WARN}This will crash all the above requests if currently still processing Clear those jobs? (Ctrl-C to cancel, any keys to continue)" diff --git a/birdhouse/scripts/create-magpie-users b/birdhouse/scripts/create-magpie-users index 1ca000a46..8c0b444e2 100755 --- a/birdhouse/scripts/create-magpie-users +++ b/birdhouse/scripts/create-magpie-users @@ -65,15 +65,12 @@ # bogus03 bvNWVWCQi8M6 409 : User name matches an already existing user name. # -RED=$(tput setaf 1) -NORMAL=$(tput sgr0) - THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" -if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then - . "$COMPOSE_DIR/read-configs.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Get MAGPIE_VERSION, PAVICS_FQDN, MAGPIE_ADMIN_PASSWORD, MAGPIE_ADMIN_USERNAME read_configs @@ -105,7 +102,7 @@ fi if [ -z "$MAGPIE_CLI_IMAGE" ]; then # MAGPIE_VERSION must be provided by 'default.env', 'env.local' or directly if [ -z "${MAGPIE_VERSION}" ]; then - echo "${RED}Error${NORMAL}: Required MAGPIE_VERSION is undefined or empty." + echo "${MSG_ERROR}Required MAGPIE_VERSION is undefined or empty." exit 1 fi MAGPIE_CLI_IMAGE="pavics/magpie:${MAGPIE_VERSION}" diff --git a/birdhouse/scripts/deprecated/trigger-pavicscrawler b/birdhouse/scripts/deprecated/trigger-pavicscrawler index e9f883043..7e1d13609 100755 --- a/birdhouse/scripts/deprecated/trigger-pavicscrawler +++ b/birdhouse/scripts/deprecated/trigger-pavicscrawler @@ -18,8 +18,8 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" -if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then - . "$COMPOSE_DIR/read-configs.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Get PAVICS_FQDN read_configs diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env index a8e3663bf..96f25ad67 100755 --- a/birdhouse/scripts/detect-user-install-in-jupyter-env +++ b/birdhouse/scripts/detect-user-install-in-jupyter-env @@ -10,7 +10,12 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" -. "$THIS_DIR/../read-configs.include.sh" +. "$THIS_DIR/read-configs.include.sh" + +if [ -f "${THIS_DIR}/logging.include.sh" ]; then + . "${THIS_DIR}/logging.include.sh" +fi + # Get JUPYTERHUB_USER_DATA_DIR read_configs @@ -28,7 +33,7 @@ ls -a */.home/.local/bin set +x -echo " +echo "${MSG_INFO} Errors like ls: cannot access */.home/.local/lib/python*/site-packages: No such file or directory ls: cannot access */.home/.local/bin: No such file or directory diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index bf8679e3b..8409465b7 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -17,10 +17,17 @@ # } # +THIS_FILE="`realpath "$0"`" +THIS_DIR="`dirname "$THIS_FILE"`" + +if [ -f "${THIS_DIR}/logging.include.sh" ]; then + . "${THIS_DIR}/logging.include.sh" +fi + # default value in case of error or missing definitions export BIRDHOUSE_DEPLOY_COMPONENTS_JSON='{"components": []}' if [ -z "${ALL_CONF_DIRS}" ]; then - echo "No components in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. Components JSON list will be empty!" + echo "${MSG_WARN}No components in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. Components JSON list will be empty!" return fi @@ -41,7 +48,7 @@ BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN="$( \ | sed -E 's/^|[[:space:]]+/ -e /' \ )" if [ -z "${BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN}" ]; then - echo "[WARNING]" \ + echo "${MSG_WARN}" \ "Could not resolve known birdhouse-deploy components." \ "Aborting to avoid potentially leaking sensible details." \ "Components will not be reported on the platform's JSON endpoint." diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index fd5a9d257..bfd9e635b 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -1,5 +1,12 @@ #!/bin/sh +THIS_FILE="`realpath "$0"`" +THIS_DIR="`dirname "$THIS_FILE"`" + +if [ -f "${THIS_DIR}/logging.include.sh" ]; then + . "${THIS_DIR}/logging.include.sh" +fi + # default value in case of error or missing definitions for adir in ${ALL_CONF_DIRS}; do @@ -8,6 +15,6 @@ for adir in ${ALL_CONF_DIRS}; do done if [ -z "${SERVICES}" ]; then - echo "${YELLOW}Warning: ${NORMAL}No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" + echo "${MSG_WARN}No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" fi export BIRDHOUSE_DEPLOY_SERVICES_JSON="{\"services\": [${SERVICES}]}" diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh new file mode 100644 index 000000000..910aebf32 --- /dev/null +++ b/birdhouse/scripts/logging.include.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +BIRDHOUSE_COLOR=${BIRDHOUSE_COLOR:-1} +if [ "${BIRDHOUSE_COLOR}" -eq "1" ]; then + BLUE=$(tput setaf 12) + GRAY=$(tput setaf 8) + CYAN=$(tput setaf 6) + PURPLE=$(tput setaf 5) + YELLOW=$(tput setaf 3) + GREEN=$(tput setaf 2) + RED=$(tput setaf 1) + NORMAL=$(tput sgr0) +fi + +export MSG_DEBUG="${GRAY}DEBUG${NORMAL}: " +export MSG_INFO="${BLUE}INFO${NORMAL}: " +export MSG_WARN="${YELLOW}WARNING${NORMAL}: " +export MSG_ERROR="${RED}ERROR${NORMAL}: " diff --git a/birdhouse/scripts/migrate-jupyterhub-user-persistence b/birdhouse/scripts/migrate-jupyterhub-user-persistence index 6059d20b1..6af4722dd 100755 --- a/birdhouse/scripts/migrate-jupyterhub-user-persistence +++ b/birdhouse/scripts/migrate-jupyterhub-user-persistence @@ -8,7 +8,7 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/scripts/read-configs.include.sh" # Get JUPYTERHUB_USER_DATA_DIR read_configs diff --git a/birdhouse/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh similarity index 86% rename from birdhouse/read-configs.include.sh rename to birdhouse/scripts/read-configs.include.sh index 6e9c56007..9d3b24436 100644 --- a/birdhouse/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -16,15 +16,26 @@ # # # Source the script providing function read_configs. # # read_configs uses COMPOSE_DIR to find default.env and env.local. -# . $COMPOSE_DIR/read-configs.include.sh +# . ${COMPOSE_DIR}/scripts/read-configs.include.sh # # # Call function read_configs to read the various config files in the # # appropriate order and process delayed eval vars properly. # read_configs +THIS_FILE="`realpath "$0"`" +THIS_DIR="`dirname "$THIS_FILE"`" + +if [ -f "${THIS_DIR}/logging.include.sh" ]; then + . "${THIS_DIR}/logging.include.sh" +fi +if [ -f "${THIS_DIR}/scripts/logging.include.sh" ]; then + . "${THIS_DIR}/scripts/logging.include.sh" +fi + + # Derive COMPOSE_DIR from the most probable locations. -# This is NOT meant to be exhautive. +# This is NOT meant to be exhaustive. # Assume the checkout is named "birdhouse-deploy", which might NOT be true. # Caller of this file can simply set COMPOSE_DIR itself, this is the safest way. discover_compose_dir() { @@ -55,7 +66,7 @@ discover_compose_dir() { # Case of sub-subdir of sibling checkout at same level as birdhouse-deploy. COMPOSE_DIR="$(realpath "../../../birdhouse-deploy/birdhouse")" fi - echo "$COMPOSE_DIR" + echo "${MSG_INFO}Resolved docker-compose directory: [${COMPOSE_DIR}]" export COMPOSE_DIR fi } @@ -82,7 +93,7 @@ read_default_env() { . "$COMPOSE_DIR/default.env" else - echo "WARNING: '$COMPOSE_DIR/default.env' not found" 1>&2 + echo "${MSG_WARN}'$COMPOSE_DIR/default.env' not found" 1>&2 fi } @@ -90,7 +101,7 @@ read_default_env() { read_env_local() { # we don't use usual .env filename, because docker-compose uses it - echo "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV}" + echo "${MSG_INFO}Using local environment file at: ${BIRDHOUSE_LOCAL_ENV}" if [ -e "$BIRDHOUSE_LOCAL_ENV" ]; then saved_shell_options="$(set +o)" @@ -102,7 +113,7 @@ read_env_local() { eval "$saved_shell_options" else - echo "WARNING: '$BIRDHOUSE_LOCAL_ENV' not found" 1>&2 + echo "${MSG_WARN}'$BIRDHOUSE_LOCAL_ENV' not found" 1>&2 fi } @@ -134,7 +145,7 @@ source_conf_files() { # corresponding PR are merged and old component names can be removed # after the corresponding PR are merge without any impact on the # autodeploy process. - echo "WARNING: '$adir' in $conf_locations does not exist" 1>&2 + echo "${MSG_WARN}'$adir' in $conf_locations does not exist" 1>&2 fi if [ -f "$adir/default.env" ]; then # Source config settings of dependencies first if they haven't been sourced previously. @@ -146,7 +157,7 @@ source_conf_files() { # reset the adir variable in case it was changed in a recursive call adir="$(printf '%b' "$_adir_stack" | tail -1)" fi - echo "reading '$adir/default.env'" + echo "${MSG_DEBUG}reading '$adir/default.env'" . "$adir/default.env" fi if echo "$ALL_CONF_DIRS" | grep -qE "^\s*$adir\s*$"; then @@ -166,7 +177,7 @@ read_components_default_env() { # EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS relative paths are relative to COMPOSE_DIR. if [ -d "$COMPOSE_DIR" ]; then - cd "$COMPOSE_DIR" + cd "$COMPOSE_DIR" >/dev/null fi source_conf_files "$DEFAULT_CONF_DIRS" 'DEFAULT_CONF_DIRS' @@ -174,11 +185,32 @@ read_components_default_env() { # Return to previous pwd. if [ -d "$COMPOSE_DIR" ]; then - cd - + cd - >/dev/null fi } +check_optional_vars() { + + for i in ${OPTIONAL_VARS} + do + v="${i}" + d=`eval echo "$v"` + n="${i#\$}" + default="\${__DEFAULT__${n}}" + result=`echo "${d}" | grep -c "${default}"` + if [ -z "`eval "echo ${v}"`" ] + then + echo "${MSG_WARN}Optional variable [${n}] is not set. Check env.local file." + fi + if [ "${result}" -gt 0 ] + then + echo "${MSG_WARN}Optional variable [${n}] employs a default recommended for override. Check env.local file." + fi + done +} + + # All scripts sourcing default.env and env.local and needing to use any vars # in DELAYED_EVAL list need to call this function to actually resolve the # value of each var in DELAYED_EVAL list. @@ -191,7 +223,7 @@ process_delayed_eval() { fi v="`eval "echo \\$${i}"`" eval 'export ${i}="`eval "echo ${v}"`"' - echo "delayed eval '$(env |grep "${i}=")'" + echo "${MSG_DEBUG}delayed eval '$(env | grep -e "^${i}=")'" ALREADY_EVALED=" $ALREADY_EVALED $i" @@ -214,6 +246,7 @@ create_compose_conf_list() { # ALL_CONF_DIRS relative paths are relative to COMPOSE_DIR. discover_compose_dir if [ -d "$COMPOSE_DIR" ]; then + echo "${MSG_INFO}Found compose directory [${COMPOSE_DIR}]" cd "$COMPOSE_DIR" || return fi @@ -248,7 +281,8 @@ create_compose_conf_list() { # Return to previous pwd. if [ -d "$COMPOSE_DIR" ]; then - cd - || return + echo "${MSG_INFO}Moving to [${COMPOSE_DIR}]" + cd - >/dev/null || return fi } @@ -262,6 +296,7 @@ read_configs() { read_env_local # for EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, need discover_env_local read_components_default_env # uses EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, sets ALL_CONF_DIRS read_env_local # again to override components default.env, need discover_env_local + check_optional_vars process_delayed_eval } diff --git a/birdhouse/scripts/sync-data b/birdhouse/scripts/sync-data index ad6cc4ea5..f08be1f21 100755 --- a/birdhouse/scripts/sync-data +++ b/birdhouse/scripts/sync-data @@ -28,7 +28,7 @@ SOURCE_HOST="$1"; shift FORCE_MODE="$1" if [ -z "$SOURCE_HOST" ]; then - echo "ERROR: no source host provided" 1>&2 + echo "${MSG_ERROR}no source host provided" 1>&2 exit 2 fi @@ -58,7 +58,7 @@ for item in $GEOSERVER_DATA_DIR/ $JUPYTERHUB_USER_DATA_DIR/ $MAGPIE_PERSIST_DIR/ done if [ ! x"$FORCE_MODE" = xforce ]; then - echo "Dry-run mode, not executing '$COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm' and other permission fixup" + echo "${MSG_INFO}Dry-run mode, not executing '$COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm' and other permission fixup" else $COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm From 4a5ed09fce927aa745cf52fce37b9c7d4fcbf154 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 14 Dec 2023 20:44:36 -0500 Subject: [PATCH 02/34] remove old config --- .../conf.extra-service.d/canarie-api.conf | 6 - .../conf.extra-service.d/canarie-api.conf | 6 - .../canarie-api/docker_configuration.py | 229 -------------- birdhouse/config/catalog/catalog.cfg | 34 -- .../canarie-api/canarie_api_monitoring.py | 34 -- .../catalog/config/magpie/permissions.cfg | 5 - .../catalog/config/magpie/providers.cfg | 43 --- .../config/proxy/canarie_api_monitoring.py | 5 - .../canarie-api/canarie_api_monitoring.py | 112 ------- .../config/finch/config/magpie/providers.cfg | 9 - .../config/proxy/canarie_api_monitoring.py | 71 ----- birdhouse/config/finch/service-config.json | 25 -- birdhouse/config/finch/wps.cfg | 15 - .../config/magpie/permissions.cfg | 5 - .../flyingpigeon/config/magpie/providers.cfg | 8 - .../config/flyingpigeon/service-config.json | 25 -- birdhouse/config/flyingpigeon/wps.cfg | 10 - .../canarie-api/canarie_api_monitoring.py | 5 - .../proxy/conf.extra-service.d/frontend.conf | 6 - birdhouse/config/frontend/frontend.env | 8 - .../canarie-api/canarie_api_monitoring.py | 40 --- .../config/proxy/canarie_api_monitoring.py | 6 - .../proxy/conf.extra-service.d/geoserver.conf | 30 -- .../config/geoserver/service-config.json | 23 -- .../canarie-api/canarie_api_monitoring.py | 38 --- .../hummingbird/config/magpie/providers.cfg | 8 - birdhouse/config/hummingbird/custom.cfg | 8 - .../config/hummingbird/service-config.json | 25 -- .../canarie-api/canarie_api_monitoring.py | 35 --- .../jupyterhub/config/magpie/providers.cfg | 10 - .../config/proxy/canarie_api_monitoring.py | 7 - .../conf.extra-service.d/jupyterhub.conf | 11 - .../jupyterhub/custom_templates/login.html | 45 --- .../config/jupyterhub/jupyterhub_config.py | 296 ------------------ .../config/jupyterhub/service-config.json | 20 -- .../canarie-api/canarie_api_monitoring.py | 42 --- .../config/proxy/canarie_api_monitoring.py | 9 - .../proxy/conf.extra-service.d/magpie.conf | 6 - .../docker-entrypoint-initdb.d/create-db.sql | 1 - birdhouse/config/magpie/magpie.ini | 183 ----------- birdhouse/config/magpie/permissions.cfg | 22 -- .../config/magpie/postgres-credentials.env | 7 - birdhouse/config/magpie/providers.cfg | 1 - .../canarie-api/canarie_api_monitoring.py | 34 -- .../malleefowl/config/magpie/providers.cfg | 8 - .../config/proxy/canarie_api_monitoring.py | 5 - birdhouse/config/malleefowl/custom.cfg | 30 -- birdhouse/config/ncops/ncops.cfg | 2 - .../config/ncwms2/config/magpie/providers.cfg | 8 - .../proxy/conf.extra-service.d/ncwms2.conf | 7 - birdhouse/config/ncwms2/custom.cfg | 11 - .../canarie-api/canarie_api_monitoring.py | 34 -- .../config/proxy/canarie_api_monitoring.py | 5 - birdhouse/config/phoenix/custom.cfg | 10 - .../proxy/conf.extra-service.d/portainer.conf | 5 - birdhouse/config/postgres/credentials.env | 3 - .../canarie-api/canarie_api_monitoring.py | 6 - .../conf.extra-service.d/project-api.conf | 6 - .../config/proxy/conf.d/all-services.include | 28 -- birdhouse/config/proxy/conf.d/frontend.conf | 48 --- birdhouse/config/proxy/nginx.conf | 41 --- birdhouse/config/proxy/static/components.json | 26 -- birdhouse/config/proxy/static/services.json | 221 ------------- birdhouse/config/proxy/static/version.json | 1 - .../canarie-api/canarie_api_monitoring.py | 39 --- .../config/raven/config/magpie/providers.cfg | 9 - .../config/proxy/canarie_api_monitoring.py | 39 --- birdhouse/config/raven/service-config.json | 25 -- birdhouse/config/raven/wps.cfg | 15 - .../canarie-api/canarie_api_monitoring.py | 39 --- .../config/proxy/canarie_api_monitoring.py | 8 - birdhouse/config/thredds/catalog.xml | 53 ---- .../canarie-api/canarie_api_monitoring.py | 72 ----- .../thredds/config/magpie/providers.cfg | 35 --- .../config/proxy/canarie_api_monitoring.py | 41 --- .../proxy/conf.extra-service.d/thredds.conf | 10 - birdhouse/config/thredds/service-config.json | 26 -- birdhouse/config/thredds/threddsConfig.xml | 137 -------- .../canarie-api/canarie_api_monitoring.py | 56 ---- .../config/proxy/canarie_api_monitoring.py | 9 - .../proxy/conf.extra-service.d/twitcher.conf | 8 - birdhouse/config/twitcher/twitcher.ini | 141 --------- .../canarie-api/canarie_api_monitoring.py | 1 - .../config/proxy/canarie_api_monitoring.py | 1 - .../wps_outputs-volume.conf | 5 - 85 files changed, 2842 deletions(-) delete mode 100644 birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf delete mode 100644 birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf delete mode 100644 birdhouse/config/canarie-api/docker_configuration.py delete mode 100644 birdhouse/config/catalog/catalog.cfg delete mode 100644 birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/catalog/config/magpie/permissions.cfg delete mode 100644 birdhouse/config/catalog/config/magpie/providers.cfg delete mode 100644 birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/finch/config/magpie/providers.cfg delete mode 100644 birdhouse/config/finch/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/finch/service-config.json delete mode 100644 birdhouse/config/finch/wps.cfg delete mode 100644 birdhouse/config/flyingpigeon/config/magpie/permissions.cfg delete mode 100644 birdhouse/config/flyingpigeon/config/magpie/providers.cfg delete mode 100644 birdhouse/config/flyingpigeon/service-config.json delete mode 100644 birdhouse/config/flyingpigeon/wps.cfg delete mode 100644 birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf delete mode 100644 birdhouse/config/frontend/frontend.env delete mode 100644 birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf delete mode 100644 birdhouse/config/geoserver/service-config.json delete mode 100644 birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/hummingbird/config/magpie/providers.cfg delete mode 100644 birdhouse/config/hummingbird/custom.cfg delete mode 100644 birdhouse/config/hummingbird/service-config.json delete mode 100644 birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/jupyterhub/config/magpie/providers.cfg delete mode 100644 birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf delete mode 100644 birdhouse/config/jupyterhub/custom_templates/login.html delete mode 100644 birdhouse/config/jupyterhub/jupyterhub_config.py delete mode 100644 birdhouse/config/jupyterhub/service-config.json delete mode 100644 birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf delete mode 100644 birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql delete mode 100644 birdhouse/config/magpie/magpie.ini delete mode 100644 birdhouse/config/magpie/permissions.cfg delete mode 100644 birdhouse/config/magpie/postgres-credentials.env delete mode 100644 birdhouse/config/magpie/providers.cfg delete mode 100644 birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/malleefowl/config/magpie/providers.cfg delete mode 100644 birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/malleefowl/custom.cfg delete mode 100644 birdhouse/config/ncops/ncops.cfg delete mode 100644 birdhouse/config/ncwms2/config/magpie/providers.cfg delete mode 100644 birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf delete mode 100644 birdhouse/config/ncwms2/custom.cfg delete mode 100644 birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/phoenix/custom.cfg delete mode 100644 birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf delete mode 100644 birdhouse/config/postgres/credentials.env delete mode 100644 birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf delete mode 100644 birdhouse/config/proxy/conf.d/all-services.include delete mode 100644 birdhouse/config/proxy/conf.d/frontend.conf delete mode 100644 birdhouse/config/proxy/nginx.conf delete mode 100644 birdhouse/config/proxy/static/components.json delete mode 100644 birdhouse/config/proxy/static/services.json delete mode 100644 birdhouse/config/proxy/static/version.json delete mode 100644 birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/raven/config/magpie/providers.cfg delete mode 100644 birdhouse/config/raven/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/raven/service-config.json delete mode 100644 birdhouse/config/raven/wps.cfg delete mode 100644 birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/solr/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/thredds/catalog.xml delete mode 100644 birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/thredds/config/magpie/providers.cfg delete mode 100644 birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf delete mode 100644 birdhouse/config/thredds/service-config.json delete mode 100644 birdhouse/config/thredds/threddsConfig.xml delete mode 100644 birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf delete mode 100644 birdhouse/config/twitcher/twitcher.ini delete mode 100644 birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py delete mode 100644 birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py delete mode 100644 birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf diff --git a/birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf b/birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf deleted file mode 100644 index e843f54f8..000000000 --- a/birdhouse/config/canarie-api/conf.extra-service.d/canarie-api.conf +++ /dev/null @@ -1,6 +0,0 @@ - location /canarie/ { - proxy_pass http://127.0.0.1:2000; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header X-Script-Name /canarie; - } diff --git a/birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf b/birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf deleted file mode 100644 index e843f54f8..000000000 --- a/birdhouse/config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf +++ /dev/null @@ -1,6 +0,0 @@ - location /canarie/ { - proxy_pass http://127.0.0.1:2000; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header X-Script-Name /canarie; - } diff --git a/birdhouse/config/canarie-api/docker_configuration.py b/birdhouse/config/canarie-api/docker_configuration.py deleted file mode 100644 index 4f03e1528..000000000 --- a/birdhouse/config/canarie-api/docker_configuration.py +++ /dev/null @@ -1,229 +0,0 @@ -# coding: utf-8 -import datetime -import os -import logging - -# note: -# Can only be dependencies also available in CanarieAPI docker image. -# Otherwise, they must be installed after the fact with the entrypoint script. -import requests_cache # see entrypoint script - -logger = logging.getLogger("canarie-api-config") - -MY_SERVER_NAME = 'https://10.0.2.15/canarie' - -DATABASE = { - 'filename': '/opt/local/src/CanarieAPI/stats.db', - 'access_log': '/var/log/nginx/access_file.log', - 'log_pid': '/var/run/nginx.pid' -} - - -# Below is to avoid rate-limiting as much as possible on multiple runs. -# Because the logparser and monitoring must load the application context and configuration, -# the requests performed to retrieve version numbers and release times occurs on every cron interval. -REQUEST_CACHE = "/tmp/canarieapi-config-cache" -os.makedirs(REQUEST_CACHE, exist_ok=True) -SESSION = requests_cache.CachedSession( - "canarieapi-config-cache", - use_temp=True, - backend="filesystem", - expire_after=datetime.timedelta(days=1), - cache_control=False, # prefer our request cache options, not ones provided by responses - stale_if_error=True, - allowable_codes=[200], - allowable_methods=["GET"], -) - - -def _get_release_time_from_github_tag(repository_reference: str, tagged_version: str) -> str: - repo = repository_reference.split("github.com/")[-1] - url = f"https://api.github.com/repos/{repo}/tags" - resp = SESSION.get(url, headers={"Accept": "application/json"}) - if resp.status_code != 200: - raise ValueError( - f"HTTP status from tags list: {resp.status_code} from [{url}] with detail:\n{resp.text!s}" - ) - - for tag_info in resp.json(): - tag_ver = tag_info.get("name") - tag_commit_url = tag_info.get("commit", {}).get("url") - if tag_ver == tagged_version or tag_ver == "v" + tagged_version and tag_commit_url: - break - else: - raise ValueError(f"No matching version in tagged releases under [{url}].") - - resp = SESSION.get(tag_commit_url, headers={"Accept": "application/json"}) - if resp.status_code != 200: - raise ValueError( - f"HTTP status from tag info: {resp.status_code} from [{tag_commit_url}] with detail:\n{resp.text!s}" - ) - - commit = resp.json()["commit"] - commit_date = commit["committer"]["date"] - return commit_date - - -def _get_release_time_from_docker_tag(repository_reference: str, tagged_version: str) -> str: - ns, repo = repository_reference.split("/", 1) - url = f"https://hub.docker.com/v2/namespaces/{ns}/repositories/{repo}/tags/{tagged_version}" - resp = SESSION.get(url, headers={"Accept": "application/json"}) - if resp.status_code != 200: - raise ValueError( - f"HTTP status from tag info: {resp.status_code} from [{url}] with detail:\n{resp.text!s}" - ) - - updated = resp.json()["last_updated"] - return updated - - -def get_release_time_from_repo_tag(repository_type: str, repository_reference: str, tagged_version: str) -> str: - try: - if repository_type == "github": - release = _get_release_time_from_github_tag(repository_reference, tagged_version) - elif repository_type == "docker": - release = _get_release_time_from_docker_tag(repository_reference, tagged_version) - else: - raise ValueError(f"Unknown repository parsing type: [{repository_type}]") - if release: - logger.error("canarie-api: found version [%s] release time [%s] from [%s: %s]", - tagged_version, release, repository_type, repository_reference) - return release - raise ValueError("no release time value found") - except Exception as exc: - logger.error("canarie-api: failed retrieving version [%s] release time from [%s: %s]", - tagged_version, repository_type, repository_reference, exc_info=exc) - return "unknown" - - -SERVICES = { - 'node': { # partial duplicate of first item from 'PLATFORM' for backward compatibility - 'info': { - 'name': 'Node', - 'synopsis': ( - 'Nodes are data, compute and index endpoints accessed through the platform or external clients. ' - 'The Node service is the backend that allows: data storage, harvesting, indexation and discovery of ' - 'local and federated data; authentication and authorization; server registration and management. ' - 'Node service is therefore composed of several other services.' - ), - # NOTE: - # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. - # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.40.0', - 'releaseTime': '2023-11-30T18:27:41Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Resource/Cloud Management', - 'tags': ['Climatology'] - }, - 'stats': { - 'method': '.*', - 'route': '(?!)' # this will be set by CANARIE_STATS_ROUTES (see below) - }, - 'redirect': { - 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', - 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md', - 'support': 'https://github.com/bird-house/birdhouse-deploy/issues', - 'source': 'https://github.com/bird-house/birdhouse-deploy', - 'tryme': 'https://10.0.2.15', - 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', - 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' - }, - 'monitoring': {} # filled in after processing everything, see end of script - } -} - -PLATFORMS = { - 'server': { - 'info': { - 'name': 'PAVICS', - 'synopsis': """ -The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of -climate analysis services served through Open Geospatial Consortium (OGC) protocols. -These services include data access, processing and visualization. Both data and algorithms -can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, -or a custom web interface. -""".replace("\n", " ").strip(), - # NOTE: - # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. - # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.40.0', - 'releaseTime': '2023-11-30T18:27:41Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'tags': ['Climatology', 'Cloud'] - }, - 'stats': { - 'method': '.*', - 'route': '(?!)' # this can be set by individual components (eg: the legacy project-api component) - }, - 'redirect': { - 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', - 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/releases', - 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', - 'source': 'https://github.com/Ouranosinc/pavics-sdi', - 'tryme': 'https://pavics.ouranos.ca', - 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', - 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html', - 'factsheet': 'http://www.canarie.ca/software/pavics' - }, - 'monitoring': {} - } -} - -CANARIE_MONITORING_EXTRA_CONF_DIR = os.environ.get( - 'CANARIE_MONITORING_EXTRA_CONF_DIR', '/bogus-notexist' -) - - -def exec_file(filepath, **kwargs): - """ - Backward compatible ``execfile`` equivalent for Python 3.x that allows keyword arguments. - - See https://stackoverflow.com/a/41658338/5936364. - """ - _globals = kwargs.get("globals") or globals() - _locals = kwargs.get("locals") or _globals # as per exec/execfile documentation - _globals.update({ - "__file__": filepath, - "__name__": "__main__", - }) - with open(filepath, 'rb') as file: - exec(compile(file.read(), filepath, "exec"), _globals, _locals) - - -CANARIE_STATS_ROUTES = [] - -if os.path.exists(CANARIE_MONITORING_EXTRA_CONF_DIR): - # alphabetically sorted for reproducible override precedence - for extra_conf in sorted(os.listdir(CANARIE_MONITORING_EXTRA_CONF_DIR)): - extra_conf_full_path = f"{CANARIE_MONITORING_EXTRA_CONF_DIR}/{extra_conf}" - # only handle files ending with .py - if os.path.isfile(extra_conf_full_path) and extra_conf_full_path.endswith(".py"): - logger.info("canarie-api: loading extra config '%s'", extra_conf_full_path) - exec_file(extra_conf_full_path, locals=locals()) - else: - logger.info("canarie-api: ignoring extra config '%s'", extra_conf_full_path) - - -# For historical reasons, the "node" service was used to contain all "services" of the current "platform". -# However, those services should instead be listed directly at the root as individual services of the current platform, -# to directly obtain their status instead of going through the "node" service sub-page. This also allows us to provide -# the individual links (docs, release, etc.) for the services themselves. The services can also be represented nested -# under the "Platform" section as well. This "Platform" section could be extended with other known platforms part of a -# DACCS network. For backward-compatibility, repopulate the "node" service using moved definitions. -_NODE = SERVICES["node"]["monitoring"] -_PLATFORM = PLATFORMS["server"]["monitoring"] -for svc_name, svc_config in SERVICES.items(): - if svc_name == "node": - continue - for mon_name, mon_config in svc_config["monitoring"].items(): - _NODE.setdefault(mon_name, mon_config) - _PLATFORM.setdefault(mon_name, mon_config) # type: ignore - -if CANARIE_STATS_ROUTES: - SERVICES['node']['stats']['route'] = '/(' + '|'.join(r.strip('/') for r in CANARIE_STATS_ROUTES) + ')/.*' - -# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/config/catalog/catalog.cfg b/birdhouse/config/catalog/catalog.cfg deleted file mode 100644 index b16b4a3ca..000000000 --- a/birdhouse/config/catalog/catalog.cfg +++ /dev/null @@ -1,34 +0,0 @@ -[catalog] -solr_host=http://10.0.2.15:8983/solr/birdhouse/ - -# Multiple thredds hosts can be given, comma separated -# note: this URL is also used as prefix when comparing authorizations from magpie -thredds_host=https://10.0.2.15/twitcher/ows/proxy/thredds - -# Multiple esgf nodes can be given, comma separated -esgf_nodes=https://esgf-node.llnl.gov/esg-search - -# Provide a magpie host to filter results based on access permissions. -# Must also provide credentials with read access so that the crawler can parse the thredds host(s) -# Leave as a comment for a public catalog. -magpie_host=https://10.0.2.15/magpie -magpie_user=admin-catalog -magpie_pw=qwerty -# SSL verification (true or false) -verify=true -# Comma separated list of magpie service name for the thredds hosts listed above -thredds_host_magpie_svc_name=thredds - -# WMS service url with replaced by each instance of the thredds_host, -# without the port number and replaced by the base url in thredds. -# Leave as comment to use the default WMS service -wms_alternate_server=https://10.0.2.15/twitcher/ows/proxy/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ - -[pywps] -outputurl=https://10.0.2.15/wpsoutputs/catalog -parallelprocesses=30 - -[logging] -#level=DEBUG -#file=/tmp/wps.log -database=postgresql://postgres-pavics:postgres-qwerty@postgres/catalog diff --git a/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 0ca41f89c..000000000 --- a/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,34 +0,0 @@ -SERVICES['Catalog'] = { - 'info': { - 'name': 'Catalog', - 'synopsis': 'Catalog', - 'version': "0.6.11", - 'institution': 'Ouranos', - 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/pavics-datacatalog", "0.6.11"), - 'researchSubject': 'Climate', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Catalog', - 'tags': ['Catalog', 'Data'] - }, - 'stats': { - 'method': '.*', - 'route': 'http://10.0.2.15:8086/.*' - }, - 'redirect': { - 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#pavics-datacatalog', - 'releasenotes': 'https://github.com/Ouranosinc/PAVICS-DataCatalog/blob/master/CHANGES.md', - 'support': 'https://github.com/ouranosinc/PAVICS-DataCatalog/issues', - 'source': 'https://github.com/ouranosinc/PAVICS-DataCatalog', - 'tryme': 'http://10.0.2.15:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities', - 'licence': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#credits', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#pavics-datacatalog' - }, - "monitoring": { - "Catalog": { - 'request': { - # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) - 'url': 'http://10.0.2.15:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities' - } - } - } -} diff --git a/birdhouse/config/catalog/config/magpie/permissions.cfg b/birdhouse/config/catalog/config/magpie/permissions.cfg deleted file mode 100644 index 79f098e2e..000000000 --- a/birdhouse/config/catalog/config/magpie/permissions.cfg +++ /dev/null @@ -1,5 +0,0 @@ -permissions: - - service: thredds - permission: read - user: admin-catalog - action: create diff --git a/birdhouse/config/catalog/config/magpie/providers.cfg b/birdhouse/config/catalog/config/magpie/providers.cfg deleted file mode 100644 index c094415b1..000000000 --- a/birdhouse/config/catalog/config/magpie/providers.cfg +++ /dev/null @@ -1,43 +0,0 @@ -providers: - catalog: - url: http://10.0.2.15:8086/pywps - title: Catalog - public: true - c4i: false - type: wps - sync_type: wps - - thredds: - url: http://10.0.2.15:8083/twitcher/ows/proxy/thredds - title: Thredds - public: true - c4i: false - type: thredds - sync_type: thredds - # below is a custom config to indicate how magpie should convert thredds path elements into resources/permissions - # see: https://pavics-magpie.readthedocs.io/en/latest/services.html#servicethredds - configuration: - skip_prefix: "thredds" # prefix to ignore, below prefixes will be matched against whatever comes after in path - file_patterns: - # note: make sure to employ quotes and double escapes to avoid parsing YAML error - - ".+\\.ncml" # match longest extension first to avoid tuncating it by match of sorter '.nc' - - ".+\\.nc" - metadata_type: - prefixes: - - null # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed - - "\\w+\\.gif" # threddsIcon, folder icon, etc. - - "\\w+\\.ico" # favicon - - "\\w+\\.txt" # licence - - "\\w+\\.css" # tds.css - - "catalog\\.\\w+" # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE` - - catalog - - ncml - - uddc - - iso - data_type: - prefixes: - - fileServer - - dodsC - - wcs - - wms - - ncss diff --git a/birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py b/birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 3fc1bbc26..000000000 --- a/birdhouse/config/catalog/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,5 +0,0 @@ -SERVICES['node']['monitoring']['Catalog'] = { - 'request': { - 'url': 'http://10.0.2.15:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities' - } -} diff --git a/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 9f59a0e74..000000000 --- a/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,112 +0,0 @@ -# used more than once, avoid unnecessary requests -FINCH_VERSION = "0.9.2" -FINCH_RELEASE_TIME = get_release_time_from_repo_tag("github", "bird-house/finch", FINCH_VERSION) - -SERVICES['indices'] = { - 'info': { - 'name': 'Climate indices', - 'synopsis': ( - 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, ' - 'the number of degree-days of cooling, the duration of heatwaves, etc. ' - 'This returns annual values of the most popular climate indices.' - ), - 'version': FINCH_VERSION, - 'institution': 'Ouranos', - 'releaseTime': FINCH_RELEASE_TIME, - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology', 'Cloud'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/finch.*' - }, - 'redirect': { - 'doc': 'https://finch.readthedocs.io/en/latest/index.html', - 'releasenotes': 'https://github.com/bird-house/finch/releases', - 'support': 'https://github.com/bird-house/finch/issues', - 'source': 'https://github.com/bird-house/finch', - 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', - 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Finch': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} -SERVICES['slicer'] = { - 'info': { - 'name': 'Spatial and temporal data slicer', - 'synopsis': 'These services allow the subsetting and download of NetCDF over user-defined regions and periods.', - 'version': '0.7.1', - 'institution': 'Ouranos', - 'releaseTime': '2021-03-25T00:00:00Z', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology', 'Cloud'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/finch.*' - }, - 'redirect': { - 'doc': 'https://finch.readthedocs.io/en/latest/index.html', - 'releasenotes': 'https://github.com/bird-house/finch/releases', - 'support': 'https://github.com/bird-house/finch/issues', - 'source': 'https://github.com/bird-house/finch', - 'tryme': 'https://pavics-sdi.readthedocs.io/en/latest/notebooks/regridding.html', - 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Finch': { - 'request': { - 'url': 'http://finch:5000/?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} - -SERVICES['Finch'] = { - 'info': { - 'name': 'Finch', - 'synopsis': ( - 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, ' - 'the number of degree-days of cooling, the duration of heatwaves, etc. This returns annual values of ' - 'the most popular climate indices.' - ), - 'version': FINCH_VERSION, - 'institution': 'Ouranos', - 'releaseTime': FINCH_RELEASE_TIME, - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology', 'Cloud'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/finch.*' - }, - 'redirect': { - 'doc': 'https://finch.readthedocs.io/en/latest/index.html', - 'releasenotes': 'https://github.com/bird-house/finch/releases', - 'support': 'https://github.com/bird-house/finch/issues', - 'source': 'https://github.com/bird-house/finch', - 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', - 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Finch': { - 'request': { - 'url': 'http://finch:5000/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - }, -} diff --git a/birdhouse/config/finch/config/magpie/providers.cfg b/birdhouse/config/finch/config/magpie/providers.cfg deleted file mode 100644 index e9eebd1a8..000000000 --- a/birdhouse/config/finch/config/magpie/providers.cfg +++ /dev/null @@ -1,9 +0,0 @@ -providers: - finch: - url: http://finch:5000/wps - title: Finch - public: true - c4i: false - type: wps - sync_type: wps - diff --git a/birdhouse/config/finch/config/proxy/canarie_api_monitoring.py b/birdhouse/config/finch/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 19f4d2b26..000000000 --- a/birdhouse/config/finch/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,71 +0,0 @@ -SERVICES['node']['monitoring']['Finch'] = { - 'request': { - 'url': 'http://10.0.2.15:8095/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } -} -SERVICES['indices'] = { - 'info': { - 'name': 'Climate indices', - 'synopsis': 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, the number of degree-days of cooling, the duration of heatwaves, etc. This returns annual values of the most popular climate indices.', - 'version': '0.7.1', - 'institution': 'Ouranos', - 'releaseTime': '2021-03-25T00:00:00Z', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology', 'Cloud'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/finch.*' - }, - 'redirect': { - 'doc': 'https://finch.readthedocs.io/en/latest/index.html', - 'releasenotes': 'https://github.com/bird-house/finch/releases', - 'support': 'https://github.com/bird-house/finch/issues', - 'source': 'https://github.com/bird-house/finch', - 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', - 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Finch': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} -SERVICES['slicer'] = { - 'info': { - 'name': 'Spatial and temporal data slicer', - 'synopsis': 'These services allows the subsetting and download of NetCDF over user-defined regions and periods.', - 'version': '0.7.1', - 'institution': 'Ouranos', - 'releaseTime': '2021-03-25T00:00:00Z', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology', 'Cloud'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/finch.*' - }, - 'redirect': { - 'doc': 'https://finch.readthedocs.io/en/latest/index.html', - 'releasenotes': 'https://github.com/bird-house/finch/releases', - 'support': 'https://github.com/bird-house/finch/issues', - 'source': 'https://github.com/bird-house/finch', - 'tryme': 'https://pavics-sdi.readthedocs.io/en/latest/notebooks/regridding.html', - 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Finch': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} diff --git a/birdhouse/config/finch/service-config.json b/birdhouse/config/finch/service-config.json deleted file mode 100644 index 7b0fde606..000000000 --- a/birdhouse/config/finch/service-config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "finch", - "keywords": [ - "service-wps" - ], - "description": "A Web Processing Service for Climate Indicators.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://finch.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" - } - ] -} diff --git a/birdhouse/config/finch/wps.cfg b/birdhouse/config/finch/wps.cfg deleted file mode 100644 index baf222d42..000000000 --- a/birdhouse/config/finch/wps.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[server] -outputurl = https://10.0.2.15/wpsoutputs/finch -outputpath = /data/wpsoutputs/finch - -# default 3mb, fix "Broken pipe" between the proxy and the wps service -maxrequestsize = 400mb - -# default 2, too low for a production server with capable CPUs -parallelprocesses = 10 - -[logging] -level = INFO -database=postgresql://postgres-pavics:postgres-qwerty@postgres/finch - - diff --git a/birdhouse/config/flyingpigeon/config/magpie/permissions.cfg b/birdhouse/config/flyingpigeon/config/magpie/permissions.cfg deleted file mode 100644 index 9db2a983a..000000000 --- a/birdhouse/config/flyingpigeon/config/magpie/permissions.cfg +++ /dev/null @@ -1,5 +0,0 @@ -permissions: - - service: flyingpigeon - permission: getcapabilities - group: administrators - action: create diff --git a/birdhouse/config/flyingpigeon/config/magpie/providers.cfg b/birdhouse/config/flyingpigeon/config/magpie/providers.cfg deleted file mode 100644 index 08f74cebd..000000000 --- a/birdhouse/config/flyingpigeon/config/magpie/providers.cfg +++ /dev/null @@ -1,8 +0,0 @@ -providers: - flyingpigeon: - url: http://flyingpigeon:8093/wps - title: Flyingpigeon - public: true - c4i: false - type: wps - sync_type: wps diff --git a/birdhouse/config/flyingpigeon/service-config.json b/birdhouse/config/flyingpigeon/service-config.json deleted file mode 100644 index c303d439e..000000000 --- a/birdhouse/config/flyingpigeon/service-config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/DACCS-node-registry/main/node_registry.schema.json#service", - "name": "flyingpigeon", - "keywords": [ - "service-wps" - ], - "description": "WPS for climate model data, indices and extreme events.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/flyingpigeon?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://flyingpigeon.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/flyingpigeon?service=WPS&request=GetCapabilities" - } - ] -} diff --git a/birdhouse/config/flyingpigeon/wps.cfg b/birdhouse/config/flyingpigeon/wps.cfg deleted file mode 100644 index b2587b600..000000000 --- a/birdhouse/config/flyingpigeon/wps.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[server] -outputurl = https://10.0.2.15/wpsoutputs/flyingpigeon -outputpath = /data/wpsoutputs/flyingpigeon -maxsingleinputsize = 2097152000.0 - -[logging] -level = INFO -database=postgresql://postgres-pavics:postgres-qwerty@postgres/flyingpigeon - - diff --git a/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index e70b9c487..000000000 --- a/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,5 +0,0 @@ -PLATFORMS['server']['monitoring']['Frontend'] = { - 'request': { - 'url': 'https://10.0.2.15' - } -} diff --git a/birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf b/birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf deleted file mode 100644 index e15a9e280..000000000 --- a/birdhouse/config/frontend/config/proxy/conf.extra-service.d/frontend.conf +++ /dev/null @@ -1,6 +0,0 @@ - - location /frontend/ { - proxy_pass http://10.0.2.15:3000; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - } diff --git a/birdhouse/config/frontend/frontend.env b/birdhouse/config/frontend/frontend.env deleted file mode 100644 index feab043c9..000000000 --- a/birdhouse/config/frontend/frontend.env +++ /dev/null @@ -1,8 +0,0 @@ -PAVICS_FRONTEND_IP=10.0.2.15 -PAVICS_FRONTEND_PORT=443 -PAVICS_FRONTEND_PROTO=https -BIRDHOUSE_HOST=10.0.2.15 -NODE_TLS_REJECT_UNAUTHORIZED=0 -NCWMS_HOST=https://10.0.2.15/twitcher/ows/proxy/ncWMS2/wms -CATALOG_HOST=https://10.0.2.15/twitcher/ows/proxy/catalog/pywps -MALLEEFOWL_HOST=https://10.0.2.15/twitcher/ows/proxy/malleefowl/wps diff --git a/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 2a098004c..000000000 --- a/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,40 +0,0 @@ -SERVICES['GeoServer'] = { - 'info': { - 'name': 'GeoServer', - 'synopsis': ( - 'GeoServer is the reference implementation of the Open Geospatial Consortium (OGC) ' - 'Web Feature Service (WFS) and Web Coverage Service (WCS) standards, as well as a high performance ' - 'certified compliant Web Map Service (WMS), compliant Catalog Service for the Web (CSW) and ' - 'implementing Web Processing Service (WPS). GeoServer forms a core component of the Geospatial Web.' - ), - 'version': "2.22.2", - 'institution': 'Ouranos', - 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/geoserver", "2.22.2-kartoza-build20230226-r7-allow-change-context-root-and-fix-missing-stable-plugins-and-avoid-chown-datadir"), - 'researchSubject': 'Geospatial', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Catalog', - 'tags': ['Data', 'Geospatial', 'Catalog', 'OGC', 'WFS', 'WMS', 'WPS'] - }, - 'stats': { - 'method': '.*', - 'route': "/geoserver/.*" - }, - 'redirect': { - 'doc': 'https://docs.geoserver.org/', - 'releasenotes': 'https://geoserver.org/release/2.22.2/', - 'support': 'https://github.com/kartoza/docker-geoserver/issues', - 'source': 'https://github.com/kartoza/docker-geoserver', - 'tryme': 'https://10.0.2.15/geoserver/', - 'licence': 'https://github.com/geoserver/geoserver/blob/2.22.2/LICENSE.txt', - 'provenance': 'https://github.com/kartoza/docker-geoserver' - }, - "monitoring": { - "GeoServer": { - 'request': { - 'url': 'https://10.0.2.15/geoserver/web/' - } - } - } -} - -CANARIE_STATS_ROUTES.append('geoserver') diff --git a/birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py b/birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index c5ebc7483..000000000 --- a/birdhouse/config/geoserver/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,6 +0,0 @@ -SERVICES['node']['monitoring'].update({'Geoserver': { - 'request': { - 'url': 'https://10.0.2.15/geoserver/web/' - } -}}) -CANARIE_STATS_ROUTES.append('geoserver') diff --git a/birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf b/birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf deleted file mode 100644 index 2d8abd4b7..000000000 --- a/birdhouse/config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf +++ /dev/null @@ -1,30 +0,0 @@ - location /geoserver/ { - auth_request /secure-geoserver-auth; - auth_request_set $auth_status $upstream_status; - - proxy_pass http://geoserver:8080/geoserver/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header Accept-Encoding ""; - gzip_proxied any; - gzip on; - gzip_comp_level 1; - gzip_types application/json text/plain application/xml text/html; - } - - location = /secure-geoserver-auth { - internal; - - - # If GEOSERVER_SKIP_AUTH is "True" then the following section is skipped and this - # location block will always return 200 (which means that the /geoserver/ location, above, - # will be publicly available. - proxy_pass https://10.0.2.15/twitcher/ows/verify/geoserver$request_uri; - proxy_pass_request_body off; - proxy_set_header Host $host; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host:$server_port; - } diff --git a/birdhouse/config/geoserver/service-config.json b/birdhouse/config/geoserver/service-config.json deleted file mode 100644 index 71cdc4455..000000000 --- a/birdhouse/config/geoserver/service-config.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "geoserver", - "keywords": [ - "data", - "service-wms", - "service-wfs", - "service-wps" - ], - "description": "GeoServer is a server that allows users to view and edit geospatial data.", - "links": [ - { - "rel": "service", - "type": "application/json", - "href": "https://10.0.2.15/geoserver/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://docs.geoserver.org/" - } - ] -} diff --git a/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 903c217ff..000000000 --- a/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,38 +0,0 @@ -HUMMINGBIRD_VERSION = "0.5_dev" -HUMMINGBIRD_RELEASE = get_release_time_from_repo_tag("docker", "pavics/hummingbird", HUMMINGBIRD_VERSION) - -SERVICES['hummingbird'] = { - 'info': { - 'name': 'Climatology compliance checker.', - 'synopsis': ( - 'A Web Processing Service for compliance checks used in the climate science community.' - ), - 'version': HUMMINGBIRD_VERSION, - 'institution': 'bird-house', - 'releaseTime': HUMMINGBIRD_RELEASE, - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Processing', - 'tags': ['Climatology', 'Checker', 'Compliance', 'CF-conventions', 'WPS', 'OGC'], - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/hummingbird.*' - }, - 'redirect': { - 'doc': 'https://hummingbird.readthedocs.io/', - 'releasenotes': 'https://github.com/bird-house/hummingbird/blob/master/CHANGES.rst', - 'support': 'https://github.com/bird-house/hummingbird/issues', - 'source': 'https://github.com/bird-house/hummingbird', - 'tryme': 'https://10.0.2.15/twitcher/ows/proxy/hummingbird/wps?service=WPS&version=1.0.0&request=GetCapabilities', - 'licence': 'https://github.com/bird-house/hummingbird/blob/master/LICENSE.txt', - 'provenance': 'https://github.com/bird-house/hummingbird' - }, - 'monitoring': { - 'Hummingbird': { - 'request': { - 'url': 'http://hummingbird:8080/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} diff --git a/birdhouse/config/hummingbird/config/magpie/providers.cfg b/birdhouse/config/hummingbird/config/magpie/providers.cfg deleted file mode 100644 index f2b6855ac..000000000 --- a/birdhouse/config/hummingbird/config/magpie/providers.cfg +++ /dev/null @@ -1,8 +0,0 @@ -providers: - hummingbird: - url: http://hummingbird:8080/wps - title: Hummingbird - public: true - c4i: false - type: wps - sync_type: wps diff --git a/birdhouse/config/hummingbird/custom.cfg b/birdhouse/config/hummingbird/custom.cfg deleted file mode 100644 index 11ee63b92..000000000 --- a/birdhouse/config/hummingbird/custom.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[buildout] -extends = profiles/docker.cfg - -[pywps] -enable-https = true -https-port = 443 -https-output-port = 443 -database=postgresql://postgres-pavics:postgres-qwerty@postgres/hummingbird diff --git a/birdhouse/config/hummingbird/service-config.json b/birdhouse/config/hummingbird/service-config.json deleted file mode 100644 index 4d17f2cc9..000000000 --- a/birdhouse/config/hummingbird/service-config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "hummingbird", - "keywords": [ - "service-wps" - ], - "description": "A WPS for metadata compliance checks used in the climate science community.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://birdhouse-hummingbird.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" - } - ] -} diff --git a/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 5c8e285ba..000000000 --- a/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,35 +0,0 @@ -SERVICES['Jupyter'] = { - 'info': { - 'name': 'Jupyter', - 'synopsis': 'Jupyter notebooks portal.', - 'version': "${JUPYTER_VERSION}", - 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/jupyterhub", "latest"), - 'institution': 'Ouranos', - 'researchSubject': 'Any', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Research', - 'tags': ['Development', 'Research', 'Notebooks'] - }, - 'stats': { - 'method': '.*', - 'route': '/jupyter/.*' - }, - 'redirect': { - 'doc': 'https://jupyter.org/hub', - 'releasenotes': 'https://github.com/Ouranosinc/jupyterhub/tags', # no CHANGES file available - 'support': 'https://github.com/Ouranosinc/jupyterhub/issues', - 'source': 'https://github.com/Ouranosinc/jupyterhub', - 'tryme': 'https://10.0.2.15/jupyter/', - 'licence': 'https://github.com/Ouranosinc/jupyterhub/blob/latest/LICENSE', - 'provenance': '' - }, - "monitoring": { - "Jupyter": { - 'request': { - 'url': 'https://10.0.2.15/jupyter/hub/login' - }, - } - } -} - -CANARIE_STATS_ROUTES.append('jupyter') diff --git a/birdhouse/config/jupyterhub/config/magpie/providers.cfg b/birdhouse/config/jupyterhub/config/magpie/providers.cfg deleted file mode 100644 index f768d7837..000000000 --- a/birdhouse/config/jupyterhub/config/magpie/providers.cfg +++ /dev/null @@ -1,10 +0,0 @@ -providers: - jupyterhub: - # below URL is only used to fill in the required location in Magpie - # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL - url: http://proxy:80 - title: Jupyter - public: true - c4i: false - type: api - sync_type: api diff --git a/birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py b/birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 65ffb99de..000000000 --- a/birdhouse/config/jupyterhub/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,7 +0,0 @@ -SERVICES['node']['monitoring']['Jupyter'] = { - 'request': { - 'url': 'https://10.0.2.15/jupyter/hub/login' - } -} -CANARIE_STATS_ROUTES.append('jupyter') - diff --git a/birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf b/birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf deleted file mode 100644 index 7da931a62..000000000 --- a/birdhouse/config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf +++ /dev/null @@ -1,11 +0,0 @@ - location /jupyter/ { - proxy_pass http://jupyterhub:8000/jupyter/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # websocket headers - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - } diff --git a/birdhouse/config/jupyterhub/custom_templates/login.html b/birdhouse/config/jupyterhub/custom_templates/login.html deleted file mode 100644 index 384107ceb..000000000 --- a/birdhouse/config/jupyterhub/custom_templates/login.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends "templates/login.html" %} {% set announcement_login = ' - - - -

Public demo login: demo

-

- Given this public nature, anyone can tamper with your notebooks so please - export your valuable notebooks elsewhere if you want to - preverve them. This public demo account also have limitted computing - resources. -

-

- Contact helpdesk@example.com for information on how to - get an account and a private workspace or the password of the - public demo account. -

-

- The only writable folder is writable-workspace - (/notebook_dir/writable-workspace in the terminal) and it is persisted - between sessions. -

-

- Please be considerate with the amount of - disk space usage on this Jupyter instance. -

-

- Please shutdown the kernel and close un-used - notebooks to avoid wasting computing resources. -

-

- This Jupyter instance can restart every day. - Long running processes will be killed without notice. -

- - -' %} - -{% set login_term_url = '' %} - -{# Example overriding the actual login terms and condition text #} -{# -{% block login_terms_text %} -Your own text here terms and conditions. -{% endblock login_terms_text %} -#} diff --git a/birdhouse/config/jupyterhub/jupyterhub_config.py b/birdhouse/config/jupyterhub/jupyterhub_config.py deleted file mode 100644 index 575a3b394..000000000 --- a/birdhouse/config/jupyterhub/jupyterhub_config.py +++ /dev/null @@ -1,296 +0,0 @@ -import os -from os.path import join -import logging -import subprocess - -from dockerspawner import DockerSpawner - -c = get_config() # noqa # can be called directy without import because injected by IPython - -c.JupyterHub.bind_url = 'http://:8000/jupyter' - -## Whether to shutdown single-user servers when the Hub shuts down. -c.JupyterHub.cleanup_servers = False - -c.JupyterHub.hub_ip = 'jupyterhub' - -c.JupyterHub.authenticator_class = 'jupyterhub_magpie_authenticator.MagpieAuthenticator' -c.MagpieAuthenticator.magpie_url = "http://magpie:2001" -c.MagpieAuthenticator.public_fqdn = "10.0.2.15" -c.MagpieAuthenticator.authorization_url = "http://twitcher:8000/ows/verify/jupyterhub" - -if os.getenv("JUPYTERHUB_CRYPT_KEY"): - c.MagpieAuthenticator.enable_auth_state = True - c.MagpieAuthenticator.refresh_pre_spawn = True - c.MagpieAuthenticator.auth_refresh_age = int("60") - -c.JupyterHub.cookie_secret_file = '/persist/jupyterhub_cookie_secret' -c.JupyterHub.db_url = '/persist/jupyterhub.sqlite' - -c.JupyterHub.template_paths = ['/custom_templates'] - -class CustomDockerSpawner(DockerSpawner): - @property - def escaped_name(self): - """ - Return the username without escaping. This ensures that mounted directories on the - host machine are discovered properly since we expect the username to match the username - set by Magpie. - """ - return self.user.name - - async def start(self): - if(os.environ['MOUNT_IMAGE_SPECIFIC_NOTEBOOKS'] == 'true'): - host_dir = join(os.environ['JUPYTERHUB_USER_DATA_DIR'], 'tutorial-notebooks-specific-images') - - # Mount a volume with a tutorial-notebook subfolder corresponding to the image name, if it exists - # The names are defined in the JUPYTERHUB_IMAGE_SELECTION_NAMES variable. - image_name = self.user_options.get('image') - if(os.path.isdir(join(host_dir, image_name))): - self.volumes[join(host_dir, image_name)] = { - "bind": '/notebook_dir/tutorial-notebooks', - "mode": "ro" - } - else: - # Try again, removing any colons and any following text. Useful if the image name contains - # the version number, which should not be used in the directory name. - image_name = image_name.split(':')[0] - if(os.path.isdir(join(host_dir, image_name))): - self.volumes[join(host_dir, image_name)] = { - "bind": '/notebook_dir/tutorial-notebooks', - "mode": "ro" - } - else: - # Mount the entire tutorial-notebooks directory - self.volumes[join(os.environ['JUPYTERHUB_USER_DATA_DIR'], "tutorial-notebooks")] = { - "bind": "/notebook_dir/tutorial-notebooks", - "mode": "ro" - } - return await super().start() - -c.JupyterHub.spawner_class = CustomDockerSpawner - -# Selects the first image from the list by default -c.DockerSpawner.image = os.environ['DOCKER_NOTEBOOK_IMAGES'].split()[0] -c.DockerSpawner.use_internal_ip = True -c.DockerSpawner.network_name = os.environ['DOCKER_NETWORK_NAME'] - -notebook_dir = '/notebook_dir' -jupyterhub_data_dir = os.environ['JUPYTERHUB_USER_DATA_DIR'] -container_workspace_dir = join(notebook_dir, "writable-workspace") -container_home_dir = join(container_workspace_dir, ".home") - -c.DockerSpawner.notebook_dir = notebook_dir -c.DockerSpawner.environment = { - "HOME": container_home_dir, - # https://docs.bokeh.org/en/latest/docs/user_guide/jupyter.html#jupyterhub - # Issue https://github.com/bokeh/bokeh/issues/12090 - # Post on Panel forum: - # https://discourse.holoviz.org/t/how-to-customize-the-display-url-from-panel-serve-for-use-behind-jupyterhub-with-jupyter-server-proxy/3571 - # Issue about Panel Preview: https://github.com/holoviz/panel/issues/3440 - "PAVICS_HOST_URL": "https://10.0.2.15", - # https://docs.dask.org/en/stable/configuration.html - # https://jupyterhub-on-hadoop.readthedocs.io/en/latest/dask.html - "DASK_DISTRIBUTED__DASHBOARD__LINK": "https://10.0.2.15{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" -} - -host_user_data_dir = join(os.environ['WORKSPACE_DIR'], "{username}") -c.DockerSpawner.volumes = {host_user_data_dir: container_workspace_dir} - -# Case for the cowbird setup, where the workspace_dir contains a symlink to the jupyterhub dir. -# The jupyterhub dir must also be mounted in this case. -if os.environ['WORKSPACE_DIR'] != jupyterhub_data_dir: - c.DockerSpawner.volumes[join(jupyterhub_data_dir, "{username}")] = { - "bind": join(jupyterhub_data_dir, "{username}"), - "mode": "rw" - } - c.DockerSpawner.volumes[join(os.environ['WORKSPACE_DIR'], os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR'])] = { - "bind": join(notebook_dir, os.environ['PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR']), - "mode": "ro" - } - -container_gdrive_settings_path = join(container_home_dir, ".jupyter/lab/user-settings/@jupyterlab/google-drive/drive.jupyterlab-settings") -host_gdrive_settings_path = os.environ['JUPYTER_GOOGLE_DRIVE_SETTINGS'] - -if len(host_gdrive_settings_path) > 0: - c.DockerSpawner.volumes[host_gdrive_settings_path] = { - "bind": container_gdrive_settings_path, - "mode": "ro" - } - -readme = os.environ.get('JUPYTERHUB_README', default="") -if readme != "": - c.DockerSpawner.volumes[readme] = { - "bind": join(notebook_dir, "README.ipynb"), - "mode": "ro" - } - -def create_dir_hook(spawner): - username = spawner.user.name - jupyterhub_user_dir = join(jupyterhub_data_dir, username) - - if not os.path.exists(jupyterhub_user_dir): - os.mkdir(jupyterhub_user_dir, 0o755) - - subprocess.call(["chown", "-R", f"{os.environ['USER_WORKSPACE_UID']}:{os.environ['USER_WORKSPACE_GID']}", - jupyterhub_user_dir]) - - if os.environ['WORKSPACE_DIR'] != jupyterhub_data_dir: - # Case for cowbird setup. The workspace directory should also have the user's ownership, - # to have working volume mounts with the DockerSpawner. - workspace_user_dir = join(os.environ['WORKSPACE_DIR'], username) - if not os.path.exists(workspace_user_dir): - raise FileNotFoundError(f"The user {username}'s workspace doesn't exist in the workspace directory, " - "but should have been created by Cowbird already.") - subprocess.call(["chown", f"{os.environ['USER_WORKSPACE_UID']}:{os.environ['USER_WORKSPACE_GID']}", - workspace_user_dir]) - - if username == os.environ['JUPYTER_DEMO_USER']: - # Restrict resources for the public demo user - # CPU limit, seems not honored by DockerSpawner - spawner.cpu_limit = float(os.environ['JUPYTER_DEMO_USER_CPU_LIMIT']) - spawner.mem_limit = os.environ['JUPYTER_DEMO_USER_MEM_LIMIT'] - -c.Spawner.pre_spawn_hook = create_dir_hook - -## Disable per-user configuration of single-user servers. -c.Spawner.disable_user_config = True - -c.DockerSpawner.default_url = '/lab' -c.DockerSpawner.remove = True # delete containers when servers are stopped - -c.DockerSpawner.image_whitelist = { - 'jupyter/scipy-notebook': 'jupyter/scipy-notebook', - 'jupyter/r-notebook': 'jupyter/r-notebook', - 'jupyter/tensorflow-notebook': 'jupyter/tensorflow-notebook', - 'jupyter/datascience-notebook': 'jupyter/datascience-notebook', - 'jupyter/pyspark-notebook': 'jupyter/pyspark-notebook', - 'jupyter/all-spark-notebook': 'jupyter/all-spark-notebook', -} - # noqa -c.DockerSpawner.pull_policy = "always" # for images not using pinned version -c.DockerSpawner.debug = True -c.JupyterHub.log_level = logging.DEBUG - -c.Spawner.debug = True - -## Timeout (in seconds) to wait for spawners to initialize -c.JupyterHub.init_spawners_timeout = 20 # default 10 - -## Timeout (in seconds) before giving up on a spawned HTTP server -c.Spawner.http_timeout = 60 # default 30 - -## Timeout (in seconds) before giving up on starting of single-user server. -c.Spawner.start_timeout = 120 # default 60 - -## Extra arguments to be passed to the single-user server. -c.Spawner.args = [ - # Allow non-empty directory deletion which enable recursive dir deletion. - # https://jupyter-server.readthedocs.io/en/latest/other/full-config.html - "--FileContentsManager.always_delete_dir=True", - ] - -c.DockerSpawner.extra_host_config = { - # start init pid 1 process to reap defunct processes - 'init': True, - } - -c.Authenticator.admin_users = {'admin'} # noqa - -## Force refresh of auth prior to spawn. -# Do nothing right now, pending implementation of -# MagpieAuthenticator.refresh_user() (see -# https://github.com/Ouranosinc/jupyterhub/issues/2) -c.Authenticator.refresh_pre_spawn = True - -## Blacklist of usernames that are not allowed to log in. -# https://jupyterhub.readthedocs.io/en/stable/api/auth.html -# -# For security reasons, block user with known hardcoded public password or -# non real Jupyter users. -blocked_users = {'authtest', '${CATALOG_USERNAME}', 'anonymous'} -c.Authenticator.blacklist = blocked_users # v0.9+ -c.Authenticator.blocked_users = blocked_users # v1.2+ - - -# ------------------------------------------------------------------------------ -# Shutdown idle user server based on configured timeouts. -# ------------------------------------------------------------------------------ -# Timeout (in seconds, default: 3 days) to shut down the user server when no kernels or terminals -# are running and there is no activity. If undefined or set to zero, the feature will not be enabled. -jupyter_idle_server_cull_timeout = int("600" or 0) -if jupyter_idle_server_cull_timeout: - c.Spawner.args.append('--NotebookApp.shutdown_no_activity_timeout={}'.format(jupyter_idle_server_cull_timeout)) -# Timeout (in seconds, default: 1 day) after which individual -# user kernels/terminals are considered idle and ready to be culled. -jupyter_idle_kernel_cull_timeout = int("10" or 0) -# Interval (in seconds, default: half of timeout) on which to check for idle kernels exceeding the cull timeout value. -jupyter_idle_kernel_cull_interval = int("" or 0) -if jupyter_idle_kernel_cull_timeout: - if not jupyter_idle_kernel_cull_interval or jupyter_idle_kernel_cull_interval > jupyter_idle_kernel_cull_timeout: - jupyter_idle_kernel_cull_interval = max(1, int(jupyter_idle_kernel_cull_timeout / 2)) - c.Spawner.args.extend([ - '--MappingKernelManager.cull_idle_timeout={}'.format(jupyter_idle_kernel_cull_timeout), - '--MappingKernelManager.cull_interval={}'.format(jupyter_idle_kernel_cull_interval), - '--TerminalManager.cull_inactive_timeout={}'.format(jupyter_idle_kernel_cull_timeout), - '--TerminalManager.cull_interval={}'.format(jupyter_idle_kernel_cull_interval), - ]) -# Culling kernels which have one or more connections for idle but open notebooks and/or terminals. -# Otherwise, browser tabs, notebooks and terminals all have to be closed for culling to work. -if jupyter_idle_server_cull_timeout or jupyter_idle_kernel_cull_timeout: - c.Spawner.args.extend([ - '--MappingKernelManager.cull_connected=True', - '--TerminalManager.cull_connected=True', - ]) - -# ------------------------------------------------------------------------------ -# Configuration overrides -# ------------------------------------------------------------------------------ - - -# do not pull docker iamge updates each time -c.DockerSpawner.pull_policy = "ifnotpresent" - -# allow HTTP requests to /jupyter/hub/api using the following token -# {Authorization: Token } -c.JupyterHub.services = [ - { - "name": "service-admin", - "api_token": "admin-token", - }, -] -c.JupyterHub.load_roles = [ - { - "name": "service-role", - "scopes": [ - # specify the permissions the token should have - "admin:users", - "admin:servers", - "access:servers", - "proxy" - ], - "services": [ - # assign the service the above permissions - "service-admin", - ], - } -] - -# mount additional local notebook locations to imitate the results from auto-deploy script: -# - birdhouse/pavics-jupyter-base/scheduler-jobs/deploy_data_pavics_jupyter.env -# - birdhouse/pavics-jupyter-base/scheduler-jobs/deploy_data_specific_image -class OverrideDockerSpawner(CustomDockerSpawner): - async def start(self): - self.volumes["/home/francis/dev/daccs/pavics-sdi/"] = { - "bind": "/home/francis/dev/daccs/pavics-sdi/", - "mode": "ro", - } - self.volumes["/home/francis/dev/daccs/pavics-jupyter-images/"] = { - "bind": "/home/francis/dev/daccs/pavics-jupyter-images/", - "mode": "ro", - } - return await super().start() - -c.JupyterHub.spawner_class = OverrideDockerSpawner - - # noqa diff --git a/birdhouse/config/jupyterhub/service-config.json b/birdhouse/config/jupyterhub/service-config.json deleted file mode 100644 index 0ac71a4f6..000000000 --- a/birdhouse/config/jupyterhub/service-config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "jupyterhub", - "keywords": [ - "jupyterhub" - ], - "description": "An interactive development environment use to create and share documents that contain live code.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://10.0.2.15/jupyter" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://jupyter.org/hub" - } - ] -} diff --git a/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 56f7cf4a0..000000000 --- a/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,42 +0,0 @@ -SERVICES['Magpie'] = { - 'info': { - 'name': 'Magpie', - 'synopsis': ( - 'Magpie is service for AuthN/AuthZ accessible via a REST API. ' - 'It allows you to manage User/Group/Service/Resource/Permission management ' - 'and integrates with Twitcher.' - ), - 'version': "3.38.0", - 'institution': 'Ouranos', - 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/Magpie", "3.38.0"), - 'researchSubject': 'Security', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Security', - 'tags': ['Security', 'Management', 'Access', 'Policy Decision Point'] - }, - 'stats': { - 'method': '.*', - 'route': "/magpie/.*" - }, - 'redirect': { - 'doc': 'https://pavics-magpie.readthedocs.io/', - 'releasenotes': 'https://github.com/Ouranosinc/Magpie/blob/master/CHANGES.rst', - 'support': 'https://github.com/Ouranosinc/Magpie/issues', - 'source': 'https://github.com/Ouranosinc/Magpie', - 'tryme': 'https://10.0.2.15/magpie/', - 'licence': 'https://github.com/Ouranosinc/Magpie/blob/3.38.0/LICENSE', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - "monitoring": { - "Magpie": { - 'request': { - 'url': 'https://10.0.2.15/magpie/version' - }, - 'response': { - 'text': r'\{.*"code": 200.*"type": "application/json".*\}' - } - } - } -} - -CANARIE_STATS_ROUTES.append('magpie') diff --git a/birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py b/birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 36157bfd3..000000000 --- a/birdhouse/config/magpie/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,9 +0,0 @@ -SERVICES['node']['monitoring'].update({'Magpie': { - 'request': { - 'url': 'https://10.0.2.15/magpie/version' - }, - 'response': { - 'text': '\{.*"code": 200.*"type": "application/json".*\}' - } -}}) -CANARIE_STATS_ROUTES.append('magpie') diff --git a/birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf b/birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf deleted file mode 100644 index cd043e908..000000000 --- a/birdhouse/config/magpie/config/proxy/conf.extra-service.d/magpie.conf +++ /dev/null @@ -1,6 +0,0 @@ - location /magpie/ { - proxy_pass http://magpie:2001/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - include /etc/nginx/conf.d/cors.include; - } diff --git a/birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql b/birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql deleted file mode 100644 index 4d7a6c031..000000000 --- a/birdhouse/config/magpie/docker-entrypoint-initdb.d/create-db.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE magpiedb; diff --git a/birdhouse/config/magpie/magpie.ini b/birdhouse/config/magpie/magpie.ini deleted file mode 100644 index 2aa29a947..000000000 --- a/birdhouse/config/magpie/magpie.ini +++ /dev/null @@ -1,183 +0,0 @@ -### -# app configuration -# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html -### -# From sample -# https://github.com/Ouranosinc/Magpie/blob/master/config/magpie.ini -### - -[composite:main] -use = egg:Paste#urlmap -/ = magpie_app -#/api = api_app -#/magpie/api = api_app - -[app:magpie_app] -use = egg:magpie -filter-with = urlprefix - -pyramid.reload_templates = false -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.default_locale_name = en -pyramid.includes = pyramid_tm ziggurat_foundations.ext.pyramid.sign_in ziggurat_foundations.ext.pyramid.get_user - -# magpie -# -# other overridable variables available in magpie/constants.py -# -magpie.port = 2001 -magpie.url = https://10.0.2.15/magpie -magpie.max_restart = 5 -magpie.push_phoenix = true -# This secret should be the same in Twitcher ! -magpie.secret = magpie -magpie.admin_user = admin -magpie.admin_password = qwertyqwerty! - -# ziggurat -ziggurat_foundations.model_locations.User = magpie.models:User -ziggurat_foundations.sign_in.username_key = user_name -ziggurat_foundations.sign_in.password_key = password -ziggurat_foundations.sign_in.came_from_key = came_from -ziggurat_foundations.sign_in.sign_in_pattern = /signin_internal -ziggurat_foundations.sign_in.sign_out_pattern = /signout -ziggurat_foundations.session_provider_callable = magpie.models:get_session_callable - -# github (login provider) -github.client_id=Iv1.afcc0662862307a4 -github.client_secret=840a968b219f10b8bc3591abd8a2d37ed98a0642 - -# temporary token definition (defaults below if omitted) -# note: token here refers to uuids employed in temporary URL endpoints, not security auth tokens -# Lifetime (in seconds) of the temporary token : -magpie.token_expire = 86400 - -# user registration and approvals - -# below values are the defaults, adjust accordingly for desired behavior -# email template overrides defaults in: magpie/api/templates -magpie.user_registration_enabled = false -magpie.user_registration_submission_email_template = -magpie.user_registration_approval_enabled = false -magpie.user_registration_approval_email_recipient = -magpie.user_registration_approval_email_template = -magpie.user_registration_approved_email_template = -magpie.user_registration_declined_email_template = -magpie.user_registration_notify_enabled = false -magpie.user_registration_notify_email_recipient = -magpie.user_registration_notify_email_template = - -# user assignment to groups with terms & conditions -magpie.group_terms_submission_email_template = -magpie.group_terms_approved_email_template = - -# smtp server configuration -magpie.smtp_user = Magpie -magpie.smtp_from = -magpie.smtp_host = -magpie.smtp_port = -magpie.smtp_ssl = -magpie.smtp_password = - -# See https://pavics-magpie.readthedocs.io/en/latest/configuration.html#envvar-MAGPIE_USER_NAME_EXTRA_REGEX for details -# User names must match the regular expression defined here as well (whether this variable is set or not): -# https://github.com/Ouranosinc/Magpie/blob/595602/magpie/api/management/user/user_utils.py#L66 -#magpie.user_name_extra_regex = - -# Caching settings for specific sections/functions - improves performance response times of recurring requests -# See Twitcher INI configuration for caching that takes effect when resolving access enforcement to actual services. -# Following cache settings apply only during Magpie API requests. -# -# NOTE: Caching will only take effect with Magpie >= 3.7 -# Detail: -# Region 'acl' takes effect whenever computing "effective permissions" of user/group onto a service/resource. -# Because Magpie is employed to manage those permissions and that computing the effective resolution of the -# complete resource hierarchy and full user-group membership inheritance is fairly rare, caching is not specifically -# needed here. It is actually more often then not useful to always refresh the latest states to be sure of which -# permissions are actually applied when the administrator manages them. Therefore, all caches are disabled here, but -# this is not the case on Twitcher side. -cache.regions = acl, service -cache.type = memory -cache.enabled = false -cache.acl.enabled = false -cache.service.enabled = false - -[app:api_app] -use = egg:Paste#static -document_root = %(here)s/ui/swagger - -[filter:urlprefix] -use = egg:PasteDeploy#prefix -# prefix must be adjusted according to 'MAGPIE_URL' / 'magpie.url' -prefix = /magpie - -### -# wsgi server configuration -### - -[server:main] -use = egg:gunicorn#main -host = 0.0.0.0 -port=2001 -timeout=10 -workers=3 -threads=4 - -# used by magpie/alembic with symlink, %(here)s corresponds to this directory -[alembic] -script_location = %(here)s/../magpie/alembic -#sqlalchemy.url = postgresql://postgres:postgres@localhost/magpie - -### -# logging configuration -# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html -### - -[loggers] -keys = root, magpie, sqlalchemy, alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = INFO -handlers = console -formatter = generic - -[logger_magpie] -# "level = DEBUG" logs detailed information about operations/settings (not for production, will leak sensitive data) -# "level = INFO" reports useful information, not leaking details about settings -# "level = WARN" only potential problems are reported -level = DEBUG -handlers = -qualname = magpie -formatter = generic - -[logger_sqlalchemy] -# "level = DEBUG" logs SQL queries, transactions and results -# "level = INFO" logs SQL queries (data can be identified from query field values) -# "level = WARN" logs neither (recommended for production systems, avoid anything below unless for dev/debug system) -level = WARN -handlers = -qualname = sqlalchemy.engine -formatter = generic - -[logger_alembic] -level = INFO -handlers = -qualname = alembic -formatter = generic - -[handler_console] -class = StreamHandler -args = (sys.stdout,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = [%(asctime)s] %(levelname)-10.10s [%(threadName)s][%(name)s] %(message)s diff --git a/birdhouse/config/magpie/permissions.cfg b/birdhouse/config/magpie/permissions.cfg deleted file mode 100644 index dc5155ffc..000000000 --- a/birdhouse/config/magpie/permissions.cfg +++ /dev/null @@ -1,22 +0,0 @@ -# -# Parameters: -# service: service name to receive the permission (directly on it if no 'resource' mentioned, must exist) -# resource (optional): tree path of the service's resource (ex: /res1/sub-res2/sub-sub-res3) -# user and/or group: user/group to apply the permission on (user/group must exist) -# permission: name of the permission to be applied (see 'magpie/permissions.py' for supported values) -# action: one of [create, remove] (default: create) -# -# Default behaviour: -# - create missing resources if supported by the service (and tree automatically resolvable), then apply permissions. -# - create missing user/group if required (default user created: (group: anonymous, password: 12345). -# - applicable service, user or group is missing, corresponding permissions are ignored and not updated. -# - unknown actions are ignored and corresponding permission are not updated, unspecified action resolves to 'create'. -# - already satisfied permission configurations are left as is. -# - -permissions: - - service: api - resource: /api - permission: read - group: anonymous - action: create diff --git a/birdhouse/config/magpie/postgres-credentials.env b/birdhouse/config/magpie/postgres-credentials.env deleted file mode 100644 index 4f1914a5f..000000000 --- a/birdhouse/config/magpie/postgres-credentials.env +++ /dev/null @@ -1,7 +0,0 @@ -POSTGRES_USER=postgres-magpie -POSTGRES_PASSWORD=postgres-qwerty -POSTGRES_DB=magpiedb -MAGPIE_POSTGRES_USER=postgres-magpie -MAGPIE_POSTGRES_PASSWORD=postgres-qwerty -MAGPIE_POSTGRES_DB=magpiedb -MAGPIE_POSTGRES_HOST=postgres-magpie diff --git a/birdhouse/config/magpie/providers.cfg b/birdhouse/config/magpie/providers.cfg deleted file mode 100644 index 768fdd8b1..000000000 --- a/birdhouse/config/magpie/providers.cfg +++ /dev/null @@ -1 +0,0 @@ -providers: {} diff --git a/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index dfeb38790..000000000 --- a/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,34 +0,0 @@ -SERVICES['Malleefowl'] = { - 'info': { - 'name': 'Malleefowl', - 'synopsis': 'A Web Processing Service for Climate Data Access and Workflows.', - 'version': "pavics-0.3.5", - 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/malleefowl", "pavics-0.3.5"), - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Resource/Cloud Management', - 'tags': ['Climatology'] - }, - 'stats': { - 'method': '.*', - 'route': '/malleefowl/.*' - }, - 'redirect': { - 'doc': 'https://malleefowl.readthedocs.io/en/latest/', - 'releasenotes': 'https://github.com/Ouranosinc/malleefowl/blob/master/CHANGES.rst', - 'support': 'https://github.com/Ouranosinc/malleefowl/issues', - 'source': 'https://github.com/Ouranosinc/malleefowl', - 'tryme': 'https://10.0.2.15/malleefowl/', - 'licence': '', - 'provenance': '' - }, - 'monitoring': { - "Malleefowl": { - 'request': { - # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) - 'url': 'http://10.0.2.15:8091/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } - } - } -} \ No newline at end of file diff --git a/birdhouse/config/malleefowl/config/magpie/providers.cfg b/birdhouse/config/malleefowl/config/magpie/providers.cfg deleted file mode 100644 index e4214fbb4..000000000 --- a/birdhouse/config/malleefowl/config/magpie/providers.cfg +++ /dev/null @@ -1,8 +0,0 @@ -providers: - malleefowl: - url: http://10.0.2.15:8091/wps - title: Malleefowl - public: true - c4i: false - type: wps - sync_type: wps diff --git a/birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py b/birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 2d12e767f..000000000 --- a/birdhouse/config/malleefowl/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,5 +0,0 @@ -SERVICES['node']['monitoring']['Malleefowl'] = { - 'request': { - 'url': 'http://10.0.2.15:8091/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } -} diff --git a/birdhouse/config/malleefowl/custom.cfg b/birdhouse/config/malleefowl/custom.cfg deleted file mode 100644 index e8f1a1b38..000000000 --- a/birdhouse/config/malleefowl/custom.cfg +++ /dev/null @@ -1,30 +0,0 @@ -[buildout] -extends=profiles/docker.cfg - -[settings] -persist-path = /pavics-data -archive-root = / -thredds_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/fileServer/birdhouse -wps_url = https://${environment:PAVICS_FQDN_PUBLIC}/wpsoutputs -wps_url2 = https://${environment:PAVICS_FQDN_PUBLIC}:443/wpsoutputs -ncwms_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs -opendap_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/dodsC/birdhouse -magpie_hostname = https://${environment:HOSTNAME}/magpie - -[pywps] -enable-https = true -https-port = 443 -https-output-port = 443 -workers = 10 -parallelprocesses = 10 -maxprocesses = 100 -database=postgresql://postgres-pavics:postgres-qwerty@postgres/malleefowl -extra-options = - archive_root=${settings:archive-root} - persist_path=${settings:persist-path} - thredds_url=${settings:thredds_url} - authz_url=${settings:magpie_hostname} - authz_thredds_service_name=thredds - known_extensions=nc,nc4 - wms_mapping=${settings:wps_url},${settings:ncwms_url}/wps_outputs,${settings:wps_url2},${settings:ncwms_url}/wps_outputs,${settings:thredds_url},${settings:ncwms_url} - opendap_mapping=${settings:wps_url},${settings:opendap_url}/wps_outputs,${settings:wps_url2},${settings:opendap_url}/wps_outputs,${settings:thredds_url},${settings:opendap_url} diff --git a/birdhouse/config/ncops/ncops.cfg b/birdhouse/config/ncops/ncops.cfg deleted file mode 100644 index 6dedf3e03..000000000 --- a/birdhouse/config/ncops/ncops.cfg +++ /dev/null @@ -1,2 +0,0 @@ -GEOSERVER_HOST=10.0.2.15:8087 -WPS_HOST=10.0.2.15:8079 diff --git a/birdhouse/config/ncwms2/config/magpie/providers.cfg b/birdhouse/config/ncwms2/config/magpie/providers.cfg deleted file mode 100644 index f78c0d350..000000000 --- a/birdhouse/config/ncwms2/config/magpie/providers.cfg +++ /dev/null @@ -1,8 +0,0 @@ -providers: - ncWMS2: - url: http://10.0.2.15:8080/ncWMS2 - title: ncWMS2 - public: true - c4i: false - type: ncwms - sync_type: ncwms diff --git a/birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf b/birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf deleted file mode 100644 index 6281babd1..000000000 --- a/birdhouse/config/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf +++ /dev/null @@ -1,7 +0,0 @@ - location /ncWMS2/ { - # proxy_pass http://10.0.2.15:8080; - # proxy_set_header Host $host; - # proxy_set_header X-Forwarded-Proto $real_scheme; - # include /etc/nginx/conf.d/cors.include; - return 302 /twitcher/ows/proxy$request_uri; - } diff --git a/birdhouse/config/ncwms2/custom.cfg b/birdhouse/config/ncwms2/custom.cfg deleted file mode 100644 index 8e6ce602e..000000000 --- a/birdhouse/config/ncwms2/custom.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[buildout] -extends = buildout.cfg - -supervisor-host = * -supervisor-port = 9001 - -[settings] -tomcat-ncwms-password = ncwmspass - -[ncwms] -data_dir = /pavics-data diff --git a/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index ad08683e9..000000000 --- a/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,34 +0,0 @@ -SERVICES['Phoenix'] = { - 'info': { - 'name': 'Phoenix', - 'synopsis': 'Legacy authentication. See Magpie/Twitcher instead.', - 'version': "pavics-0.2.3", - 'institution': 'Ouranos', - 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/pyramid-phoenix", "pavics-0.2.3"), - 'researchSubject': 'Authentication', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Authentication', - 'tags': ['Authentication', 'Legacy'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/geoserver/web/.*' # FIXME: original value doesn't make sense - }, - 'redirect': { - 'doc': 'http://pyramid-phoenix.readthedocs.io/en/latest/index.html', - 'releasenotes': 'https://github.com/ouranosinc/pyramid-phoenix/CHANGES.rst', - 'support': 'https://github.com/ouranosinc/pyramid-phoenix/issues', - 'source': 'https://github.com/ouranosinc/pyramid-phoenix', - 'tryme': 'https://10.0.2.15:8443/', - 'licence': 'https://github.com/ouranosinc/pyramid-phoenix/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - "monitoring": { - "Phoenix": { - 'request': { - # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) - 'url': 'https://10.0.2.15:8443/' - } - } - } -} \ No newline at end of file diff --git a/birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py b/birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 508d9175b..000000000 --- a/birdhouse/config/phoenix/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,5 +0,0 @@ -SERVICES['node']['monitoring']['Phoenix'] = { - 'request': { - 'url': 'https://10.0.2.15:8443/' - } -} diff --git a/birdhouse/config/phoenix/custom.cfg b/birdhouse/config/phoenix/custom.cfg deleted file mode 100644 index 7d91ddd76..000000000 --- a/birdhouse/config/phoenix/custom.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[buildout] -extends=profiles/docker.cfg -parts += pytest - -[settings] -esgf-search-url = http://136.172.30.96/esg-search -phoenix-password = sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 -phoenix-solr = true -phoenix-wms = true -phoenix-require-csrf = false diff --git a/birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf b/birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf deleted file mode 100644 index d0fa2db86..000000000 --- a/birdhouse/config/portainer/config/proxy/conf.extra-service.d/portainer.conf +++ /dev/null @@ -1,5 +0,0 @@ - location /portainer/ { - proxy_pass http://portainer:9000/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - } diff --git a/birdhouse/config/postgres/credentials.env b/birdhouse/config/postgres/credentials.env deleted file mode 100644 index 672bc4ae0..000000000 --- a/birdhouse/config/postgres/credentials.env +++ /dev/null @@ -1,3 +0,0 @@ -POSTGRES_USER=postgres-pavics -POSTGRES_PASSWORD=postgres-qwerty -POSTGRES_DB=pavics diff --git a/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index f4fab156a..000000000 --- a/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,6 +0,0 @@ -PLATFORMS['server']['monitoring']['Project'] = { - 'request': { - 'url': 'https://10.0.2.15/project-api/explorer/' - } -} -PLATFORMS['server']['stats']['route'] = '/project-api/.*' diff --git a/birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf b/birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf deleted file mode 100644 index e35658447..000000000 --- a/birdhouse/config/project-api/config/proxy/conf.extra-service.d/project-api.conf +++ /dev/null @@ -1,6 +0,0 @@ - - location /project-api/ { - proxy_pass http://10.0.2.15:3005/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - } diff --git a/birdhouse/config/proxy/conf.d/all-services.include b/birdhouse/config/proxy/conf.d/all-services.include deleted file mode 100644 index e6cff734d..000000000 --- a/birdhouse/config/proxy/conf.d/all-services.include +++ /dev/null @@ -1,28 +0,0 @@ - location / { - return 302 https://10.0.2.15/stac/; - } - - location /components { - default_type application/json; - root /static; - try_files /components.json =404; - } - - location /services { - default_type application/json; - root /static; - try_files /services.json =404; - } - - location /version { - default_type application/json; - root /static; - try_files /version.json =404; - } - - location /doc { - return 302 https://www.example.com/; - } - - # for other extra components to extend Nginx - include /etc/nginx/conf.extra-service.d/*/*.conf; diff --git a/birdhouse/config/proxy/conf.d/frontend.conf b/birdhouse/config/proxy/conf.d/frontend.conf deleted file mode 100644 index 0d18f014b..000000000 --- a/birdhouse/config/proxy/conf.d/frontend.conf +++ /dev/null @@ -1,48 +0,0 @@ -# top-level http config for websocket headers -# If Upgrade is defined, Connection = upgrade -# If Upgrade is empty, Connection = close -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -# Sets a $real_scheme variable whose value is the scheme passed by the load -# balancer in X-Forwarded-Proto (if any), defaulting to $scheme. -# Similar to how the HttpRealIp module treats X-Forwarded-For. -map $http_x_forwarded_proto $real_scheme { - default $http_x_forwarded_proto; - '' $scheme; -} - -server { - listen 80; - server_name localhost; - - include /etc/nginx/conf.d/all-services.include; - - # redirect server error pages to the static page /50x.html - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} - -server { - listen 443; - server_name localhost; - proxy_buffering off; - - resolver 127.0.0.11; - - ssl on; - ssl_certificate /etc/nginx/cert.pem; - ssl_certificate_key /etc/nginx/cert.pem; - - include /etc/nginx/conf.d/all-services.include; - - # redirect server error pages to the static page /50x.html - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/birdhouse/config/proxy/nginx.conf b/birdhouse/config/proxy/nginx.conf deleted file mode 100644 index aad412e7f..000000000 --- a/birdhouse/config/proxy/nginx.conf +++ /dev/null @@ -1,41 +0,0 @@ - -user root; -worker_processes 4; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_iso8601] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access_file.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - client_max_body_size 1000m; - client_body_timeout 600s; - - # timeout for reading a response from the proxied server - proxy_read_timeout 240s; # default 60s - - include /etc/nginx/conf.d/*.conf; - - # for other extra components to extend Nginx - include /etc/nginx/conf.extra.d/*/*.conf; - -} diff --git a/birdhouse/config/proxy/static/components.json b/birdhouse/config/proxy/static/components.json deleted file mode 100644 index e370c72d0..000000000 --- a/birdhouse/config/proxy/static/components.json +++ /dev/null @@ -1,26 +0,0 @@ -{"components": ["bird-house/birdhouse-deploy:config/proxy", -"bird-house/birdhouse-deploy:config/canarie-api", -"bird-house/birdhouse-deploy:config/geoserver", -"bird-house/birdhouse-deploy:config/wps_outputs-volume", -"bird-house/birdhouse-deploy:config/postgres", -"bird-house/birdhouse-deploy:config/finch", -"bird-house/birdhouse-deploy:config/raven", -"bird-house/birdhouse-deploy:config/data-volume", -"bird-house/birdhouse-deploy:config/hummingbird", -"bird-house/birdhouse-deploy:config/thredds", -"bird-house/birdhouse-deploy:config/portainer", -"bird-house/birdhouse-deploy:config/magpie", -"bird-house/birdhouse-deploy:config/twitcher", -"bird-house/birdhouse-deploy:config/jupyterhub", -"bird-house/birdhouse-deploy:optional-components/canarie-api-full-monitoring", -"bird-house/birdhouse-deploy:optional-components/wps-healthchecks", -"bird-house/birdhouse-deploy:optional-components/secure-thredds", -"bird-house/birdhouse-deploy:optional-components/testthredds", -"bird-house/birdhouse-deploy:components/weaver", -"bird-house/birdhouse-deploy:optional-components/test-weaver", -"bird-house/birdhouse-deploy:optional-components/secure-data-proxy", -"bird-house/birdhouse-deploy:optional-components/x-robots-tag-header", -"bird-house/birdhouse-deploy:components/stac", -"bird-house/birdhouse-deploy:optional-components/stac-public-access", -"bird-house/birdhouse-deploy:components/cowbird", -"bird-house/birdhouse-deploy:optional-components/stac-data-proxy"]} diff --git a/birdhouse/config/proxy/static/services.json b/birdhouse/config/proxy/static/services.json deleted file mode 100644 index 5cc667101..000000000 --- a/birdhouse/config/proxy/static/services.json +++ /dev/null @@ -1,221 +0,0 @@ -{"services": [ { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "geoserver", - "keywords": [ - "data", - "service-wms", - "service-wfs", - "service-wps" - ], - "description": "GeoServer is a server that allows users to view and edit geospatial data.", - "links": [ - { - "rel": "service", - "type": "application/json", - "href": "https://10.0.2.15/geoserver/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://docs.geoserver.org/" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "finch", - "keywords": [ - "service-wps" - ], - "description": "A Web Processing Service for Climate Indicators.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://finch.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/finch?service=WPS&request=GetCapabilities" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "raven", - "keywords": [ - "service-wps" - ], - "description": "A WPS providing hydrological modeling and analysis services.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://pavics-raven.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "hummingbird", - "keywords": [ - "service-wps" - ], - "description": "A WPS for metadata compliance checks used in the climate science community.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://birdhouse-hummingbird.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/hummingbird?service=WPS&request=GetCapabilities" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "thredds", - "keywords": [ - "data", - "catalog" - ], - "description": "A server that provides catalog, metadata, and data access services for scientific data.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://10.0.2.15/thredds/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://www.unidata.ucar.edu/software/tds/" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/thredds/catalog.xml" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "jupyterhub", - "keywords": [ - "jupyterhub" - ], - "description": "An interactive development environment use to create and share documents that contain live code.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://10.0.2.15/jupyter" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://jupyter.org/hub" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "weaver", - "keywords": [ - "service-ogcapi_processes" - ], - "description": "An OGC-API flavored Execution Management Service", - "links": [ - { - "rel": "service", - "type": "application/json", - "href": "https://10.0.2.15/weaver/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://pavics-weaver.readthedocs.io/" - }, - { - "rel": "service-desc", - "type": "application/json", - "href": "https://10.0.2.15/weaver/" - }, - { - "rel": "conformance", - "type": "application/json", - "href": "https://10.0.2.15/weaver/conformance/" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "stac", - "keywords": [ - "catalog" - ], - "description": "Searchable spatiotemporal metadata describing climate and Earth observation datasets.", - "links": [ - { - "rel": "service", - "type": "application/json", - "href": "https://10.0.2.15/stac/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://github.com/radiantearth/stac-api-spec/tree/v1.0.0" - }, - { - "rel": "service-desc", - "type": "application/yaml", - "href": "https://raw.githubusercontent.com/radiantearth/stac-api-spec/main/core/openapi.yaml" - }, - { - "rel": "alternate", - "type": "text/html", - "href": "https://10.0.2.15/stac-browser/" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "stac-browser", - "keywords": [ - "catalog" - ], - "description": "A web interface to browse the STAC catalogs.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://10.0.2.15/stac-browser/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://github.com/radiantearth/stac-browser" - }, - { - "rel": "alternate", - "type": "application/json", - "href": "https://10.0.2.15/stac/" - } - ] -}]} diff --git a/birdhouse/config/proxy/static/version.json b/birdhouse/config/proxy/static/version.json deleted file mode 100644 index f870a933b..000000000 --- a/birdhouse/config/proxy/static/version.json +++ /dev/null @@ -1 +0,0 @@ -{"version": "1.40.0", "release_time": "2023-11-30T18:27:41Z"} diff --git a/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 50aae7063..000000000 --- a/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,39 +0,0 @@ -RAVEN_VERSION = "0.18.1" -RAVEN_RELEASE = get_release_time_from_repo_tag("github", "Ouranosinc/raven", RAVEN_VERSION) - -SERVICES['raven'] = { - 'info': { - 'name': 'Hydrological modeling and analysis', - 'synopsis': ( - 'A suite of WPS processes to calibrate and run hydrological models, ' - 'including geographical information retrieval and processing as well as time series analysis.' - ), - 'version': RAVEN_VERSION, - 'institution': 'Ouranos', - 'releaseTime': RAVEN_RELEASE, - 'researchSubject': 'Hydrology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Hydrology'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/raven.*' - }, - 'redirect': { - 'doc': 'https://pavics-raven.readthedocs.io/en/latest/', - 'releasenotes': 'https://github.com/Ouranosinc/raven/releases', - 'support': 'https://github.com/Ouranosinc/raven/issues', - 'source': 'https://github.com/Ouranosinc/raven', - 'tryme': 'https://pavics-raven.readthedocs.io/en/latest/notebooks/index.html', - 'licence': 'https://github.com/Ouranosinc/raven/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Raven': { - 'request': { - 'url': 'http://raven:9099/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} diff --git a/birdhouse/config/raven/config/magpie/providers.cfg b/birdhouse/config/raven/config/magpie/providers.cfg deleted file mode 100644 index f72469649..000000000 --- a/birdhouse/config/raven/config/magpie/providers.cfg +++ /dev/null @@ -1,9 +0,0 @@ -providers: - raven: - url: http://raven:9099/wps - title: Raven - public: true - c4i: false - type: wps - sync_type: wps - diff --git a/birdhouse/config/raven/config/proxy/canarie_api_monitoring.py b/birdhouse/config/raven/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 76c0775c8..000000000 --- a/birdhouse/config/raven/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,39 +0,0 @@ -SERVICES['node']['monitoring']['Raven'] = { - 'request': { - 'url': 'http://10.0.2.15:8096/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } -} -SERVICES['raven'] = { - 'info': { - 'name': 'Hydrological modeling and analysis', - 'synopsis': ('A suite of WPS processes to calibrate and run hydrological models, including geographical' - 'information retrieval and processing as well as time series analysis.'), - 'version': '0.11.1', - 'institution': 'Ouranos', - 'releaseTime': '2021-02-01T00:00:00Z', - 'researchSubject': 'Hydrology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Hydrology'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/raven.*' - }, - 'redirect': { - 'doc': 'https://pavics-raven.readthedocs.io/en/latest/', - 'releasenotes': 'https://github.com/Ouranosinc/raven/releases', - 'support': 'https://github.com/Ouranosinc/raven/issues', - 'source': 'https://github.com/Ouranosinc/raven', - 'tryme': 'https://pavics-raven.readthedocs.io/en/latest/notebooks/index.html', - 'licence': 'https://github.com/Ouranosinc/raven/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'Raven': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&version=1.0.0&request=GetCapabilities' - } - }, - } -} diff --git a/birdhouse/config/raven/service-config.json b/birdhouse/config/raven/service-config.json deleted file mode 100644 index 43c3f775c..000000000 --- a/birdhouse/config/raven/service-config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "raven", - "keywords": [ - "service-wps" - ], - "description": "A WPS providing hydrological modeling and analysis services.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://pavics-raven.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/twitcher/ows/proxy/raven?service=WPS&request=GetCapabilities" - } - ] -} diff --git a/birdhouse/config/raven/wps.cfg b/birdhouse/config/raven/wps.cfg deleted file mode 100644 index ef7382950..000000000 --- a/birdhouse/config/raven/wps.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[server] -outputurl = https://10.0.2.15/wpsoutputs/raven -outputpath = /data/wpsoutputs/raven - -# default 3mb, fix "Broken pipe" between the proxy and the wps service -maxrequestsize = 100mb - -# default 2, too low for a production server with capable CPUs -parallelprocesses = 10 - -[logging] -level = INFO -database=postgresql://postgres-pavics:postgres-qwerty@postgres/raven - - diff --git a/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 6371110f5..000000000 --- a/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,39 +0,0 @@ -SERVICES['Solr'] = { - 'info': { - 'name': 'Solr', - 'synopsis': ( - 'SOLR is a search platform part of the Apache Lucene project. ' - 'It is used in this project for its faceted search capability. ' - 'Search queries are relayed from the UI or WPS processes to the SOLR database, ' - 'which returns a json file with the links to matching files.' - ), - 'version': "5.2.1", - 'institution': 'Ouranos', - 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/solr", "5.2.1"), - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Indexation', 'Search'] - }, - 'stats': { - 'method': '.*', - 'route': '/solr/.*' - }, - 'redirect': { - 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/backend.html#indexation', - 'releasenotes': 'https://github.com/Ouranosinc/PAVICS/tags', - 'support': 'https://github.com/Ouranosinc/PAVICS/issues', - 'source': 'https://github.com/Ouranosinc/PAVICS/tree/master/birdhouse/docker/solr', - 'tryme': 'http://10.0.2.15:8983/solr/', - 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/arch/backend.html#indexation' - }, - "monitoring": { - 'Solr': { - 'request': { - # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) - 'url': 'http://10.0.2.15:8983/solr/birdhouse/select' - } - }, - } -} diff --git a/birdhouse/config/solr/config/proxy/canarie_api_monitoring.py b/birdhouse/config/solr/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 8f61b06f5..000000000 --- a/birdhouse/config/solr/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,8 +0,0 @@ -SERVICES['node']['monitoring']['Solr'] = { - 'request': { - 'url': 'http://10.0.2.15:8983/solr/birdhouse/select?q=CMIP5&fq=model:MPI-ESM-MR&fq=experiment:rcp45&fq=variable:tasmax&fq=institute:MPI-M&fq=frequency:mon&wt=json' - }, - 'response': { - 'text': '.*catalog_url\":\".+/testdata/flyingpigeon/cmip5.*/catalog.xml.*' - } -} diff --git a/birdhouse/config/thredds/catalog.xml b/birdhouse/config/thredds/catalog.xml deleted file mode 100644 index aba2d4fe8..000000000 --- a/birdhouse/config/thredds/catalog.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - all - - - - - - - - - - - - - - - - - all - - - - - - - - - - - - - - - - diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index dcc606354..000000000 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,72 +0,0 @@ -SERVICES['renderer'] = { - 'info': { - 'name': 'High-resolution spatial gridded data renderer', - 'synopsis': ( - 'This service renders gridded data on the server and sends images to the client for display within ' - 'mapping applications using Open Geospatial Consortium (OGC) Web Mappping Service (WMS) standard.' - ), - 'version': '4.6.15', - 'institution': 'Unidata', - 'releaseTime': '2020-06-16T00:00:00Z', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology'] - }, - 'stats': { - 'method': '.*', - 'route': '/thredds/.*' - }, - 'redirect': { - 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/frontend.html#gridded-data-rendering', - 'releasenotes': 'https://github.com/Unidata/tds/releases', - 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', - 'source': 'https://github.com/Unidata/tds', - 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/rendering.html', - 'licence': 'https://github.com/Unidata/tds/blob/master/LICENSE', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'ncWMS': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' - } - }, - } -} - -SERVICES['Thredds'] = { - 'info': { - 'name': 'Thredds', - 'synopsis': 'Climate Data Catalog and Format Renderers', - 'version': "4.6.18-unidata-2022-01", - 'releaseTime': get_release_time_from_repo_tag("docker", "pavics/thredds-docker", "4.6.18-unidata-2022-01"), - 'institution': 'Ouranos', - 'researchSubject': 'Catalog', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Catalog', - 'tags': ['Catalog', 'Climate Data'] - }, - 'stats': { - 'method': '.*', - 'route': '/twitcher/ows/proxy/thredds/.*' - }, - 'redirect': { - 'doc': 'https://www.unidata.ucar.edu/software/tds/', - 'releasenotes': 'https://docs.unidata.ucar.edu/tds/current/userguide/upgrade.html', - 'support': 'https://www.unidata.ucar.edu/software/tds/#help', - 'source': 'https://github.com/Unidata/tds', - 'tryme': 'https://10.0.2.15/twitcher/ows/proxy/thredds/', - 'licence': 'https://github.com/Unidata/tds/blob/main/LICENSE', - 'provenance': 'https://downloads.unidata.ucar.edu/tds/' - }, - "monitoring": { - "Thredds": { - 'request': { - 'url': 'http://thredds:8080//twitcher/ows/proxy/thredds/catalog.html' - } - } - } -} - -CANARIE_STATS_ROUTES.append('thredds') diff --git a/birdhouse/config/thredds/config/magpie/providers.cfg b/birdhouse/config/thredds/config/magpie/providers.cfg deleted file mode 100644 index 3ca5f9d84..000000000 --- a/birdhouse/config/thredds/config/magpie/providers.cfg +++ /dev/null @@ -1,35 +0,0 @@ -providers: - thredds: - url: http://thredds:8080/twitcher/ows/proxy/thredds - title: Thredds - public: true - c4i: false - type: thredds - sync_type: thredds - # below is a custom config to indicate how magpie should convert thredds path elements into resources/permissions - # see: https://pavics-magpie.readthedocs.io/en/latest/services.html#servicethredds - configuration: - skip_prefix: "thredds" # prefix to ignore, below prefixes will be matched against whatever comes after in path - file_patterns: - # note: make sure to employ quotes and double escapes to avoid parsing YAML error - - ".+\\.ncml" # match longest extension first to avoid tuncating it by match of sorter '.nc' - - ".+\\.nc" - metadata_type: - prefixes: - - null # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed - - "\\w+\\.gif" # threddsIcon, folder icon, etc. - - "\\w+\\.ico" # favicon - - "\\w+\\.txt" # licence - - "\\w+\\.css" # tds.css - - "catalog\\.\\w+" # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE` - - catalog - - ncml - - uddc - - iso - data_type: - prefixes: - - fileServer - - dodsC - - wcs - - wms - - ncss diff --git a/birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py b/birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 34d4ed188..000000000 --- a/birdhouse/config/thredds/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,41 +0,0 @@ -SERVICES['node']['monitoring'].update({'Thredds': { - 'request': { - 'url': 'http://10.0.2.15:8083/twitcher/ows/proxy/thredds/catalog.html' - } - }}) - -SERVICES['renderer'] = { - 'info': { - 'name': 'High-resolution spatial gridded data renderer', - 'synopsis': 'This service renders gridded data on the server and sends images to the client for display within mapping applications using Open Geospatial Consortium (OGC) Web Mappping Service (WMS) standard.', - 'version': '4.6.15', - 'institution': 'Unidata', - 'releaseTime': '2020-06-16T00:00:00Z', - 'researchSubject': 'Climatology', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Data Manipulation', - 'tags': ['Climatology'] - }, - 'stats': { - 'method': '.*', - 'route': '/thredds/.*' - }, - 'redirect': { - 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/frontend.html#gridded-data-rendering', - 'releasenotes': 'https://github.com/Unidata/tds/releases', - 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', - 'source': 'https://github.com/Unidata/tds', - 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/rendering.html', - 'licence': 'https://github.com/Unidata/tds/blob/master/LICENSE', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - 'monitoring': { - 'ncWMS': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' - } - }, - } -} - -CANARIE_STATS_ROUTES.append('thredds') diff --git a/birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf b/birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf deleted file mode 100644 index 1e5cb1911..000000000 --- a/birdhouse/config/thredds/config/proxy/conf.extra-service.d/thredds.conf +++ /dev/null @@ -1,10 +0,0 @@ - - location /thredds/ { - #return 302 /twitcher/ows/proxy$request_uri; - proxy_pass https://10.0.2.15/twitcher/ows/proxy/thredds/; - # direct hit Thredds, bypassing twitcher, for debugging only - # proxy_pass http://thredds:8080/twitcher/ows/proxy/thredds/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } diff --git a/birdhouse/config/thredds/service-config.json b/birdhouse/config/thredds/service-config.json deleted file mode 100644 index 5f30820fc..000000000 --- a/birdhouse/config/thredds/service-config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "thredds", - "keywords": [ - "data", - "catalog" - ], - "description": "A server that provides catalog, metadata, and data access services for scientific data.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://10.0.2.15/thredds/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://www.unidata.ucar.edu/software/tds/" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://10.0.2.15/thredds/catalog.xml" - } - ] -} diff --git a/birdhouse/config/thredds/threddsConfig.xml b/birdhouse/config/thredds/threddsConfig.xml deleted file mode 100644 index 11f037b37..000000000 --- a/birdhouse/config/thredds/threddsConfig.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - - Birdhouse Thredds Data Server - /twitcher/ows/proxy/thredds/threddsIcon.gif - TDS - - Scientific Climate Data - meteorology, atmosphere, climate, ocean, earth science, birdhouse - - - Support - Birdhouse - - - - - - Birdhouse - - - Birdhouse - - - - - true - - - - - true - false - true - 5000 - 5000 - - - - - true - false - 15 min - 30 min - - - - - true - true - true - - - - - true - 1 hour - 10 Gb - - - - - true - - - diff --git a/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index fe0835e19..000000000 --- a/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1,56 +0,0 @@ - -def get_twitcher_magpie_adapter_version() -> str: - dockerfile_url = "https://raw.githubusercontent.com/Ouranosinc/Magpie/3.38.0/Dockerfile.adapter" - resp = SESSION.get(dockerfile_url) - if resp.status_code != 200: - raise ValueError( - f"HTTP status: {resp.status_code} from [{dockerfile_url}] with detail:\n{resp.text!s}" - ) - tag_line = [line for line in resp.text.splitlines() if line.startswith("FROM birdhouse/twitcher")] - if not tag_line: - raise ValueError(f"Could not locate Twitcher version from [{dockerfile_url}]") - tag_version = tag_line[0].split(":v", 1)[-1].strip() - if not tag_version: - raise ValueError(f"Could not locate Twitcher version from [{dockerfile_url}]") - return tag_version - - -TWITCHER_VERSION = get_twitcher_magpie_adapter_version() -TWITCHER_RELEASE = get_release_time_from_repo_tag("github", "bird-house/twitcher", TWITCHER_VERSION) -SERVICES['Twitcher'] = { - 'info': { - 'name': 'Twitcher', - 'synopsis': ( - 'Twitcher is a security proxy for OWS services like Web Processing Services (WPS). ' - 'The proxy service uses OAuth2 access tokens to protect the OWS service access using ' - 'Magpie permissions.' - ), - 'version': TWITCHER_VERSION, - 'institution': 'Ouranos', - 'releaseTime': TWITCHER_RELEASE, - 'researchSubject': 'Security', - 'supportEmail': 'helpdesk@example.com', - 'category': 'Security', - 'tags': ['Security', 'Proxy', 'Access', 'Policy Enforcement Point'] - }, - 'stats': { - 'method': '.*', - 'route': "/twitcher/(?!ows).*" - }, - 'redirect': { - 'doc': 'https://twitcher.readthedocs.io/', - 'releasenotes': 'https://github.com/bird-house/twitcher/blob/master/CHANGES.rst', - 'support': 'https://github.com/bird-house/twitcher/issues', - 'source': 'https://github.com/bird-house/twitcher', - 'tryme': 'https://10.0.2.15/twitcher/', - 'licence': 'https://github.com/bird-house/twitcher/blob/master/LICENSE.txt', - 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' - }, - "monitoring": { - "Twitcher": { - 'request': { - 'url': 'https://10.0.2.15/twitcher/' - } - } - } -} diff --git a/birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py b/birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index bca45380d..000000000 --- a/birdhouse/config/twitcher/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1,9 +0,0 @@ -# We can only monitor twitcher if there is an endpoint that it is protecting that we can try to access -# If there is at least one other service that provides a route protected by twitcher, monitor that route; -# otherwise do nothing. -if 'flyingpigeon': - SERVICES['node']['monitoring'].update({'Twitcher': { - 'request': { - 'url': 'https://10.0.2.15/twitcher/ows/proxy/flyingpigeon?service=WPS&version=1.0.0&request=GetCapabilities' - } - }}) diff --git a/birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf b/birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf deleted file mode 100644 index becccfd9d..000000000 --- a/birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf +++ /dev/null @@ -1,8 +0,0 @@ - location /twitcher/ { - proxy_pass http://twitcher:8000/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $real_scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host:$server_port; - proxy_set_header Forwarded "proto=https;host=10.0.2.15"; # Helps the STAC component to craft URLs containing the full PAVICS_FQDN_PUBLIC - } diff --git a/birdhouse/config/twitcher/twitcher.ini b/birdhouse/config/twitcher/twitcher.ini deleted file mode 100644 index 64c58329e..000000000 --- a/birdhouse/config/twitcher/twitcher.ini +++ /dev/null @@ -1,141 +0,0 @@ -### -# app configuration -# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html -### -# From sample -# https://github.com/bird-house/twitcher/blob/master/development.ini -### - -[app:main] -use = egg:pyramid_twitcher - -pyramid.reload_templates = true -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.default_locale_name = en -pyramid.includes = -# twitcher not using db, instead employ magpie adapter -sqlalchemy.url = sqlite:///%(here)s/twitcher.sqlite -#sqlalchemy.url = postgresql://${POSTGIS_USER}:${POSTGIS_PASSWORD}@postgis:${POSTGIS_PORT}/twitcher - -retry.attempts = 3 - -# Caching settings for specific sections/functions - improves performance response times of recurring requests -# For caching related to Magpie API endpoints themselves, instead refer to Magpie INI configuration file. -# -# Although related to Magpie code, cache regions settings executed by 'MagpieAdapter' are running under Twitcher -# to resolve Access Control Lists (ACL) to services/resources, and must therefore be placed in Twitcher configuration. -# Caching that takes effect in Twitcher via 'MagpieAdapter' is when requests use the URL endpoint: -# -# //proxy/[/...] -# -# NOTE: Caching will only take effect with Magpie >= 3.7 -# Detail: -# Both 'acl' and 'service' scopes occur on every permission resolution for a given user requesting any access. -# The difference is that 'acl' applies for every combination of (user/group, service/resource, permissions), -# while 'service' limits itself to the service name/type resolution from the request path. -# Since services are not expected to change often, 'service' cache can be safely increased at a much higher refresh -# interval than 'acl' which should re-validate any permission changes more frequently. -# Caching can be forced reset/ignored by using the 'Cache-Control: no-cache' header during any corresponding request. -cache.regions = acl, service -cache.type = memory -cache.enabled = true -cache.acl.enabled = true -cache.acl.expire = 20 -cache.service.enabled = true -cache.service.expire = 60 - -# By default, the toolbar only appears for clients from IP addresses -# '127.0.0.1' and '::1'. -# debugtoolbar.hosts = 127.0.0.1 ::1 - -# twitcher -twitcher.url = https://10.0.2.15/twitcher -twitcher.adapter = magpie.adapter.MagpieAdapter -twitcher.rpcinterface = false -twitcher.username = -twitcher.password = -twitcher.ows_security = true -twitcher.ows_proxy = true -twitcher.ows_proxy_delegate = false -twitcher.ows_proxy_protected_path = /ows -twitcher.ows_proxy_ssl_verify = false -twitcher.workdir = -twitcher.prefix = - -# magpie (for twitcher.adapter) -magpie.url = https://10.0.2.15/magpie -magpie.secret = magpie -magpie.admin_user = admin -magpie.admin_password = qwertyqwerty! - -### -# wsgi server configuration -### - -[alembic] -# path to migration scripts -script_location = twitcher/alembic -file_template = %%(year)d%%(month).2d%%(day).2d_%%(rev)s -# file_template = %%(rev)s_%%(slug)s - -[server:main] -use = egg:gunicorn#main -host = 0.0.0.0 -port = 8000 -timeout = 10 -workers = 10 -threads = 4 - -### -# logging configuration -# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html -### - -[loggers] -keys = root, twitcher, magpie, sqlalchemy - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = INFO -handlers = console - -[logger_twitcher] -# "level = DEBUG" logs detailed information about operations/settings (not for production, will leak sensitive data) -# "level = INFO" reports useful information, not leaking details about settings -# "level = WARN" only potential problems/unexpected results reported, such as when caching is employed -level = DEBUG -handlers = -qualname = twitcher - -# MagpieAdapter or any other Magpie utilities it employs through Twitcher proxy-adapter security check -[logger_magpie] -# "level = DEBUG" logs detailed information about operations/settings (not for production, will leak sensitive data) -# "level = INFO" reports useful information about operations, not leaking details about settings -# "level = WARN" only potential problems are reported such as missing settings in configuration -level = DEBUG -handlers = -qualname = magpie - -[logger_sqlalchemy] -# "level = DEBUG" logs SQL queries, transactions and results -# "level = INFO" logs SQL queries (data can be identified from query field values) -# "level = WARN" logs neither (recommended for production systems, avoid anything below unless for dev/debug system) -level = WARN -handlers = -qualname = sqlalchemy.engine - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py b/birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py deleted file mode 100644 index 949946a09..000000000 --- a/birdhouse/config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py +++ /dev/null @@ -1 +0,0 @@ -CANARIE_STATS_ROUTES.append('wpsoutputs') diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py b/birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py deleted file mode 100644 index 949946a09..000000000 --- a/birdhouse/config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py +++ /dev/null @@ -1 +0,0 @@ -CANARIE_STATS_ROUTES.append('wpsoutputs') diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf deleted file mode 100644 index 990bdfbb8..000000000 --- a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf +++ /dev/null @@ -1,5 +0,0 @@ - location /wpsoutputs/ { - include /etc/nginx/conf.extra-service.d/secure-data-proxy/secure-data-auth.include; - - alias /data/wps_outputs/; - } From adda409969dd5d71614c11aba99a6291289748e1 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 14 Dec 2023 20:46:33 -0500 Subject: [PATCH 03/34] update changes --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a64bd3439..1184027b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,9 @@ * Replace all explicit color "logging" related `echo` in scripts by utility variables `MSG_DEBUG`, `MSG_INFO`, `MSG_WARN` and `MSG_ERROR` as applicable per respective messages. * Move `read-configs.include.sh` into `birdhouse/scripts` along other include scripts. + * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call. + This can be used to run a "dry-run" of the command and validate that was is loaded is as expected, by inspecting + provided log messages. - Defaults: * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS. From 6be39d8fde230590ef45ca20f51e57181501e3ed Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 14 Dec 2023 20:58:31 -0500 Subject: [PATCH 04/34] update unittests with new birdhouse/scripts/read-configs.include.sh path --- tests/test_read_configs_include.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 646b5d604..157a7f36e 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -3,6 +3,7 @@ import tempfile import pytest import subprocess +from typing import Union ENV_SPLIT_STR: str = "#env for testing#" @@ -25,10 +26,10 @@ def run_in_compose_dir(root_dir): @pytest.fixture(scope="module") def read_config_include_file(root_dir) -> str: - return os.path.join(root_dir, "birdhouse", "read-configs.include.sh") + return os.path.join(root_dir, "birdhouse", "scripts", "read-configs.include.sh") -def set_local_env(env_file: io.FileIO, content: str | dict) -> None: +def set_local_env(env_file: io.FileIO, content: Union[str, dict]) -> None: env_file.truncate() if isinstance(content, dict): env_file.write("\n".join(f"{k}={v}" for k, v in content.items())) @@ -82,7 +83,7 @@ class TestReadConfigs: ] def run_func( - self, include_file: str, local_env: str | dict, command_suffix: str = "" + self, include_file: str, local_env: Union[str, dict], command_suffix: str = "" ) -> subprocess.CompletedProcess: try: with tempfile.NamedTemporaryFile(delete=False, mode="w") as f: From 805e5daabddc9f353d5956235e71f0614706344e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 15 Dec 2023 11:58:10 -0500 Subject: [PATCH 05/34] adjust cd exit 1 code --- birdhouse/pavics-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index e9ff36f0c..badb707f5 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -42,7 +42,7 @@ OPTIONAL_VARS=' # we switch to the real directory of the script, so it still works when used from $PATH # tip: ln -s /path/to/pavics-compose.sh ~/bin/ # Setup PWD for sourcing env.local. -cd "$(dirname "$(readlink -f "$0" || realpath "$0")")" || (echo "Unresolved path [$0]" && exit 1) +cd "$(dirname "$(readlink -f "$0" || realpath "$0")")" || exit 1 # Setup COMPOSE_DIR for sourcing env.local. # Prevent un-expected difference when this script is run inside autodeploy From 816292b94b034b048311553ceb171e4898222e36 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 15 Dec 2023 12:17:11 -0500 Subject: [PATCH 06/34] replace all THIS_FILE with readlink|realpath cmd --- .gitignore | 1 + birdhouse/components/geoserver/pre-docker-compose-up | 2 +- birdhouse/components/proxy/pre-docker-compose-up | 5 ++--- birdhouse/deployment/certbotwrapper | 2 +- birdhouse/deployment/fix-geoserver-data-dir-perm | 2 +- birdhouse/deployment/fix-write-perm | 2 +- birdhouse/scripts/bootstrap-instance-for-testsuite | 2 +- birdhouse/scripts/check-autodeploy-repos | 2 +- birdhouse/scripts/check-instance-ready | 2 +- birdhouse/scripts/check-wps-database.sh | 2 +- birdhouse/scripts/clear-running-wps-jobs-in-db.sh | 2 +- birdhouse/scripts/create-magpie-authtest-user | 2 +- birdhouse/scripts/create-magpie-users | 2 +- birdhouse/scripts/deprecated/trigger-pavicscrawler | 2 +- birdhouse/scripts/detect-user-install-in-jupyter-env | 2 +- birdhouse/scripts/extract-jupyter-users-from-magpie-db | 2 +- birdhouse/scripts/get-components-json.include.sh | 2 +- birdhouse/scripts/get-services-json.include.sh | 2 +- birdhouse/scripts/migrate-jupyterhub-user-persistence | 2 +- birdhouse/scripts/read-configs.include.sh | 2 +- birdhouse/scripts/sync-data | 2 +- 21 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 84d36d61b..128ede050 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ venv/ ## Testing .pytest_cache/ +*.log diff --git a/birdhouse/components/geoserver/pre-docker-compose-up b/birdhouse/components/geoserver/pre-docker-compose-up index f3ac3f686..b86140046 100755 --- a/birdhouse/components/geoserver/pre-docker-compose-up +++ b/birdhouse/components/geoserver/pre-docker-compose-up @@ -1,6 +1,6 @@ #!/bin/sh -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="$THIS_DIR/../.." diff --git a/birdhouse/components/proxy/pre-docker-compose-up b/birdhouse/components/proxy/pre-docker-compose-up index 0d5acbe8c..cefe55588 100755 --- a/birdhouse/components/proxy/pre-docker-compose-up +++ b/birdhouse/components/proxy/pre-docker-compose-up @@ -3,12 +3,11 @@ # Create JSON files containing the version information, available services, and # enabled components. These files will be served by the nginx proxy as static files. -THIS_FILE="$(realpath "$0")" -THIS_DIR="$(dirname "$THIS_FILE")" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" +THIS_DIR="`dirname "$THIS_FILE"`" mkdir -p "${THIS_DIR}/static" echo "${BIRDHOUSE_VERSION_JSON}" > "${THIS_DIR}/static/version.json" echo "${BIRDHOUSE_DEPLOY_SERVICES_JSON}" > "${THIS_DIR}/static/services.json" echo "${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}" > "${THIS_DIR}/static/components.json" - diff --git a/birdhouse/deployment/certbotwrapper b/birdhouse/deployment/certbotwrapper index 4cc3769dc..004e024ff 100755 --- a/birdhouse/deployment/certbotwrapper +++ b/birdhouse/deployment/certbotwrapper @@ -46,7 +46,7 @@ certbotwrapper START_TIME=$START_TIME" set -x -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" SAVED_PWD="`pwd`" diff --git a/birdhouse/deployment/fix-geoserver-data-dir-perm b/birdhouse/deployment/fix-geoserver-data-dir-perm index d5616961c..484b601e4 100755 --- a/birdhouse/deployment/fix-geoserver-data-dir-perm +++ b/birdhouse/deployment/fix-geoserver-data-dir-perm @@ -8,7 +8,7 @@ # global.xml will exist and this script will not execute. Without # FIRST_RUN_ONLY, this script will always execute. -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" # Go to repo root. diff --git a/birdhouse/deployment/fix-write-perm b/birdhouse/deployment/fix-write-perm index a68a7b444..17f9e148a 100755 --- a/birdhouse/deployment/fix-write-perm +++ b/birdhouse/deployment/fix-write-perm @@ -37,7 +37,7 @@ # So the setfacl solution is the simplest, most portable/generic and most # localized (only the directories we need) solution. -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" # Go to repo root. diff --git a/birdhouse/scripts/bootstrap-instance-for-testsuite b/birdhouse/scripts/bootstrap-instance-for-testsuite index dac2acdbb..36ee85f34 100755 --- a/birdhouse/scripts/bootstrap-instance-for-testsuite +++ b/birdhouse/scripts/bootstrap-instance-for-testsuite @@ -9,7 +9,7 @@ # been called). # -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" set -x diff --git a/birdhouse/scripts/check-autodeploy-repos b/birdhouse/scripts/check-autodeploy-repos index 865193502..5a6547404 100755 --- a/birdhouse/scripts/check-autodeploy-repos +++ b/birdhouse/scripts/check-autodeploy-repos @@ -2,7 +2,7 @@ # Check Autodeploy repos status. # If there are changes or uncommitted, autodeploy will not trigger. -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/check-instance-ready b/birdhouse/scripts/check-instance-ready index e024bf84f..406aafac6 100755 --- a/birdhouse/scripts/check-instance-ready +++ b/birdhouse/scripts/check-instance-ready @@ -7,7 +7,7 @@ # also enabled. # -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/check-wps-database.sh b/birdhouse/scripts/check-wps-database.sh index dedcc9e58..caa903f9f 100755 --- a/birdhouse/scripts/check-wps-database.sh +++ b/birdhouse/scripts/check-wps-database.sh @@ -1,6 +1,6 @@ #!/bin/bash -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" if [ -f "${THIS_DIR}/logging.include.sh" ]; then diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index 41483d091..267c5ae39 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -1,6 +1,6 @@ #!/bin/sh -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/create-magpie-authtest-user b/birdhouse/scripts/create-magpie-authtest-user index 231d5da96..265f1d8a9 100755 --- a/birdhouse/scripts/create-magpie-authtest-user +++ b/birdhouse/scripts/create-magpie-authtest-user @@ -7,7 +7,7 @@ # Options: # -d: delete user 'authtest' instead of creating it -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" TMP_CONFIG_FILE="/tmp/create-magpie-authtest-user.yml" diff --git a/birdhouse/scripts/create-magpie-users b/birdhouse/scripts/create-magpie-users index 8c0b444e2..b06444264 100755 --- a/birdhouse/scripts/create-magpie-users +++ b/birdhouse/scripts/create-magpie-users @@ -65,7 +65,7 @@ # bogus03 bvNWVWCQi8M6 409 : User name matches an already existing user name. # -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/deprecated/trigger-pavicscrawler b/birdhouse/scripts/deprecated/trigger-pavicscrawler index 7e1d13609..33f6b1c50 100755 --- a/birdhouse/scripts/deprecated/trigger-pavicscrawler +++ b/birdhouse/scripts/deprecated/trigger-pavicscrawler @@ -14,7 +14,7 @@ # # Set env var PAVICS_CRAWLER_HOST to target different PAVICS host. -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env index 96f25ad67..9d5b1a2f4 100755 --- a/birdhouse/scripts/detect-user-install-in-jupyter-env +++ b/birdhouse/scripts/detect-user-install-in-jupyter-env @@ -7,7 +7,7 @@ # version is pinned at the user installed version. # -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" . "$THIS_DIR/read-configs.include.sh" diff --git a/birdhouse/scripts/extract-jupyter-users-from-magpie-db b/birdhouse/scripts/extract-jupyter-users-from-magpie-db index 9d0c4dfbf..31984717d 100755 --- a/birdhouse/scripts/extract-jupyter-users-from-magpie-db +++ b/birdhouse/scripts/extract-jupyter-users-from-magpie-db @@ -19,7 +19,7 @@ # (4 rows) -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index 8409465b7..33653db7b 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -17,7 +17,7 @@ # } # -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" if [ -f "${THIS_DIR}/logging.include.sh" ]; then diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index bfd9e635b..e5eaa94b9 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -1,6 +1,6 @@ #!/bin/sh -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" if [ -f "${THIS_DIR}/logging.include.sh" ]; then diff --git a/birdhouse/scripts/migrate-jupyterhub-user-persistence b/birdhouse/scripts/migrate-jupyterhub-user-persistence index 6af4722dd..2bc7eac66 100755 --- a/birdhouse/scripts/migrate-jupyterhub-user-persistence +++ b/birdhouse/scripts/migrate-jupyterhub-user-persistence @@ -4,7 +4,7 @@ # location JUPYTERHUB_USER_DATA_DIR/PUBLIC_USERNAME. # -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index 9d3b24436..35ea60419 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -23,7 +23,7 @@ # read_configs -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" if [ -f "${THIS_DIR}/logging.include.sh" ]; then diff --git a/birdhouse/scripts/sync-data b/birdhouse/scripts/sync-data index f08be1f21..62c2959e1 100755 --- a/birdhouse/scripts/sync-data +++ b/birdhouse/scripts/sync-data @@ -13,7 +13,7 @@ # Assume # * ssh passwordless to source host is setup properly -THIS_FILE="`realpath "$0"`" +THIS_FILE="`readlink -f "$0" || realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" From a41dd91f49e57249bd3a497d6e487ba0d3324048 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 19 Dec 2023 11:48:50 -0500 Subject: [PATCH 07/34] replace backticks by $(...) for path resolution operations --- birdhouse/components/geoserver/pre-docker-compose-up | 4 ++-- birdhouse/components/proxy/pre-docker-compose-up | 4 ++-- birdhouse/deployment/certbotwrapper | 4 ++-- birdhouse/deployment/deploy.sh | 4 ++-- birdhouse/deployment/fix-geoserver-data-dir-perm | 4 ++-- birdhouse/deployment/fix-write-perm | 4 ++-- birdhouse/deployment/triggerdeploy.sh | 2 +- birdhouse/pavics-compose.sh | 2 +- birdhouse/scripts/bootstrap-instance-for-testsuite | 4 ++-- birdhouse/scripts/bootstrap-testdata | 2 +- birdhouse/scripts/check-autodeploy-repos | 4 ++-- birdhouse/scripts/check-instance-ready | 4 ++-- birdhouse/scripts/check-wps-database.sh | 4 ++-- birdhouse/scripts/clear-running-wps-jobs-in-db.sh | 6 +++--- birdhouse/scripts/create-magpie-authtest-user | 4 ++-- birdhouse/scripts/create-magpie-users | 4 ++-- birdhouse/scripts/deprecated/trigger-pavicscrawler | 4 ++-- birdhouse/scripts/detect-user-install-in-jupyter-env | 4 ++-- birdhouse/scripts/extract-jupyter-users-from-magpie-db | 4 ++-- birdhouse/scripts/get-components-json.include.sh | 4 ++-- birdhouse/scripts/get-services-json.include.sh | 4 ++-- birdhouse/scripts/migrate-jupyterhub-user-persistence | 4 ++-- birdhouse/scripts/read-configs.include.sh | 4 ++-- birdhouse/scripts/send-dummy-alert.sh | 2 +- birdhouse/scripts/sync-data | 4 ++-- 25 files changed, 47 insertions(+), 47 deletions(-) diff --git a/birdhouse/components/geoserver/pre-docker-compose-up b/birdhouse/components/geoserver/pre-docker-compose-up index b86140046..d12752b5b 100755 --- a/birdhouse/components/geoserver/pre-docker-compose-up +++ b/birdhouse/components/geoserver/pre-docker-compose-up @@ -1,7 +1,7 @@ #!/bin/sh -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="$THIS_DIR/../.." if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then diff --git a/birdhouse/components/proxy/pre-docker-compose-up b/birdhouse/components/proxy/pre-docker-compose-up index cefe55588..a2103b7d3 100755 --- a/birdhouse/components/proxy/pre-docker-compose-up +++ b/birdhouse/components/proxy/pre-docker-compose-up @@ -3,8 +3,8 @@ # Create JSON files containing the version information, available services, and # enabled components. These files will be served by the nginx proxy as static files. -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" mkdir -p "${THIS_DIR}/static" diff --git a/birdhouse/deployment/certbotwrapper b/birdhouse/deployment/certbotwrapper index 004e024ff..cb985d82a 100755 --- a/birdhouse/deployment/certbotwrapper +++ b/birdhouse/deployment/certbotwrapper @@ -46,8 +46,8 @@ certbotwrapper START_TIME=$START_TIME" set -x -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" SAVED_PWD="`pwd`" . "$THIS_DIR/../read-configs.include.sh" diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh index 9917efece..d18fe6d4c 100755 --- a/birdhouse/deployment/deploy.sh +++ b/birdhouse/deployment/deploy.sh @@ -79,8 +79,8 @@ else shift fi -COMPOSE_DIR="`realpath "$COMPOSE_DIR"`" -REPO_ROOT="`realpath "$COMPOSE_DIR/.."`" +COMPOSE_DIR="$(realpath "$COMPOSE_DIR")" +REPO_ROOT="$(realpath "$COMPOSE_DIR/..")" if [ ! -f "$COMPOSE_DIR/docker-compose.yml" -o \ ! -f "$COMPOSE_DIR/pavics-compose.sh" ]; then diff --git a/birdhouse/deployment/fix-geoserver-data-dir-perm b/birdhouse/deployment/fix-geoserver-data-dir-perm index 484b601e4..d9c7eace2 100755 --- a/birdhouse/deployment/fix-geoserver-data-dir-perm +++ b/birdhouse/deployment/fix-geoserver-data-dir-perm @@ -8,8 +8,8 @@ # global.xml will exist and this script will not execute. Without # FIRST_RUN_ONLY, this script will always execute. -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" # Go to repo root. cd $THIS_DIR/../.. diff --git a/birdhouse/deployment/fix-write-perm b/birdhouse/deployment/fix-write-perm index 17f9e148a..ecb35d83f 100755 --- a/birdhouse/deployment/fix-write-perm +++ b/birdhouse/deployment/fix-write-perm @@ -37,8 +37,8 @@ # So the setfacl solution is the simplest, most portable/generic and most # localized (only the directories we need) solution. -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" # Go to repo root. cd $THIS_DIR/../.. diff --git a/birdhouse/deployment/triggerdeploy.sh b/birdhouse/deployment/triggerdeploy.sh index bdb4f6347..25e16458b 100755 --- a/birdhouse/deployment/triggerdeploy.sh +++ b/birdhouse/deployment/triggerdeploy.sh @@ -59,7 +59,7 @@ else shift fi -COMPOSE_DIR="`realpath "$COMPOSE_DIR"`" +COMPOSE_DIR="$(realpath "$COMPOSE_DIR")" if [ ! -f "$COMPOSE_DIR/docker-compose.yml" ]; then echo "ERROR: missing docker-compose.yml in '$COMPOSE_DIR'" 1>&2 diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index badb707f5..d74f53554 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -47,7 +47,7 @@ cd "$(dirname "$(readlink -f "$0" || realpath "$0")")" || exit 1 # Setup COMPOSE_DIR for sourcing env.local. # Prevent un-expected difference when this script is run inside autodeploy # container and manually from the host. -COMPOSE_DIR="`pwd`" +COMPOSE_DIR="$(pwd)" . "${COMPOSE_DIR}/scripts/read-configs.include.sh" read_configs # this sets ALL_CONF_DIRS diff --git a/birdhouse/scripts/bootstrap-instance-for-testsuite b/birdhouse/scripts/bootstrap-instance-for-testsuite index 36ee85f34..6d5af0382 100755 --- a/birdhouse/scripts/bootstrap-instance-for-testsuite +++ b/birdhouse/scripts/bootstrap-instance-for-testsuite @@ -9,8 +9,8 @@ # been called). # -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" set -x # Populate test .nc file on Thredds. diff --git a/birdhouse/scripts/bootstrap-testdata b/birdhouse/scripts/bootstrap-testdata index 4a10fb437..4b200ee97 100755 --- a/birdhouse/scripts/bootstrap-testdata +++ b/birdhouse/scripts/bootstrap-testdata @@ -31,7 +31,7 @@ cccma/CanESM2/historical/fx/atmos/r0i0p0/sftlf/sftlf_fx_CanESM2_historical_r0i0p testdata/xclim/NRCANdaily/nrcan_canada_daily_tasmin_1990.nc " -cd "$DATASET_ROOT" +cd "$DATASET_ROOT" || exit 1 for afile in $FILE_LIST; do if [ ! -f "$afile" ]; then diff --git a/birdhouse/scripts/check-autodeploy-repos b/birdhouse/scripts/check-autodeploy-repos index 5a6547404..870226e4e 100755 --- a/birdhouse/scripts/check-autodeploy-repos +++ b/birdhouse/scripts/check-autodeploy-repos @@ -2,8 +2,8 @@ # Check Autodeploy repos status. # If there are changes or uncommitted, autodeploy will not trigger. -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then diff --git a/birdhouse/scripts/check-instance-ready b/birdhouse/scripts/check-instance-ready index 406aafac6..a775b4d49 100755 --- a/birdhouse/scripts/check-instance-ready +++ b/birdhouse/scripts/check-instance-ready @@ -7,8 +7,8 @@ # also enabled. # -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then diff --git a/birdhouse/scripts/check-wps-database.sh b/birdhouse/scripts/check-wps-database.sh index caa903f9f..aeab5910f 100755 --- a/birdhouse/scripts/check-wps-database.sh +++ b/birdhouse/scripts/check-wps-database.sh @@ -1,7 +1,7 @@ #!/bin/bash -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" if [ -f "${THIS_DIR}/logging.include.sh" ]; then . "${THIS_DIR}/logging.include.sh" diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index 267c5ae39..201923381 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -1,8 +1,8 @@ #!/bin/sh -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" -COMPOSE_DIR="`dirname "$THIS_DIR"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" +COMPOSE_DIR="$(dirname "$THIS_DIR")" if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then . "${COMPOSE_DIR}/scripts/logging.include.sh" diff --git a/birdhouse/scripts/create-magpie-authtest-user b/birdhouse/scripts/create-magpie-authtest-user index 265f1d8a9..a71c3b796 100755 --- a/birdhouse/scripts/create-magpie-authtest-user +++ b/birdhouse/scripts/create-magpie-authtest-user @@ -7,8 +7,8 @@ # Options: # -d: delete user 'authtest' instead of creating it -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" TMP_CONFIG_FILE="/tmp/create-magpie-authtest-user.yml" diff --git a/birdhouse/scripts/create-magpie-users b/birdhouse/scripts/create-magpie-users index b06444264..219a60349 100755 --- a/birdhouse/scripts/create-magpie-users +++ b/birdhouse/scripts/create-magpie-users @@ -65,8 +65,8 @@ # bogus03 bvNWVWCQi8M6 409 : User name matches an already existing user name. # -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then diff --git a/birdhouse/scripts/deprecated/trigger-pavicscrawler b/birdhouse/scripts/deprecated/trigger-pavicscrawler index 33f6b1c50..e4956cec4 100755 --- a/birdhouse/scripts/deprecated/trigger-pavicscrawler +++ b/birdhouse/scripts/deprecated/trigger-pavicscrawler @@ -14,8 +14,8 @@ # # Set env var PAVICS_CRAWLER_HOST to target different PAVICS host. -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env index 9d5b1a2f4..887252e9d 100755 --- a/birdhouse/scripts/detect-user-install-in-jupyter-env +++ b/birdhouse/scripts/detect-user-install-in-jupyter-env @@ -7,8 +7,8 @@ # version is pinned at the user installed version. # -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" . "$THIS_DIR/read-configs.include.sh" diff --git a/birdhouse/scripts/extract-jupyter-users-from-magpie-db b/birdhouse/scripts/extract-jupyter-users-from-magpie-db index 31984717d..8e1b81e52 100755 --- a/birdhouse/scripts/extract-jupyter-users-from-magpie-db +++ b/birdhouse/scripts/extract-jupyter-users-from-magpie-db @@ -19,8 +19,8 @@ # (4 rows) -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index 33653db7b..2b2ebad9a 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -17,8 +17,8 @@ # } # -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" if [ -f "${THIS_DIR}/logging.include.sh" ]; then . "${THIS_DIR}/logging.include.sh" diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index e5eaa94b9..68b2a1d53 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -1,7 +1,7 @@ #!/bin/sh -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" if [ -f "${THIS_DIR}/logging.include.sh" ]; then . "${THIS_DIR}/logging.include.sh" diff --git a/birdhouse/scripts/migrate-jupyterhub-user-persistence b/birdhouse/scripts/migrate-jupyterhub-user-persistence index 2bc7eac66..05bc0a06f 100755 --- a/birdhouse/scripts/migrate-jupyterhub-user-persistence +++ b/birdhouse/scripts/migrate-jupyterhub-user-persistence @@ -4,8 +4,8 @@ # location JUPYTERHUB_USER_DATA_DIR/PUBLIC_USERNAME. # -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" . "${COMPOSE_DIR}/scripts/read-configs.include.sh" diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index 35ea60419..16fd43fb8 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -23,8 +23,8 @@ # read_configs -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" if [ -f "${THIS_DIR}/logging.include.sh" ]; then . "${THIS_DIR}/logging.include.sh" diff --git a/birdhouse/scripts/send-dummy-alert.sh b/birdhouse/scripts/send-dummy-alert.sh index 0edf503a3..fe51b7b8f 100755 --- a/birdhouse/scripts/send-dummy-alert.sh +++ b/birdhouse/scripts/send-dummy-alert.sh @@ -1,6 +1,6 @@ #!/bin/bash # https://gist.githubusercontent.com/cherti/61ec48deaaab7d288c9fcf17e700853a/raw/a69ddd1d96507f6d94059071d500fe499631e739/alert.sh -# Useful to test receving alert on UI and via email notif. +# Useful to test receiving alert on UI and via email notif. name=$RANDOM url='http://localhost:9093/api/v1/alerts' diff --git a/birdhouse/scripts/sync-data b/birdhouse/scripts/sync-data index 62c2959e1..9d94ebb62 100755 --- a/birdhouse/scripts/sync-data +++ b/birdhouse/scripts/sync-data @@ -13,8 +13,8 @@ # Assume # * ssh passwordless to source host is setup properly -THIS_FILE="`readlink -f "$0" || realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="`dirname "$THIS_DIR"`" if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then From eff7dd349434d8bb661db41e5e4ac68fdad64e58 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 8 Jan 2024 12:17:24 -0500 Subject: [PATCH 08/34] update changes --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 52c24bf13..35e708b14 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,8 @@ * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call. This can be used to run a "dry-run" of the command and validate that was is loaded is as expected, by inspecting provided log messages. + * Replaced older backtick (``` ` ```) executions by `$(...)` representation except for `eval` calls that require + them for backward compatibility of `sh` on some server instances. - Defaults: * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS. From 1d301038f2bb9a31ff77806945cb1ad72a855f95 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 8 Jan 2024 13:58:18 -0500 Subject: [PATCH 09/34] add log function and logging level support --- CHANGES.md | 11 +++-- birdhouse/pavics-compose.sh | 17 +++---- birdhouse/scripts/check-wps-database.sh | 2 +- .../scripts/clear-running-wps-jobs-in-db.sh | 2 +- .../scripts/get-components-json.include.sh | 4 +- .../scripts/get-services-json.include.sh | 2 +- birdhouse/scripts/logging.include.sh | 47 +++++++++++++++++-- birdhouse/scripts/read-configs.include.sh | 22 ++++----- 8 files changed, 75 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 35e708b14..e638b003a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,13 +18,16 @@ ## Changes - Compose script utilities: * Add `BIRDHOUSE_COLOR` option and various logging/messaging definitions in `birdhouse/scripts/logging.include.sh`. - * Replace all explicit color "logging" related `echo` in scripts by utility variables - `MSG_DEBUG`, `MSG_INFO`, `MSG_WARN` and `MSG_ERROR` as applicable per respective messages. + * Replace all explicit color "logging" related `echo` in scripts by a utility `log {LEVEL} {message}` function + that employs variables `LOG_DEBUG`, `LOG_INFO`, `LOG_WARN`, `LOG_ERROR` and `LOG_CRITICAL` as applicable per + respective messages to report logging messages in a standard approach. + Colors can be disabled with `BIRDHOUSE_COLOR=0` and logging level can be set with `BIRDHOUSE_LOG_LEVEL={LEVEL}` + where all levels above or equal to the configured one will be displayed (default logging level is `INFO`). * Move `read-configs.include.sh` into `birdhouse/scripts` along other include scripts. * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call. - This can be used to run a "dry-run" of the command and validate that was is loaded is as expected, by inspecting + This can be used to perform a "dry-run" of the command and validate that was is loaded is as expected, by inspecting provided log messages. - * Replaced older backtick (``` ` ```) executions by `$(...)` representation except for `eval` calls that require + * Replace older backtick (``` ` ```) executions by `$(...)` representation except for `eval` calls that require them for backward compatibility of `sh` on some server instances. - Defaults: diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index d74f53554..ef40a9075 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -61,7 +61,7 @@ do v="${i}" if [ -z "`eval "echo ${v}"`" ] then - echo "${MSG_ERROR}Required variable $v is not set. Check env.local file." + log ERROR "Required variable $v is not set. Check env.local file." exit 1 fi done @@ -70,14 +70,14 @@ done # will add delay # if [ ! -f $SSL_CERTIFICATE ] # then -# echo "${MSG_ERROR}SSL certificate file $SSL_CERTIFICATE is missing" +# log ERROR "SSL certificate file $SSL_CERTIFICATE is missing" # exit 1 # fi TIMEWAIT_REUSE=$(/sbin/sysctl -n net.ipv4.tcp_tw_reuse) if [ "${TIMEWAIT_REUSE}" -eq 0 ] then - echo "${MSG_WARN}the sysctl net.ipv4.tcp_tw_reuse is not enabled. " \ + log WARN "the sysctl net.ipv4.tcp_tw_reuse is not enabled. " \ "It it suggested to set it to 1, otherwise the pavicscrawler may fail." fi @@ -90,7 +90,7 @@ done export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES # we apply all the templates -find $ALL_CONF_DIRS -name '*.template' | +find $ALL_CONF_DIRS -name '*.template' 2>/dev/null | while read FILE do DEST=${FILE%.template} @@ -101,18 +101,19 @@ if [ x"$1" = x"up" ]; then for adir in $ALL_CONF_DIRS; do COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then - echo "${MSG_INFO}executing '$COMPONENT_PRE_COMPOSE_UP'" + log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" sh -x "$COMPONENT_PRE_COMPOSE_UP" fi done fi create_compose_conf_list # this sets COMPOSE_CONF_LIST +log INFO "Displaying resolved compose configurations:" echo "COMPOSE_CONF_LIST=" echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' if [ x"$1" = x"info" ]; then - echo "${MSG_INFO}Stopping before execution of docker-compose command." + log INFO "Stopping before execution of docker-compose command." exit 0 fi @@ -124,7 +125,7 @@ ERR=$? type post-compose 2>&1 | grep 'post-compose is a function' > /dev/null if [ $? -eq 0 ] then - [ ${ERR} -gt 0 ] && { echo "${MSG_ERROR}Error occurred with docker-compose, not running post-compose"; exit $?; } + [ ${ERR} -gt 0 ] && { log ERROR "Error occurred with docker-compose, not running post-compose"; exit $?; } post-compose $* fi @@ -145,7 +146,7 @@ do for adir in $ALL_CONF_DIRS; do COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up" if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then - echo "${MSG_INFO}executing '$COMPONENT_POST_COMPOSE_UP'" + log INFO "Executing '$COMPONENT_POST_COMPOSE_UP'" sh -x "$COMPONENT_POST_COMPOSE_UP" fi done diff --git a/birdhouse/scripts/check-wps-database.sh b/birdhouse/scripts/check-wps-database.sh index aeab5910f..9fc53e647 100755 --- a/birdhouse/scripts/check-wps-database.sh +++ b/birdhouse/scripts/check-wps-database.sh @@ -66,7 +66,7 @@ case $2 in docker run -ti --rm -v birdhouse_data:/data birdhouse/bird-base sqlite3 $DB ;; *) - echo "${MSG_ERROR}unknown operation: $2" + log ERROR "unknown operation: $2" usage ;; esac diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index 201923381..bac3a4781 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -11,7 +11,7 @@ fi # eg: DB_NAME=finch DB_NAME="$1" if [ -z "$DB_NAME" ]; then - echo "${MSG_ERROR}please provide a database name, ex: finch" 1>&2 + log ERROR "please provide a database name, ex: finch" 1>&2 exit 2 fi shift diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index 2b2ebad9a..508526cdd 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -27,7 +27,7 @@ fi # default value in case of error or missing definitions export BIRDHOUSE_DEPLOY_COMPONENTS_JSON='{"components": []}' if [ -z "${ALL_CONF_DIRS}" ]; then - echo "${MSG_WARN}No components in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. Components JSON list will be empty!" + log WARN "No components in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. Components JSON list will be empty!" return fi @@ -48,7 +48,7 @@ BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN="$( \ | sed -E 's/^|[[:space:]]+/ -e /' \ )" if [ -z "${BIRDHOUSE_DEPLOY_COMPONENTS_LIST_KNOWN}" ]; then - echo "${MSG_WARN}" \ + log WARN "" \ "Could not resolve known birdhouse-deploy components." \ "Aborting to avoid potentially leaking sensible details." \ "Components will not be reported on the platform's JSON endpoint." diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index 68b2a1d53..e3164f7a2 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -15,6 +15,6 @@ for adir in ${ALL_CONF_DIRS}; do done if [ -z "${SERVICES}" ]; then - echo "${MSG_WARN}No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" + log WARN "No services in DEFAULT_CONF_DIRS and EXTRA_CONF_DIRS. SERVICES JSON list will be empty!" fi export BIRDHOUSE_DEPLOY_SERVICES_JSON="{\"services\": [${SERVICES}]}" diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh index 910aebf32..273730501 100644 --- a/birdhouse/scripts/logging.include.sh +++ b/birdhouse/scripts/logging.include.sh @@ -9,10 +9,49 @@ if [ "${BIRDHOUSE_COLOR}" -eq "1" ]; then YELLOW=$(tput setaf 3) GREEN=$(tput setaf 2) RED=$(tput setaf 1) + REG_BG_BOLD="$(tput sgr0)$(tput setab 1)$(tput bold)" NORMAL=$(tput sgr0) fi -export MSG_DEBUG="${GRAY}DEBUG${NORMAL}: " -export MSG_INFO="${BLUE}INFO${NORMAL}: " -export MSG_WARN="${YELLOW}WARNING${NORMAL}: " -export MSG_ERROR="${RED}ERROR${NORMAL}: " +BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL:-INFO} +export LOG_DEBUG="${GRAY}DEBUG${NORMAL}: " +export LOG_INFO="${BLUE}INFO${NORMAL}: " +export LOG_WARN="${YELLOW}WARNING${NORMAL}: " +export LOG_ERROR="${RED}ERROR${NORMAL}: " +export LOG_CRITICAL="${REG_BG_BOLD}CRITICAL${NORMAL}: " # to report misuse of functions + +# Usage: log {LEVEL} "{message}" +log() { + if [ "${BIRDHOUSE_LOG_LEVEL}" != DEBUG ] \ + && [ "${BIRDHOUSE_LOG_LEVEL}" != INFO ] \ + && [ "${BIRDHOUSE_LOG_LEVEL}" != WARN ] \ + && [ "${BIRDHOUSE_LOG_LEVEL}" != ERROR ]; then + echo "${LOG_CRITICAL}Invalid log level setting: [BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL}]." + exit 2 + fi + if [ "$2" = "" ]; then + echo "${LOG_CRITICAL}Invalid log message is missing." + exit 2 + fi + if [ "$1" = "DEBUG" ]; then + if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ]; then + echo "${LOG_DEBUG}$2" + fi + elif [ "$1" = "INFO" ]; then + if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ + || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ]; then + echo "${LOG_INFO}$2" + fi + elif [ "$1" = "WARN" ]; then + if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ + || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ] \ + || [ "${BIRDHOUSE_LOG_LEVEL}" = WARN ]; then + echo "${LOG_WARN}$2" + fi + elif [ "$1" = "ERROR" ]; then + echo "${LOG_ERROR}$2" + else + echo "${LOG_CRITICAL}Invalid log level: [$1]" + exit 2 + fi +} diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index 16fd43fb8..7892e3735 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -66,7 +66,7 @@ discover_compose_dir() { # Case of sub-subdir of sibling checkout at same level as birdhouse-deploy. COMPOSE_DIR="$(realpath "../../../birdhouse-deploy/birdhouse")" fi - echo "${MSG_INFO}Resolved docker-compose directory: [${COMPOSE_DIR}]" + log INFO "Resolved docker-compose directory: [${COMPOSE_DIR}]" export COMPOSE_DIR fi } @@ -93,7 +93,7 @@ read_default_env() { . "$COMPOSE_DIR/default.env" else - echo "${MSG_WARN}'$COMPOSE_DIR/default.env' not found" 1>&2 + log WARN "'$COMPOSE_DIR/default.env' not found" 1>&2 fi } @@ -101,7 +101,7 @@ read_default_env() { read_env_local() { # we don't use usual .env filename, because docker-compose uses it - echo "${MSG_INFO}Using local environment file at: ${BIRDHOUSE_LOCAL_ENV}" + log INFO "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV}" if [ -e "$BIRDHOUSE_LOCAL_ENV" ]; then saved_shell_options="$(set +o)" @@ -113,7 +113,7 @@ read_env_local() { eval "$saved_shell_options" else - echo "${MSG_WARN}'$BIRDHOUSE_LOCAL_ENV' not found" 1>&2 + log WARN "'$BIRDHOUSE_LOCAL_ENV' not found" 1>&2 fi } @@ -145,7 +145,7 @@ source_conf_files() { # corresponding PR are merged and old component names can be removed # after the corresponding PR are merge without any impact on the # autodeploy process. - echo "${MSG_WARN}'$adir' in $conf_locations does not exist" 1>&2 + log WARN "'$adir' in $conf_locations does not exist" 1>&2 fi if [ -f "$adir/default.env" ]; then # Source config settings of dependencies first if they haven't been sourced previously. @@ -157,7 +157,7 @@ source_conf_files() { # reset the adir variable in case it was changed in a recursive call adir="$(printf '%b' "$_adir_stack" | tail -1)" fi - echo "${MSG_DEBUG}reading '$adir/default.env'" + log DEBUG "reading '$adir/default.env'" . "$adir/default.env" fi if echo "$ALL_CONF_DIRS" | grep -qE "^\s*$adir\s*$"; then @@ -201,11 +201,11 @@ check_optional_vars() { result=`echo "${d}" | grep -c "${default}"` if [ -z "`eval "echo ${v}"`" ] then - echo "${MSG_WARN}Optional variable [${n}] is not set. Check env.local file." + log WARN "Optional variable [${n}] is not set. Check env.local file." fi if [ "${result}" -gt 0 ] then - echo "${MSG_WARN}Optional variable [${n}] employs a default recommended for override. Check env.local file." + log WARN "Optional variable [${n}] employs a default recommended for override. Check env.local file." fi done } @@ -223,7 +223,7 @@ process_delayed_eval() { fi v="`eval "echo \\$${i}"`" eval 'export ${i}="`eval "echo ${v}"`"' - echo "${MSG_DEBUG}delayed eval '$(env | grep -e "^${i}=")'" + log DEBUG "delayed eval '$(env | grep -e "^${i}=")'" ALREADY_EVALED=" $ALREADY_EVALED $i" @@ -246,7 +246,7 @@ create_compose_conf_list() { # ALL_CONF_DIRS relative paths are relative to COMPOSE_DIR. discover_compose_dir if [ -d "$COMPOSE_DIR" ]; then - echo "${MSG_INFO}Found compose directory [${COMPOSE_DIR}]" + log INFO "Found compose directory [${COMPOSE_DIR}]" cd "$COMPOSE_DIR" || return fi @@ -281,7 +281,7 @@ create_compose_conf_list() { # Return to previous pwd. if [ -d "$COMPOSE_DIR" ]; then - echo "${MSG_INFO}Moving to [${COMPOSE_DIR}]" + log INFO "Moving to [${COMPOSE_DIR}]" cd - >/dev/null || return fi } From 6a75a1082e8f0e0f7b621983bb70289e501ffdc3 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 11 Jan 2024 14:18:10 -0500 Subject: [PATCH 10/34] add security defaults check for required variables that should be modified --- birdhouse/default.env | 18 ++++++++++ birdhouse/pavics-compose.sh | 10 +----- birdhouse/scripts/logging.include.sh | 26 ++++++++------- birdhouse/scripts/read-configs.include.sh | 40 +++++++++++++++++++++-- 4 files changed, 72 insertions(+), 22 deletions(-) diff --git a/birdhouse/default.env b/birdhouse/default.env index 1cbdb15f7..45cf1c334 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -80,6 +80,24 @@ export SERVER_RELEASE_NOTES_URL='${__DEFAULT__SERVER_RELEASE_NOTES_URL}' export SERVER_SUPPORT_URL='${__DEFAULT__SERVER_SUPPORT_URL}' export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' +# Defaults for required variables recommended for override for security reasons. +# Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. +# These values would be detected only if the instance was configured using a copy of 'env.local.example'. +export __DEFAULT__MAGPIE_SECRET=itzaseekrit +#export __DEFAULT__MAGPIE_ADMIN_USERNAME=admin +export __DEFAULT__MAGPIE_ADMIN_PASSWORD=qwertyqwerty! +#export __DEFAULT__POSTGRES_PAVICS_USERNAME=postgres-pavics +export __DEFAULT__POSTGRES_PAVICS_PASSWORD=postgres-qwerty +#export __DEFAULT__POSTGRES_MAGPIE_USERNAME=postgres-magpie +export __DEFAULT__POSTGRES_MAGPIE_PASSWORD=postgres-qwerty +#export __DEFAULT__GEOSERVER_ADMIN_USER=admingeo +export __DEFAULT__GEOSERVER_ADMIN_PASSWORD=geoserverpass +export __DEFAULT__TOMCAT_NCWMS_PASSWORD=ncwmspass +#export __DEFAULT__CATALOG_USERNAME=admin-catalog +export __DEFAULT__CATALOG_PASSWORD=qwerty +export __DEFAULT__PHOENIX_PASSWORD=phoenix_pass +export __DEFAULT__PHOENIX_PASSWORD_HASH=sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 + export DEFAULT_CONF_DIRS=' ./components/proxy ./components/magpie diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index ef40a9075..a24c69e20 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -56,15 +56,7 @@ read_configs # this sets ALL_CONF_DIRS . "${COMPOSE_DIR}/scripts/get-services-json.include.sh" . "${COMPOSE_DIR}/scripts/get-version-json.include.sh" -for i in ${VARS} -do - v="${i}" - if [ -z "`eval "echo ${v}"`" ] - then - log ERROR "Required variable $v is not set. Check env.local file." - exit 1 - fi -done +check_required_vars ## check fails when root access is required to access this file.. workaround possible by going through docker daemon... but # will add delay diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh index 273730501..fcce426bd 100644 --- a/birdhouse/scripts/logging.include.sh +++ b/birdhouse/scripts/logging.include.sh @@ -20,7 +20,9 @@ export LOG_WARN="${YELLOW}WARNING${NORMAL}: " export LOG_ERROR="${RED}ERROR${NORMAL}: " export LOG_CRITICAL="${REG_BG_BOLD}CRITICAL${NORMAL}: " # to report misuse of functions -# Usage: log {LEVEL} "{message}" + +# Usage: log {LEVEL} "{message}" [...] +# Any amount of messages can be passed to the function. log() { if [ "${BIRDHOUSE_LOG_LEVEL}" != DEBUG ] \ && [ "${BIRDHOUSE_LOG_LEVEL}" != INFO ] \ @@ -29,29 +31,31 @@ log() { echo "${LOG_CRITICAL}Invalid log level setting: [BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL}]." exit 2 fi - if [ "$2" = "" ]; then + level="$1" + shift + if [ "$*" = "" ]; then echo "${LOG_CRITICAL}Invalid log message is missing." exit 2 fi - if [ "$1" = "DEBUG" ]; then + if [ "${level}" = "DEBUG" ]; then if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ]; then - echo "${LOG_DEBUG}$2" + echo "${LOG_DEBUG}$*" fi - elif [ "$1" = "INFO" ]; then + elif [ "${level}" = "INFO" ]; then if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ]; then - echo "${LOG_INFO}$2" + echo "${LOG_INFO}$*" fi - elif [ "$1" = "WARN" ]; then + elif [ "${level}" = "WARN" ]; then if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ] \ || [ "${BIRDHOUSE_LOG_LEVEL}" = WARN ]; then - echo "${LOG_WARN}$2" + echo "${LOG_WARN}$*" fi - elif [ "$1" = "ERROR" ]; then - echo "${LOG_ERROR}$2" + elif [ "${level}" = "ERROR" ]; then + echo "${LOG_ERROR}$*" else - echo "${LOG_CRITICAL}Invalid log level: [$1]" + echo "${LOG_CRITICAL}Invalid log level: [${level}]" exit 2 fi } diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index 7892e3735..ee24fe0f6 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -190,8 +190,30 @@ read_components_default_env() { } -check_optional_vars() { +# Check that all optional variables are defined with a different value than the default to emit a warning log message. +# Also check that required variables do not use generic defaults to indicate possible security issues. +check_default_vars() { + # for required variables, do not check for omitted override + # those will be flagged as error (check_required_vars) + # only indicate if there is a possible security concern + # note that the defaults of required variables are not define in those variables + # (ie: __DEFAULT__MAGPIE_ADMIN_PASSWORD exists, but MAGPIE_ADMIN_PASSWORD is not set, must have explicit override) + for i in ${VARS} + do + n="${i#\$}" + v=`eval echo "${i}" 2>/dev/null` + default="\${__DEFAULT__${n}}" + d=`eval echo "${default}" 2>/dev/null` + if [ ! -z "${d}" ]; then + if [ "${d}" = "${v}" ]; then + log WARN \ + "Required variable [${n}] employs a default recommended for override." \ + "The security of your deployment may be compromised unless it is changed. Check env.local file." + fi + fi + done + # for optional variables, warn about possibility omitted override or when defaults are employed for i in ${OPTIONAL_VARS} do v="${i}" @@ -211,6 +233,20 @@ check_optional_vars() { } +# Verify that all required variables are set, and error out otherwise with an error log message. +check_required_vars() { + for i in ${VARS} + do + v="${i}" + if [ -z "`eval "echo ${v}"`" ] + then + log ERROR "Required variable $v is not set. Check env.local file." + exit 1 + fi + done +} + + # All scripts sourcing default.env and env.local and needing to use any vars # in DELAYED_EVAL list need to call this function to actually resolve the # value of each var in DELAYED_EVAL list. @@ -296,7 +332,7 @@ read_configs() { read_env_local # for EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, need discover_env_local read_components_default_env # uses EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, sets ALL_CONF_DIRS read_env_local # again to override components default.env, need discover_env_local - check_optional_vars + check_default_vars process_delayed_eval } From 86922012149a0e2a17ceac9dd8b2c26a9e9a7e45 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 12 Jan 2024 18:00:47 -0500 Subject: [PATCH 11/34] add note about deprecated vars defaults --- birdhouse/default.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/birdhouse/default.env b/birdhouse/default.env index 45cf1c334..d538718d8 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -92,6 +92,9 @@ export __DEFAULT__POSTGRES_PAVICS_PASSWORD=postgres-qwerty export __DEFAULT__POSTGRES_MAGPIE_PASSWORD=postgres-qwerty #export __DEFAULT__GEOSERVER_ADMIN_USER=admingeo export __DEFAULT__GEOSERVER_ADMIN_PASSWORD=geoserverpass +############################################################################# +# Deprecated vars (for components in the ./deprecated-components directory) +############################################################################# export __DEFAULT__TOMCAT_NCWMS_PASSWORD=ncwmspass #export __DEFAULT__CATALOG_USERNAME=admin-catalog export __DEFAULT__CATALOG_PASSWORD=qwerty From c92e6a0040673bab78fe8d5d71f72a78a707e522 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 16 Jan 2024 11:30:13 -0500 Subject: [PATCH 12/34] jupyterhub: ensure username variables are respected everywhere --- CHANGES.md | 3 +++ birdhouse/README.rst | 2 +- birdhouse/components/README.rst | 2 +- birdhouse/components/jupyterhub/default.env | 4 ++++ birdhouse/default.env | 5 ----- birdhouse/env.local.example | 7 ++++++- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e638b003a..9fb24822c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,6 +43,9 @@ * Replace the LICENSE URL of the server node pointing at [Ouranosinc/pavics-sdi](https://github.com/Ouranosinc/pavics-sdi) instead of intended [bird-house/birdhouse-deploy](https://github.com/bird-house/birdhouse-deploy). +- Magpie: ensure that the `MAGPIE_ADMIN_USERNAME` variable is respected + * When determining the `JUPYTERHUB_ADMIN_USERS` variable + * Double check that it is being respected everywhere else [2.0.2](https://github.com/bird-house/birdhouse-deploy/tree/2.0.2) (2023-12-15) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/README.rst b/birdhouse/README.rst index 3d9f8557a..d9f53f5fe 100644 --- a/birdhouse/README.rst +++ b/birdhouse/README.rst @@ -148,7 +148,7 @@ instructions below. Manual instructions: * Go to - ``https:///magpie/ui/login`` and login with the ``admin`` user. The password should be in ``env.local``. + ``https:///magpie/ui/login`` and login with the ``MAGPIE_ADMIN_USERNAME`` user. The password should be in ``env.local``. * Then go to ``https:///magpie/ui/users/add``. diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index 80f25bf73..a56e460ac 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -304,7 +304,7 @@ birdhouse-deploy software stack and the machine that it is running on. It is hig make these routes available to anyone who does not have proper access permissions. Add existing users to the ``monitoring`` group to allow them access to the various monitoring WebUI. -This way, we do not need to share the ``admin`` user account and do not have to add them to the +This way, we do not need to share the ``MAGPIE_ADMIN_USERNAME`` user account and do not have to add them to the ``administrators`` group, which would give them too much permissions. diff --git a/birdhouse/components/jupyterhub/default.env b/birdhouse/components/jupyterhub/default.env index 6b746cf25..80eb8476d 100644 --- a/birdhouse/components/jupyterhub/default.env +++ b/birdhouse/components/jupyterhub/default.env @@ -68,9 +68,13 @@ export JUPYTERHUB_CRYPT_KEY= # JUPYTERHUB_CRYPT_KEY is set. export JUPYTERHUB_AUTHENTICATOR_REFRESH_AGE=60 +# Usernames that should be given admin access in jupyterhub +export JUPYTERHUB_ADMIN_USERS='{"${MAGPIE_ADMIN_USERNAME}"}' # python set syntax + export DELAYED_EVAL=" $DELAYED_EVAL JUPYTERHUB_USER_DATA_DIR + JUPYTERHUB_ADMIN_USERS " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here diff --git a/birdhouse/default.env b/birdhouse/default.env index d538718d8..98d7bc473 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -84,19 +84,14 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. # These values would be detected only if the instance was configured using a copy of 'env.local.example'. export __DEFAULT__MAGPIE_SECRET=itzaseekrit -#export __DEFAULT__MAGPIE_ADMIN_USERNAME=admin export __DEFAULT__MAGPIE_ADMIN_PASSWORD=qwertyqwerty! -#export __DEFAULT__POSTGRES_PAVICS_USERNAME=postgres-pavics export __DEFAULT__POSTGRES_PAVICS_PASSWORD=postgres-qwerty -#export __DEFAULT__POSTGRES_MAGPIE_USERNAME=postgres-magpie export __DEFAULT__POSTGRES_MAGPIE_PASSWORD=postgres-qwerty -#export __DEFAULT__GEOSERVER_ADMIN_USER=admingeo export __DEFAULT__GEOSERVER_ADMIN_PASSWORD=geoserverpass ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# export __DEFAULT__TOMCAT_NCWMS_PASSWORD=ncwmspass -#export __DEFAULT__CATALOG_USERNAME=admin-catalog export __DEFAULT__CATALOG_PASSWORD=qwerty export __DEFAULT__PHOENIX_PASSWORD=phoenix_pass export __DEFAULT__PHOENIX_PASSWORD_HASH=sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 02b4a5c7f..1ec1690a1 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -24,7 +24,6 @@ export MAGPIE_ADMIN_PASSWORD=qwertyqwerty! export TWITCHER_PROTECTED_PATH=/twitcher/ows/proxy export SUPPORT_EMAIL=helpdesk@example.com export CMIP5_THREDDS_ROOT=birdhouse/CMIP5/CCCMA -export JUPYTERHUB_ADMIN_USERS="{'admin'}" # python set syntax export POSTGRES_PAVICS_USERNAME=postgres-pavics export POSTGRES_PAVICS_PASSWORD=postgres-qwerty export POSTGRES_MAGPIE_USERNAME=postgres-magpie @@ -426,6 +425,12 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass #c.Spawner.pre_spawn_hook = custom_create_dir_hook #" +# Usernames that should be given admin access in jupyterhub +# By default, only the MAGPIE_ADMIN_USERNAME user is given admin access. Update this variable only if you wish +# to give additional users admin access by default. +# Note that you can also give users admin access through the jupyterhub UI. +#export JUPYTERHUB_ADMIN_USERS='{"${MAGPIE_ADMIN_USERNAME}", "othername"}' # python set syntax + # Extra PyWPS config for **all** WPS services (currently only Flyingpigeon, Finch and Raven supported). # export EXTRA_PYWPS_CONFIG=" # [logging] From 4fd9d0129a5f90cc371a8e252e8e1d4ccb97e001 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:43:43 -0500 Subject: [PATCH 13/34] ensure change is backwards compatible --- birdhouse/components/jupyterhub/default.env | 2 +- birdhouse/default.env | 5 +++++ birdhouse/env.local.example | 2 +- birdhouse/scripts/read-configs.include.sh | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/birdhouse/components/jupyterhub/default.env b/birdhouse/components/jupyterhub/default.env index 80eb8476d..f8e0cc30e 100644 --- a/birdhouse/components/jupyterhub/default.env +++ b/birdhouse/components/jupyterhub/default.env @@ -69,7 +69,7 @@ export JUPYTERHUB_CRYPT_KEY= export JUPYTERHUB_AUTHENTICATOR_REFRESH_AGE=60 # Usernames that should be given admin access in jupyterhub -export JUPYTERHUB_ADMIN_USERS='{"${MAGPIE_ADMIN_USERNAME}"}' # python set syntax +export JUPYTERHUB_ADMIN_USERS='{\"${MAGPIE_ADMIN_USERNAME}\"}' # python set syntax export DELAYED_EVAL=" $DELAYED_EVAL diff --git a/birdhouse/default.env b/birdhouse/default.env index 98d7bc473..69aa69139 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -84,14 +84,19 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. # These values would be detected only if the instance was configured using a copy of 'env.local.example'. export __DEFAULT__MAGPIE_SECRET=itzaseekrit +export __DEFAULT__MAGPIE_ADMIN_USERNAME=admin export __DEFAULT__MAGPIE_ADMIN_PASSWORD=qwertyqwerty! +export __DEFAULT__POSTGRES_PAVICS_USERNAME=postgres-pavics export __DEFAULT__POSTGRES_PAVICS_PASSWORD=postgres-qwerty +export __DEFAULT__POSTGRES_MAGPIE_USERNAME=postgres-magpie export __DEFAULT__POSTGRES_MAGPIE_PASSWORD=postgres-qwerty +export __DEFAULT__GEOSERVER_ADMIN_USER=admingeo export __DEFAULT__GEOSERVER_ADMIN_PASSWORD=geoserverpass ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# export __DEFAULT__TOMCAT_NCWMS_PASSWORD=ncwmspass +export __DEFAULT__CATALOG_USERNAME=admin-catalog export __DEFAULT__CATALOG_PASSWORD=qwerty export __DEFAULT__PHOENIX_PASSWORD=phoenix_pass export __DEFAULT__PHOENIX_PASSWORD_HASH=sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 1ec1690a1..efa584f3b 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -429,7 +429,7 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # By default, only the MAGPIE_ADMIN_USERNAME user is given admin access. Update this variable only if you wish # to give additional users admin access by default. # Note that you can also give users admin access through the jupyterhub UI. -#export JUPYTERHUB_ADMIN_USERS='{"${MAGPIE_ADMIN_USERNAME}", "othername"}' # python set syntax +#export JUPYTERHUB_ADMIN_USERS='{\"${MAGPIE_ADMIN_USERNAME}\", \"othername\"}' # python set syntax # Extra PyWPS config for **all** WPS services (currently only Flyingpigeon, Finch and Raven supported). # export EXTRA_PYWPS_CONFIG=" diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index ee24fe0f6..b49f0ba32 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -258,7 +258,8 @@ process_delayed_eval() { continue fi v="`eval "echo \\$${i}"`" - eval 'export ${i}="`eval "echo ${v}"`"' + value=`eval "echo \"${v}\""` + eval 'export ${i}="${value}"' log DEBUG "delayed eval '$(env | grep -e "^${i}=")'" ALREADY_EVALED=" $ALREADY_EVALED From 739024579b0c9ba9bd2191adb54a1f52a225f898 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:25:24 -0500 Subject: [PATCH 14/34] unittest --- tests/test_read_configs_include.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 157a7f36e..41eb4ee70 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -183,6 +183,12 @@ def test_delayed_eval_custom_value(self, read_config_include_file) -> None: assert (split_and_strip(get_command_stdout(proc))[-1] == "public.example.com - /my-data-root/jupyterhub_user_data - /my-geoserver-data") + def test_delayed_eval_quoting(self, read_config_include_file) -> None: + """Test that the delayed evaluation functions resolve quotation marks and braces properly""" + extra = {"EXTRA_TEST_VAR": "\"{'123'}\"", "DELAYED_EVAL": "$DELAYED_EVAL EXTRA_TEST_VAR"} + proc = self.run_func(read_config_include_file, extra, 'echo "${EXTRA_TEST_VAR}"') + assert split_and_strip(get_command_stdout(proc))[-1] == "{'123'}" + class TestCreateComposeConfList: default_conf_list_order: list[str] = [ From 7ea0f633c127e2e85be7a2e1c25bc97943f0d1da Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Jan 2024 10:35:29 -0500 Subject: [PATCH 15/34] adjust sourced config file to standardize across scripts --- birdhouse/scripts/check-wps-database.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/birdhouse/scripts/check-wps-database.sh b/birdhouse/scripts/check-wps-database.sh index 9fc53e647..27a8fc61f 100755 --- a/birdhouse/scripts/check-wps-database.sh +++ b/birdhouse/scripts/check-wps-database.sh @@ -3,8 +3,8 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" -if [ -f "${THIS_DIR}/logging.include.sh" ]; then - . "${THIS_DIR}/logging.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" fi function usage(){ From 2d120a1e7b5be8ec2942496e09131ffdd4dd699e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Jan 2024 10:38:38 -0500 Subject: [PATCH 16/34] remove duplicate nested include of logging utils --- birdhouse/scripts/check-wps-database.sh | 4 ++-- birdhouse/scripts/detect-user-install-in-jupyter-env | 6 ++---- birdhouse/scripts/get-components-json.include.sh | 4 ++-- birdhouse/scripts/get-services-json.include.sh | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/birdhouse/scripts/check-wps-database.sh b/birdhouse/scripts/check-wps-database.sh index 27a8fc61f..73e6a964b 100755 --- a/birdhouse/scripts/check-wps-database.sh +++ b/birdhouse/scripts/check-wps-database.sh @@ -3,8 +3,8 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" -if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then - . "${COMPOSE_DIR}/scripts/read-configs.include.sh" +if [ -f "${THIS_DIR}/read-configs.include.sh" ]; then + . "${THIS_DIR}/read-configs.include.sh" fi function usage(){ diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env index 887252e9d..5e6b60b97 100755 --- a/birdhouse/scripts/detect-user-install-in-jupyter-env +++ b/birdhouse/scripts/detect-user-install-in-jupyter-env @@ -10,10 +10,8 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" -. "$THIS_DIR/read-configs.include.sh" - -if [ -f "${THIS_DIR}/logging.include.sh" ]; then - . "${THIS_DIR}/logging.include.sh" +if [ -f "${THIS_DIR}/read-configs.include.sh" ]; then + . "${THIS_DIR}/read-configs.include.sh" fi diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index 508526cdd..1bc80def6 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -20,8 +20,8 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" -if [ -f "${THIS_DIR}/logging.include.sh" ]; then - . "${THIS_DIR}/logging.include.sh" +if [ -f "${THIS_DIR}/read-configs.include.sh" ]; then + . "${THIS_DIR}/read-configs.include.sh" fi # default value in case of error or missing definitions diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index e3164f7a2..bf28c0818 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -3,8 +3,8 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" -if [ -f "${THIS_DIR}/logging.include.sh" ]; then - . "${THIS_DIR}/logging.include.sh" +if [ -f "${THIS_DIR}/read-configs.include.sh" ]; then + . "${THIS_DIR}/read-configs.include.sh" fi # default value in case of error or missing definitions From 73140053044ad4091fc2095a3f2d1cdea799c4f7 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Jan 2024 10:54:07 -0500 Subject: [PATCH 17/34] adjust pre/post compose script to include -x command display only on debug log level --- CHANGES.md | 2 ++ birdhouse/pavics-compose.sh | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e638b003a..0798d8999 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,8 @@ provided log messages. * Replace older backtick (``` ` ```) executions by `$(...)` representation except for `eval` calls that require them for backward compatibility of `sh` on some server instances. + * Modify the `sh -x` calls to scripts listed in `COMPONENT_PRE_COMPOSE_UP` and `COMPONENT_POST_COMPOSE_UP` to employ + the `-x` flag (showing commands) only when `BIRDHOUSE_LOG_LEVEL=DEBUG`. - Defaults: * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS. diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index a24c69e20..16f309458 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -89,12 +89,17 @@ find $ALL_CONF_DIRS -name '*.template' 2>/dev/null | cat "${FILE}" | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > "${DEST}" done +SHELL_EXEC_FLAGS= +if [ "${BIRDHOUSE_LOG_LEVEL}" = "DEBUG" ]; then + SHELL_EXEC_FLAGS=-x +fi + if [ x"$1" = x"up" ]; then for adir in $ALL_CONF_DIRS; do COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" - sh -x "$COMPONENT_PRE_COMPOSE_UP" + sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP" fi done fi @@ -139,7 +144,7 @@ do COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up" if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then log INFO "Executing '$COMPONENT_POST_COMPOSE_UP'" - sh -x "$COMPONENT_POST_COMPOSE_UP" + sh ${SHELL_EXEC_FLAGS} "$COMPONENT_POST_COMPOSE_UP" fi done From 17c5a94ac0190f1e95a6ba6bfddf8f44949fb0c6 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Jan 2024 10:57:01 -0500 Subject: [PATCH 18/34] add double quotes around default values --- birdhouse/default.env | 44 ++++++++++++++++++------------------- birdhouse/env.local.example | 36 +++++++++++++++--------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/birdhouse/default.env b/birdhouse/default.env index d538718d8..fe881382a 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -47,7 +47,7 @@ export DELAYED_EVAL=" # Previous defaults are defined for backward-compatibility. # If not overridden explicitly by their non '__' prefixed variant, # a WARN message will be displayed by pavics-compose. -export __DEFAULT__SERVER_NAME=PAVICS +export __DEFAULT__SERVER_NAME="PAVICS" export __DEFAULT__SERVER_DESCRIPTION=" The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of climate analysis services served through Open Geospatial Consortium (OGC) protocols. @@ -55,17 +55,17 @@ These services include data access, processing and visualization. Both data and can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, or a custom web interface. " -export __DEFAULT__SERVER_INSTITUTION=Ouranos -export __DEFAULT__SERVER_SUBJECT=Climatology +export __DEFAULT__SERVER_INSTITUTION="Ouranos" +export __DEFAULT__SERVER_SUBJECT="Climatology" # below can be a CSV list of tags -export __DEFAULT__SERVER_TAGS=Climatology -export __DEFAULT__SERVER_DOCUMENTATION_URL=https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html -export __DEFAULT__SERVER_RELEASE_NOTES_URL=https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md -export __DEFAULT__SERVER_SUPPORT_URL=https://github.com/bird-house/birdhouse-deploy/issues +export __DEFAULT__SERVER_TAGS="Climatology" +export __DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html" +export __DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md" +export __DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues" # NOTE: # This value does not use the previously hard coded default. # Previous default pointed at the wrong repository with a mismatching LICENSE file. -export __DEFAULT__SERVER_LICENSE_URL=https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE +export __DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE" # apply overrides or fallback above defaults with delayed evaluation # see also: 'SUPPORT_EMAIL' @@ -83,23 +83,23 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' # Defaults for required variables recommended for override for security reasons. # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. # These values would be detected only if the instance was configured using a copy of 'env.local.example'. -export __DEFAULT__MAGPIE_SECRET=itzaseekrit -#export __DEFAULT__MAGPIE_ADMIN_USERNAME=admin -export __DEFAULT__MAGPIE_ADMIN_PASSWORD=qwertyqwerty! -#export __DEFAULT__POSTGRES_PAVICS_USERNAME=postgres-pavics -export __DEFAULT__POSTGRES_PAVICS_PASSWORD=postgres-qwerty -#export __DEFAULT__POSTGRES_MAGPIE_USERNAME=postgres-magpie -export __DEFAULT__POSTGRES_MAGPIE_PASSWORD=postgres-qwerty -#export __DEFAULT__GEOSERVER_ADMIN_USER=admingeo -export __DEFAULT__GEOSERVER_ADMIN_PASSWORD=geoserverpass +export __DEFAULT__MAGPIE_SECRET="itzaseekrit" +#export __DEFAULT__MAGPIE_ADMIN_USERNAME="admin" +export __DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" +#export __DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics" +export __DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty" +#export __DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie" +export __DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty" +#export __DEFAULT__GEOSERVER_ADMIN_USER="admingeo" +export __DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass" ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# -export __DEFAULT__TOMCAT_NCWMS_PASSWORD=ncwmspass -#export __DEFAULT__CATALOG_USERNAME=admin-catalog -export __DEFAULT__CATALOG_PASSWORD=qwerty -export __DEFAULT__PHOENIX_PASSWORD=phoenix_pass -export __DEFAULT__PHOENIX_PASSWORD_HASH=sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 +export __DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass" +#export __DEFAULT__CATALOG_USERNAME="admin-catalog" +export __DEFAULT__CATALOG_PASSWORD="qwerty" +export __DEFAULT__PHOENIX_PASSWORD="phoenix_pass" +export __DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234" export DEFAULT_CONF_DIRS=' ./components/proxy diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 02b4a5c7f..aa2e6baa7 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -16,21 +16,21 @@ export SSL_CERTIFICATE="/path/to/ssl/cert.pem" # *absolute* path to the nginx ssl certificate, path and key bundle export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation export DOC_URL="https://www.example.com/" # URL where /doc gets redirected -export MAGPIE_SECRET=itzaseekrit -export MAGPIE_ADMIN_USERNAME=admin +export MAGPIE_SECRET="itzaseekrit" +export MAGPIE_ADMIN_USERNAME="admin" # Magpie now requires a password length of at least 12 characters # For initial bootstrap only, change in the Magpie Web UI after initial boostrap. -export MAGPIE_ADMIN_PASSWORD=qwertyqwerty! -export TWITCHER_PROTECTED_PATH=/twitcher/ows/proxy -export SUPPORT_EMAIL=helpdesk@example.com -export CMIP5_THREDDS_ROOT=birdhouse/CMIP5/CCCMA +export MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" +export TWITCHER_PROTECTED_PATH="/twitcher/ows/proxy" +export SUPPORT_EMAIL="helpdesk@example.com" +export CMIP5_THREDDS_ROOT="birdhouse/CMIP5/CCCMA" export JUPYTERHUB_ADMIN_USERS="{'admin'}" # python set syntax -export POSTGRES_PAVICS_USERNAME=postgres-pavics -export POSTGRES_PAVICS_PASSWORD=postgres-qwerty -export POSTGRES_MAGPIE_USERNAME=postgres-magpie -export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty -export GEOSERVER_ADMIN_USER=admingeo -export GEOSERVER_ADMIN_PASSWORD=geoserverpass +export POSTGRES_PAVICS_USERNAME="postgres-pavics" +export POSTGRES_PAVICS_PASSWORD="postgres-qwerty" +export POSTGRES_MAGPIE_USERNAME="postgres-magpie" +export POSTGRES_MAGPIE_PASSWORD="postgres-qwerty" +export GEOSERVER_ADMIN_USER="admingeo" +export GEOSERVER_ADMIN_PASSWORD="geoserverpass" ############################################################################# # Optional vars @@ -573,9 +573,9 @@ export THREDDS_ADDITIONAL_CATALOG="" ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# -export TOMCAT_NCWMS_PASSWORD=ncwmspass -export CATALOG_USERNAME=admin-catalog -export CATALOG_PASSWORD=qwerty -export CATALOG_THREDDS_SERVICE=thredds -export PHOENIX_PASSWORD=phoenix_pass -export PHOENIX_PASSWORD_HASH=sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 +export TOMCAT_NCWMS_PASSWORD="ncwmspass" +export CATALOG_USERNAME="admin-catalog" +export CATALOG_PASSWORD="qwerty" +export CATALOG_THREDDS_SERVICE="thredds" +export PHOENIX_PASSWORD="phoenix_pass" +export PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234" From 58b2c4c6b924d1e484f3e826f47ba684b6d7041a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Jan 2024 11:06:26 -0500 Subject: [PATCH 19/34] revert canarie config invalid template vars values --- .../docker_configuration.py.template | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index bf7662c19..70c7a4e17 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -109,10 +109,10 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.0.4', - 'releaseTime': '2024-01-18T14:27:21Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', + 'version': '2.0.5', + 'releaseTime': '2024-01-22T04:23:33Z', + 'institution': '${SERVER_INSTITUTION}', + 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', 'category': 'Resource/Cloud Management', 'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()], @@ -142,12 +142,12 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.0.4', - 'releaseTime': '2024-01-18T14:27:21Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', + 'version': '2.0.5', + 'releaseTime': '2024-01-22T04:23:33Z', + 'institution': '${SERVER_INSTITUTION}', + 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', - 'tags': ['Climatology', 'Cloud'] + 'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()], }, 'stats': { 'method': '.*', From 3cae72816343d0ab0310d9548fe2d4dbd67c371d Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Jan 2024 10:56:39 -0500 Subject: [PATCH 20/34] replace missed logging call --- birdhouse/components/geoserver/pre-docker-compose-up | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/components/geoserver/pre-docker-compose-up b/birdhouse/components/geoserver/pre-docker-compose-up index d12752b5b..8c43a5e4d 100755 --- a/birdhouse/components/geoserver/pre-docker-compose-up +++ b/birdhouse/components/geoserver/pre-docker-compose-up @@ -9,6 +9,6 @@ if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then fi if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then - echo "${MSG_INFO}fix GeoServer data dir permission on first run only, when data dir do not exist yet." + log INFO "fix GeoServer data dir permission on first run only, when data dir do not exist yet." FIRST_RUN_ONLY=1 "$COMPOSE_DIR"/deployment/fix-geoserver-data-dir-perm fi From adb54c7b9cadd72c0fe59d78d6bcc70d0740b095 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Jan 2024 11:26:11 -0500 Subject: [PATCH 21/34] add check_default_vars to read_basic_configs_only --- birdhouse/scripts/read-configs.include.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index ee24fe0f6..6301deaf4 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -345,5 +345,6 @@ read_basic_configs_only() { discover_env_local read_default_env read_env_local # need discover_env_local + check_default_vars process_delayed_eval } From 22a161feecacc6fd636a6255005794248b52ffd7 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Jan 2024 11:35:15 -0500 Subject: [PATCH 22/34] replace check_required_vars exit by return --- birdhouse/pavics-compose.sh | 2 +- birdhouse/scripts/read-configs.include.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 16f309458..503fcbd75 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -56,7 +56,7 @@ read_configs # this sets ALL_CONF_DIRS . "${COMPOSE_DIR}/scripts/get-services-json.include.sh" . "${COMPOSE_DIR}/scripts/get-version-json.include.sh" -check_required_vars +check_required_vars || exit $? ## check fails when root access is required to access this file.. workaround possible by going through docker daemon... but # will add delay diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index b8e467c37..818801493 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -241,7 +241,7 @@ check_required_vars() { if [ -z "`eval "echo ${v}"`" ] then log ERROR "Required variable $v is not set. Check env.local file." - exit 1 + return 1 fi done } From e17649b226fdcacf9ae4ae09a8d1c902120138ce Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Jan 2024 11:44:51 -0500 Subject: [PATCH 23/34] add comments from PR review --- birdhouse/default.env | 1 + birdhouse/env.local.example | 3 +++ birdhouse/scripts/read-configs.include.sh | 9 +++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/birdhouse/default.env b/birdhouse/default.env index d664ba550..7cd30f60d 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -83,6 +83,7 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' # Defaults for required variables recommended for override for security reasons. # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. # These values would be detected only if the instance was configured using a copy of 'env.local.example'. +# Any default value change here should be synchronized with the corresponding value in 'env.local.example'. export __DEFAULT__MAGPIE_SECRET="itzaseekrit" export __DEFAULT__MAGPIE_ADMIN_USERNAME="admin" export __DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 41b70ce4e..082945334 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -4,6 +4,9 @@ # # Do NOT use environment variables in here since when pavics-compose.sh runs # inside a container, the environment vars do not have the same value. +# +# Any default value change here should be synchronized with the corresponding +# value in 'default.env' for the corresponding '__DEFAULT__{var}' definition. ############################################################################# # Override data persistence root directory diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index 818801493..5c7d83c4f 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -193,10 +193,11 @@ read_components_default_env() { # Check that all optional variables are defined with a different value than the default to emit a warning log message. # Also check that required variables do not use generic defaults to indicate possible security issues. check_default_vars() { - # for required variables, do not check for omitted override - # those will be flagged as error (check_required_vars) - # only indicate if there is a possible security concern - # note that the defaults of required variables are not define in those variables + # For required variables, do not check for omitted override, + # since those will be flagged as error anyway (see 'check_required_vars'). + # Only indicate if there is a possible security concern. + # Note that the defaults of required variables are not actually set in those variables, but + # are listed in 'env.local.example', hence why they pose a possible security concern. # (ie: __DEFAULT__MAGPIE_ADMIN_PASSWORD exists, but MAGPIE_ADMIN_PASSWORD is not set, must have explicit override) for i in ${VARS} do From bcfe94cd8413df43dd6da66bfa175ceba0c70708 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Jan 2024 11:51:46 -0500 Subject: [PATCH 24/34] move discover_compose_dir to resolve COMPOSE_DIR and reuse it to source logging utilities --- birdhouse/scripts/read-configs.include.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index 5c7d83c4f..b5580128f 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -23,21 +23,11 @@ # read_configs -THIS_FILE="$(readlink -f "$0" || realpath "$0")" -THIS_DIR="$(dirname "$THIS_FILE")" - -if [ -f "${THIS_DIR}/logging.include.sh" ]; then - . "${THIS_DIR}/logging.include.sh" -fi -if [ -f "${THIS_DIR}/scripts/logging.include.sh" ]; then - . "${THIS_DIR}/scripts/logging.include.sh" -fi - - # Derive COMPOSE_DIR from the most probable locations. # This is NOT meant to be exhaustive. # Assume the checkout is named "birdhouse-deploy", which might NOT be true. # Caller of this file can simply set COMPOSE_DIR itself, this is the safest way. +# WARNING: cannot use 'log' calls within this function until the following logging script gets resolved and sourced. discover_compose_dir() { if [ -z "$COMPOSE_DIR" ] || [ ! -e "$COMPOSE_DIR" ]; then if [ -f "./pavics-compose.sh" ]; then @@ -66,12 +56,18 @@ discover_compose_dir() { # Case of sub-subdir of sibling checkout at same level as birdhouse-deploy. COMPOSE_DIR="$(realpath "../../../birdhouse-deploy/birdhouse")" fi - log INFO "Resolved docker-compose directory: [${COMPOSE_DIR}]" export COMPOSE_DIR fi } +discover_compose_dir +if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/logging.include.sh" +fi +log INFO "Resolved docker-compose directory: [${COMPOSE_DIR}]" + + discover_env_local() { if [ -z "$BIRDHOUSE_LOCAL_ENV" ]; then BIRDHOUSE_LOCAL_ENV="$COMPOSE_DIR/env.local" From f59a1e12fb614454febd8f746dd7aada403ed364 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Jan 2024 15:38:43 -0500 Subject: [PATCH 25/34] fix missing echo -> log conversions --- birdhouse/scripts/clear-running-wps-jobs-in-db.sh | 7 +++---- birdhouse/scripts/create-magpie-users | 2 +- birdhouse/scripts/detect-user-install-in-jupyter-env | 2 +- birdhouse/scripts/sync-data | 5 +++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index bac3a4781..f93fa5e79 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -31,10 +31,9 @@ set -x docker exec $POSTGRES_CONTAINER_NAME psql -U $POSTGRES_USER $DB_NAME -c "select * from pywps_requests where percent_done > -1 and percent_done < 100.0;" set +x -echo " -${MSG_WARN}This will crash all the above requests if currently still processing - -Clear those jobs? (Ctrl-C to cancel, any keys to continue)" +log WARN \ + "This will crash all the above requests if currently still processing.\n" \ + "Clear those jobs? (Ctrl-C to cancel, any keys to continue)" read a diff --git a/birdhouse/scripts/create-magpie-users b/birdhouse/scripts/create-magpie-users index 219a60349..fd91839c2 100755 --- a/birdhouse/scripts/create-magpie-users +++ b/birdhouse/scripts/create-magpie-users @@ -102,7 +102,7 @@ fi if [ -z "$MAGPIE_CLI_IMAGE" ]; then # MAGPIE_VERSION must be provided by 'default.env', 'env.local' or directly if [ -z "${MAGPIE_VERSION}" ]; then - echo "${MSG_ERROR}Required MAGPIE_VERSION is undefined or empty." + log ERROR "Required MAGPIE_VERSION is undefined or empty." exit 1 fi MAGPIE_CLI_IMAGE="pavics/magpie:${MAGPIE_VERSION}" diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env index 5e6b60b97..c399df452 100755 --- a/birdhouse/scripts/detect-user-install-in-jupyter-env +++ b/birdhouse/scripts/detect-user-install-in-jupyter-env @@ -31,7 +31,7 @@ ls -a */.home/.local/bin set +x -echo "${MSG_INFO} +log INFO " Errors like ls: cannot access */.home/.local/lib/python*/site-packages: No such file or directory ls: cannot access */.home/.local/bin: No such file or directory diff --git a/birdhouse/scripts/sync-data b/birdhouse/scripts/sync-data index 9d94ebb62..23b0dbd41 100755 --- a/birdhouse/scripts/sync-data +++ b/birdhouse/scripts/sync-data @@ -28,7 +28,7 @@ SOURCE_HOST="$1"; shift FORCE_MODE="$1" if [ -z "$SOURCE_HOST" ]; then - echo "${MSG_ERROR}no source host provided" 1>&2 + log ERROR "no source host provided" 1>&2 exit 2 fi @@ -58,8 +58,9 @@ for item in $GEOSERVER_DATA_DIR/ $JUPYTERHUB_USER_DATA_DIR/ $MAGPIE_PERSIST_DIR/ done if [ ! x"$FORCE_MODE" = xforce ]; then - echo "${MSG_INFO}Dry-run mode, not executing '$COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm' and other permission fixup" + log INFO "Dry-run mode, not executing '$COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm' and other permission fixup" else + log INFO "Executing '$COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm' and other permission fixup" $COMPOSE_DIR/deployment/fix-geoserver-data-dir-perm docker run --rm --name fix-jupyter-data-dir-perm \ From 151c8a53e71d0ca493a18d0d38993e34f2c63f92 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Jan 2024 15:50:27 -0500 Subject: [PATCH 26/34] adjustments from PR review --- birdhouse/components/geoserver/pre-docker-compose-up | 9 ++++++--- birdhouse/scripts/clear-running-wps-jobs-in-db.sh | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/birdhouse/components/geoserver/pre-docker-compose-up b/birdhouse/components/geoserver/pre-docker-compose-up index 8c43a5e4d..50580776e 100755 --- a/birdhouse/components/geoserver/pre-docker-compose-up +++ b/birdhouse/components/geoserver/pre-docker-compose-up @@ -4,11 +4,14 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="$THIS_DIR/../.." -if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then - . "${COMPOSE_DIR}/scripts/logging.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" + + # resolve GEOSERVER_DATA_DIR + read_configs fi if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then log INFO "fix GeoServer data dir permission on first run only, when data dir do not exist yet." - FIRST_RUN_ONLY=1 "$COMPOSE_DIR"/deployment/fix-geoserver-data-dir-perm + FIRST_RUN_ONLY=1 "${COMPOSE_DIR}"/deployment/fix-geoserver-data-dir-perm fi diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index f93fa5e79..2a0e9be04 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -4,8 +4,8 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "$THIS_FILE")" COMPOSE_DIR="$(dirname "$THIS_DIR")" -if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then - . "${COMPOSE_DIR}/scripts/logging.include.sh" +if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/scripts/read-configs.include.sh" fi # eg: DB_NAME=finch From 13392c866ddeda5372a0459657597ab31d172fa0 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 26 Jan 2024 15:52:06 -0500 Subject: [PATCH 27/34] move compose list resolution and 'info' mode handling before pre-docker-compose-up script handling --- birdhouse/pavics-compose.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 503fcbd75..729a2936b 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -94,16 +94,6 @@ if [ "${BIRDHOUSE_LOG_LEVEL}" = "DEBUG" ]; then SHELL_EXEC_FLAGS=-x fi -if [ x"$1" = x"up" ]; then - for adir in $ALL_CONF_DIRS; do - COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" - if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then - log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" - sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP" - fi - done -fi - create_compose_conf_list # this sets COMPOSE_CONF_LIST log INFO "Displaying resolved compose configurations:" echo "COMPOSE_CONF_LIST=" @@ -114,6 +104,16 @@ if [ x"$1" = x"info" ]; then exit 0 fi +if [ x"$1" = x"up" ]; then + for adir in $ALL_CONF_DIRS; do + COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" + if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then + log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" + sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP" + fi + done +fi + # the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script PROXY_SECURE_PORT=443 HOSTNAME=${PAVICS_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ERR=$? From 85d291056155e8062e34ea2198557e37a71c7edc Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 20 Feb 2024 13:48:05 -0500 Subject: [PATCH 28/34] do not export __DEFAULT__ variables --- birdhouse/default.env | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/birdhouse/default.env b/birdhouse/default.env index 7cd30f60d..25c8b3e76 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -47,25 +47,25 @@ export DELAYED_EVAL=" # Previous defaults are defined for backward-compatibility. # If not overridden explicitly by their non '__' prefixed variant, # a WARN message will be displayed by pavics-compose. -export __DEFAULT__SERVER_NAME="PAVICS" -export __DEFAULT__SERVER_DESCRIPTION=" +__DEFAULT__SERVER_NAME="PAVICS" +__DEFAULT__SERVER_DESCRIPTION=" The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of climate analysis services served through Open Geospatial Consortium (OGC) protocols. These services include data access, processing and visualization. Both data and algorithms can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, or a custom web interface. " -export __DEFAULT__SERVER_INSTITUTION="Ouranos" -export __DEFAULT__SERVER_SUBJECT="Climatology" +__DEFAULT__SERVER_INSTITUTION="Ouranos" +__DEFAULT__SERVER_SUBJECT="Climatology" # below can be a CSV list of tags -export __DEFAULT__SERVER_TAGS="Climatology" -export __DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html" -export __DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md" -export __DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues" +__DEFAULT__SERVER_TAGS="Climatology" +__DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html" +__DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md" +__DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues" # NOTE: # This value does not use the previously hard coded default. # Previous default pointed at the wrong repository with a mismatching LICENSE file. -export __DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE" +__DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE" # apply overrides or fallback above defaults with delayed evaluation # see also: 'SUPPORT_EMAIL' @@ -84,23 +84,23 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. # These values would be detected only if the instance was configured using a copy of 'env.local.example'. # Any default value change here should be synchronized with the corresponding value in 'env.local.example'. -export __DEFAULT__MAGPIE_SECRET="itzaseekrit" -export __DEFAULT__MAGPIE_ADMIN_USERNAME="admin" -export __DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" -export __DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics" -export __DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty" -export __DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie" -export __DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty" -export __DEFAULT__GEOSERVER_ADMIN_USER="admingeo" -export __DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass" +__DEFAULT__MAGPIE_SECRET="itzaseekrit" +__DEFAULT__MAGPIE_ADMIN_USERNAME="admin" +__DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" +__DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics" +__DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty" +__DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie" +__DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty" +__DEFAULT__GEOSERVER_ADMIN_USER="admingeo" +__DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass" ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# -export __DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass" -export __DEFAULT__CATALOG_USERNAME="admin-catalog" -export __DEFAULT__CATALOG_PASSWORD="qwerty" -export __DEFAULT__PHOENIX_PASSWORD="phoenix_pass" -export __DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234" +__DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass" +__DEFAULT__CATALOG_USERNAME="admin-catalog" +__DEFAULT__CATALOG_PASSWORD="qwerty" +__DEFAULT__PHOENIX_PASSWORD="phoenix_pass" +__DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234" export DEFAULT_CONF_DIRS=' ./components/proxy From dbc79a3372b6f4865e466d070b70b74121740af1 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 20 Feb 2024 14:22:22 -0500 Subject: [PATCH 29/34] reuse __DEFAULT__ values for env.local.example --- birdhouse/default.env | 15 ++++++++++++--- birdhouse/env.local.example | 36 ++++++++++++++++++------------------ birdhouse/pavics-compose.sh | 1 + 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/birdhouse/default.env b/birdhouse/default.env index 25c8b3e76..aa6c40fb8 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -29,6 +29,9 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}' export DELAYED_EVAL=" $DELAYED_EVAL PAVICS_FQDN_PUBLIC + DOC_URL + SUPPORT_EMAIL + SSL_CERTIFICATE DATA_PERSIST_SHARED_ROOT WPS_OUTPUTS_DIR SERVER_NAME @@ -66,10 +69,16 @@ __DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/is # This value does not use the previously hard coded default. # Previous default pointed at the wrong repository with a mismatching LICENSE file. __DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE" +__DEFAULT__SUPPORT_EMAIL="helpdesk@example.com" +__DEFAULT__DOC_URL="https://www.example.com/" +__DEFAULT__PAVICS_FQDN="hostname.domainname" +__DEFAULT__SSL_CERTIFICATE="/path/to/ssl/cert.pem" # apply overrides or fallback above defaults with delayed evaluation -# see also: 'SUPPORT_EMAIL' -# This parameter is relevant to server details, but uses the old name for backward compatibility. +# exceptions for 'SUPPORT_EMAIL' and 'DOC_URL' using the old name for backward compatibility. +export SUPPORT_EMAIL='${__DEFAULT__SUPPORT_EMAIL}' +export DOC_URL='${__DEFAULT__DOC_URL}' +export SSL_CERTIFICATE='${__DEFAULT__SSL_CERTIFICATE}' export SERVER_NAME='${__DEFAULT__SERVER_NAME}' export SERVER_DESCRIPTION='${__DEFAULT__SERVER_DESCRIPTION}' export SERVER_INSTITUTION='${__DEFAULT__SERVER_INSTITUTION}' @@ -83,7 +92,7 @@ export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' # Defaults for required variables recommended for override for security reasons. # Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. # These values would be detected only if the instance was configured using a copy of 'env.local.example'. -# Any default value change here should be synchronized with the corresponding value in 'env.local.example'. +# Any default value should use the corresponding '${__DEFAULT__{var}}' reference in 'env.local.example'. __DEFAULT__MAGPIE_SECRET="itzaseekrit" __DEFAULT__MAGPIE_ADMIN_USERNAME="admin" __DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index d27a5ebc0..0cebdd563 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -16,23 +16,23 @@ # are "shared" between subdirectories). This means that the subdirectory structure is fixed. #export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' # otherwise use the value from 'default.env', must exist -export SSL_CERTIFICATE="/path/to/ssl/cert.pem" # *absolute* path to the nginx ssl certificate, path and key bundle -export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation -export DOC_URL="https://www.example.com/" # URL where /doc gets redirected -export MAGPIE_SECRET="itzaseekrit" -export MAGPIE_ADMIN_USERNAME="admin" +export SSL_CERTIFICATE="${__DEFAULT__SSL_CERTIFICATE}" # *absolute* path to the nginx ssl certificate, path and key bundle +export PAVICS_FQDN="${__DEFAULT__PAVICS_FQDN}" # Fully qualified domain name of this Pavics installation +export DOC_URL="${__DEFAULT__DOC_URL}" # URL where /doc gets redirected +export MAGPIE_SECRET="${__DEFAULT__MAGPIE_SECRET}" +export MAGPIE_ADMIN_USERNAME="${__DEFAULT__MAGPIE_ADMIN_USERNAME}" # Magpie now requires a password length of at least 12 characters # For initial bootstrap only, change in the Magpie Web UI after initial boostrap. -export MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" +export MAGPIE_ADMIN_PASSWORD="${__DEFAULT__MAGPIE_ADMIN_PASSWORD}" export TWITCHER_PROTECTED_PATH="/twitcher/ows/proxy" -export SUPPORT_EMAIL="helpdesk@example.com" +export SUPPORT_EMAIL="${__DEFAULT__SUPPORT_EMAIL}" export CMIP5_THREDDS_ROOT="birdhouse/CMIP5/CCCMA" -export POSTGRES_PAVICS_USERNAME="postgres-pavics" -export POSTGRES_PAVICS_PASSWORD="postgres-qwerty" -export POSTGRES_MAGPIE_USERNAME="postgres-magpie" -export POSTGRES_MAGPIE_PASSWORD="postgres-qwerty" -export GEOSERVER_ADMIN_USER="admingeo" -export GEOSERVER_ADMIN_PASSWORD="geoserverpass" +export POSTGRES_PAVICS_USERNAME="${__DEFAULT__POSTGRES_PAVICS_USERNAME}" +export POSTGRES_PAVICS_PASSWORD="${__DEFAULT__POSTGRES_PAVICS_PASSWORD}" +export POSTGRES_MAGPIE_USERNAME="${__DEFAULT__POSTGRES_MAGPIE_USERNAME}" +export POSTGRES_MAGPIE_PASSWORD="${__DEFAULT__POSTGRES_MAGPIE_PASSWORD}" +export GEOSERVER_ADMIN_USER="${__DEFAULT__GEOSERVER_ADMIN_USER}" +export GEOSERVER_ADMIN_PASSWORD="${__DEFAULT__GEOSERVER_ADMIN_PASSWORD}" ############################################################################# # Optional vars @@ -584,9 +584,9 @@ export THREDDS_ADDITIONAL_CATALOG="" ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# -export TOMCAT_NCWMS_PASSWORD="ncwmspass" -export CATALOG_USERNAME="admin-catalog" -export CATALOG_PASSWORD="qwerty" +export TOMCAT_NCWMS_PASSWORD="${__DEFAULT__TOMCAT_NCWMS_PASSWORD}" +export CATALOG_USERNAME="${__DEFAULT__CATALOG_USERNAME}" +export CATALOG_PASSWORD="${__DEFAULT__CATALOG_PASSWORD}" export CATALOG_THREDDS_SERVICE="thredds" -export PHOENIX_PASSWORD="phoenix_pass" -export PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234" +export PHOENIX_PASSWORD="${__DEFAULT__PHOENIX_PASSWORD}" +export PHOENIX_PASSWORD_HASH="${__DEFAULT__PHOENIX_PASSWORD_HASH}" diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 729a2936b..61e68f5df 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -27,6 +27,7 @@ VARS=' # when the value provided explicitly, it will be used instead of guessing it by inferred values from other variables OPTIONAL_VARS=' $PAVICS_FQDN_PUBLIC + $SSL_CERTIFICATE $EXTRA_PYWPS_CONFIG $SERVER_NAME $SERVER_DESCRIPTION From 442d2f2a5cca646c2910e465b885f19fa4aad8d0 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 20 Feb 2024 17:14:57 -0500 Subject: [PATCH 30/34] add COMPOSE_DIR check if provided explicitly --- birdhouse/scripts/read-configs.include.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/birdhouse/scripts/read-configs.include.sh b/birdhouse/scripts/read-configs.include.sh index b5580128f..de5fb31a6 100644 --- a/birdhouse/scripts/read-configs.include.sh +++ b/birdhouse/scripts/read-configs.include.sh @@ -58,13 +58,21 @@ discover_compose_dir() { fi export COMPOSE_DIR fi + # Perform last-chance validation in case 'COMPOSE_DIR' was incorrectly set explicitly + # and that 'read-configs.include.sh' was sourced directly from an invalid location. + if [ ! -f "${COMPOSE_DIR}/pavics-compose.sh" ]; then + echo \ + "CRITICAL: [${COMPOSE_DIR}/pavics-compose.sh] not found," \ + "please set variable 'COMPOSE_DIR' to a valid location." \ + "Many features depend on this variable." 1>&2 + return 2 + fi } -discover_compose_dir -if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then - . "${COMPOSE_DIR}/scripts/logging.include.sh" -fi +# error out appropriately without closing shell according to 'sh