Weaver: add option to automatically unregister old providers (#423)

## Overview

Introduces the `WEAVER_UNREGISTER_DROPPED_PROVIDERS` variable. If set to
"True", Weaver providers that are no longer working (not responding when
deployed) and are not named in `WEAVER_WPS_PROVIDERS` will be
unregistered. This is useful when deploying Weaver with fewer providers
than a previous deployment.

For example, if the stack is deployed with the Weaver, Finch, and Raven
components. Then later deployed with just Weaver and Raven, the Finch
provider will be unregistered from weaver.

Previously, the Finch provider would have remained as a Weaver provider
despite the fact that it has been removed from the stack.

## Changes

**Non-breaking changes**
- Adds an additional config option to unregister old weaver providers 

**Breaking changes**
None

## Related Issue / Discussion

## Additional Information

Links to other issues or sources.


<!--
The test suite can be run using a different DACCS config with
``birdhouse_daccs_configs_branch: branch_name`` in the PR description.
To globally skip the test suite regardless of the commit message use
``birdhouse_skip_ci: true`` in the PR description.
-->

birdhouse_daccs_configs_branch: master
birdhouse_skip_ci: false

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.5
+current_version = 2.0.6
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2024-01-22T04:23:33Z
+search = {current_version} 2024-02-15T16:29:01Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2024-01-22T04:23:33Z
+values = 2024-02-15T16:29:01Z
 
 [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -17,6 +17,23 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[2.0.6](https://github.com/bird-house/birdhouse-deploy/tree/2.0.6) (2024-02-15)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Weaver: add option to automatically unregister old providers
+
+  Introduces the `WEAVER_UNREGISTER_DROPPED_PROVIDERS` variable. If set to "True", Weaver providers that are no longer 
+  working (not responding when deployed) and are not named in `WEAVER_WPS_PROVIDERS` will be unregistered. This is  
+  useful when deploying Weaver with fewer providers than a previous deployment.
+
+  For example, if the stack is deployed with the Weaver, Finch, and Raven components. Then later deployed with just
+  Weaver and Raven, the Finch provider will be unregistered from weaver.
+
+  Previously, the Finch provider would have remained as a Weaver provider despite the fact that it has been removed from
+  the stack.
+
 [2.0.5](https://github.com/bird-house/birdhouse-deploy/tree/2.0.5) (2024-01-22)
 ------------------------------------------------------------------------------------------------------------------
 

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.0.5
+override APP_VERSION := 2.0.6
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.0.5.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.0.6.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.0.5...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.0.6...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.0.5-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.0.6-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.0.5
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.0.6
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-2.0.5 2024-01-22T04:23:33Z
+2.0.6 2024-02-15T16:29:01Z

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -109,8 +109,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.0.5',
-            'releaseTime': '2024-01-22T04:23:33Z',
+            'version': '2.0.6',
+            'releaseTime': '2024-02-15T16:29:01Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.0.5',
-            'releaseTime': '2024-01-22T04:23:33Z',
+            'version': '2.0.6',
+            'releaseTime': '2024-02-15T16:29:01Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',

birdhouse/components/weaver/default.env

@@ -98,6 +98,10 @@ export WEAVER_WPS_PROVIDERS_RETRY_AFTER=5
 
 export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
 
+# If "True", Weaver providers that are no longer working (not responding when deployed) and are not named in
+# WEAVER_WPS_PROVIDERS will be unregistered. This is useful when deploying Weaver with fewer providers than a previous
+# deployment.
+export WEAVER_UNREGISTER_DROPPED_PROVIDERS="False"
 
 export DELAYED_EVAL="
   $DELAYED_EVAL

birdhouse/components/weaver/post-docker-compose-up

@@ -333,6 +333,37 @@ for prov in ${WEAVER_WPS_PROVIDERS}; do
 done
 echo "${PREFIX}All Weaver remote WPS providers registered successfully!"
 
+if [ x"${WEAVER_UNREGISTER_DROPPED_PROVIDERS}" = x"True" ]; then
+    # Get all registered providers whether they are working or not
+    all_providers_resp=$( \
+        curl_cmd --insecure --silent --location \
+            -m ${REQUEST_TIMEOUT} \
+            -b "${cookie}" \
+            "${WEAVER_URL}/providers?check=false&detail=false" \
+    )
+    # Get all registered working providers
+    working_providers_resp=$( \
+        curl_cmd --insecure --silent --location \
+            -m ${REQUEST_TIMEOUT} \
+            -b "${cookie}" \
+            "${WEAVER_URL}/providers?check=true&detail=false" \
+    )
+    working_providers=$(echo "$working_providers_resp" | tr '\n' ' ' | \
+                        sed -e 's/.*"providers":\[//' -e 's/\].*//' -e 's/[",]/ /g')
+    for prov in $(echo "$all_providers_resp" | sed -e 's/.*"providers":\[//' -e 's/\].*//' -e 's/[",]/ /g'); do
+        # Note: both WEAVER_WPS_PROVIDERS and working_providers are whitespace delimited with no newlines
+        if echo " ${WEAVER_WPS_PROVIDERS} " | grep -qv "[[:space:]]${prov}[[:space:]]" && \
+           echo " ${working_providers} " | grep -qv "[[:space:]]${prov}[[:space:]]"; then
+            # unregister provider that is no longer specified in WEAVER_WPS_PROVIDERS and is no longer working
+            echo "${PREFIX}Unregistering the remote WPS provider matching [${prov}] not in WEAVER_WPS_PROVIDERS."
+            curl_cmd --insecure --silent --location \
+                 -m ${REQUEST_TIMEOUT} \
+                 -b "${cookie}" \
+                 -X DELETE \
+                 "${WEAVER_URL}/providers/${prov}"
+            fi
+    done
+fi
 
 echo "${PREFIX}Starting Weaver WebApp/Worker Celery tasks validation..."
 CUR_SCRIPT_DIR="$(dirname "$(realpath "$0")")"

birdhouse/env.local.example

@@ -542,6 +542,11 @@ export THREDDS_ADDITIONAL_CATALOG=""
 # (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
 #export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
 
+# If "True", Weaver providers that are no longer working (not responding when deployed) and are not named in
+# WEAVER_WPS_PROVIDERS will be unregistered. This is useful when deploying Weaver with fewer providers than a previous
+# deployment.
+#export WEAVER_UNREGISTER_DROPPED_PROVIDERS="True"
+
 # If "True", requests to the geoserver endpoint will not be authorized through twitcher/magpie
 # (note: this is NOT recommended but will slightly improve performance when accessing geoserver endpoints)
 #export GEOSERVER_SKIP_AUTH=True

docs/source/conf.py

@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.0.5'
+version = '2.0.6'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.5'
+release = '2.0.6'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.