Have a separate cache for the OGC servers

Pipfile

@@ -87,11 +87,12 @@ typed-ast = "==1.4.3"
 typing = "==3.7.4.3"
 typing-extensions = "==3.10.0.0"
 unidecode = "==1.2.0"
-urllib3 = "==1.25.9"
+urllib3 = "==1.25.11"
 waitress = "==2.1.1"
 webob = "==1.8.6"
 wrapt = "==1.11.2"
 zipp = "==3.4.1"
+responses = "==0.21.0"
 
 [packages]
 alembic = "==1.4.2"  # geoportal

admin/c2cgeoportal_admin/views/ogc_servers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2017-2020, Camptocamp SA
+# Copyright (c) 2017-2022, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -29,19 +29,28 @@
 
 
 from functools import partial
+import logging
+import threading
+from typing import Any, Dict, List, cast
 
 from c2cgeoform.schema import GeoFormSchemaNode
-from c2cgeoform.views.abstract_views import AbstractViews, ListField
+from c2cgeoform.views.abstract_views import AbstractViews, ItemAction, ListField
 from deform.widget import FormWidget
 from pyramid.view import view_config, view_defaults
+import requests
+from sqlalchemy import inspect
 
+from c2cgeoportal_admin import _
+from c2cgeoportal_commons.models import cache_invalidate_cb
 from c2cgeoportal_commons.models.main import OGCServer
 
 _list_field = partial(ListField, OGCServer)
 
 base_schema = GeoFormSchemaNode(OGCServer, widget=FormWidget(fields_template="ogcserver_fields"))
 base_schema.add_unique_validator(OGCServer.name, OGCServer.id)
 
+LOG = logging.getLogger(__name__)
+
 
 @view_defaults(match_param="table=ogc_servers")
 class OGCServerViews(AbstractViews):
@@ -69,20 +78,68 @@ def index(self):
     def grid(self):
         return super().grid()
 
+    def _item_actions(self, item: OGCServer, readonly: bool = False) -> List[Any]:
+        actions = cast(List[Any], super()._item_actions(item, readonly))
+        if inspect(item).persistent:
+            actions.insert(
+                next((i for i, v in enumerate(actions) if v.name() == "delete")),
+                ItemAction(
+                    name="clear-cache",
+                    label=_("Clear the cache"),
+                    icon="glyphicon glyphicon-hdd",
+                    url=self._request.route_url(
+                        "ogc_server_clear_cache",
+                        id=getattr(item, self._id_field),
+                        _query={
+                            "came_from": self._request.current_route_url(),
+                        },
+                    ),
+                    confirmation=_("The current changes will be lost."),
+                ),
+            )
+        return actions
+
     @view_config(route_name="c2cgeoform_item", request_method="GET", renderer="../templates/edit.jinja2")
     def view(self):
         return super().edit()
 
     @view_config(route_name="c2cgeoform_item", request_method="POST", renderer="../templates/edit.jinja2")
     def save(self):
-        return super().save()
+        result: Dict[str, Any] = super().save()
+        self._update_cache(self._get_object())
+        return result
 
     @view_config(route_name="c2cgeoform_item", request_method="DELETE", renderer="fast_json")
     def delete(self):
-        return super().delete()
+        result: Dict[str, Any] = super().delete()
+        cache_invalidate_cb()
+        return result
 
     @view_config(
         route_name="c2cgeoform_item_duplicate", request_method="GET", renderer="../templates/edit.jinja2"
     )
     def duplicate(self):
-        return super().duplicate()
+        result: Dict[str, Any] = super().duplicate()
+        self._update_cache(self._get_object())
+        return result
+
+    def _update_cache(self, ogc_server: OGCServer) -> None:
+        try:
+            ogc_server_id = ogc_server.id
+
+            def update_cache() -> None:
+                response = requests.get(
+                    self._request.route_url(
+                        "ogc_server_clear_cache",
+                        id=ogc_server_id,
+                        _query={
+                            "came_from": self._request.current_route_url(),
+                        },
+                    )
+                )
+                if not response.ok:
+                    LOG.error("Error while cleaning the OGC server cache:\n%s", response.text)
+
+            threading.Thread(target=update_cache).start()
+        except Exception:
+            LOG.error("Error on cleaning the OGC server cache", exc_info=True)

admin/tests/conftest.py

@@ -59,6 +59,8 @@ def app(app_env, dbsession):
     config.add_request_method(lambda request: dbsession, "dbsession", reify=True)
     config.add_route("user_add", "user_add")
     config.add_route("users_nb", "users_nb")
+    config.add_route("base", "/", static=True)
+    config.add_route("ogc_server_clear_cache", "/ogc_server_clear_cache/{id}", static=True)
     config.scan(package="tests")
     app = config.make_wsgi_app()
     yield app

checks.mk

@@ -15,7 +15,7 @@ prospector:
 	prospector --version
 	mypy --version
 	pylint --version --rcfile=/dev/null
-	prospector
+	prospector --output=pylint
 
 .PHONY: bandit
 bandit:

commons/c2cgeoportal_commons/models/main.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2011-2020, Camptocamp SA
+# Copyright (c) 2011-2022, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -535,11 +535,6 @@ def __str__(self) -> str:
         return self.name or ""  # pragma: no cover
 
 
-event.listen(OGCServer, "after_insert", cache_invalidate_cb, propagate=True)
-event.listen(OGCServer, "after_update", cache_invalidate_cb, propagate=True)
-event.listen(OGCServer, "after_delete", cache_invalidate_cb, propagate=True)
-
-
 class LayerWMS(DimensionLayer):
     __tablename__ = "layer_wms"
     __table_args__ = {"schema": _schema}

geoportal/c2cgeoportal_geoportal/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2011-2021, Camptocamp SA
+# Copyright (c) 2011-2022, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -126,7 +126,9 @@ def add_interface_ngeo(config, route_name, route, renderer=None, permission=None
 def add_admin_interface(config):
     if config.get_settings().get("enable_admin_interface", False):
         config.add_request_method(
-            lambda request: c2cgeoportal_commons.models.DBSession(), "dbsession", reify=True,
+            lambda request: c2cgeoportal_commons.models.DBSession(),
+            "dbsession",
+            reify=True,
         )
         config.add_view(c2cgeoportal_geoportal.views.add_ending_slash, route_name="admin_add_ending_slash")
         config.add_route("admin_add_ending_slash", "/admin", request_method="GET")
@@ -184,7 +186,7 @@ def is_valid_referer(request, settings=None):
 
 def create_get_user_from_request(settings):
     def get_user_from_request(request, username=None):
-        """ Return the User object for the request.
+        """Return the User object for the request.
 
         Return ``None`` if:
         * user is anonymous
@@ -244,7 +246,7 @@ def get_user_from_request(request, username=None):
 
 
 def set_user_validator(config, user_validator):
-    """ Call this function to register a user validator function.
+    """Call this function to register a user validator function.
 
     The validator function is passed three arguments: ``request``,
     ``username``, and ``password``. The function should return the
@@ -287,7 +289,7 @@ def default_user_validator(request, username, password):
 
 
 class MapserverproxyRoutePredicate:
-    """ Serve as a custom route predicate function for mapserverproxy.
+    """Serve as a custom route predicate function for mapserverproxy.
     If the hide_capabilities setting is set and is true then we want to
     return 404s on GetCapabilities requests."""
 
@@ -386,13 +388,14 @@ def includeme(config: pyramid.config.Configurator):
         for name, cache_config in settings["cache"].items():
             caching.init_region(cache_config, name)
 
-            @zope.event.classhandler.handler(InvalidateCacheEvent)
-            def handle(event: InvalidateCacheEvent):  # pylint: disable=unused-variable
-                del event
-                caching.invalidate_region()
-                if caching.MEMORY_CACHE_DICT:
-                    caching.get_region("std").delete_multi(caching.MEMORY_CACHE_DICT.keys())
-                caching.MEMORY_CACHE_DICT.clear()
+        @zope.event.classhandler.handler(InvalidateCacheEvent)
+        def handle(event: InvalidateCacheEvent):  # pylint: disable=unused-variable
+            del event
+            caching.invalidate_region("std")
+            caching.invalidate_region("obj")
+            if caching.MEMORY_CACHE_DICT:
+                caching.get_region("std").delete_multi(caching.MEMORY_CACHE_DICT.keys())
+            caching.MEMORY_CACHE_DICT.clear()
 
     # Register a tween to get back the cache buster path.
     if "cache_path" not in config.get_settings():
@@ -556,6 +559,8 @@ def add_static_route(name: str, attr: str, path: str, renderer: str):
     # Used memory in caches
     config.add_route("memory", "/memory", request_method="GET")
 
+    config.add_route("ogc_server_clear_cache", "clear-ogc-server-cache/{id}", request_method="GET")
+
     # Scan view decorator for adding routes
     config.scan(
         ignore=[

geoportal/c2cgeoportal_geoportal/lib/caching.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2012-2020, Camptocamp SA
+# Copyright (c) 2012-2022, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -35,9 +35,12 @@
 from dogpile.cache.api import NO_VALUE
 from dogpile.cache.backends.redis import RedisBackend
 from dogpile.cache.region import make_region
-from dogpile.cache.util import compat, sha1_mangle_key
-from pyramid.request import Request
+from dogpile.cache.util import sha1_mangle_key
+import pyramid.interfaces
+import pyramid.request
+import pyramid.response
 from sqlalchemy.orm.util import identity_key
+import zope.interface
 
 from c2cgeoportal_commons.models import Base
 
@@ -76,8 +79,10 @@ def generate_key(*args, **kw):
         parts.extend(namespace)
         if ignore_first_argument:
             args = args[1:]
-        new_args: List[str] = [arg for arg in args if not isinstance(arg, Request)]
-        parts.extend(map(compat.text_type, map(map_dbobject, new_args)))
+        new_args: List[str] = [
+            arg for arg in args if pyramid.interfaces.IRequest not in zope.interface.implementedBy(type(arg))
+        ]
+        parts.extend(map(str, map(map_dbobject, new_args)))
         return "|".join(parts)
 
     return generate_key
@@ -94,11 +99,10 @@ def init_region(conf, region):
 
 def _configure_region(conf, cache_region):
     global MEMORY_CACHE_DICT
-    kwargs = {"replace_existing_backend": True}
+    kwargs: Dict[str, Any] = {"replace_existing_backend": True}
     backend = conf["backend"]
     kwargs.update({k: conf[k] for k in conf if k != "backend"})
-    kwargs.setdefault("arguments", {})  # type: ignore
-    kwargs["arguments"]["cache_dict"] = MEMORY_CACHE_DICT  # type: ignore
+    kwargs.setdefault("arguments", {}).setdefault("cache_dict", MEMORY_CACHE_DICT)
     cache_region.configure(backend, **kwargs)
 
 

geoportal/c2cgeoportal_geoportal/scaffolds/update/geoportal/CONST_vars.yaml_tmpl

@@ -233,7 +233,7 @@ vars:
   cache:
     std:
       backend: c2cgeoportal.hybrid
-      arguments:
+      arguments: &redis-cache-arguments
         host: '{REDIS_HOST}'
         port: '{REDIS_PORT}'
         db: '{REDIS_DB}'
@@ -242,6 +242,9 @@ vars:
         distributed_lock: True
     obj:
       backend: dogpile.cache.memory
+    ogc-server:
+      backend: dogpile.cache.redis
+      arguments: *redis-cache-arguments
 
   admin_interface: