From 22e8ae6f9f23896cc9bc8084f7aa1779677680e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Wed, 6 Jul 2022 09:51:28 +0200 Subject: [PATCH] Have a separate cache for the OGC servers --- Pipfile | 3 +- Pipfile.lock | 10 +- admin/c2cgeoportal_admin/views/ogc_servers.py | 67 ++- admin/tests/conftest.py | 2 + checks.mk | 2 +- commons/c2cgeoportal_commons/models/main.py | 7 +- geoportal/c2cgeoportal_geoportal/__init__.py | 29 +- .../c2cgeoportal_geoportal/lib/caching.py | 20 +- .../update/geoportal/CONST_vars.yaml_tmpl | 5 +- .../c2cgeoportal_geoportal/views/theme.py | 411 +++++++++++------- geoportal/tests/__init__.py | 1 + geoportal/tests/config.yaml | 2 + geoportal/tests/functional/__init__.py | 1 + .../tests/functional/test_dbreflection.py | 1 + .../tests/functional/test_dynamicview.py | 1 + .../tests/functional/test_themes_entry.py | 22 +- .../tests/functional/test_themes_loop.py | 4 +- .../tests/functional/test_themes_mixed.py | 2 +- .../test_themes_ogc_server_cache_clean.py | 319 ++++++++++++++ 19 files changed, 694 insertions(+), 215 deletions(-) create mode 100644 geoportal/tests/functional/test_themes_ogc_server_cache_clean.py diff --git a/Pipfile b/Pipfile index d774d6bf56..317922b4ad 100644 --- a/Pipfile +++ b/Pipfile @@ -87,11 +87,12 @@ typed-ast = "==1.4.3" typing = "==3.7.4.3" typing-extensions = "==3.10.0.0" unidecode = "==1.2.0" -urllib3 = "==1.25.9" +urllib3 = "==1.25.11" waitress = "==2.1.1" webob = "==1.8.6" wrapt = "==1.11.2" zipp = "==3.4.1" +responses = "==0.21.0" [packages] alembic = "==1.4.2" # geoportal diff --git a/Pipfile.lock b/Pipfile.lock index 86b8923959..e31e4ac2b3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "47354fd596ddf4575682a136aa07da2c25d9a8389ec81b31053ee1b38ed19f47" + "sha256": "d051b58ea05ff21aeb5e624c99e2a907a732828f66d185c30a3478f958a9b4f0" }, "pipfile-spec": 6, "requires": { @@ -1624,6 +1624,14 @@ "index": "pypi", "version": "==0.7" }, + "responses": { + "hashes": [ + "sha256:2dcc863ba63963c0c3d9ee3fa9507cbe36b7d7b0fccb4f0bdfd9e96c539b1487", + "sha256:b82502eb5f09a0289d8e209e7bad71ef3978334f56d09b444253d5ad67bf5253" + ], + "index": "pypi", + "version": "==0.21.0" + }, "rsa": { "hashes": [ "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5", diff --git a/admin/c2cgeoportal_admin/views/ogc_servers.py b/admin/c2cgeoportal_admin/views/ogc_servers.py index 06d4bc49f0..d1d5ab8fdf 100644 --- a/admin/c2cgeoportal_admin/views/ogc_servers.py +++ b/admin/c2cgeoportal_admin/views/ogc_servers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2017-2020, Camptocamp SA +# Copyright (c) 2017-2022, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -29,12 +29,19 @@ from functools import partial +import logging +import threading +from typing import Any, Dict, List, cast from c2cgeoform.schema import GeoFormSchemaNode -from c2cgeoform.views.abstract_views import AbstractViews, ListField +from c2cgeoform.views.abstract_views import AbstractViews, ItemAction, ListField from deform.widget import FormWidget from pyramid.view import view_config, view_defaults +import requests +from sqlalchemy import inspect +from c2cgeoportal_admin import _ +from c2cgeoportal_commons.models import cache_invalidate_cb from c2cgeoportal_commons.models.main import OGCServer _list_field = partial(ListField, OGCServer) @@ -42,6 +49,8 @@ base_schema = GeoFormSchemaNode(OGCServer, widget=FormWidget(fields_template="ogcserver_fields")) base_schema.add_unique_validator(OGCServer.name, OGCServer.id) +LOG = logging.getLogger(__name__) + @view_defaults(match_param="table=ogc_servers") class OGCServerViews(AbstractViews): @@ -69,20 +78,68 @@ def index(self): def grid(self): return super().grid() + def _item_actions(self, item: OGCServer, readonly: bool = False) -> List[Any]: + actions = cast(List[Any], super()._item_actions(item, readonly)) + if inspect(item).persistent: + actions.insert( + next((i for i, v in enumerate(actions) if v.name() == "delete")), + ItemAction( + name="clear-cache", + label=_("Clear the cache"), + icon="glyphicon glyphicon-hdd", + url=self._request.route_url( + "ogc_server_clear_cache", + id=getattr(item, self._id_field), + _query={ + "came_from": self._request.current_route_url(), + }, + ), + confirmation=_("The current changes will be lost."), + ), + ) + return actions + @view_config(route_name="c2cgeoform_item", request_method="GET", renderer="../templates/edit.jinja2") def view(self): return super().edit() @view_config(route_name="c2cgeoform_item", request_method="POST", renderer="../templates/edit.jinja2") def save(self): - return super().save() + result: Dict[str, Any] = super().save() + self._update_cache(self._get_object()) + return result @view_config(route_name="c2cgeoform_item", request_method="DELETE", renderer="fast_json") def delete(self): - return super().delete() + result: Dict[str, Any] = super().delete() + cache_invalidate_cb() + return result @view_config( route_name="c2cgeoform_item_duplicate", request_method="GET", renderer="../templates/edit.jinja2" ) def duplicate(self): - return super().duplicate() + result: Dict[str, Any] = super().duplicate() + self._update_cache(self._get_object()) + return result + + def _update_cache(self, ogc_server: OGCServer) -> None: + try: + ogc_server_id = ogc_server.id + + def update_cache() -> None: + response = requests.get( + self._request.route_url( + "ogc_server_clear_cache", + id=ogc_server_id, + _query={ + "came_from": self._request.current_route_url(), + }, + ) + ) + if not response.ok: + LOG.error("Error while cleaning the OGC server cache:\n%s", response.text) + + threading.Thread(target=update_cache).start() + except Exception: + LOG.error("Error on cleaning the OGC server cache", exc_info=True) diff --git a/admin/tests/conftest.py b/admin/tests/conftest.py index eaf65500f8..ab4e758e9f 100644 --- a/admin/tests/conftest.py +++ b/admin/tests/conftest.py @@ -59,6 +59,8 @@ def app(app_env, dbsession): config.add_request_method(lambda request: dbsession, "dbsession", reify=True) config.add_route("user_add", "user_add") config.add_route("users_nb", "users_nb") + config.add_route("base", "/", static=True) + config.add_route("ogc_server_clear_cache", "/ogc_server_clear_cache/{id}", static=True) config.scan(package="tests") app = config.make_wsgi_app() yield app diff --git a/checks.mk b/checks.mk index 905c838f56..467d90404d 100644 --- a/checks.mk +++ b/checks.mk @@ -15,7 +15,7 @@ prospector: prospector --version mypy --version pylint --version --rcfile=/dev/null - prospector + prospector --output=pylint .PHONY: bandit bandit: diff --git a/commons/c2cgeoportal_commons/models/main.py b/commons/c2cgeoportal_commons/models/main.py index 2d5dcb59c9..1c2d55e9c8 100644 --- a/commons/c2cgeoportal_commons/models/main.py +++ b/commons/c2cgeoportal_commons/models/main.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2011-2020, Camptocamp SA +# Copyright (c) 2011-2022, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -535,11 +535,6 @@ def __str__(self) -> str: return self.name or "" # pragma: no cover -event.listen(OGCServer, "after_insert", cache_invalidate_cb, propagate=True) -event.listen(OGCServer, "after_update", cache_invalidate_cb, propagate=True) -event.listen(OGCServer, "after_delete", cache_invalidate_cb, propagate=True) - - class LayerWMS(DimensionLayer): __tablename__ = "layer_wms" __table_args__ = {"schema": _schema} diff --git a/geoportal/c2cgeoportal_geoportal/__init__.py b/geoportal/c2cgeoportal_geoportal/__init__.py index eaaeaba388..cbdccb55be 100644 --- a/geoportal/c2cgeoportal_geoportal/__init__.py +++ b/geoportal/c2cgeoportal_geoportal/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2011-2021, Camptocamp SA +# Copyright (c) 2011-2022, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -126,7 +126,9 @@ def add_interface_ngeo(config, route_name, route, renderer=None, permission=None def add_admin_interface(config): if config.get_settings().get("enable_admin_interface", False): config.add_request_method( - lambda request: c2cgeoportal_commons.models.DBSession(), "dbsession", reify=True, + lambda request: c2cgeoportal_commons.models.DBSession(), + "dbsession", + reify=True, ) config.add_view(c2cgeoportal_geoportal.views.add_ending_slash, route_name="admin_add_ending_slash") config.add_route("admin_add_ending_slash", "/admin", request_method="GET") @@ -184,7 +186,7 @@ def is_valid_referer(request, settings=None): def create_get_user_from_request(settings): def get_user_from_request(request, username=None): - """ Return the User object for the request. + """Return the User object for the request. Return ``None`` if: * user is anonymous @@ -244,7 +246,7 @@ def get_user_from_request(request, username=None): def set_user_validator(config, user_validator): - """ Call this function to register a user validator function. + """Call this function to register a user validator function. The validator function is passed three arguments: ``request``, ``username``, and ``password``. The function should return the @@ -287,7 +289,7 @@ def default_user_validator(request, username, password): class MapserverproxyRoutePredicate: - """ Serve as a custom route predicate function for mapserverproxy. + """Serve as a custom route predicate function for mapserverproxy. If the hide_capabilities setting is set and is true then we want to return 404s on GetCapabilities requests.""" @@ -386,13 +388,14 @@ def includeme(config: pyramid.config.Configurator): for name, cache_config in settings["cache"].items(): caching.init_region(cache_config, name) - @zope.event.classhandler.handler(InvalidateCacheEvent) - def handle(event: InvalidateCacheEvent): # pylint: disable=unused-variable - del event - caching.invalidate_region() - if caching.MEMORY_CACHE_DICT: - caching.get_region("std").delete_multi(caching.MEMORY_CACHE_DICT.keys()) - caching.MEMORY_CACHE_DICT.clear() + @zope.event.classhandler.handler(InvalidateCacheEvent) + def handle(event: InvalidateCacheEvent): # pylint: disable=unused-variable + del event + caching.invalidate_region("std") + caching.invalidate_region("obj") + if caching.MEMORY_CACHE_DICT: + caching.get_region("std").delete_multi(caching.MEMORY_CACHE_DICT.keys()) + caching.MEMORY_CACHE_DICT.clear() # Register a tween to get back the cache buster path. if "cache_path" not in config.get_settings(): @@ -556,6 +559,8 @@ def add_static_route(name: str, attr: str, path: str, renderer: str): # Used memory in caches config.add_route("memory", "/memory", request_method="GET") + config.add_route("ogc_server_clear_cache", "clear-ogc-server-cache/{id}", request_method="GET") + # Scan view decorator for adding routes config.scan( ignore=[ diff --git a/geoportal/c2cgeoportal_geoportal/lib/caching.py b/geoportal/c2cgeoportal_geoportal/lib/caching.py index 2e0cb642fd..fb66d5153a 100644 --- a/geoportal/c2cgeoportal_geoportal/lib/caching.py +++ b/geoportal/c2cgeoportal_geoportal/lib/caching.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2012-2020, Camptocamp SA +# Copyright (c) 2012-2022, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -35,9 +35,12 @@ from dogpile.cache.api import NO_VALUE from dogpile.cache.backends.redis import RedisBackend from dogpile.cache.region import make_region -from dogpile.cache.util import compat, sha1_mangle_key -from pyramid.request import Request +from dogpile.cache.util import sha1_mangle_key +import pyramid.interfaces +import pyramid.request +import pyramid.response from sqlalchemy.orm.util import identity_key +import zope.interface from c2cgeoportal_commons.models import Base @@ -76,8 +79,10 @@ def generate_key(*args, **kw): parts.extend(namespace) if ignore_first_argument: args = args[1:] - new_args: List[str] = [arg for arg in args if not isinstance(arg, Request)] - parts.extend(map(compat.text_type, map(map_dbobject, new_args))) + new_args: List[str] = [ + arg for arg in args if pyramid.interfaces.IRequest not in zope.interface.implementedBy(type(arg)) + ] + parts.extend(map(str, map(map_dbobject, new_args))) return "|".join(parts) return generate_key @@ -94,11 +99,10 @@ def init_region(conf, region): def _configure_region(conf, cache_region): global MEMORY_CACHE_DICT - kwargs = {"replace_existing_backend": True} + kwargs: Dict[str, Any] = {"replace_existing_backend": True} backend = conf["backend"] kwargs.update({k: conf[k] for k in conf if k != "backend"}) - kwargs.setdefault("arguments", {}) # type: ignore - kwargs["arguments"]["cache_dict"] = MEMORY_CACHE_DICT # type: ignore + kwargs.setdefault("arguments", {}).setdefault("cache_dict", MEMORY_CACHE_DICT) cache_region.configure(backend, **kwargs) diff --git a/geoportal/c2cgeoportal_geoportal/scaffolds/update/geoportal/CONST_vars.yaml_tmpl b/geoportal/c2cgeoportal_geoportal/scaffolds/update/geoportal/CONST_vars.yaml_tmpl index ed5411019a..ffca3ed210 100644 --- a/geoportal/c2cgeoportal_geoportal/scaffolds/update/geoportal/CONST_vars.yaml_tmpl +++ b/geoportal/c2cgeoportal_geoportal/scaffolds/update/geoportal/CONST_vars.yaml_tmpl @@ -233,7 +233,7 @@ vars: cache: std: backend: c2cgeoportal.hybrid - arguments: + arguments: &redis-cache-arguments host: '{REDIS_HOST}' port: '{REDIS_PORT}' db: '{REDIS_DB}' @@ -242,6 +242,9 @@ vars: distributed_lock: True obj: backend: dogpile.cache.memory + ogc-server: + backend: dogpile.cache.redis + arguments: *redis-cache-arguments admin_interface: diff --git a/geoportal/c2cgeoportal_geoportal/views/theme.py b/geoportal/c2cgeoportal_geoportal/views/theme.py index d4fef45987..043227b038 100644 --- a/geoportal/c2cgeoportal_geoportal/views/theme.py +++ b/geoportal/c2cgeoportal_geoportal/views/theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2011-2021, Camptocamp SA +# Copyright (c) 2011-2022, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -35,19 +35,25 @@ import re import sys import time -from typing import Any, Dict, List, Set, Union, cast +from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast import urllib.parse from c2cwsgiutils.auth import auth_view from defusedxml import lxml +import dogpile.cache.api +from lxml import etree # nosec from owslib.wms import WebMapService +import pyramid.httpexceptions +import pyramid.request from pyramid.view import view_config import requests +import sqlalchemy from sqlalchemy.orm import subqueryload from sqlalchemy.orm.exc import NoResultFound +import sqlalchemy.orm.query from c2cgeoportal_commons import models -from c2cgeoportal_commons.models import main +from c2cgeoportal_commons.models import cache_invalidate_cb, main from c2cgeoportal_geoportal.lib import ( add_url_params, get_roles_id, @@ -64,21 +70,28 @@ get_protected_layers_query, ) from c2cgeoportal_geoportal.lib.wmstparsing import TimeInformation, parse_extent -from c2cgeoportal_geoportal.views import restrict_headers from c2cgeoportal_geoportal.views.layers import get_layer_metadatas LOG = logging.getLogger(__name__) CACHE_REGION = get_region("std") +CACHE_OGC_SERVER_REGION = get_region("ogc-server") -def get_http_cached(http_options, url, headers): - @CACHE_REGION.cache_on_arguments() - def do_get_http_cached(url): +def get_http_cached( + http_options: Dict[str, Any], url: str, headers: Dict[str, str], cache: bool = True +) -> Tuple[bytes, str]: + """Get the content of the URL with a cash (dogpile).""" + + @CACHE_OGC_SERVER_REGION.cache_on_arguments() + def do_get_http_cached(url: str) -> Tuple[bytes, str]: response = requests.get(url, headers=headers, timeout=300, **http_options) + response.raise_for_status() LOG.info("Get url '%s' in %.1fs.", url, response.elapsed.total_seconds()) - return response + return response.content, response.headers.get("Content-Type", "") - return do_get_http_cached(url) + if cache: + return do_get_http_cached(url) + return do_get_http_cached.refresh(url) class DimensionInformation: @@ -138,8 +151,6 @@ def __init__(self, request): self.request = request self.settings = request.registry.settings self.http_options = self.settings.get("http_options", {}) - self.headers_whitelist = self.settings.get("headers_whitelist", []) - self.headers_blacklist = self.settings.get("headers_blacklist", []) self.metadata_type = get_types_map( self.settings.get("admin_interface", {}).get("available_metadata", []) ) @@ -180,26 +191,24 @@ def _get_metadatas(self, item, errors): return metadatas - async def _wms_getcap(self, ogc_server, preload=False): - url, content, errors = await self._wms_getcap_cached( - ogc_server, self._get_capabilities_cache_role_key(ogc_server) - ) - - if errors or preload: - return None, errors + async def _wms_getcap( + self, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True + ) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]: + LOG.debug("Get the WMS Capabilities of %s, preload: %s, cache: %s", ogc_server.name, preload, cache) - @CACHE_REGION.cache_on_arguments() - def build_web_map_service(ogc_server_id): + @CACHE_OGC_SERVER_REGION.cache_on_arguments() + def build_web_map_service(ogc_server_id: int) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]: del ogc_server_id # Just for cache - version = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query)['VERSION'][0] + version = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query)["VERSION"][0] # type: ignore layers = {} try: wms = WebMapService(None, xml=content, version=version) except Exception as e: # pragma: no cover error = ( - "WARNING! an error '{}' occurred while trying to read the mapfile and recover the themes." - "\nURL: {}\n{}".format(e, url, content) + "WARNING! an error '{!r}' occurred while trying to read the mapfile and " + "recover the themes." + "\nURL: {}\n{}".format(e, url, None if content is None else content.decode()) ) LOG.error(error, exc_info=True) return None, {error} @@ -227,11 +236,28 @@ def build_web_map_service(ogc_server_id): return {"layers": layers}, set() - return build_web_map_service(ogc_server.id) + if cache: + result = build_web_map_service.get(ogc_server.id) + if result != dogpile.cache.api.NO_VALUE: + return result - async def _wms_getcap_cached(self, ogc_server, _): - """ _ is just for cache on the role id """ + try: + url, content, errors = await self._wms_getcap_cached(ogc_server, cache=cache) + except requests.exceptions.RequestException as exception: + error = ( + f"Unable to get the WMS Capabilities for OGC server '{ogc_server.name}', " + f"return the error: {exception.response.status_code} {exception.response.reason}" + ) + LOG.exception(error) + return None, {error} + if errors or preload: + return None, errors + + return build_web_map_service.refresh(ogc_server.id) + async def _wms_getcap_cached( + self, ogc_server: main.OGCServer, cache: bool = True + ) -> Tuple[Optional[str], Optional[bytes], Set[str]]: errors: Set[str] = set() url = get_url2("The OGC server '{}'".format(ogc_server.name), ogc_server.url, self.request, errors) if errors or url is None: # pragma: no cover @@ -257,50 +283,34 @@ async def _wms_getcap_cached(self, ogc_server, _): LOG.debug("Get WMS GetCapabilities for url: %s", url) - # Forward request to target (without Host Header) - headers = dict(self.request.headers) + headers = {} # Add headers for Geoserver if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER: headers["sec-username"] = "root" headers["sec-roles"] = "root" - if urllib.parse.urlsplit(url).hostname != "localhost" and "Host" in headers: # pragma: no cover - headers.pop("Host") - - headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist) - try: - response = await asyncio.get_event_loop().run_in_executor( - None, get_http_cached, self.http_options, url, headers - ) - except Exception: # pragma: no cover - error = "Unable to GetCapabilities from URL {}".format(url) + content, content_type = get_http_cached(self.http_options, url, headers, cache=cache) + except Exception: + error = f"Unable to GetCapabilities from URL {url}" errors.add(error) LOG.error(error, exc_info=True) return url, None, errors - if not response.ok: # pragma: no cover - error = "GetCapabilities from URL {} return the error: {:d} {}".format( - url, response.status_code, response.reason - ) - errors.add(error) - LOG.error(error) - return url, None, errors - # With wms 1.3 it returns text/xml also in case of error :-( - if response.headers.get("Content-Type", "").split(";")[0].strip() not in [ + if content_type.split(";")[0].strip() not in [ "application/vnd.ogc.wms_xml", "text/xml", ]: error = "GetCapabilities from URL {} returns a wrong Content-Type: {}\n{}".format( - url, response.headers.get("Content-Type", ""), response.text + url, content_type, content.decode() ) errors.add(error) LOG.error(error) return url, None, errors - return url, response.content, errors + return url, content, errors def _create_layer_query(self, interface): """ @@ -374,7 +384,7 @@ def _get_layer_resolution_hint(self, layer): 999999999 if resolution_hint_max is None else resolution_hint_max, ) - def _layer(self, layer, time_=None, dim=None, mixed=True): + async def _layer(self, layer, time_=None, dim=None, mixed=True): errors: Set[str] = set() layer_info = {"id": layer.id, "name": layer.name, "metadata": self._get_metadatas(layer, errors)} if re.search("[/?#]", layer.name): # pragma: no cover @@ -391,7 +401,7 @@ def _layer(self, layer, time_=None, dim=None, mixed=True): errors |= dim.merge(layer, layer_info, mixed) if isinstance(layer, main.LayerWMS): - wms, wms_errors = self._wms_layers(layer.ogc_server) + wms, wms_errors = await self._wms_layers(layer.ogc_server) errors |= wms_errors if wms is None: return None if errors else layer_info, errors @@ -400,7 +410,7 @@ def _layer(self, layer, time_=None, dim=None, mixed=True): return None, errors layer_info["type"] = "WMS" layer_info["layers"] = layer.layer - self._fill_wms(layer_info, layer, errors, mixed=mixed) + await self._fill_wms(layer_info, layer, errors, mixed=mixed) errors |= self._merge_time(time_, layer_info, layer, wms) elif isinstance(layer, main.LayerWMTS): @@ -419,9 +429,7 @@ def _merge_time(time_, layer_theme, layer, wms): wmslayer = layer.layer def merge_time(wms_layer_obj): - extent = parse_extent( - wms_layer_obj["timepositions"], wms_layer_obj["defaulttimeposition"] - ) + extent = parse_extent(wms_layer_obj["timepositions"], wms_layer_obj["defaulttimeposition"]) time_.merge(layer_theme, extent, layer.time_mode, layer.time_widget) try: @@ -474,8 +482,8 @@ def _fill_editable(self, layer_theme, layer): errors.add(str(exception)) return errors - def _fill_wms(self, layer_theme, layer, errors, mixed): - wms, wms_errors = self._wms_layers(layer.ogc_server) + async def _fill_wms(self, layer_theme, layer, errors, mixed): + wms, wms_errors = await self._wms_layers(layer.ogc_server) errors |= wms_errors if wms is None: return @@ -567,7 +575,7 @@ def _layer_included(tree_item): return isinstance(tree_item, main.Layer) def _get_ogc_servers(self, group, depth): - """ Recurse on all children to get unique identifier for each child. """ + """Recurse on all children to get unique identifier for each child.""" ogc_servers: Set[Union[str, bool]] = set() @@ -593,7 +601,7 @@ def _get_ogc_servers(self, group, depth): def is_mixed(ogc_servers): return len(ogc_servers) != 1 or ogc_servers[0] is False - def _group( + async def _group( self, path, group, @@ -634,7 +642,7 @@ def _group( for tree_item in group.children: if isinstance(tree_item, main.LayerGroup): - group_theme, gp_errors = self._group( + group_theme, gp_errors = await self._group( "{}/{}".format(path, tree_item.name), tree_item, layers, @@ -656,7 +664,7 @@ def _group( if isinstance(tree_item, main.LayerWMS): wms_layers.extend(tree_item.layer.split(",")) - layer_theme, l_errors = self._layer(tree_item, mixed=mixed, time_=time_, dim=dim) + layer_theme, l_errors = await self._layer(tree_item, mixed=mixed, time_=time_, dim=dim) errors |= l_errors if layer_theme is not None: if depth < min_levels: @@ -700,9 +708,9 @@ def _layers(self, interface): query = self._create_layer_query(interface=interface) return [name for (name,) in query.all()] - def _wms_layers(self, ogc_server): + async def _wms_layers(self, ogc_server): # retrieve layers metadata via GetCapabilities - wms, wms_errors = asyncio.run(self._wms_getcap(ogc_server)) + wms, wms_errors = await self._wms_getcap(ogc_server) if wms_errors: return None, wms_errors @@ -737,7 +745,7 @@ def _load_tree_items(self) -> None: .all() ) - def _themes(self, interface="desktop", filter_themes=True, min_levels=1): + async def _themes(self, interface="desktop", filter_themes=True, min_levels=1): """ This function returns theme information for the role identified by ``role_id``. @@ -767,7 +775,7 @@ def _themes(self, interface="desktop", filter_themes=True, min_levels=1): errors.add("The theme has an unsupported name '{}'.".format(theme.name)) continue - children, children_errors = self._get_children(theme, layers, min_levels) + children, children_errors = await self._get_children(theme, layers, min_levels) errors |= children_errors # Test if the theme is visible for the current user @@ -806,12 +814,12 @@ def invalidate_cache(self): # pragma: no cover main.cache_invalidate_cb() return {"success": True} - def _get_children(self, theme, layers, min_levels): + async def _get_children(self, theme, layers, min_levels): children = [] errors: Set[str] = set() for item in theme.children: if isinstance(item, main.LayerGroup): - group_theme, gp_errors = self._group( + group_theme, gp_errors = await self._group( "{}/{}".format(theme.name, item.name), item, layers, min_levels=min_levels ) errors |= gp_errors @@ -825,7 +833,7 @@ def _get_children(self, theme, layers, min_levels): ) ) elif item.name in layers: - layer_theme, l_errors = self._layer(item, dim=DimensionInformation()) + layer_theme, l_errors = await self._layer(item, dim=DimensionInformation()) errors |= l_errors if layer_theme is not None: children.append(layer_theme) @@ -850,7 +858,9 @@ def _get_layers_enum(self): def _get_role_ids(self): return None if self.request.user is None else {role.id for role in self.request.user.roles} - async def _wfs_get_features_type(self, wfs_url, preload=False): + async def _wfs_get_features_type( + self, wfs_url: str, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True + ) -> Tuple[Optional[etree.Element], Set[str]]: errors = set() params = { @@ -864,38 +874,41 @@ async def _wfs_get_features_type(self, wfs_url, preload=False): LOG.debug("WFS DescribeFeatureType for base url: %s", wfs_url) - # forward request to target (without Host Header) - headers = dict(self.request.headers) - if urllib.parse.urlsplit(wfs_url).hostname != "localhost" and "Host" in headers: - headers.pop("Host") # pragma nocover - - headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist) + headers: Dict[str, str] = {} try: - response = await asyncio.get_event_loop().run_in_executor( - None, get_http_cached, self.http_options, wfs_url, headers + content, _ = get_http_cached(self.http_options, wfs_url, headers, cache) + except requests.exceptions.RequestException as exception: + error = ( + f"Unable to get WFS DescribeFeatureType from the URL '{wfs_url}' for " + f"OGC server {ogc_server.name}, " + + ( + f"return the error: {exception.response.status_code} {exception.response.reason}" + if exception.response is not None + else f"{exception}" + ) ) - except Exception: # pragma: no cover - errors.add("Unable to get DescribeFeatureType from URL {}".format(wfs_url)) + errors.add(error) + LOG.exception(error) return None, errors - - if not response.ok: # pragma: no cover - errors.add( - "DescribeFeatureType from URL {} return the error: {:d} {}".format( - wfs_url, response.status_code, response.reason - ) + except Exception: + error = ( + f"Unable to get WFS DescribeFeatureType from the URL {wfs_url} for " + f"OGC server {ogc_server.name}" ) + errors.add(error) + LOG.exception(error) return None, errors if preload: return None, errors try: - return lxml.XML(response.text.encode("utf-8")), errors + return lxml.XML(content), errors except Exception as e: # pragma: no cover errors.add( "Error '{}' on reading DescribeFeatureType from URL {}:\n{}".format( - str(e), wfs_url, response.text + str(e), wfs_url, content.decode() ) ) return None, errors @@ -928,86 +941,117 @@ def get_url_internal_wfs(self, ogc_server, errors): url_internal_wfs = url_wfs return url_internal_wfs, url, url_wfs - async def preload(self, errors): + async def preload(self, errors: Set[str]) -> None: tasks = set() for ogc_server in models.DBSession.query(main.OGCServer).all(): - url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors) - if ogc_server.wfs_support: - tasks.add(self._wfs_get_features_type(url_internal_wfs, True)) - tasks.add(self._wms_getcap(ogc_server, True)) + # Don't load unused OGC servers, required for landing page, because the related OGC server + # will be on error in those functions. + nb_layers = ( + models.DBSession.query(sqlalchemy.func.count(main.LayerWMS.id)) + .filter(main.LayerWMS.ogc_server_id == ogc_server.id) + .one() + ) + LOG.debug("%i layers for OGC server '%s'", nb_layers[0], ogc_server.name) + if nb_layers[0] > 0: + LOG.debug("Preload OGC server '%s'", ogc_server.name) + url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors) + if url_internal_wfs is not None: + tasks.add(self.preload_ogc_server(ogc_server, url_internal_wfs)) await asyncio.gather(*tasks) - @CACHE_REGION.cache_on_arguments() - def _get_features_attributes(self, url_internal_wfs): - all_errors: Set[str] = set() - feature_type, errors = asyncio.run(self._wfs_get_features_type(url_internal_wfs)) - LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)])) - if errors: - all_errors |= errors - return None, None, all_errors - namespace = feature_type.attrib.get("targetNamespace") - types = {} - elements = {} - for child in feature_type.getchildren(): - if child.tag == "{http://www.w3.org/2001/XMLSchema}element": - name = child.attrib["name"] - type_namespace, type_ = child.attrib["type"].split(":") - if type_namespace not in child.nsmap: - LOG.info( - "The namespace '%s' of the type '%s' is not found in the available namespaces: %s", - type_namespace, + async def preload_ogc_server( + self, ogc_server: main.OGCServer, url_internal_wfs: str, cache: bool = True + ) -> None: + if ogc_server.wfs_support: + await self._get_features_attributes(url_internal_wfs, ogc_server, cache=cache) + await self._wms_getcap(ogc_server, False, cache=cache) + + async def _get_features_attributes( + self, url_internal_wfs: str, ogc_server: main.OGCServer, cache: bool = True + ) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]: + @CACHE_OGC_SERVER_REGION.cache_on_arguments() + def _get_features_attributes_cache( + url_internal_wfs: str, ogc_server_name: str + ) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]: + del url_internal_wfs, ogc_server_name # Just for cache + all_errors: Set[str] = set() + LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)])) + if errors: + all_errors |= errors + return None, None, all_errors + assert feature_type is not None + namespace: str = feature_type.attrib.get("targetNamespace") + types: Dict[Any, Dict[str, Any]] = {} + elements = {} + for child in feature_type.getchildren(): + if child.tag == "{http://www.w3.org/2001/XMLSchema}element": + name = child.attrib["name"] + type_namespace, type_ = child.attrib["type"].split(":") + if type_namespace not in child.nsmap: + LOG.info( + "The namespace '%s' of the type '%s' is not found in the " + "available namespaces: %s", + type_namespace, + name, + ", ".join(child.nsmap.keys()), + ) + if child.nsmap[type_namespace] != namespace: + LOG.info( + "The namespace '%s' of the type '%s' should be '%s'.", + child.nsmap[type_namespace], + name, + namespace, + ) + elements[name] = type_ + + if child.tag == "{http://www.w3.org/2001/XMLSchema}complexType": + sequence = child.find(".//{http://www.w3.org/2001/XMLSchema}sequence") + attrib = {} + for children in sequence.getchildren(): + type_namespace = None + type_ = children.attrib["type"] + if len(type_.split(":")) == 2: + type_namespace, type_ = type_.split(":") + type_namespace = children.nsmap[type_namespace] + name = children.attrib["name"] + attrib[name] = {"namespace": type_namespace, "type": type_} + for key, value in children.attrib.items(): + if key not in ("name", "type", "namespace"): + attrib[name][key] = value + types[child.attrib["name"]] = attrib + attributes: Dict[str, Dict[Any, Dict[str, Any]]] = {} + for name, type_ in elements.items(): + if type_ in types: + attributes[name] = types[type_] + elif (type_ == "Character") and (name + "Type") in types: + LOG.debug( + 'Due to MapServer weird behavior when using METADATA "gml_types" "auto"' + "the type 'ms:Character' is returned as type '%sType' for feature '%s'.", name, - ", ".join(child.nsmap.keys()), - ) - if child.nsmap[type_namespace] != namespace: - LOG.info( - "The namespace '%s' of the thye '%s' should be '%s'.", - child.nsmap[type_namespace], name, - namespace, ) - elements[name] = type_ - - if child.tag == "{http://www.w3.org/2001/XMLSchema}complexType": - sequence = child.find(".//{http://www.w3.org/2001/XMLSchema}sequence") - attrib = {} - for children in sequence.getchildren(): - type_namespace = None - type_ = children.attrib["type"] - if len(type_.split(":")) == 2: - type_namespace, type_ = type_.split(":") - type_namespace = children.nsmap[type_namespace] - name = children.attrib["name"] - attrib[name] = {"namespace": type_namespace, "type": type_} - for key, value in children.attrib.items(): - if key not in ("name", "type", "namespace"): - attrib[name][key] = value - types[child.attrib["name"]] = attrib - attributes = {} - for name, type_ in elements.items(): - if type_ in types: - attributes[name] = types[type_] - elif (type_ == "Character") and (name + "Type") in types: - LOG.debug( - "Due mapserver strange result the type 'ms:Character' is fallbacked to type '%sType'" - " for feature '%s', This is a stange comportement of mapserver when we use the " - 'METADATA "gml_types" "auto"', - name, - name, - ) - attributes[name] = types[name + "Type"] - else: - LOG.warning( - "The provided type '%s' does not exist, available types are %s.", - type_, - ", ".join(types.keys()), - ) + attributes[name] = types[name + "Type"] + else: + LOG.warning( + "The provided type '%s' does not exist, available types are %s.", + type_, + ", ".join(types.keys()), + ) + + return attributes, namespace, all_errors + + if cache: + result = _get_features_attributes_cache.get(url_internal_wfs, ogc_server.name) + if result != dogpile.cache.api.NO_VALUE: + return result + + feature_type, errors = await self._wfs_get_features_type(url_internal_wfs, ogc_server, False, cache) - return attributes, namespace, all_errors + return _get_features_attributes_cache.refresh(url_internal_wfs, ogc_server.name) @view_config(route_name="themes", renderer="json") - def themes(self): + def themes(self) -> Dict[str, Union[Dict[str, Dict[str, Any]], List[str]]]: interface = self.request.params.get("interface", "desktop") sets = self.request.params.get("set", "all") min_levels = int(self.request.params.get("min_levels", 1)) @@ -1016,7 +1060,7 @@ def themes(self): set_common_headers(self.request, "themes", PRIVATE_CACHE) - def get_theme(): + async def get_theme(): export_themes = sets in ("all", "themes") export_group = group is not None and sets in ("all", "group") export_background = background_layers_group is not None and sets in ("all", "background") @@ -1025,7 +1069,7 @@ def get_theme(): all_errors: Set[str] = set() LOG.debug("Start preload") start_time = time.time() - asyncio.run(self.preload(all_errors)) + await self.preload(all_errors) LOG.debug("End preload") LOG.info("Do preload in %.3fs.", time.time() - start_time) result["ogcServers"] = {} @@ -1035,7 +1079,9 @@ def get_theme(): attributes = None namespace = None if ogc_server.wfs_support: - attributes, namespace, errors = self._get_features_attributes(url_internal_wfs) + attributes, namespace, errors = await self._get_features_attributes( + url_internal_wfs, ogc_server + ) # Create a local copy (don't modify the cache) if attributes is not None: attributes = dict(attributes) @@ -1068,19 +1114,19 @@ def get_theme(): "attributes": attributes, } if export_themes: - themes, errors = self._themes(interface, True, min_levels) + themes, errors = await self._themes(interface, True, min_levels) result["themes"] = themes all_errors |= errors if export_group: - exported_group, errors = self._get_group(group, interface) + exported_group, errors = await self._get_group(group, interface) if exported_group is not None: result["group"] = exported_group all_errors |= errors if export_background: - exported_group, errors = self._get_group(background_layers_group, interface) + exported_group, errors = await self._get_group(background_layers_group, interface) result["background_layers"] = exported_group["children"] if exported_group is not None else [] all_errors |= errors @@ -1093,7 +1139,7 @@ def get_theme(): def get_theme_anonymous(intranet, interface, sets, min_levels, group, background_layers_group, host): # Only for cache key del intranet, interface, sets, min_levels, group, background_layers_group, host - return get_theme() + return asyncio.run(get_theme()) if self.request.user is None: return get_theme_anonymous( @@ -1105,13 +1151,13 @@ def get_theme_anonymous(intranet, interface, sets, min_levels, group, background background_layers_group, self.request.headers.get("Host"), ) - return get_theme() + return asyncio.run(get_theme()) - def _get_group(self, group, interface): + async def _get_group(self, group, interface): layers = self._layers(interface) try: group_db = models.DBSession.query(main.LayerGroup).filter(main.LayerGroup.name == group).one() - return self._group(group_db.name, group_db, layers, depth=2, dim=DimensionInformation()) + return await self._group(group_db.name, group_db, layers, depth=2, dim=DimensionInformation()) except NoResultFound: # pragma: no cover return ( None, @@ -1124,3 +1170,36 @@ def _get_group(self, group, interface): ] ), ) + + @view_config(route_name="ogc_server_clear_cache", renderer="json") + def ogc_server_clear_cache_view(self) -> Dict[str, Any]: + + self._ogc_server_clear_cache( + models.DBSession.query(main.OGCServer).filter_by(id=self.request.matchdict.get("id")).one() + ) + came_from = self.request.params.get("came_from") + if came_from: + raise pyramid.httpexceptions.HTTPFound(location=came_from) + return {"success": True} + + def _ogc_server_clear_cache(self, ogc_server: main.OGCServer) -> None: + errors: Set[str] = set() + url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors) + if errors: + LOG.error( + "Error while getting the URL of the OGC Server %s:\n%s", ogc_server.id, "\n".join(errors) + ) + return + if url_internal_wfs is None: + return + + asyncio.run(self._async_cache_invalidate_ogc_server_cb(ogc_server, url_internal_wfs)) + + async def _async_cache_invalidate_ogc_server_cb( + self, ogc_server: main.OGCServer, url_internal_wfs: str + ) -> None: + + # Fill the cache + await self.preload_ogc_server(ogc_server, url_internal_wfs, False) + + cache_invalidate_cb() diff --git a/geoportal/tests/__init__.py b/geoportal/tests/__init__.py index de2e8ad6bc..9c07bd3126 100644 --- a/geoportal/tests/__init__.py +++ b/geoportal/tests/__init__.py @@ -50,6 +50,7 @@ def __init__(self, *args, **kwargs): def setup_common(): caching.init_region({"backend": "dogpile.cache.null"}, "std") caching.init_region({"backend": "dogpile.cache.null"}, "obj") + caching.init_region({"backend": "dogpile.cache.null"}, "ogc-server") def create_dummy_request(additional_settings=None, *args, **kargs): diff --git a/geoportal/tests/config.yaml b/geoportal/tests/config.yaml index c895f7c20a..74419f91cb 100644 --- a/geoportal/tests/config.yaml +++ b/geoportal/tests/config.yaml @@ -11,3 +11,5 @@ vars: backend: dogpile.cache.memory obj: backend: dogpile.cache.memory + ogc-server: + backend: dogpile.cache.memory diff --git a/geoportal/tests/functional/__init__.py b/geoportal/tests/functional/__init__.py index c0a21b9ad4..416b51b6b0 100644 --- a/geoportal/tests/functional/__init__.py +++ b/geoportal/tests/functional/__init__.py @@ -97,6 +97,7 @@ def setup_db(): caching.init_region({"backend": "dogpile.cache.null"}, "std") caching.init_region({"backend": "dogpile.cache.null"}, "obj") + caching.init_region({"backend": "dogpile.cache.null"}, "ogc-server") caching.invalidate_region() diff --git a/geoportal/tests/functional/test_dbreflection.py b/geoportal/tests/functional/test_dbreflection.py index 397407f22d..858a3e1385 100644 --- a/geoportal/tests/functional/test_dbreflection.py +++ b/geoportal/tests/functional/test_dbreflection.py @@ -112,6 +112,7 @@ def test_get_class(self): init_region({"backend": "dogpile.cache.memory"}, "std") init_region({"backend": "dogpile.cache.memory"}, "obj") + init_region({"backend": "dogpile.cache.memory"}, "ogc-server") self._create_table("table_a") modelclass = get_class("table_a") diff --git a/geoportal/tests/functional/test_dynamicview.py b/geoportal/tests/functional/test_dynamicview.py index 0026ce3d36..e1383a52c2 100644 --- a/geoportal/tests/functional/test_dynamicview.py +++ b/geoportal/tests/functional/test_dynamicview.py @@ -77,6 +77,7 @@ def setup_method(self, _): init_region({"backend": "dogpile.cache.memory"}, "std") init_region({"backend": "dogpile.cache.memory"}, "obj") + init_region({"backend": "dogpile.cache.memory"}, "ogc-server") def teardown_method(self, _): testing.tearDown() diff --git a/geoportal/tests/functional/test_themes_entry.py b/geoportal/tests/functional/test_themes_entry.py index b18e3f553c..91fd7f2bc0 100644 --- a/geoportal/tests/functional/test_themes_entry.py +++ b/geoportal/tests/functional/test_themes_entry.py @@ -243,7 +243,7 @@ def _create_request_obj(username=None, params=None, **kwargs): return request - def test_theme(self): + async def test_theme(self): from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.static import User from c2cgeoportal_geoportal.views.theme import Theme @@ -252,7 +252,7 @@ def test_theme(self): theme_view = Theme(request) # unautenticated - themes, errors = theme_view._themes() + themes, errors = await theme_view._themes() assert {e[:90] for e in errors} == set() assert len(themes) == 1 groups = {g["name"] for g in themes[0]["children"]} @@ -263,7 +263,7 @@ def test_theme(self): # authenticated request.params = {} request.user = DBSession.query(User).filter_by(username="__test_user1").one() - themes, errors = theme_view._themes() + themes, errors = await theme_view._themes() assert {e[:90] for e in errors} == set() assert len(themes) == 1 groups = {g["name"] for g in themes[0]["children"]} @@ -271,7 +271,7 @@ def test_theme(self): layers = {l["name"] for l in themes[0]["children"][0]["children"]} assert layers == {"__test_private_layer_edit", "__test_public_layer", "__test_private_layer"} - def test_no_layers(self): + async def test_no_layers(self): # mapfile error from c2cgeoportal_geoportal.views.theme import Theme @@ -280,46 +280,46 @@ def test_no_layers(self): request.params = {} invalidate_region() - themes, errors = theme_view._themes("interface_no_layers") + themes, errors = await theme_view._themes("interface_no_layers") assert themes == [] assert {e[:90] for e in errors} == { "The layer '__test_public_layer_no_layers' do not have any layers" } - def test_not_in_mapfile(self): + async def test_not_in_mapfile(self): # mapfile error from c2cgeoportal_geoportal.views.theme import Theme theme_view = Theme(self._create_request_obj()) invalidate_region() - themes, errors = theme_view._themes("interface_not_in_mapfile") + themes, errors = await theme_view._themes("interface_not_in_mapfile") assert len(themes) == 0 assert {e[:90] for e in errors} == { "The layer '__test_public_layer_not_in_mapfile' (__test_public_layer_not_in_mapfile) is not" } - def test_notmapfile(self): + async def test_notmapfile(self): # mapfile error from c2cgeoportal_geoportal.views.theme import Theme theme_view = Theme(self._create_request_obj()) invalidate_region() - themes, errors = theme_view._themes("interface_notmapfile") + themes, errors = await theme_view._themes("interface_notmapfile") assert len(themes) == 0 assert {e[:90] for e in errors} == { "GetCapabilities from URL http://mapserver:8080/?map=not_a_mapfile&SERVICE=WMS&VERSION=1.1.", } - def test_theme_geoserver(self): + async def test_theme_geoserver(self): from c2cgeoportal_geoportal.views.theme import Theme request = self._create_request_obj() theme_view = Theme(request) # unautenticated v1 - themes, errors = theme_view._themes("interface_geoserver") + themes, errors = await theme_view._themes("interface_geoserver") assert {e[:90] for e in errors} == set() assert len(themes) == 1 layers = {l["name"] for l in themes[0]["children"][0]["children"]} diff --git a/geoportal/tests/functional/test_themes_loop.py b/geoportal/tests/functional/test_themes_loop.py index 3c2a1bf0ff..f7f81dc01d 100644 --- a/geoportal/tests/functional/test_themes_loop.py +++ b/geoportal/tests/functional/test_themes_loop.py @@ -84,7 +84,7 @@ def teardown_method(self, _): transaction.commit() - def test_theme(self): + async def test_theme(self): from c2cgeoportal_geoportal.views.theme import Theme request = DummyRequest() @@ -92,7 +92,7 @@ def test_theme(self): request.route_url = lambda url, _query={}: mapserv_url request.user = None theme_view = Theme(request) - _, errors = theme_view._themes("desktop2", True, 2) + _, errors = await theme_view._themes("desktop2", True, 2) self.assertEqual( len([e for e in errors if e == "Too many recursions with group '__test_layer_group'"]), 1 ) diff --git a/geoportal/tests/functional/test_themes_mixed.py b/geoportal/tests/functional/test_themes_mixed.py index b911b08c45..2a271c2409 100644 --- a/geoportal/tests/functional/test_themes_mixed.py +++ b/geoportal/tests/functional/test_themes_mixed.py @@ -218,7 +218,7 @@ def test_theme_mixed(self): self.assertEqual( self._get_filtered_errors(themes), { - "WARNING! an error 'The WMS version (1.0.0) you requested is not implemented. Please use 1.1.1 or 1.3", + "WARNING! an error 'NotImplementedError('The WMS version (1.0.0) you requested is not implemented. Pl" }, ) self.assertEqual( diff --git a/geoportal/tests/functional/test_themes_ogc_server_cache_clean.py b/geoportal/tests/functional/test_themes_ogc_server_cache_clean.py new file mode 100644 index 0000000000..25f0e59100 --- /dev/null +++ b/geoportal/tests/functional/test_themes_ogc_server_cache_clean.py @@ -0,0 +1,319 @@ +# Copyright (c) 2022, Camptocamp SA +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +# pylint: disable=missing-docstring,attribute-defined-outside-init,protected-access + +import asyncio +from unittest import TestCase + +from pyramid import testing +import pyramid.url +import responses +from tests.functional import create_default_ogcserver, create_dummy_request +from tests.functional import setup_common as setup_module # noqa +from tests.functional import teardown_common as teardown_module # noqa +import transaction + +from c2cgeoportal_geoportal.lib import caching + +CAP = """ + + + + + + demo + EPSG:2056 + + + + + __test_layer_internal_wms + EPSG:2056 + + + + text/xml + + + + + {name2} + EPSG:2056 + + + + text/xml + + + + + + + +""" + +DFT = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +class TestThemesView(TestCase): + def setup_method(self, _): + # Always see the diff + # https://docs.python.org/2/library/unittest.html#unittest.TestCase.maxDiff + self.maxDiff = None + + from c2cgeoportal_commons.models import DBSession + from c2cgeoportal_commons.models.main import Interface, LayerWMS, Theme + + main = Interface(name="desktop") + + ogc_server_internal = create_default_ogcserver() + + layer_internal_wms = LayerWMS(name="__test_layer_internal_wms", public=True) + layer_internal_wms.layer = "__test_layer_internal_wms" + layer_internal_wms.interfaces = [main] + layer_internal_wms.ogc_server = ogc_server_internal + + theme = Theme(name="__test_theme") + theme.interfaces = [ + main, + ] + theme.children = [layer_internal_wms] + + DBSession.add_all([theme]) + + self.std_cache = {} + self.ogc_cache = {} + caching.MEMORY_CACHE_DICT.clear() + + caching.init_region( + {"backend": "dogpile.cache.memory", "arguments": {"cache_dict": self.std_cache}}, "std" + ) + caching.init_region({"backend": "dogpile.cache.memory"}, "obj") + caching.init_region( + {"backend": "dogpile.cache.memory", "arguments": {"cache_dict": self.ogc_cache}}, "ogc-server" + ) + + transaction.commit() + + def teardown_method(self, _): + testing.tearDown() + + from c2cgeoportal_commons.models import DBSession + from c2cgeoportal_commons.models.main import Dimension, Interface, Metadata, OGCServer, TreeItem + + for item in DBSession.query(TreeItem).all(): + DBSession.delete(item) + DBSession.query(Interface).filter(Interface.name == "main").delete() + DBSession.query(OGCServer).delete() + + transaction.commit() + + @responses.activate + def test_ogc_server_cache_clean(self): + from c2cgeoportal_commons.models import DBSession + from c2cgeoportal_commons.models.main import OGCServer + from c2cgeoportal_geoportal.views.theme import Theme + + ogc_server = DBSession.query(OGCServer).one() + + request = create_dummy_request() + request.route_url = ( + lambda url, _query=None: "/dummy/route/url/{}".format(url) + if _query is None + else "/dummy/route/url/{}?{}".format(url, pyramid.url.urlencode(_query)) + ) + theme = Theme(request) + all_errors = set() + url_internal_wfs, _, _ = theme.get_url_internal_wfs(ogc_server, all_errors) + + responses.get( + "http://mapserver:8080/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetCapabilities&ROLE_ID=0&USER_ID=0", + content_type="application/vnd.ogc.wms_xml", + body=CAP.format(name2="__test_layer_internal_wms2"), + ) + responses.get( + "http://mapserver:8080/?SERVICE=WFS&VERSION=1.0.0&REQUEST=DescribeFeatureType&ROLE_ID=0&USER_ID=0", + content_type="application/vnd.ogc.wms_xml", + body=DFT.format(name2="police1"), + ) + asyncio.run(theme.preload(set())) + responses.reset() + + layers, err = asyncio.run(theme._wms_getcap(ogc_server)) + assert err == set() + assert set(layers["layers"].keys()) == { + "__test_layer_internal_wms", + "__test_layer_internal_wms2", + "demo", + } + attributes, namespace, err = asyncio.run(theme._get_features_attributes(url_internal_wfs, ogc_server)) + assert err == set() + assert namespace == "http://mapserver.gis.umn.edu/mapserver" + assert set(attributes.keys()) == {"hotel_label", "police1"} + + assert set(self.std_cache.keys()) == set() + assert set(caching.MEMORY_CACHE_DICT.keys()) == { + "c2cgeoportal_geoportal.lib.functionality|_get_role|anonymous", + "c2cgeoportal_geoportal.lib.functionality|_get_functionalities_type", + "c2cgeoportal_geoportal.lib|_get_intranet_networks", + } + assert set(self.ogc_cache.keys()) == { + "c2cgeoportal_geoportal.views.theme|_get_features_attributes_cache|http://mapserver:8080/|__test_ogc_server", + f"c2cgeoportal_geoportal.views.theme|build_web_map_service|{ogc_server.id}", + "c2cgeoportal_geoportal.views.theme|do_get_http_cached|http://mapserver:8080/?SERVICE=WFS&VERSION=1.0.0&REQUEST=DescribeFeatureType&ROLE_ID=0&USER_ID=0", + "c2cgeoportal_geoportal.views.theme|do_get_http_cached|http://mapserver:8080/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetCapabilities&ROLE_ID=0&USER_ID=0", + } + + responses.get( + "http://mapserver:8080/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetCapabilities&ROLE_ID=0&USER_ID=0", + content_type="application/vnd.ogc.wms_xml", + body=CAP.format(name2="__test_layer_internal_wms3"), + ) + responses.get( + "http://mapserver:8080/?SERVICE=WFS&VERSION=1.0.0&REQUEST=DescribeFeatureType&ROLE_ID=0&USER_ID=0", + content_type="application/vnd.ogc.wms_xml", + body=DFT.format(name2="police2"), + ) + theme._ogc_server_clear_cache(ogc_server) + responses.reset() + + layers, err = asyncio.run(theme._wms_getcap(ogc_server)) + assert err == set() + assert set(layers["layers"].keys()) == { + "__test_layer_internal_wms", + "__test_layer_internal_wms3", + "demo", + } + + attributes, namespace, err = asyncio.run(theme._get_features_attributes(url_internal_wfs, ogc_server)) + assert err == set() + assert namespace == "http://mapserver.gis.umn.edu/mapserver" + assert set(attributes.keys()) == {"hotel_label", "police2"} + + assert set(self.std_cache.keys()) == set() + #assert set(caching.MEMORY_CACHE_DICT.keys()) == { + # "c2cgeoportal_geoportal.lib.functionality|_get_role|anonymous", + # "c2cgeoportal_geoportal.lib.functionality|_get_functionalities_type", + # "c2cgeoportal_geoportal.lib|_get_intranet_networks", + #} + assert set(self.ogc_cache.keys()) == { + "c2cgeoportal_geoportal.views.theme|_get_features_attributes_cache|http://mapserver:8080/|__test_ogc_server", + f"c2cgeoportal_geoportal.views.theme|build_web_map_service|{ogc_server.id}", + "c2cgeoportal_geoportal.views.theme|do_get_http_cached|http://mapserver:8080/?SERVICE=WFS&VERSION=1.0.0&REQUEST=DescribeFeatureType&ROLE_ID=0&USER_ID=0", + "c2cgeoportal_geoportal.views.theme|do_get_http_cached|http://mapserver:8080/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetCapabilities&ROLE_ID=0&USER_ID=0", + }