From ea8c40a0c62a09e13d15f7c89eaea2a2127f7125 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 30 May 2024 17:48:48 -0500 Subject: [PATCH 1/2] Add "Pelican cache" and "Pelican origin" services Most cache/origin endpoints work the same for the Pelican cache/origin service types as they do for the XRootD cache/origin services The exception is the namespaces JSON which explicitly excludes Pelican caches/origins. (SOFTWARE-5867) --- src/app.py | 6 ++-- src/stashcache.py | 65 ++++++++++++++++++++--------------- src/webapp/common.py | 2 ++ src/webapp/data_federation.py | 10 +++--- src/webapp/exceptions.py | 15 +++++--- src/webapp/topology.py | 19 +++++++++- topology/services.yaml | 2 ++ 7 files changed, 79 insertions(+), 40 deletions(-) diff --git a/src/app.py b/src/app.py index 30e7d4b25..3a17aebc9 100755 --- a/src/app.py +++ b/src/app.py @@ -22,7 +22,7 @@ from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \ escape, cache_control_private, PreJSON, is_true, GRIDTYPE_1, GRIDTYPE_2, NamespacesFilters from webapp.flask_common import create_accepted_response -from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService +from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingServices from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm from webapp.models import GlobalData from webapp.oasis_managers import get_oasis_manager_endpoint_info @@ -603,7 +603,7 @@ def _get_cache_authfile(public_only): fqdn=cache_fqdn, legacy=app.config["STASHCACHE_LEGACY_AUTH"], suppress_errors=False) - except (ResourceNotRegistered, ResourceMissingService) as e: + except (ResourceNotRegistered, ResourceMissingServices) as e: return Response("# {}\n" "# Please check your query or contact help@osg-htc.org\n" .format(str(e)), @@ -627,7 +627,7 @@ def _get_origin_authfile(public_only): try: auth = stashcache.generate_origin_authfile(global_data=global_data, fqdn=request.args['fqdn'], suppress_errors=False, public_origin=public_only) - except (ResourceNotRegistered, ResourceMissingService) as e: + except (ResourceNotRegistered, ResourceMissingServices) as e: return Response("# {}\n" "# Please check your query or contact help@osg-htc.org\n" .format(str(e)), diff --git a/src/stashcache.py b/src/stashcache.py index a9ac3a5a7..494cda645 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -1,8 +1,9 @@ from collections import defaultdict -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Sequence -from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, NamespacesFilters -from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService +from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, PELICAN_CACHE, PELICAN_ORIGIN, \ + NamespacesFilters +from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingServices from webapp.models import GlobalData from webapp.topology import Resource, ResourceGroup, Topology from webapp.vos_data import VOsData @@ -22,47 +23,52 @@ def _log_or_raise(suppress_errors: bool, an_exception: BaseException, logmethod= raise an_exception -def _resource_has_cache(resource: Resource) -> bool: - return XROOTD_CACHE_SERVER in resource.service_names - -def _resource_has_origin(resource: Resource) -> bool: - return XROOTD_ORIGIN_SERVER in resource.service_names - - -def _get_resource_with_service(fqdn: Optional[str], service_name: str, topology: Topology, - suppress_errors: bool) -> Optional[Resource]: - """If given an FQDN, returns the Resource _if it has the given service. +def _get_resource_with_services(fqdn: Optional[str], service_names: Sequence[str], topology: Topology, + suppress_errors: bool) -> Optional[Resource]: + """ + If given an FQDN, returns the Resource if it has one of the given services. If given None, returns None. If multiple Resources have the same FQDN, checks the first one. If suppress_errors is False, raises an exception on the following conditions: - no Resource matching FQDN (ResourceNotRegistered) - - Resource does not provide a SERVICE_NAME (ResourceMissingService) + - Resource does not provide any of the services in SERVICE_NAMES (ResourceMissingServices) If suppress_errors is True, logs the error and returns None on the above conditions. - """ resource = None + if isinstance(service_names, str): + service_names = [service_names] if fqdn: resource = topology.safe_get_resource_by_fqdn(fqdn) if not resource: _log_or_raise(suppress_errors, ResourceNotRegistered(fqdn=fqdn)) return None - if service_name not in resource.service_names: + + for service_name in service_names: + if service_name in resource.service_names: + return resource + else: _log_or_raise( suppress_errors, - ResourceMissingService(resource, service_name) + ResourceMissingServices(resource, service_names) ) return None return resource def _get_cache_resource(fqdn: Optional[str], topology: Topology, suppress_errors: bool) -> Optional[Resource]: - """Convenience wrapper around _get_resource-with-service() for a cache""" - return _get_resource_with_service(fqdn, XROOTD_CACHE_SERVER, topology, suppress_errors) + """ + Convenience wrapper around _get_resource_with_services() for an xrootd + or pelican cache + """ + return _get_resource_with_services(fqdn, [XROOTD_CACHE_SERVER, PELICAN_CACHE], topology, suppress_errors) def _get_origin_resource(fqdn: Optional[str], topology: Topology, suppress_errors: bool) -> Optional[Resource]: - """Convenience wrapper around _get_resource-with-service() for an origin""" - return _get_resource_with_service(fqdn, XROOTD_ORIGIN_SERVER, topology, suppress_errors) + """ + Convenience wrapper around _get_resource_with_services() for an xrootd + or pelican origin + """ + return _get_resource_with_services(fqdn, [XROOTD_ORIGIN_SERVER, PELICAN_ORIGIN], topology, suppress_errors) def resource_allows_namespace(resource: Resource, namespace: Optional[Namespace]) -> bool: @@ -120,7 +126,8 @@ def get_supported_caches_for_namespace(namespace: Namespace, topology: Topology) all_caches = [resource for group in resource_groups for resource in group.resources - if _resource_has_cache(resource)] + if (resource.has_xrootd_cache or + resource.has_pelican_cache)] return [cache for cache in all_caches if namespace_allows_cache_resource(namespace, cache) @@ -546,6 +553,8 @@ def get_namespaces_info(global_data: GlobalData, filters: Optional[NamespacesFil If `include_downed` is True, caches/origins in downtime are also included. If `include_inactive` is True, caches/origins that are not marked as active are also included. + + NOTE: This is specific to XRootD caches and origins; Pelican caches and origins are not included. """ if filters is None: filters = NamespacesFilters() @@ -579,10 +588,10 @@ def _service_resource_dict( "production": production, } - def _cache_resource_dict(r: Resource): + def _xrootd_cache_resource_dict(r: Resource): return _service_resource_dict(r=r, service_name=XROOTD_CACHE_SERVER, auth_port_default=8443, unauth_port_default=8000) - def _origin_resource_dict(r: Resource): + def _xrootd_origin_resource_dict(r: Resource): return _service_resource_dict(r=r, service_name=XROOTD_ORIGIN_SERVER, auth_port_default=1095, unauth_port_default=1094) def _namespace_dict(ns: Namespace): @@ -653,12 +662,12 @@ def _resource_has_downed_origin(r: Resource, t: Topology): if group.itb and not filters.itb: continue for resource in group.resources: - if (_resource_has_cache(resource) + if (resource.has_xrootd_cache and (filters.include_inactive or resource.is_active) and (filters.include_downed or not _resource_has_downed_cache(resource, topology)) ): cache_resource_objs[resource.name] = resource - cache_resource_dicts[resource.name] = _cache_resource_dict(resource) + cache_resource_dicts[resource.name] = _xrootd_cache_resource_dict(resource) # Build a dict of origin resources @@ -671,12 +680,12 @@ def _resource_has_downed_origin(r: Resource, t: Topology): if group.itb and not filters.itb: continue for resource in group.resources: - if (_resource_has_origin(resource) + if (resource.has_xrootd_origin and (filters.include_inactive or resource.is_active) and (filters.include_downed or not _resource_has_downed_origin(resource, topology)) ): origin_resource_objs[resource.name] = resource - origin_resource_dicts[resource.name] = _origin_resource_dict(resource) + origin_resource_dicts[resource.name] = _xrootd_origin_resource_dict(resource) result_namespaces = [] for stashcache_obj in vos_data.stashcache_by_vo_name.values(): diff --git a/src/webapp/common.py b/src/webapp/common.py index d11c8e232..311ccca6c 100644 --- a/src/webapp/common.py +++ b/src/webapp/common.py @@ -396,5 +396,7 @@ def wrapped(): XROOTD_CACHE_SERVER = "XRootD cache server" XROOTD_ORIGIN_SERVER = "XRootD origin server" +PELICAN_CACHE = "Pelican cache" +PELICAN_ORIGIN = "Pelican origin" GRIDTYPE_1 = "OSG Production Resource" GRIDTYPE_2 = "OSG Integration Test Bed Resource" diff --git a/src/webapp/data_federation.py b/src/webapp/data_federation.py index 8542f527f..f78b1afc3 100644 --- a/src/webapp/data_federation.py +++ b/src/webapp/data_federation.py @@ -4,7 +4,7 @@ from collections import OrderedDict from typing import Optional, List, Dict, Tuple, Union, Set -from .common import XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, ParsedYaml, is_null +from .common import PELICAN_CACHE, PELICAN_ORIGIN, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, ParsedYaml, is_null try: from .x509 import generate_dn_hash except ImportError: # if asn1 is unavailable @@ -89,14 +89,16 @@ def __str__(self): f"map_subject={self.map_subject}" def get_scitokens_conf_block(self, service_name: str): - if service_name not in [XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER]: - raise ValueError(f"service_name must be '{XROOTD_CACHE_SERVER}' or '{XROOTD_ORIGIN_SERVER}'") + if service_name not in {PELICAN_CACHE, PELICAN_ORIGIN, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER}: + raise ValueError( + f"service_name must be one of: '{PELICAN_CACHE}', '{PELICAN_ORIGIN}', " + f"'{XROOTD_CACHE_SERVER}', or '{XROOTD_ORIGIN_SERVER}'") block = (f"[Issuer {self.issuer}]\n" f"issuer = {self.issuer}\n" f"base_path = {self.base_path}\n") if self.restricted_path: block += f"restricted_path = {self.restricted_path}\n" - if service_name == XROOTD_ORIGIN_SERVER: + if service_name in {PELICAN_ORIGIN, XROOTD_ORIGIN_SERVER}: block += f"map_subject = {self.map_subject}\n" return block diff --git a/src/webapp/exceptions.py b/src/webapp/exceptions.py index 7fd827cae..613075468 100644 --- a/src/webapp/exceptions.py +++ b/src/webapp/exceptions.py @@ -25,10 +25,17 @@ def __init__(self, resource: "Resource", text: str): super().__init__(f"Resource {resource.name}, FQDN {resource.fqdn}: {text}") -class ResourceMissingService(ResourceDataError): - def __init__(self, resource: "Resource", service_name: str): - self.service_name = service_name - super().__init__(resource=resource, text=f"Missing expected service {service_name}") +class ResourceMissingServices(ResourceDataError): + def __init__(self, resource: "Resource", service_names): + if isinstance(service_names, str): + service_names = [service_names] + self.service_names = service_names + self.service_names_str = ", ".join(service_names) + super().__init__( + resource=resource, + text="None of the following expected services are available: " + + self.service_names_str + ) class VODataError(DataError): diff --git a/src/webapp/topology.py b/src/webapp/topology.py index d09ae511c..6c589f689 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -9,7 +9,7 @@ from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml, \ is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, \ - gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true + gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true, PELICAN_ORIGIN, PELICAN_CACHE from .contacts_reader import ContactsData, User from .exceptions import DataError @@ -111,8 +111,25 @@ def __init__(self, name: str, yaml_data: ParsedYaml, common_data: CommonData, rg self.name = name self.service_types = common_data.service_types self.common_data = common_data + # Some "indexes" to speed up data lookup + self.has_xrootd_cache = False + self.has_xrootd_origin = False + self.has_pelican_cache = False + self.has_pelican_origin = False + self.service_names = [] if not is_null(yaml_data, "Services"): self.services = self._expand_services(yaml_data["Services"]) + for svc in self.services: + if "Name" in svc: + self.service_names.append(svc["Name"]) + if svc["Name"] == XROOTD_CACHE_SERVER: + self.has_xrootd_cache = True + elif svc["Name"] == XROOTD_ORIGIN_SERVER: + self.has_xrootd_origin = True + elif svc["Name"] == PELICAN_CACHE: + self.has_pelican_cache = True + elif svc["Name"] == PELICAN_ORIGIN: + self.has_pelican_origin = True else: self.services = [] self.service_names = [n["Name"] for n in self.services if "Name" in n] diff --git a/topology/services.yaml b/topology/services.yaml index 16b3362f4..e5c834b1e 100644 --- a/topology/services.yaml +++ b/topology/services.yaml @@ -26,6 +26,8 @@ perfsonar data store: 145 Connect: 149 XRootD origin server: 147 XRootD cache server: 156 +Pelican origin: 163 +Pelican cache: 164 XRootD HA component: 148 Virtual Machine Host: 150 Certificate Provider: 151 From 53595057aa0bacfd85411d51c651831c1b000aaf Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Wed, 19 Jun 2024 14:24:20 -0500 Subject: [PATCH 2/2] Use the new "Pelican cache" and "Pelican origin" services for the ITB OSDF cache and origin --- topology/University of Wisconsin/CHTC/CHTC-ITB.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/topology/University of Wisconsin/CHTC/CHTC-ITB.yaml b/topology/University of Wisconsin/CHTC/CHTC-ITB.yaml index fc6d114db..ee3f278c7 100644 --- a/topology/University of Wisconsin/CHTC/CHTC-ITB.yaml +++ b/topology/University of Wisconsin/CHTC/CHTC-ITB.yaml @@ -406,11 +406,8 @@ Resources: FQDN: itb-osdf-pelican-origin.osdf-dev.chtc.io DN: /CN=itb-osdf-pelican-origin.osdf-dev.chtc.io Services: - XRootD origin server: + Pelican origin: Description: ITB OSDF Pelican Origin - Details: - endpoint_override: itb-osdf-pelican-origin.osdf-dev.chtc.io:8443 - auth_endpoint_override: itb-osdf-pelican-origin.osdf-dev.chtc.io:8443 AllowedVOs: - GLOW @@ -443,10 +440,7 @@ Resources: FQDN: itb-osdf-pelican-cache.osdf-dev.chtc.io DN: /CN=itb-osdf-pelican-cache.osdf-dev.chtc.io Services: - XRootD cache server: + Pelican cache: Description: ITB OSDF Pelican Cache - Details: - endpoint_override: itb-osdf-pelican-cache.osdf-dev.chtc.io:8443 - auth_endpoint_override: itb-osdf-pelican-cache.osdf-dev.chtc.io:8443 AllowedVOs: - GLOW