Merge pull request #3980 from matyasselmeci/pr/pelican-service-types.…

…SOFTWARE-5867

ITB: Add "Pelican cache" and "Pelican origin" services (SOFTWARE-5867)

src/app.py

@@ -22,7 +22,7 @@
 from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \
     escape, cache_control_private, PreJSON, is_true, GRIDTYPE_1, GRIDTYPE_2, NamespacesFilters
 from webapp.flask_common import create_accepted_response
-from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
+from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingServices
 from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
 from webapp.models import GlobalData
 from webapp.oasis_managers import get_oasis_manager_endpoint_info
@@ -603,7 +603,7 @@ def _get_cache_authfile(public_only):
                                  fqdn=cache_fqdn,
                                  legacy=app.config["STASHCACHE_LEGACY_AUTH"],
                                  suppress_errors=False)
-    except (ResourceNotRegistered, ResourceMissingService) as e:
+    except (ResourceNotRegistered, ResourceMissingServices) as e:
         return Response("# {}\n"
                         "# Please check your query or contact [email protected]\n"
                         .format(str(e)),
@@ -627,7 +627,7 @@ def _get_origin_authfile(public_only):
     try:
         auth = stashcache.generate_origin_authfile(global_data=global_data, fqdn=request.args['fqdn'],
                                                    suppress_errors=False, public_origin=public_only)
-    except (ResourceNotRegistered, ResourceMissingService) as e:
+    except (ResourceNotRegistered, ResourceMissingServices) as e:
         return Response("# {}\n"
                         "# Please check your query or contact [email protected]\n"
                         .format(str(e)),

src/stashcache.py

@@ -1,8 +1,9 @@
 from collections import defaultdict
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Sequence
 
-from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, NamespacesFilters
-from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
+from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, PELICAN_CACHE, PELICAN_ORIGIN, \
+    NamespacesFilters
+from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingServices
 from webapp.models import GlobalData
 from webapp.topology import Resource, ResourceGroup, Topology
 from webapp.vos_data import VOsData
@@ -22,47 +23,52 @@ def _log_or_raise(suppress_errors: bool, an_exception: BaseException, logmethod=
         raise an_exception
 
 
-def _resource_has_cache(resource: Resource) -> bool:
-    return XROOTD_CACHE_SERVER in resource.service_names
-
-def _resource_has_origin(resource: Resource) -> bool:
-    return XROOTD_ORIGIN_SERVER in resource.service_names
-
-
-def _get_resource_with_service(fqdn: Optional[str], service_name: str, topology: Topology,
-                               suppress_errors: bool) -> Optional[Resource]:
-    """If given an FQDN, returns the Resource _if it has the given service.
+def _get_resource_with_services(fqdn: Optional[str], service_names: Sequence[str], topology: Topology,
+                                suppress_errors: bool) -> Optional[Resource]:
+    """
+    If given an FQDN, returns the Resource if it has one of the given services.
     If given None, returns None.
     If multiple Resources have the same FQDN, checks the first one.
     If suppress_errors is False, raises an exception on the following conditions:
     - no Resource matching FQDN (ResourceNotRegistered)
-    - Resource does not provide a SERVICE_NAME (ResourceMissingService)
+    - Resource does not provide any of the services in SERVICE_NAMES (ResourceMissingServices)
     If suppress_errors is True, logs the error and returns None on the above conditions.
-
     """
     resource = None
+    if isinstance(service_names, str):
+        service_names = [service_names]
     if fqdn:
         resource = topology.safe_get_resource_by_fqdn(fqdn)
         if not resource:
             _log_or_raise(suppress_errors, ResourceNotRegistered(fqdn=fqdn))
             return None
-        if service_name not in resource.service_names:
+
+        for service_name in service_names:
+            if service_name in resource.service_names:
+                return resource
+        else:
             _log_or_raise(
                 suppress_errors,
-                ResourceMissingService(resource, service_name)
+                ResourceMissingServices(resource, service_names)
             )
             return None
     return resource
 
 
 def _get_cache_resource(fqdn: Optional[str], topology: Topology, suppress_errors: bool) -> Optional[Resource]:
-    """Convenience wrapper around _get_resource-with-service() for a cache"""
-    return _get_resource_with_service(fqdn, XROOTD_CACHE_SERVER, topology, suppress_errors)
+    """
+    Convenience wrapper around _get_resource_with_services() for an xrootd
+    or pelican cache
+    """
+    return _get_resource_with_services(fqdn, [XROOTD_CACHE_SERVER, PELICAN_CACHE], topology, suppress_errors)
 
 
 def _get_origin_resource(fqdn: Optional[str], topology: Topology, suppress_errors: bool) -> Optional[Resource]:
-    """Convenience wrapper around _get_resource-with-service() for an origin"""
-    return _get_resource_with_service(fqdn, XROOTD_ORIGIN_SERVER, topology, suppress_errors)
+    """
+    Convenience wrapper around _get_resource_with_services() for an xrootd
+    or pelican origin
+    """
+    return _get_resource_with_services(fqdn, [XROOTD_ORIGIN_SERVER, PELICAN_ORIGIN], topology, suppress_errors)
 
 
 def resource_allows_namespace(resource: Resource, namespace: Optional[Namespace]) -> bool:
@@ -120,7 +126,8 @@ def get_supported_caches_for_namespace(namespace: Namespace, topology: Topology)
     all_caches = [resource
                   for group in resource_groups
                   for resource in group.resources
-                  if _resource_has_cache(resource)]
+                  if (resource.has_xrootd_cache or
+                      resource.has_pelican_cache)]
     return [cache
             for cache in all_caches
             if namespace_allows_cache_resource(namespace, cache)
@@ -546,6 +553,8 @@ def get_namespaces_info(global_data: GlobalData, filters: Optional[NamespacesFil
 
     If `include_downed` is True, caches/origins in downtime are also included.
     If `include_inactive` is True, caches/origins that are not marked as active are also included.
+
+    NOTE: This is specific to XRootD caches and origins; Pelican caches and origins are not included.
     """
     if filters is None:
         filters = NamespacesFilters()
@@ -579,10 +588,10 @@ def _service_resource_dict(
             "production": production,
         }
 
-    def _cache_resource_dict(r: Resource):
+    def _xrootd_cache_resource_dict(r: Resource):
         return _service_resource_dict(r=r, service_name=XROOTD_CACHE_SERVER, auth_port_default=8443, unauth_port_default=8000)
 
-    def _origin_resource_dict(r: Resource):
+    def _xrootd_origin_resource_dict(r: Resource):
         return _service_resource_dict(r=r, service_name=XROOTD_ORIGIN_SERVER, auth_port_default=1095, unauth_port_default=1094)
 
     def _namespace_dict(ns: Namespace):
@@ -653,12 +662,12 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
         if group.itb and not filters.itb:
             continue
         for resource in group.resources:
-            if (_resource_has_cache(resource)
+            if (resource.has_xrootd_cache
                     and (filters.include_inactive or resource.is_active)
                     and (filters.include_downed or not _resource_has_downed_cache(resource, topology))
             ):
                 cache_resource_objs[resource.name] = resource
-                cache_resource_dicts[resource.name] = _cache_resource_dict(resource)
+                cache_resource_dicts[resource.name] = _xrootd_cache_resource_dict(resource)
 
     # Build a dict of origin resources
 
@@ -671,12 +680,12 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
         if group.itb and not filters.itb:
             continue
         for resource in group.resources:
-            if (_resource_has_origin(resource)
+            if (resource.has_xrootd_origin
                     and (filters.include_inactive or resource.is_active)
                     and (filters.include_downed or not _resource_has_downed_origin(resource, topology))
             ):
                 origin_resource_objs[resource.name] = resource
-                origin_resource_dicts[resource.name] = _origin_resource_dict(resource)
+                origin_resource_dicts[resource.name] = _xrootd_origin_resource_dict(resource)
 
     result_namespaces = []
     for stashcache_obj in vos_data.stashcache_by_vo_name.values():

src/webapp/common.py

@@ -396,5 +396,7 @@ def wrapped():
 
 XROOTD_CACHE_SERVER = "XRootD cache server"
 XROOTD_ORIGIN_SERVER = "XRootD origin server"
+PELICAN_CACHE = "Pelican cache"
+PELICAN_ORIGIN = "Pelican origin"
 GRIDTYPE_1 = "OSG Production Resource"
 GRIDTYPE_2 = "OSG Integration Test Bed Resource"

src/webapp/data_federation.py

@@ -4,7 +4,7 @@
 from collections import OrderedDict
 from typing import Optional, List, Dict, Tuple, Union, Set
 
-from .common import XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, ParsedYaml, is_null
+from .common import PELICAN_CACHE, PELICAN_ORIGIN, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, ParsedYaml, is_null
 try:
     from .x509 import generate_dn_hash
 except ImportError:  # if asn1 is unavailable
@@ -89,14 +89,16 @@ def __str__(self):
                 f"map_subject={self.map_subject}"
 
     def get_scitokens_conf_block(self, service_name: str):
-        if service_name not in [XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER]:
-            raise ValueError(f"service_name must be '{XROOTD_CACHE_SERVER}' or '{XROOTD_ORIGIN_SERVER}'")
+        if service_name not in {PELICAN_CACHE, PELICAN_ORIGIN, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER}:
+            raise ValueError(
+                f"service_name must be one of: '{PELICAN_CACHE}', '{PELICAN_ORIGIN}', "
+                f"'{XROOTD_CACHE_SERVER}', or '{XROOTD_ORIGIN_SERVER}'")
         block = (f"[Issuer {self.issuer}]\n"
                  f"issuer = {self.issuer}\n"
                  f"base_path = {self.base_path}\n")
         if self.restricted_path:
             block += f"restricted_path = {self.restricted_path}\n"
-        if service_name == XROOTD_ORIGIN_SERVER:
+        if service_name in {PELICAN_ORIGIN, XROOTD_ORIGIN_SERVER}:
             block += f"map_subject = {self.map_subject}\n"
 
         return block

src/webapp/exceptions.py

@@ -25,10 +25,17 @@ def __init__(self, resource: "Resource", text: str):
         super().__init__(f"Resource {resource.name}, FQDN {resource.fqdn}: {text}")
 
 
-class ResourceMissingService(ResourceDataError):
-    def __init__(self, resource: "Resource", service_name: str):
-        self.service_name = service_name
-        super().__init__(resource=resource, text=f"Missing expected service {service_name}")
+class ResourceMissingServices(ResourceDataError):
+    def __init__(self, resource: "Resource", service_names):
+        if isinstance(service_names, str):
+            service_names = [service_names]
+        self.service_names = service_names
+        self.service_names_str = ", ".join(service_names)
+        super().__init__(
+            resource=resource,
+            text="None of the following expected services are available: " +
+                 self.service_names_str
+        )
 
 
 class VODataError(DataError):

src/webapp/topology.py

@@ -9,7 +9,7 @@
 
 from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml, \
     is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, \
-    gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true
+    gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true, PELICAN_ORIGIN, PELICAN_CACHE
 from .contacts_reader import ContactsData, User
 from .exceptions import DataError
 
@@ -111,8 +111,25 @@ def __init__(self, name: str, yaml_data: ParsedYaml, common_data: CommonData, rg
         self.name = name
         self.service_types = common_data.service_types
         self.common_data = common_data
+        # Some "indexes" to speed up data lookup
+        self.has_xrootd_cache = False
+        self.has_xrootd_origin = False
+        self.has_pelican_cache = False
+        self.has_pelican_origin = False
+        self.service_names = []
         if not is_null(yaml_data, "Services"):
             self.services = self._expand_services(yaml_data["Services"])
+            for svc in self.services:
+                if "Name" in svc:
+                    self.service_names.append(svc["Name"])
+                    if svc["Name"] == XROOTD_CACHE_SERVER:
+                        self.has_xrootd_cache = True
+                    elif svc["Name"] == XROOTD_ORIGIN_SERVER:
+                        self.has_xrootd_origin = True
+                    elif svc["Name"] == PELICAN_CACHE:
+                        self.has_pelican_cache = True
+                    elif svc["Name"] == PELICAN_ORIGIN:
+                        self.has_pelican_origin = True
         else:
             self.services = []
         self.service_names = [n["Name"] for n in self.services if "Name" in n]

topology/University of Wisconsin/CHTC/CHTC-ITB.yaml

@@ -406,11 +406,8 @@ Resources:
     FQDN: itb-osdf-pelican-origin.osdf-dev.chtc.io
     DN: /CN=itb-osdf-pelican-origin.osdf-dev.chtc.io
     Services:
-      XRootD origin server:
+      Pelican origin:
         Description: ITB OSDF Pelican Origin
-        Details:
-          endpoint_override: itb-osdf-pelican-origin.osdf-dev.chtc.io:8443
-          auth_endpoint_override: itb-osdf-pelican-origin.osdf-dev.chtc.io:8443
     AllowedVOs:
       - GLOW
 
@@ -443,10 +440,7 @@ Resources:
     FQDN: itb-osdf-pelican-cache.osdf-dev.chtc.io
     DN: /CN=itb-osdf-pelican-cache.osdf-dev.chtc.io
     Services:
-      XRootD cache server:
+      Pelican cache:
         Description: ITB OSDF Pelican Cache
-        Details:
-          endpoint_override: itb-osdf-pelican-cache.osdf-dev.chtc.io:8443
-          auth_endpoint_override: itb-osdf-pelican-cache.osdf-dev.chtc.io:8443
     AllowedVOs:
       - GLOW

topology/services.yaml

@@ -26,6 +26,8 @@ perfsonar data store: 145
 Connect: 149
 XRootD origin server: 147
 XRootD cache server: 156
+Pelican origin: 163
+Pelican cache: 164
 XRootD HA component: 148
 Virtual Machine Host: 150
 Certificate Provider: 151