Merge branch 'master' into 2024-july-catchup

projects/LSU_Chakraborty.yaml

@@ -0,0 +1,13 @@
+Department: Physics and Astronomy
+Description: 'Work in the interdisciplinary fields in theoretical condensed matter
+  physics, quantum information science and statistical physics. My research focuses
+  on non-equilibrium dynamics of quantum many-particle systems, especially open quantum
+  systems and disordered systems. My "Quantum Dynamics and Information" group at LSU
+  will focus on three integrated research areas: (1) entanglement dynamics in monitored
+  quantum circuits, (2) manipulation of quantum material by light and (3) dynamics
+  of disordered systems and thermalization.'
+FieldOfScience: Physics
+FieldOfScienceID: '40.0808'
+InstitutionID: https://osg-htc.org/iid/lk45ajqlj7f1
+Organization: Louisiana State University
+PIName: Ahana Chakraborty

projects/SparseNeuro.yaml

@@ -0,0 +1,9 @@
+Description: >-
+  Using machine learning techniques that discover solutions with anatomically and temporally structured sparsity,
+  we aim to test representational predictions from cognitive psychology using whole-brain neuroimaging datasets. 
+Department: Department of Psychology
+FieldOfScience: Behavioral Science
+Organization: Louisiana State University
+PIName: Christopher Cox 
+InstitutionID: 'https://osg-htc.org/iid/lk45ajqlj7f1'
+FieldOfScienceID: '30.1701'

src/app.py

@@ -22,7 +22,7 @@
 from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \
     escape, cache_control_private, PreJSON, is_true, GRIDTYPE_1, GRIDTYPE_2, NamespacesFilters
 from webapp.flask_common import create_accepted_response
-from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
+from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingServices
 from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
 from webapp.models import GlobalData
 from webapp.oasis_managers import get_oasis_manager_endpoint_info
@@ -603,7 +603,7 @@ def _get_cache_authfile(public_only):
                                  fqdn=cache_fqdn,
                                  legacy=app.config["STASHCACHE_LEGACY_AUTH"],
                                  suppress_errors=False)
-    except (ResourceNotRegistered, ResourceMissingService) as e:
+    except (ResourceNotRegistered, ResourceMissingServices) as e:
         return Response("# {}\n"
                         "# Please check your query or contact [email protected]\n"
                         .format(str(e)),
@@ -627,7 +627,7 @@ def _get_origin_authfile(public_only):
     try:
         auth = stashcache.generate_origin_authfile(global_data=global_data, fqdn=request.args['fqdn'],
                                                    suppress_errors=False, public_origin=public_only)
-    except (ResourceNotRegistered, ResourceMissingService) as e:
+    except (ResourceNotRegistered, ResourceMissingServices) as e:
         return Response("# {}\n"
                         "# Please check your query or contact [email protected]\n"
                         .format(str(e)),

src/stashcache.py

@@ -1,8 +1,9 @@
 from collections import defaultdict
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Sequence
 
-from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, NamespacesFilters
-from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
+from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, PELICAN_CACHE, PELICAN_ORIGIN, \
+    NamespacesFilters
+from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingServices
 from webapp.models import GlobalData
 from webapp.topology import Resource, ResourceGroup, Topology
 from webapp.vos_data import VOsData
@@ -22,47 +23,52 @@ def _log_or_raise(suppress_errors: bool, an_exception: BaseException, logmethod=
         raise an_exception
 
 
-def _resource_has_cache(resource: Resource) -> bool:
-    return XROOTD_CACHE_SERVER in resource.service_names
-
-def _resource_has_origin(resource: Resource) -> bool:
-    return XROOTD_ORIGIN_SERVER in resource.service_names
-
-
-def _get_resource_with_service(fqdn: Optional[str], service_name: str, topology: Topology,
-                               suppress_errors: bool) -> Optional[Resource]:
-    """If given an FQDN, returns the Resource _if it has the given service.
+def _get_resource_with_services(fqdn: Optional[str], service_names: Sequence[str], topology: Topology,
+                                suppress_errors: bool) -> Optional[Resource]:
+    """
+    If given an FQDN, returns the Resource if it has one of the given services.
     If given None, returns None.
     If multiple Resources have the same FQDN, checks the first one.
     If suppress_errors is False, raises an exception on the following conditions:
     - no Resource matching FQDN (ResourceNotRegistered)
-    - Resource does not provide a SERVICE_NAME (ResourceMissingService)
+    - Resource does not provide any of the services in SERVICE_NAMES (ResourceMissingServices)
     If suppress_errors is True, logs the error and returns None on the above conditions.
-
     """
     resource = None
+    if isinstance(service_names, str):
+        service_names = [service_names]
     if fqdn:
         resource = topology.safe_get_resource_by_fqdn(fqdn)
         if not resource:
             _log_or_raise(suppress_errors, ResourceNotRegistered(fqdn=fqdn))
             return None
-        if service_name not in resource.service_names:
+
+        for service_name in service_names:
+            if service_name in resource.service_names:
+                return resource
+        else:
             _log_or_raise(
                 suppress_errors,
-                ResourceMissingService(resource, service_name)
+                ResourceMissingServices(resource, service_names)
             )
             return None
     return resource
 
 
 def _get_cache_resource(fqdn: Optional[str], topology: Topology, suppress_errors: bool) -> Optional[Resource]:
-    """Convenience wrapper around _get_resource-with-service() for a cache"""
-    return _get_resource_with_service(fqdn, XROOTD_CACHE_SERVER, topology, suppress_errors)
+    """
+    Convenience wrapper around _get_resource_with_services() for an xrootd
+    or pelican cache
+    """
+    return _get_resource_with_services(fqdn, [XROOTD_CACHE_SERVER, PELICAN_CACHE], topology, suppress_errors)
 
 
 def _get_origin_resource(fqdn: Optional[str], topology: Topology, suppress_errors: bool) -> Optional[Resource]:
-    """Convenience wrapper around _get_resource-with-service() for an origin"""
-    return _get_resource_with_service(fqdn, XROOTD_ORIGIN_SERVER, topology, suppress_errors)
+    """
+    Convenience wrapper around _get_resource_with_services() for an xrootd
+    or pelican origin
+    """
+    return _get_resource_with_services(fqdn, [XROOTD_ORIGIN_SERVER, PELICAN_ORIGIN], topology, suppress_errors)
 
 
 def resource_allows_namespace(resource: Resource, namespace: Optional[Namespace]) -> bool:
@@ -120,7 +126,8 @@ def get_supported_caches_for_namespace(namespace: Namespace, topology: Topology)
     all_caches = [resource
                   for group in resource_groups
                   for resource in group.resources
-                  if _resource_has_cache(resource)]
+                  if (resource.has_xrootd_cache or
+                      resource.has_pelican_cache)]
     return [cache
             for cache in all_caches
             if namespace_allows_cache_resource(namespace, cache)
@@ -546,6 +553,8 @@ def get_namespaces_info(global_data: GlobalData, filters: Optional[NamespacesFil
 
     If `include_downed` is True, caches/origins in downtime are also included.
     If `include_inactive` is True, caches/origins that are not marked as active are also included.
+
+    NOTE: This is specific to XRootD caches and origins; Pelican caches and origins are not included.
     """
     if filters is None:
         filters = NamespacesFilters()
@@ -579,10 +588,10 @@ def _service_resource_dict(
             "production": production,
         }
 
-    def _cache_resource_dict(r: Resource):
+    def _xrootd_cache_resource_dict(r: Resource):
         return _service_resource_dict(r=r, service_name=XROOTD_CACHE_SERVER, auth_port_default=8443, unauth_port_default=8000)
 
-    def _origin_resource_dict(r: Resource):
+    def _xrootd_origin_resource_dict(r: Resource):
         return _service_resource_dict(r=r, service_name=XROOTD_ORIGIN_SERVER, auth_port_default=1095, unauth_port_default=1094)
 
     def _namespace_dict(ns: Namespace):
@@ -653,12 +662,12 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
         if group.itb and not filters.itb:
             continue
         for resource in group.resources:
-            if (_resource_has_cache(resource)
+            if (resource.has_xrootd_cache
                     and (filters.include_inactive or resource.is_active)
                     and (filters.include_downed or not _resource_has_downed_cache(resource, topology))
             ):
                 cache_resource_objs[resource.name] = resource
-                cache_resource_dicts[resource.name] = _cache_resource_dict(resource)
+                cache_resource_dicts[resource.name] = _xrootd_cache_resource_dict(resource)
 
     # Build a dict of origin resources
 
@@ -671,12 +680,12 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
         if group.itb and not filters.itb:
             continue
         for resource in group.resources:
-            if (_resource_has_origin(resource)
+            if (resource.has_xrootd_origin
                     and (filters.include_inactive or resource.is_active)
                     and (filters.include_downed or not _resource_has_downed_origin(resource, topology))
             ):
                 origin_resource_objs[resource.name] = resource
-                origin_resource_dicts[resource.name] = _origin_resource_dict(resource)
+                origin_resource_dicts[resource.name] = _xrootd_origin_resource_dict(resource)
 
     result_namespaces = []
     for stashcache_obj in vos_data.stashcache_by_vo_name.values():

src/webapp/common.py

@@ -396,5 +396,7 @@ def wrapped():
 
 XROOTD_CACHE_SERVER = "XRootD cache server"
 XROOTD_ORIGIN_SERVER = "XRootD origin server"
+PELICAN_CACHE = "Pelican cache"
+PELICAN_ORIGIN = "Pelican origin"
 GRIDTYPE_1 = "OSG Production Resource"
 GRIDTYPE_2 = "OSG Integration Test Bed Resource"

src/webapp/data_federation.py

@@ -4,7 +4,7 @@
 from collections import OrderedDict
 from typing import Optional, List, Dict, Tuple, Union, Set
 
-from .common import XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, ParsedYaml, is_null
+from .common import PELICAN_CACHE, PELICAN_ORIGIN, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, ParsedYaml, is_null
 try:
     from .x509 import generate_dn_hash
 except ImportError:  # if asn1 is unavailable
@@ -89,14 +89,16 @@ def __str__(self):
                 f"map_subject={self.map_subject}"
 
     def get_scitokens_conf_block(self, service_name: str):
-        if service_name not in [XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER]:
-            raise ValueError(f"service_name must be '{XROOTD_CACHE_SERVER}' or '{XROOTD_ORIGIN_SERVER}'")
+        if service_name not in {PELICAN_CACHE, PELICAN_ORIGIN, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER}:
+            raise ValueError(
+                f"service_name must be one of: '{PELICAN_CACHE}', '{PELICAN_ORIGIN}', "
+                f"'{XROOTD_CACHE_SERVER}', or '{XROOTD_ORIGIN_SERVER}'")
         block = (f"[Issuer {self.issuer}]\n"
                  f"issuer = {self.issuer}\n"
                  f"base_path = {self.base_path}\n")
         if self.restricted_path:
             block += f"restricted_path = {self.restricted_path}\n"
-        if service_name == XROOTD_ORIGIN_SERVER:
+        if service_name in {PELICAN_ORIGIN, XROOTD_ORIGIN_SERVER}:
             block += f"map_subject = {self.map_subject}\n"
 
         return block

src/webapp/exceptions.py

@@ -25,10 +25,17 @@ def __init__(self, resource: "Resource", text: str):
         super().__init__(f"Resource {resource.name}, FQDN {resource.fqdn}: {text}")
 
 
-class ResourceMissingService(ResourceDataError):
-    def __init__(self, resource: "Resource", service_name: str):
-        self.service_name = service_name
-        super().__init__(resource=resource, text=f"Missing expected service {service_name}")
+class ResourceMissingServices(ResourceDataError):
+    def __init__(self, resource: "Resource", service_names):
+        if isinstance(service_names, str):
+            service_names = [service_names]
+        self.service_names = service_names
+        self.service_names_str = ", ".join(service_names)
+        super().__init__(
+            resource=resource,
+            text="None of the following expected services are available: " +
+                 self.service_names_str
+        )
 
 
 class VODataError(DataError):

src/webapp/topology.py

@@ -9,7 +9,7 @@
 
 from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml, \
     is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, \
-    gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true
+    gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true, PELICAN_ORIGIN, PELICAN_CACHE
 from .contacts_reader import ContactsData, User
 from .exceptions import DataError
 
@@ -111,8 +111,25 @@ def __init__(self, name: str, yaml_data: ParsedYaml, common_data: CommonData, rg
         self.name = name
         self.service_types = common_data.service_types
         self.common_data = common_data
+        # Some "indexes" to speed up data lookup
+        self.has_xrootd_cache = False
+        self.has_xrootd_origin = False
+        self.has_pelican_cache = False
+        self.has_pelican_origin = False
+        self.service_names = []
         if not is_null(yaml_data, "Services"):
             self.services = self._expand_services(yaml_data["Services"])
+            for svc in self.services:
+                if "Name" in svc:
+                    self.service_names.append(svc["Name"])
+                    if svc["Name"] == XROOTD_CACHE_SERVER:
+                        self.has_xrootd_cache = True
+                    elif svc["Name"] == XROOTD_ORIGIN_SERVER:
+                        self.has_xrootd_origin = True
+                    elif svc["Name"] == PELICAN_CACHE:
+                        self.has_pelican_cache = True
+                    elif svc["Name"] == PELICAN_ORIGIN:
+                        self.has_pelican_origin = True
         else:
             self.services = []
         self.service_names = [n["Name"] for n in self.services if "Name" in n]

topology/Clemson University/Clemson-Palmetto/Clemson-Palmetto.yaml

@@ -7,14 +7,14 @@ Resources:
     ContactLists:
       Administrative Contact:
         Primary:
-          ID: 0a22bab3de2d83d723811e3fb1ebca904e924a97
+          ID: OSG1000013
           Name: Lincoln Bryant
         Secondary:
           ID: 56f7f6139ae847fc94317c466a7789acf8cdc85d
           Name: Randall Martin
       Security Contact:
         Primary:
-          ID: 0a22bab3de2d83d723811e3fb1ebca904e924a97
+          ID: OSG1000013
           Name: Lincoln Bryant
         Secondary:
           ID: 56f7f6139ae847fc94317c466a7789acf8cdc85d

topology/Compute Canada/ComputeCanada - Cedar/ComputeCanada-Cedar_downtime.yaml

@@ -42,3 +42,14 @@
   Services:
   - XRootD cache server
 # ---------------------------------------------------------
+- Class: UNSCHEDULED
+  ID: 1861822300
+  Description: HW issue
+  Severity: Outage
+  StartTime: Jul 15, 2024 19:30 +0000
+  EndTime: Dec 31, 2024 19:30 +0000
+  CreatedTime: Jul 17, 2024 02:10 +0000
+  ResourceName: ComputeCanada-Cedar-Cache
+  Services:
+  - XRootD cache server
+# ---------------------------------------------------------

topology/Fermi National Accelerator Laboratory/FNAL USCMS Tier1/USCMS-FNAL-WC1_downtime.yaml

@@ -1060,5 +1060,29 @@
   Services:
   - SRMv2
 # ---------------------------------------------------------
+- Class: SCHEDULED
+  ID: 1866535972
+  Description: dCache upgrade
+  Severity: Severe
+  StartTime: Jul 24, 2024 13:10 +0000
+  EndTime: Jul 24, 2024 22:10 +0000
+  CreatedTime: Jul 22, 2024 13:06 +0000
+  ResourceName: USCMS-FNAL-WC1-SE
+  Services:
+  - FTS
+  - SRMv2
+# ---------------------------------------------------------
+- Class: SCHEDULED
+  ID: 1866536437
+  Description: dCache upgrade
+  Severity: Severe
+  StartTime: Jul 23, 2024 13:10 +0000
+  EndTime: Jul 23, 2024 22:10 +0000
+  CreatedTime: Jul 22, 2024 13:07 +0000
+  ResourceName: USCMS-FNAL-WC1-SE2
+  Services:
+  - FTS
+  - SRMv2
+# ---------------------------------------------------------
 
 

topology/Internet2/Internet2Houston/I2HoustonInfrastructure_downtime.yaml

@@ -20,3 +20,14 @@
   Services:
   - XRootD cache server
 # ---------------------------------------------------------
+- Class: UNSCHEDULED
+  ID: 1860708585
+  Description: Mark as down until Fabio can check
+  Severity: Outage
+  StartTime: Jul 15, 2024 19:00 +0000
+  EndTime: Jul 25, 2024 19:00 +0000
+  CreatedTime: Jul 15, 2024 19:14 +0000
+  ResourceName: HOUSTON2_INTERNET2_OSDF_CACHE
+  Services:
+  - XRootD cache server
+# ---------------------------------------------------------

topology/Kansas University/KANSAS_HOSTED/KANSAS_HOSTED_BOSCO_CE.yaml

@@ -8,11 +8,11 @@ Resources:
     ContactLists:
       Administrative Contact:
         Primary:
-          ID: 0a22bab3de2d83d723811e3fb1ebca904e924a97
+          ID: OSG1000013
           Name: Lincoln Bryant
       Security Contact:
         Primary:
-          ID: 0a22bab3de2d83d723811e3fb1ebca904e924a97
+          ID: OSG1000013
           Name: Lincoln Bryant
     Description: BOSCO CE gatekeeper for the cluster at KU Community Cluster
     FQDN: hosted-ce01.grid.uchicago.edu

topology/Montana State University/Montana State RCI/MTState-Tempest_downtime.yaml

@@ -0,0 +1,11 @@
+- Class: SCHEDULED
+  ID: 1861471608
+  Description: Bianual Updates
+  Severity: Severe
+  StartTime: Jul 15, 2024 14:00 +0000
+  EndTime: Jul 18, 2024 14:00 +0000
+  CreatedTime: Jul 16, 2024 16:26 +0000
+  ResourceName: MTState-Tempest-CE1
+  Services:
+  - CE
+# ---------------------------------------------------------