Merge branch 'master' into itb

* master:
  Add downtime for FZU_STASH due to offline on checmk
  Add downtime for ComputeCanada-Cedar-Cache due to offline on checmk
  Rename the field `osg_id` to `id` in the Institution->ID mappings as per schema change in SOFTWARE-5786
  Fix response type
  Validate mappings in test_institution_ids() test
  Validate mapping for `/institution_ids` endpoint and add a test
  Add `/institution_ids` endpoint and the YAML file it's built from
  Remove unnecessary IDs
  Added new AP: comses.sol.rc.asu.edu
  Add downtime for CARDIFF_UK_OSDF_CACHE due to Access problem
  Adds downtime for Alabama CHPC due to cluster reinstall
  adding dCache tape upgrade downtime
  Add downtime for NMSU-Discovery-CE1 due to maintenance.
  Update Baylor-Kodiak_downtime.yaml
  Make Base Paths the same for the ap22 issuer
  add ND downtime
  Update AGLT2_downtime.yaml
  Make S3 origins public (request from Justin Hiemstra over Slack)
  Fixing spaces
  Bump downtime on sc-cache.chtc.wisc.edu some more
  Bring sc-cache.chtc.wisc.edu down for testing
  Add downtime for Stashcache-Manhattan due to HW problem
  Create MIT_Takei.yaml
  Fix DN of KAGRA origin

projects/MIT_Takei.yaml

@@ -0,0 +1,6 @@
+Department: Global Programs
+Description: I am working on models for quantifying the effects on the change of financial
+  regulations
+FieldOfScience: Finance
+Organization: Massachusetts Institute of Technology
+PIName: Ikuo Takei

src/app.py

@@ -17,7 +17,7 @@
 from flask_wtf.csrf import CSRFProtect
 
 from webapp import default_config
-from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape, cache_control_private
+from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape, cache_control_private, PreJSON
 from webapp.flask_common import create_accepted_response
 from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
 from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
@@ -180,6 +180,14 @@ def nsfscience_csv():
     response.headers.set("Content-Disposition", "attachment", filename="nsfscience.csv")
     return response
 
+@app.route('/institution_ids')
+def institution_ids():
+    institution_ids = global_data.get_mappings().institution_ids
+    if not institution_ids:
+        return Response("Error getting Institution/OSG ID mappings: no mappings returned", status=503)
+
+    return Response(to_json_bytes(PreJSON(institution_ids)), mimetype='application/json')
+
 
 @app.route('/organizations')
 def organizations():

src/tests/test_api.py

@@ -840,6 +840,21 @@ def test_institution_default(self, client: flask.Flask):
         # Check all projects have the project bit flipped
         assert all(x[2] for x in institutions if x[0] in projects)
 
+    def test_institution_ids(self, client: flask.Flask):
+        _ = global_data.get_mappings(strict=True)
+        institution_ids_list = client.get("/institution_ids").json
+
+        assert len(institution_ids_list) > 20, "Unexpectedly few institutions: %d" % len(institution_ids_list)
+        names_list = [i["name"] for i in institution_ids_list]
+        names_set = set(names_list)
+        duplicates = len(names_list) - len(names_set)
+        assert duplicates == 0, "%d duplicate names found in institution_ids list provided by API" % duplicates
+
+        osg_ids_list = [i["id"] for i in institution_ids_list]
+        osg_ids_set = set(osg_ids_list)
+        duplicates = len(osg_ids_list) - len(osg_ids_set)
+        assert duplicates == 0, "%d duplicate ids found in institution_ids list provided by API" % duplicates
+
 
 if __name__ == '__main__':
     pytest.main()

src/webapp/common.py

@@ -30,7 +30,7 @@
 SSH_WITH_KEY = os.path.abspath(os.path.dirname(__file__) + "/ssh_with_key.sh")
 
 ParsedYaml = NewType("ParsedYaml", Dict[str, Any])  # a complex data structure that's a result of parsing a YAML file
-PreJSON = NewType("PreJSON", Dict[str, Any])  # a complex data structure that will be converted to JSON in the webapp
+PreJSON = NewType("PreJSON", Union[List, Dict[str, Any]])  # a complex data structure that will be converted to JSON in the webapp
 T = TypeVar("T")
 
 

src/webapp/mappings.py

@@ -5,21 +5,23 @@
 import os
 import sys
 import yaml
-from typing import Dict
+from typing import Dict, List
 
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from webapp.exceptions import DataError
 from webapp.common import load_yaml_file
 
 
 log = logging.getLogger(__name__)
 
 
 class Mappings:
-    def __init__(self, nsfscience: Dict, project_institution: Dict):
+    def __init__(self, nsfscience: Dict, project_institution: Dict, institution_ids: List):
         self.nsfscience = nsfscience
         self.project_institution = project_institution
+        self.institution_ids = institution_ids
 
 
 def get_nsfscience(indir: str, strict: bool) -> Dict:
@@ -48,7 +50,58 @@ def get_project_institution(indir: str, strict: bool) -> Dict:
     return project_institution
 
 
+def validate_institution_ids(institution_ids: List) -> List[str]:
+    """Validate the institution/id mapping loaded by get_institution_ids():
+    ensure required attributes are present and nonempty, and there are no duplicates.
+    """
+    errors = []
+    institution_name_fields_set = set()
+    institution_id_fields_set = set()
+    for ii in institution_ids:
+        name = ii.get("name", None)
+        id_ = ii.get("id", None)
+        if not name:
+            errors.append("Missing 'name' in entry %r" % ii)
+            continue
+        if not id_:
+            errors.append("Missing 'id' in entry %r" % ii)
+            continue
+        if name in institution_name_fields_set:
+            errors.append("Duplicate 'name' %s in entry %r" % (name, ii))
+            continue
+        if id_ in institution_id_fields_set:
+            errors.append("Duplicate 'id' %s in entry %r" % (id_, ii))
+            continue
+        institution_name_fields_set.add(name)
+        institution_id_fields_set.add(id_)
+    return errors
+
+
+def get_institution_ids(indir: str, strict: bool) -> List:
+    institution_ids = []
+    try:
+        institution_ids = load_yaml_file(os.path.join(indir, "institution_ids.yaml"))
+    except yaml.YAMLError:
+        if strict:
+            raise
+        else:
+            # load_yaml_file() already logs the specific error
+            log.error("skipping (non-strict mode)")
+
+    errors = validate_institution_ids(institution_ids)
+    if errors:
+        message = "Errors found with institution/id mappings:\n%s" % "\n".join(errors)
+        if strict:
+            raise DataError(message)
+        else:
+            log.error(message)
+            log.error("skipping bad mappings (non-strict mode)")
+
+    return institution_ids
+
+
 def get_mappings(indir="../mappings", strict=False):
     mappings = Mappings(nsfscience=get_nsfscience(indir, strict),
-                        project_institution=get_project_institution(indir, strict))
+                        project_institution=get_project_institution(indir, strict),
+                        institution_ids=get_institution_ids(indir, strict))
     return mappings

src/webapp/models.py

@@ -315,18 +315,20 @@ def get_projects(self) -> Optional[Dict]:
 
         return self.projects.data
 
-    def get_mappings(self) -> Optional[mappings.Mappings]:
+    def get_mappings(self, strict=None) -> Optional[mappings.Mappings]:
         """
         Get mappings data.
         May return None if we fail to get the data for the first time.
         """
+        if strict is None:
+            strict = self.strict
         if self.mappings.should_update():
             ok = self._update_topology_repo()
             if ok:
                 try:
-                    self.mappings.update(mappings.get_mappings(indir=self.mappings_dir, strict=self.strict))
+                    self.mappings.update(mappings.get_mappings(indir=self.mappings_dir, strict=strict))
                 except Exception:
-                    if self.strict:
+                    if strict:
                         raise
                     log.exception("Failed to update mappings")
                     self.mappings.try_again()

topology/Arizona State University/ASU Research Computing/ASU-APs.yaml

@@ -0,0 +1,23 @@
+GroupDescription: ASU Access Points
+Production: true
+Resources:
+  ASU-Comses:
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          ID: OSG1000646
+          Name: Alan Chapman
+      Security Contact:
+        Primary:
+          ID: OSG1000646
+          Name: Alan Chapman
+    Description: Access point
+    FQDN: comses.sol.rc.asu.edu
+    Services:
+      Submit Node:
+        Description: Access point
+        Details:
+          hidden: false
+    Tags:
+      - OSPool
+SupportCenter: Self Supported

topology/Baylor University/Baylor University/Baylor-Kodiak_downtime.yaml

@@ -54,3 +54,15 @@
   - CE
   - Squid
 # ---------------------------------------------------------
+- Class: SCHEDULED
+  ID: 1696887320
+  Description: Maintenance
+  Severity: Severe
+  StartTime: Jan 08, 2024 15:00 +0000
+  EndTime: Jan 13, 2024 05:00 +0000
+  CreatedTime: Jan 08, 2024 04:38 +0000
+  ResourceName: Baylor-Kodiak-CE
+  Services:
+  - CE
+  - Squid
+# ---------------------------------------------------------

topology/Cardiff University/Cardiff Computing Cluster/CardiffPRPCachingInfrastructure_downtime.yaml

@@ -0,0 +1,10 @@
+- Class: UNSCHEDULED
+  ID: 1698308933
+  Description: Access problem
+  Severity: Outage
+  StartTime: Jan 09, 2024 20:30 +0000
+  EndTime: Jan 12, 2024 20:30 +0000
+  CreatedTime: Jan 09, 2024 20:08 +0000
+  ResourceName: CARDIFF_UK_OSDF_CACHE
+  Services:
+  - XRootD cache server

topology/Compute Canada/ComputeCanada - Cedar/ComputeCanada-Cedar_downtime.yaml

@@ -9,3 +9,14 @@
   Services:
   - XRootD cache server
 # ---------------------------------------------------------
+
+- Class: UNSCHEDULED
+  ID: 1699263279
+  Description: Offline on check
+  Severity: Outage
+  StartTime: Jan 10, 2024 20:30 +0000
+  EndTime: Jan 12, 2024 20:30 +0000
+  CreatedTime: Jan 10, 2024 22:38 +0000
+  ResourceName: ComputeCanada-Cedar-Cache
+  Services:
+  - XRootD cache server

topology/Fermi National Accelerator Laboratory/FNAL USCMS Tier1/USCMS-FNAL-WC1_downtime.yaml

@@ -1038,3 +1038,14 @@
   Services:
   - CE
 # ---------------------------------------------------------
+- Class: SCHEDULED
+  ID: 1697197635
+  Description: dCache upgrade
+  Severity: Severe
+  StartTime: Jan 09, 2024 14:00 +0000
+  EndTime: Jan 09, 2024 23:30 +0000
+  CreatedTime: Jan 08, 2024 13:16 +0000
+  ResourceName: USCMS-FNAL-WC1-SE
+  Services:
+  - SRMv2
+# ---------------------------------------------------------

topology/Institute of Physics ASCR/FZU/FZU_downtime.yaml

@@ -9,3 +9,16 @@
   Severity: Outage
   StartTime: Dec 30, 2017 23:00 +0000
 # ----------------------------------------------------------
+
+- Class: UNSCHEDULED
+  ID: 1699269131
+  Description: Offline on check
+  Severity: Outage
+  StartTime: Jan 10, 2024 20:30 +0000
+  EndTime: Jan 12, 2024 20:30 +0000
+  CreatedTime: Jan 10, 2024 22:48 +0000
+  ResourceName: FZU_STASH
+  Services:
+  - XRootD cache server
+# ---------------------------------------------------------
+

topology/Internet2/Internet2Manhattan/I2ManhattanInfrastructure_downtime.yaml

@@ -0,0 +1,11 @@
+- Class: UNSCHEDULED
+  ID: 1693052428
+  Description: HW/K8s problem
+  Severity: Intermittent Outage
+  StartTime: Jan 02, 2024 20:30 +0000
+  EndTime: Jan 05, 2024 20:30 +0000
+  CreatedTime: Jan 03, 2024 18:07 +0000
+  ResourceName: Stashcache-Manhattan
+  Services:
+  - XRootD cache server
+# ---------------------------------------------------------

topology/New Mexico State University/New Mexico State Discovery/NMSU_DISCOVERY_downtime.yaml

@@ -58,4 +58,14 @@
   ResourceName: NMSU-Discovery-CE1
   Services:
   - CE
+- Class: SCHEDULED
+  ID: 1696997558
+  Description: Scheduled biannual cluster maintenance
+  Severity: Outage
+  StartTime: Jan 08, 2024 15:00 +0000
+  EndTime: Jan 22, 2024 08:00 +0000
+  CreatedTime: Jan 08, 2024 07:42 +0000
+  ResourceName: NMSU-Discovery-CE1
+  Services:
+  - CE
 # ---------------------------------------------------------

topology/The University of Tokyo/KAGRA Observatory/KAGRA-OSDF.yaml

@@ -18,7 +18,7 @@ Resources:
           ID: 5ea6fa23f5d804890b3c22e478c6945f841e5647
           Name: Marco Meyer
     FQDN: kagra-dsr-b2.icrr.u-tokyo.ac.jp
-    DN: /DC=JP/O=KEK/OU=CRC/CN=host/kagra-dsr-b2.icrr.u-tokyo.ac.jp
+    DN: /C=JP/O=KEK/OU=CRC/CN=host/kagra-dsr-b2.icrr.u-tokyo.ac.jp
     Services:
       XRootD origin server:
         Description: KAGRA OSDF origin server

topology/University of Alabama/University of Alabama/Alabama-CHPC_downtime.yaml

@@ -0,0 +1,11 @@
+- Class: SCHEDULED
+  ID: 1697413007
+  Description: Reinstalling cluster
+  Severity: Outage
+  StartTime: Jan 09, 2024 05:00 +0000
+  EndTime: Feb 08, 2024 05:00 +0000
+  CreatedTime: Jan 08, 2024 19:15 +0000
+  ResourceName: Alabama-CHPC-CE1
+  Services:
+  - CE
+# ---------------------------------------------------------

topology/University of California San Diego/SDSC OSG/DataInfrastructure.yaml

@@ -1,19 +1,8 @@
-# Production is true if the resource is for production and not testing use
 Production: false
-# SupportCenter is one of the support centers in topology/support-centers.yaml
 SupportCenter: Community Support Center
-
-# GroupDescription is a long description of the resource group; may be multiple lines.
 GroupDescription: Data infrastructure at SDSC
-
-# If you have an up-to-date local git clone, fill GroupID with the output from `bin/next_resourcegroup_id`
-# Otherwise, leave it blank and we will fill in the appropriate value for you.
 GroupID: 494
-
-# Resources contains one or more resources in this
-# ResourceGroup. A resource provides one or more services
 Resources:
-  # Resource Name should be a short descriptor of the resource.
   SDSC-MERRA2-Origin:
     Active: false
     Description: SDSC MERRA2 StashCache XRootD origin
@@ -26,22 +15,17 @@ Resources:
         Secondary:
           ID: bc36e7fe84fffcb5cf195fe09cc42336f5cd5d1f
           Name: Diego Davila
-
       Security Contact:
         Primary:
           Name: Igor Sfiligoi
           ID: 593daf4f680b3135849201036e1055f03f89cfac
-
     FQDN: stashcache-origin-merra2.nautilus.optiputer.net
-
     Services:
       XRootD origin server:
         Description: SDSC MERRA2 Origin Server, initally for XCache testing.
-
     AllowedVOs:
       - ANY
 
-  # Resource Name should be a short descriptor of the resource.
   SDSC-test-Origin:
     Active: true
     Description: OSDF origin test
@@ -50,18 +34,14 @@ Resources:
         Primary:
           Name: Fabio Andrijauskas
           ID: OSG1000162
-
       Security Contact:
         Primary:
           Name: Fabio Andrijauskas
           ID: OSG1000162
-
     FQDN: osdftest.t2.ucsd.edu
     ID: 1307
-
     Services:
       XRootD origin server:
         Description: OSDF origin test
-
     AllowedVOs:
       - ANY