Merge branch 'master' into pr/add-eos

mappings/project_institution.yaml

@@ -50,6 +50,7 @@ NJIT: "New Jersey Institute of Technology"
 NMSU: "New Mexico State University"
 NOAA: "National Oceanic and Atmospheric Administration"
 Northeastern: "Northeastern University"
+NorthwesternMed: "Northwestern Medicine"
 NSHE: "Nevada System of Higher Education"
 OHSU: "Oregon Health & Science University"
 OSU: "The Ohio State University"

projects/Auburn_Hauck.yaml

@@ -0,0 +1,9 @@
+Department: Pathobiology
+Description: Experimental infections with avian reovirus and co-infections with other micro-organisms. We analyze bioinformatic data pertaining to microbiome, gene expression, metagenome and transcriptome obtained from these experiments.
+FieldOfScience: Agricultural Sciences specifically Poultry Science
+Organization: Auburn University
+PIName: Ruediger Hauck
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/IIT_Rosa.yaml

@@ -0,0 +1,9 @@
+Description: "Generating trajectories in high-dimensional parameter spaces using numerical continuation methods.  The software repo is available here: https://github.com/nr-codes/BipedalGaitGeneration."
+Department: Mechanical, Materials, and Aerospace Engineering Deptartment
+FieldOfScience: Mechanical Engineering
+Organization: Illinois Institute of Technology
+PIName: Nelson Rosa
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/ND_Chen.yaml

@@ -0,0 +1,5 @@
+Department: College of Engineering
+Description: Training AI models on public medical image data
+FieldOfScience: Engineering
+Organization: University of Notre Dame
+PIName: Danny Chen

projects/NorthwesternMed_Yadav.yaml

@@ -0,0 +1,6 @@
+Department: Department of Radiation Oncology
+Description: 'Monte Carlo simulations of the Boltzmann radiation transport equation to investigate radiation absorbed dose delivered from megavoltage linear accelerators. '
+FieldOfScience: Physics
+Organization: Northwestern Medicine
+PIName: Poonam Yadav
+

projects/UCLA_Huang.yaml

@@ -0,0 +1,11 @@
+Description: Studying $\Omega$-hadron correlation to search for signatures of baryon junction mechanisms at RHIC BES energies.
+Department: Physics and Astronomy
+FieldOfScience: Astronomy
+Organization: Arizona State University
+PIName: Huan Zhong Huang
+
+ID: '731'
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

projects/UTA_Jones.yaml

@@ -0,0 +1,7 @@
+Description: >
+  works on the NEXT neutrinoless double beta decay experiment: https://next.ific.uv.es/next/ which is an international collaboration. 
+  The experiment is trying to determine if the neutrino is its own anti-particle.
+Department: Physics
+FieldOfScience: Physics
+Organization: University of Texas at Arlington
+PIName: Ben Jones

projects/UWMadison_DeWerd.yaml

@@ -0,0 +1,6 @@
+Department: Medical Physics
+Description: Simulations for radiation therapy applications
+FieldOfScience: Physics
+Organization: University of Wisconsin-Madison
+PIName: Larry DeWerd
+

projects/Washington_Savage.yaml

@@ -0,0 +1,9 @@
+Description: Quantum simulations of many-body systems for nuclear and high-energy physics (https://iqus.uw.edu).
+Department: IQuS, Department of Physics
+FieldOfScience: Physics
+Organization: University of Washington
+PIName: Martin J. Savage
+
+Sponsor:
+  CampusGrid:
+    Name: OSG Connect

src/README.md

@@ -442,7 +442,7 @@ For every cache resource, add a `u <DN HASH> <PATH1> rl <PATH2> rl ...` ACL for
 The Authfile for a public cache is served at `/cache/Authfile-public?fqdn=<CACHE FQDN>`.
 
 The public Authfile is basically a giant `u *` list:
-- Explicitly deny read access to `/user/ligo` (with `-rl` permissions)
+- Explicitly deny read access to `/user/ligo` (with `-rl` permissions); this is needed, because granting access to the OSG VO `/user` path would otherwise also grant access to `/user/ligo`
 - Allow read access to the path of each namespace supported by the cache (`rl` permissions) 
 
 ### Cache authenticated Authfile generation
@@ -452,9 +452,9 @@ The Authfile for an authenticated cache is served at `/cache/Authfile?fqdn=<CACH
 - Add a `u <DN HASH> <PATH1> rl <PATH2> rl ...` for every DN listed in the Authorizations list of every namespace supported by the cache.
 - Add a `g <FQAN> <PATH1> rl <PATH2> rl ...` for every FQAN listed in the Authorizations list of every namespace supported by the cache.
 
-In addition, if the cache supports the `/user/ligo` namespace and the webapp can access LIGO's LDAP server:
+In addition, if the cache supports the LIGO VO and the webapp can access LIGO's LDAP server:
 
-- Add a `u <DN HASH> /user/ligo rl` for every DN obtained from the LIGO's LDAP server.
+- Add a `u <DN HASH> <LIGO PATH1> rl <LIGO PATH2> rl ...` for every DN obtained from the LIGO's LDAP server.
 
 
 ### Origin xrootd-scitokens config generation

src/rundev

@@ -12,9 +12,10 @@ ask_yn () {
 }
 
 export FLASK_ENV=development
+export FLASK_DEBUG=True
 cd "$(dirname "$0")"
 while true; do
-    python app.py; ret=$?
+    python3 app.py; ret=$?
     if [[ $ret -eq 0 ]]; then
         exit
     elif [[ $ret -eq 130 ]]; then  # control-C

src/stashcache.py

@@ -151,7 +151,7 @@ def fetch_ligo_authz_list_if_needed():
                     ligo_authz_list.append(parse_authz(f"DN:{dn}")[0])
             return ligo_authz_list
 
-        for stashcache_obj in vos_data.stashcache_by_vo_name.values():
+        for vo_name, stashcache_obj in vos_data.stashcache_by_vo_name.items():
             for path, namespace in stashcache_obj.namespaces.items():
                 if not namespace_allows_cache_resource(namespace, cache_resource):
                     continue
@@ -165,7 +165,7 @@ def fetch_ligo_authz_list_if_needed():
 
                 # Extend authz list with LIGO DNs if applicable
                 extended_authz_list = namespace.authz_list
-                if path == "/user/ligo":
+                if vo_name.lower() == "ligo":
                     if legacy:
                         extended_authz_list += fetch_ligo_authz_list_if_needed()
                     else:
@@ -323,11 +323,6 @@ def generate_cache_grid_mapfile(global_data: GlobalData,
         if not resource:
             return ""
 
-    ligo_authz_list: List[AuthMethod] = []
-    if legacy:
-        for dn in global_data.get_ligo_dn_list():
-            ligo_authz_list.append(parse_authz(f"DN:{dn}")[0])
-
     idns = _IdNamespaceData.for_cache(
         global_data=global_data,
         topology=topology,

src/templates/iframe.html.j2

@@ -221,7 +221,28 @@ will want to adjust the Map.data which populates the select input and legend.
     const storage_services = new Set(["Squid", "SRMv2", "XRootD cache server", "GridFtp", "XRootD component", "XRootD origin server", "SRMv1", "LFC", "Apache Server", "FTS"]);
     const network_services = new Set(["net.perfSONAR.Bandwidth", "net.perfSONAR.Latency"]);
     const ospool_resources = new Set([
-        "SU-ITS-CE3", "Expanse-PATH-EP", "SU-ITS-CE2", "Lancium-PEP", "UNL-PATH-EP", "UConn-HPC", "SDSC-PRP-OSPool-Provisioner", "IU-Jetstream2-Backfill", "Lancium-Backfill", "WISC-PATH-EP", "GPGrid", "MWT2", "GLOW", "Purdue-Geddes", "Lehigh - Hawk", "UW-IT", "NWICG_NDCMS", "UColorado_HEP", "TACC-Jetstream-Backfill", "UConn-OSG", "GPN-GP-ARGO-Backfill", "AMNH-Mendel-CE1", "BNL-ATLAS", "TCNJ-ELSA", "CUmulus-Backfill", "AMNH-HEL-CE1", "Rice-RAPID-Backfill", "OSG_US_FSU_HNPGRID", "GP-ARGO-wichita-backfill", "SDSC-PRP-CE1", "UA-HPC-CE1", "VU-AUGIE", "SIUE-CC-production", "GP-ARGO-dsu-backfill", "SLATE_US_NMSU_DISCOVERY", "UC-Denver-CE1", "UTC-Epyc", "Crane", "AGLT2", "GP-ARGO-uams-backfill", "SYRA-PATH-EP", "GP-ARGO-oru-backfill", "USCMS-FNAL-WC1", "GP-ARGO-sdsmt-backfill", "GP-ARGO-sdsu-backfill", "GP-ARGO-astate-backfill", "GP-ARGO-usd-backfill", "Georgia_Tech_PACE_CE_2", "GP-ARGO-ksu-backfill", "GP-ARGO-emporia-backfill", "Rhodes-HPC-CE1", "Swan-CE1", "GP-ARGO-langston-backfill", "GP-ARGO-uark-backfill", "GP-ARGO-creighton-backfill", "ODU-Ubuntu", "LSUHSC-Tigerfish-CE1", "WTAMU_HPC", "ASU-DELL_M420", "GP-ARGO-mst-backfill", "GP-ARGO-ku-backfill", "uprm-cms", "WSU-GRID", "USC-CARC-Artemis-Backfill", "FANDM-ITS", "Clarkson-ACRES-CE2", "NMSU-Discovery-CE1", "LC-ITS", "Rhino", "UA-LR-ITS-EP", "PDX-Coeus-CE1", "GP-ARGO-osu-backfill", "Nebraska", "cinvestav", "ISI", "Purdue-Anvil-Composable", "GP-ARGO-kanren-backfill", "GSU-ACIDS", "Clemson-Palmetto", "LSUHSC-Tigerfish-CE2", "AMNH-ARES-CE1", "GP-ARGO-doane-backfill", "LSU-DB-CE1", "CHTC", "CHTC-canary2", "NMSU-Discovery", "KSU-Beocat-CE1", "WSU-GRID-CE1", "GP-ARGO-cameron-backfill", "SPRACE", "ND_CAMLGPU", "CYBERA_EDMONTON", "UMissHEP", "USF", "Maine-ACG", "Clarkson-ACRES", "FIU-PATH-BACKFILL", "GP-ARGO-semo-backfill", "AMNH-ARES", "FIU-IRCC", "Alabama-CHPC-CE1", "TIGER-OSG-BACKFILL-PROD", "CIT_CMS_T2", "UCR-HPCC-CE1", "Tufts-Cluster", "Purdue-Anvil-CE1", "IIT_CE2", "IIT_CE1", "CHTC-GPU-EP", "Maine-ACG-CE1", "NCSU-HPC-CE1", "SLATE_US_UUTAH_LONEPEAK", "USC-CARC-Carina-Backfill", "SLATE_US_UUTAH_NOTCHPEAK", "ISI_ImageTest", "SLATE_US_UUTAH_KINGSPEAK", "Wilson Facility"
+        "UMT-Hellgate-CE1", "SU-ITS-CE3", "Expanse-PATH-EP", "SU-ITS-CE2", "Lancium-PEP", "UNL-PATH-EP", "UConn-HPC",
+        "SDSC-PRP-OSPool-Provisioner", "IU-Jetstream2-Backfill", "Lancium-Backfill", "WISC-PATH-EP", "GPGrid", "MWT2",
+        "GLOW", "Purdue-Geddes", "Lehigh - Hawk", "UW-IT", "NWICG_NDCMS", "UColorado_HEP", "TACC-Jetstream-Backfill",
+        "UConn-OSG", "GPN-GP-ARGO-Backfill", "AMNH-Mendel-CE1", "BNL-ATLAS", "TCNJ-ELSA", "CUmulus-Backfill",
+        "AMNH-HEL-CE1", "Rice-RAPID-Backfill", "OSG_US_FSU_HNPGRID", "GP-ARGO-wichita-backfill", "SDSC-PRP-CE1",
+        "UA-HPC-CE1", "VU-AUGIE", "SIUE-CC-production", "GP-ARGO-dsu-backfill", "SLATE_US_NMSU_DISCOVERY",
+        "UC-Denver-CE1", "UTC-Epyc", "Crane", "AGLT2", "GP-ARGO-uams-backfill", "SYRA-PATH-EP", "GP-ARGO-oru-backfill",
+        "USCMS-FNAL-WC1", "GP-ARGO-sdsmt-backfill", "GP-ARGO-sdsu-backfill", "GP-ARGO-astate-backfill",
+        "GP-ARGO-usd-backfill", "Georgia_Tech_PACE_CE_2", "GP-ARGO-ksu-backfill", "GP-ARGO-emporia-backfill",
+        "Rhodes-HPC-CE1", "Swan-CE1", "GP-ARGO-langston-backfill", "GP-ARGO-uark-backfill",
+        "GP-ARGO-creighton-backfill", "ODU-Ubuntu", "LSUHSC-Tigerfish-CE1", "WTAMU_HPC", "ASU-DELL_M420",
+        "GP-ARGO-mst-backfill", "GP-ARGO-ku-backfill", "uprm-cms", "WSU-GRID", "USC-CARC-Artemis-Backfill",
+        "FANDM-ITS", "Clarkson-ACRES-CE2", "NMSU-Discovery-CE1", "LC-ITS", "Rhino", "UA-LR-ITS-EP",
+        "PDX-Coeus-CE1", "GP-ARGO-osu-backfill", "Nebraska", "cinvestav", "ISI", "Purdue-Anvil-Composable",
+        "GP-ARGO-kanren-backfill", "GSU-ACIDS", "Clemson-Palmetto", "LSUHSC-Tigerfish-CE2", "AMNH-ARES-CE1",
+        "GP-ARGO-doane-backfill", "LSU-DB-CE1", "CHTC", "CHTC-canary2", "NMSU-Discovery", "KSU-Beocat-CE1",
+        "WSU-GRID-CE1", "GP-ARGO-cameron-backfill", "SPRACE", "ND_CAMLGPU", "CYBERA_EDMONTON", "UMissHEP",
+        "USF", "Maine-ACG", "Clarkson-ACRES", "FIU-PATH-BACKFILL", "GP-ARGO-semo-backfill", "AMNH-ARES",
+        "FIU-IRCC", "Alabama-CHPC-CE1", "TIGER-OSG-BACKFILL-PROD", "CIT_CMS_T2", "UCR-HPCC-CE1", "Tufts-Cluster",
+        "Purdue-Anvil-CE1", "IIT_CE2", "IIT_CE1", "CHTC-GPU-EP", "Maine-ACG-CE1", "NCSU-HPC-CE1",
+        "SLATE_US_UUTAH_LONEPEAK", "USC-CARC-Carina-Backfill", "SLATE_US_UUTAH_NOTCHPEAK", "ISI_ImageTest",
+        "SLATE_US_UUTAH_KINGSPEAK", "Wilson Facility"
     ].map(x => x.toLowerCase()))
 
     const iconConfig = {
@@ -545,7 +566,8 @@ will want to adjust the Map.data which populates the select input and legend.
 
             // Check for zoom and location inputs
             if(window.location.hash){
-                let viewportComponents = window.location.hash.split("#")[1].split("|");
+                let decodedHash = decodeURI(window.location.hash)
+                let viewportComponents = decodedHash.split("#")[1].split("|");
                 [latitude, longitude] = viewportComponents[0].split(',')
                 zoom = viewportComponents[1]
             }

src/tests/test_stashcache.py

@@ -50,14 +50,14 @@ def test_allowedVO_includes_ANY_for_ligo_inclusion(self, client: flask.Flask, mo
 
         stashcache.generate_cache_authfile(global_data, "osg-sunnyvale-stashcache.t2.ucsd.edu")
 
-        assert spy.call_count == 1
+        assert spy.call_count == 5
 
     def test_allowedVO_includes_LIGO_for_ligo_inclusion(self, client: flask.Flask, mocker: MockerFixture):
         spy = mocker.spy(global_data, "get_ligo_dn_list")
 
         stashcache.generate_cache_authfile(global_data, "stashcache.gwave.ics.psu.edu")
 
-        assert spy.call_count == 1
+        assert spy.call_count == 5
 
     def test_allowedVO_excludes_LIGO_and_ANY_for_ligo_inclusion(self, client: flask.Flask, mocker: MockerFixture):
         spy = mocker.spy(global_data, "get_ligo_dn_list")
@@ -96,8 +96,8 @@ def test_cache_grid_mapfile_nohost(self, client: flask.Flask):
             mm = GRID_MAPPING_REGEX.match(line)
             if mm:
                 dn = mm.group(1)
-                if "CN=Brian Paul Bockelman" in dn or "CN=Matyas Selmeci A148276" in dn:
-                    # HACK: these two have their FQANs explicitly allowed in some namespaces so it's OK
+                if "CN=Brian Paul Bockelman" in dn or "CN=Matyas Selmeci A148276" in dn or "CN=Judith Lorraine Stephen" in dn:
+                    # HACK: these three have their FQANs explicitly allowed in some namespaces so it's OK
                     # for them to show up in grid-mapfiles even without an FQDN
                     continue
                 else:

src/tests/verify_resources.py

@@ -141,7 +141,7 @@ def main():
     errors += test_14_vo_contacts_match(vos, vofns, contacts)
     # per SOFTWARE-3329, we are not checking support center contacts
 #   errors += test_14_sc_contacts_match(support_centers, contacts)
-    errors += test_15_facility_site_files()
+    errors += test_15_site_files()
     errors += test_16_Xrootd_DNs(rgs, rgfns)
     errors += test_17_osdf_data(rgs, rgfns)
     warnings += test_18_osdf_data_cache_warnings(rgs, rgfns, vomap)
@@ -190,7 +190,6 @@ def main():
                                " or a CILogonID",
     'UnknownContactID'       : "Contact IDs must exist in contact repo",
     'ContactNameMismatch'    : "Contact names must match in contact repo",
-    'NoFacility'             : "Facility directories must contain a FACILITY.yaml",
     'NoSite'                 : "Site directories must contain a SITE.yaml",
     'XrootdWithoutDN'        : "Xrootd cache server must provide a DN",
     'OSDFServiceVOsList'     : "OSDF Services must contain an AllowedVOs list",
@@ -663,16 +662,9 @@ def test_14_sc_contacts_match(support_centers, contacts):
     return errors
 
 
-def test_15_facility_site_files():
-    # verify the required FACILITY.yaml and SITE.yaml files
+def test_15_site_files():
+    # verify the required SITE.yaml files
     errors = 0
-
-    for facdir in glob.glob("*/"):
-        if not os.path.exists(facdir + "FACILITY.yaml"):
-            print_emsg_once('NoFacility')
-            print("ERROR: " + facdir[:-1] + " does not have required FACILITY.yaml file")
-            errors += 1
-
     for sitedir in glob.glob("*/*/"):
         if not os.path.exists(sitedir + "SITE.yaml"):
             print_emsg_once('NoSite')

src/webapp/common.py

@@ -291,7 +291,7 @@ def gen_id_from_yaml(data: dict, alternate_name: str, id_key = "ID", mod = 2 **
     Given a yaml object, return its existing ID if an ID is present, or generate a new ID for the object
     based on the md5sum of an alternate string value (usually the key of the object in its parent dictionary)
     """
-    return data.get(id_key) or gen_id(alternate_name, mod, minimum, hashfn)
+    return data[id_key] if data.get(id_key) is not None else gen_id(alternate_name, mod, minimum, hashfn)
 
 def gen_id(instr: AnyStr, mod = 2 ** 31 - 1, minimum=1, hashfn=hashlib.md5) -> int:
     """

src/webapp/rg_reader.py

@@ -69,21 +69,17 @@ def get_topology(indir="../topology", contacts_data=None, strict=False):
 
     skip_msg = "skipping (non-strict mode)"
 
-    for facility_path in root.glob("*/FACILITY.yaml"):
-        name = facility_path.parts[-2]
-        facility_data = load_yaml_file(facility_path)
+    for facility_path in root.glob("*"):
+        if not os.path.isdir(facility_path):
+            continue
+        name = facility_path.parts[-1]
+        facility_yaml_path = facility_path / 'FACILITY.yaml'
+        facility_data = load_yaml_file(facility_yaml_path) if facility_yaml_path.exists() else {}
         id_ = gen_id_from_yaml(facility_data or {}, name)
         topology.add_facility(name, id_)
     for site_path in root.glob("*/*/SITE.yaml"):
         facility, name = site_path.parts[-3:-1]
-        if facility not in topology.facilities:
-            msg = f"Missing facility {facility} for site {name}.  Is there a FACILITY.yaml?"
-            if strict:
-                raise TopologyError(msg)
-            else:
-                log.error(msg)
-                log.error(skip_msg)
-                continue
+        assert facility in topology.facilities, f"Missing facility {facility} for site {name}"
         site_info = load_yaml_file(site_path)
         id_ = gen_id_from_yaml(site_info, name)
         topology.add_site(facility, name, id_, site_info)

topology/Brookhaven National Laboratory/BNL ATLAS Tier1/BNL-ATLAS-OPP_downtime.yaml

@@ -0,0 +1,22 @@
+- Class: SCHEDULED
+  ID: 1630657310
+  Description: dCache Downtime
+  Severity: Outage
+  StartTime: Oct 31, 2023 14:00 +0000
+  EndTime: Oct 31, 2023 23:00 +0000
+  CreatedTime: Oct 23, 2023 12:55 +0000
+  ResourceName: BNL_CLOUD
+  Services:
+  - Submit Node
+# ---------------------------------------------------------
+- Class: SCHEDULED
+  ID: 1630658688
+  Description: Network downtime
+  Severity: Outage
+  StartTime: Dec 19, 2023 11:00 +0000
+  EndTime: Dec 20, 2023 02:00 +0000
+  CreatedTime: Oct 23, 2023 12:57 +0000
+  ResourceName: BNL_CLOUD
+  Services:
+  - Submit Node
+# ---------------------------------------------------------