Merge branch 'master' into pr/cache-git

opensciencegrid · May 29, 2024 · 935fd95 · 935fd95
2 parents f304c24 + c5e44e2
commit 935fd95
Show file tree

Hide file tree

Showing 28 changed files with 416 additions and 151 deletions.
diff --git a/.github/scripts/check_project_fos_precision/requirements.txt b/.github/scripts/check_project_fos_precision/requirements.txt
@@ -6,7 +6,7 @@ pandas==2.2.2
 python-dateutil==2.9.0.post0
 pytz==2024.1
 PyYAML==6.0.1
-requests==2.31.0
+requests==2.32.0
 six==1.16.0
 tzdata==2024.1
 urllib3==2.2.1
diff --git a/projects/GSU_Maimon.yaml → projects/GSU_Wang.yaml b/projects/GSU_Maimon.yaml → projects/GSU_Wang.yaml
diff --git a/projects/TNTTech_ITS.yaml → projects/TNTech_ITS.yaml b/projects/TNTTech_ITS.yaml → projects/TNTech_ITS.yaml
diff --git a/projects/UMiss_Bennett.yaml b/projects/UMiss_Bennett.yaml
@@ -0,0 +1,8 @@
+Department: Physics and Astronomy
+Description: 'In the decay of Xi_c baryon to lambda kaon pion, I am performing a likelihood
+  fitting to extract the coupling to each potential resonance.  '
+FieldOfScience: Physics and astronomy
+FieldOfScienceID: '40.0804'
+InstitutionID: https://osg-htc.org/iid/bigpgrrmxblz
+Organization: University of Mississippi
+PIName: Jake Bennett
diff --git a/projects/UNL_Turner.yaml b/projects/UNL_Turner.yaml
@@ -0,0 +1,8 @@
+Department: Mechanical and Materials Engineering
+Description: 'Working with ultrasonic wave propagation in polycrystalline materials.
+  Website: https://engineering.unl.edu/quisp/'
+FieldOfScience: Engineering
+FieldOfScienceID: '14.1901'
+InstitutionID: 'https://osg-htc.org/iid/q9k1b8dfrw25'
+Organization: University of Nebraska - Lincoln
+PIName: Joseph Turner
diff --git a/projects/UWMadison_OConnor.yaml b/projects/UWMadison_OConnor.yaml
@@ -0,0 +1,8 @@
+Department: Pathology and Laboratory Medicine
+Description: "The overarching goal of the O\u2019Connor lab is to contribute meaningfully\
+  \ to the global response to viral infections impacting human health."
+FieldOfScience: Biological and Biomedical Sciences
+FieldOfScienceID: 26.0999b
+InstitutionID: 'https://osg-htc.org/iid/fq8thqsj99zh'
+Organization: University of Wisconsin-Madison
+PIName: David O'Connor
diff --git a/src/stashcache.py b/src/stashcache.py
@@ -172,10 +172,10 @@ def fetch_ligo_authz_list_if_needed():
 
                 for authz in extended_authz_list:
                     if authz.used_in_authfile:
-                        self.id_to_paths[authz.get_authfile_id()].add(path)
-                        self.id_to_str[authz.get_authfile_id()] = str(authz)
+                        self.id_to_paths[authz.authfile_id].add(path)
+                        self.id_to_str[authz.authfile_id] = str(authz)
                     if authz.used_in_grid_mapfile:
-                        self.grid_mapfile_lines.add(authz.get_grid_mapfile_line())
+                        self.grid_mapfile_lines.add(authz.grid_mapfile_line)
 
         return self
 
@@ -220,10 +220,10 @@ def for_origin(cls, topology: Topology, vos_data: VOsData,
 
                 for authz in authz_list:
                     if authz.used_in_authfile:
-                        self.id_to_paths[authz.get_authfile_id()].add(path)
-                        self.id_to_str[authz.get_authfile_id()] = str(authz)
+                        self.id_to_paths[authz.authfile_id].add(path)
+                        self.id_to_str[authz.authfile_id] = str(authz)
                     if authz.used_in_grid_mapfile:
-                        self.grid_mapfile_lines.add(authz.get_grid_mapfile_line())
+                        self.grid_mapfile_lines.add(authz.grid_mapfile_line)
         return self
 
 
@@ -534,7 +534,7 @@ def get_credential_generation_dict_for_namespace(ns: Namespace) -> Optional[Dict
 def get_scitokens_list_for_namespace(ns: Namespace) -> List[Dict]:
     """Return the list of scitokens issuer info for the .namespaces[*].scitokens attribute in the namespaces JSON"""
     return list(
-        filter(None, (a.get_namespaces_scitokens_block() for a in ns.authz_list))
+        filter(None, (a.namespaces_scitokens_block for a in ns.authz_list))
     )
 
 

diff --git a/src/tests/test_api.py b/src/tests/test_api.py
@@ -203,7 +203,7 @@ def test_institution_accept_type(self, client: flask.Flask):
 
 
     def test_origin_grid_mapfile(self, client: flask.Flask):
-        TEST_ORIGIN = "origin-auth2001.chtc.wisc.edu"  # This origin serves protected data
+        TEST_ORIGIN = "ap20.uc.osg-htc.org"  # This origin serves protected data
         response = client.get("/origin/grid-mapfile")
         assert response.status_code == 400  # fqdn not specified
 

diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py
@@ -201,7 +201,7 @@ def test_origin_grid_mapfile_nohost(self, client: flask.Flask):
             assert EMPTY_LINE_REGEX.match(line), f'Unexpected text "{line}".\nFull text:\n{text}\n'
 
     def test_origin_grid_mapfile_with_host(self, client: flask.Flask):
-        text = stashcache.generate_origin_grid_mapfile(global_data, "origin-auth2001.chtc.wisc.edu",
+        text = stashcache.generate_origin_grid_mapfile(global_data, "ap20.uc.osg-htc.org",
                                                        suppress_errors=False)
         num_mappings = 0
         for line in text.split("\n"):

diff --git a/src/webapp/data_federation.py b/src/webapp/data_federation.py
@@ -12,22 +12,20 @@
 
 
 class AuthMethod:
+    __slots__ = ("authfile_id", "grid_mapfile_line", "namespaces_scitokens_block")
     is_public = False
     used_in_authfile = False
     used_in_scitokens_conf = False
     used_in_grid_mapfile = False
 
-    def get_authfile_id(self):
-        return ""
+    def __init__(self):
+        self.authfile_id = ""
+        self.grid_mapfile_line = ""
+        self.namespaces_scitokens_block = None
 
     def get_scitokens_conf_block(self, service_name: str):
         return ""
 
-    def get_grid_mapfile_line(self):
-        return ""
-
-    def get_namespaces_scitokens_block(self):
-        return None
 
 class NullAuth(AuthMethod):
     pass
@@ -37,54 +35,54 @@ class PublicAuth(AuthMethod):
     is_public = True
     used_in_authfile = True
 
+    def __init__(self):
+        super().__init__()
+        self.authfile_id = "u *"
+
     def __str__(self):
         return "PUBLIC"
 
-    def get_authfile_id(self):
-        return "u *"
-
 
 class DNAuth(AuthMethod):
+    __slots__ = ("dn", "dn_hash")
     used_in_authfile = True
     used_in_grid_mapfile = True
 
     def __init__(self, dn: str):
+        super().__init__()
         self.dn = dn
+        self.dn_hash = generate_dn_hash(dn)
+        self.authfile_id = f"u {self.dn_hash}"
+        self.grid_mapfile_line = f'"{self.dn}" {self.dn_hash}'
 
     def __str__(self):
         return "DN: " + self.dn
 
-    def get_dn_hash(self):
-        return generate_dn_hash(self.dn)
-
-    def get_authfile_id(self):
-        return f"u {self.get_dn_hash()}"
-
-    def get_grid_mapfile_line(self):
-        return f'"{self.dn}" {self.get_dn_hash()}'
-
 
 class FQANAuth(AuthMethod):
+    __slots__ = ("fqan",)
     used_in_authfile = True
 
     def __init__(self, fqan: str):
+        super().__init__()
         self.fqan = fqan
+        self.authfile_id = f"g {self.fqan}"
 
     def __str__(self):
         return "FQAN: " + self.fqan
 
-    def get_authfile_id(self):
-        return f"g {self.fqan}"
-
 
 class SciTokenAuth(AuthMethod):
+    __slots__ = ("issuer", "base_path", "restricted_path", "map_subject")
     used_in_scitokens_conf = True
 
     def __init__(self, issuer: str, base_path: str, restricted_path: Optional[str], map_subject: bool):
+        super().__init__()
         self.issuer = issuer
         self.base_path = base_path
         self.restricted_path = restricted_path
         self.map_subject = map_subject
+        self.namespaces_scitokens_block = self._get_namespaces_scitokens_block()
 
     def __str__(self):
         return f"SciToken: issuer={self.issuer} base_path={self.base_path} restricted_path={self.restricted_path} " \
@@ -103,7 +101,7 @@ def get_scitokens_conf_block(self, service_name: str):
 
         return block
 
-    def get_namespaces_scitokens_block(self):
+    def _get_namespaces_scitokens_block(self):
         base_path = re.split(r"\s*,\s*", self.base_path)
         restricted_path = re.split(r"\s*,\s*", self.restricted_path) if self.restricted_path else []
         return {

diff --git a/src/webapp/x509.py b/src/webapp/x509.py
@@ -1,3 +1,4 @@
+import functools
 import hashlib
 import re
 
@@ -17,6 +18,7 @@
    }
 
 
+@functools.lru_cache(maxsize=2048)
 def generate_dn_hash(dn: str) -> str:
     """
     Given a DN one-liner as commonly encoded in the grid world

diff --git a/topology/Brookhaven National Laboratory/BNL ATLAS Tier1/BNL-ATLAS.yaml b/topology/Brookhaven National Laboratory/BNL ATLAS Tier1/BNL-ATLAS.yaml
@@ -201,7 +201,7 @@ Resources:
       KSI2KMin: 0
       StorageCapacityMax: 0
       StorageCapacityMin: 0
-  BNL_ATLAS_7:
+  BNL_ATLAS_07:
     Active: true
     ContactLists:
       Administrative Contact:
@@ -219,9 +219,8 @@ Resources:
           Name: Robert Hancock
           ID: OSG1000212
     Description: An OSG CE resource at BNL, mainly serving USATLAS production jobs,
-      not open to general OSG users.
+      not open to general OSG users. EL9
     FQDN: gridgk07.sdcc.bnl.gov
-    ID: 438
     Services:
       CE:
         Description: Compute Element
@@ -242,7 +241,48 @@ Resources:
       StorageCapacityMax: 0
       StorageCapacityMin: 0
       TapeCapacity: 0
-  BNL_ATLAS_8:
+  BNL_ATLAS_7:
+    Active: false
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          ID: 1e47726bf41f2128059de31cf7bc9c8d5dcadad6
+          Name: BNL Atlas Tier 1 support
+        Secondary:
+          ID: OSG1000638
+          Name: Ivan Glushkov
+      Security Contact:
+        Primary:
+          Name: BNL Atlas Tier 1 support
+          ID: 1e47726bf41f2128059de31cf7bc9c8d5dcadad6
+        Secondary:
+          Name: Robert Hancock
+          ID: OSG1000212
+    Description: An OSG CE resource at BNL, mainly serving USATLAS production jobs,
+      not open to general OSG users.
+    FQDN: gridgk07.racf.bnl.gov
+    ID: 438
+    Services:
+      CE:
+        Description: Compute Element
+        Details:
+          hidden: false
+          sam_uri: htcondor://gridgk07.racf.bnl.gov
+    VOOwnership:
+      ATLAS: 100
+    WLCGInformation:
+      APELNormalFactor: 12.69
+      AccountingName: BNL_ATLAS_1
+      HEPSPEC: 0
+      InteropAccounting: true
+      InteropBDII: true
+      InteropMonitoring: true
+      KSI2KMax: 0
+      KSI2KMin: 0
+      StorageCapacityMax: 0
+      StorageCapacityMin: 0
+      TapeCapacity: 0
+  BNL_ATLAS_08:
     Active: true
     ContactLists:
       Administrative Contact:
@@ -262,7 +302,6 @@ Resources:
     Description: This CE will serve grid jobs submitted to the *local* USATLAS resources,
       e.g. for panda queue ANALY_BNL_LOCAL, at BNL.
     FQDN: gridgk08.sdcc.bnl.gov
-    ID: 545
     Services:
       CE:
         Description: Compute Element
@@ -282,6 +321,46 @@ Resources:
       KSI2KMin: 0
       StorageCapacityMax: 0
       StorageCapacityMin: 0
+  BNL_ATLAS_8:
+    Active: false
+    ContactLists:
+      Administrative Contact:
+        Primary:
+          ID: 1e47726bf41f2128059de31cf7bc9c8d5dcadad6
+          Name: BNL Atlas Tier 1 support
+        Secondary:
+          ID: OSG1000638
+          Name: Ivan Glushkov
+      Security Contact:
+        Primary:
+          Name: BNL Atlas Tier 1 support
+          ID: 1e47726bf41f2128059de31cf7bc9c8d5dcadad6
+        Secondary:
+          Name: Robert Hancock
+          ID: OSG1000212
+    Description: This CE will serve grid jobs submitted to the *local* USATLAS resources,
+      e.g. for panda queue ANALY_BNL_LOCAL, at BNL.
+    FQDN: gridgk08.racf.bnl.gov
+    ID: 545
+    Services:
+      CE:
+        Description: Compute Element
+        Details:
+          hidden: false
+          sam_uri: htcondor://gridgk08.racf.bnl.gov
+    VOOwnership:
+      ATLAS: 100
+    WLCGInformation:
+      APELNormalFactor: 12.69
+      AccountingName: BNL_ATLAS_1
+      HEPSPEC: 0
+      InteropAccounting: false
+      InteropBDII: false
+      InteropMonitoring: false
+      KSI2KMax: 0
+      KSI2KMin: 0
+      StorageCapacityMax: 0
+      StorageCapacityMin: 0
   BNL_ATLAS_Frontier_Squid:
     Active: true
     ContactLists:

diff --git a/topology/Compute Canada/ComputeCanada - Cedar/ComputeCanada-Cedar_downtime.yaml b/topology/Compute Canada/ComputeCanada - Cedar/ComputeCanada-Cedar_downtime.yaml
@@ -31,3 +31,14 @@
   Services:
   - XRootD cache server
 # ---------------------------------------------------------
+- Class: UNSCHEDULED
+  ID: 1812395767
+  Description: overloaded
+  Severity: Outage
+  StartTime: May 19, 2024 08:01 +0000
+  EndTime: May 27, 2024 19:30 +0000
+  CreatedTime: May 20, 2024 21:12 +0000
+  ResourceName: ComputeCanada-Cedar-Cache
+  Services:
+  - XRootD cache server
+# ---------------------------------------------------------
diff --git a/topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml b/topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml
@@ -22,4 +22,4 @@ Resources:
       XRootD cache server:
         Description: ESnet Amsterdam Cache
     AllowedVOs:
-      - LIGO
+      - ANY
diff --git a/topology/Energy Sciences Network/London/ESnetLondon.yaml b/topology/Energy Sciences Network/London/ESnetLondon.yaml
@@ -22,4 +22,4 @@ Resources:
       XRootD cache server:
         Description: Internet2 London Cache
     AllowedVOs:
-      - LIGO
+      - ANY
diff --git a/topology/IN2P3/IN2P3-CC/IN2P3-OSDF.yaml b/topology/IN2P3/IN2P3-CC/IN2P3-OSDF.yaml
@@ -15,7 +15,7 @@ Resources:
         Primary:
           Name: IN2P3 Security
           ID: 33fcdfcbfc32afbe7691bb90d29d1a48bcda8918
-    DN: /DC=org/DC=terena/DC=tcs/C=FR/ST=Paris/O=Centre national de la recherche scientifique/CN=ccstashcache01.in2p3.fr
+    DN: /DC=org/DC=terena/DC=tcs/C=FR/ST=Paris/O=Centre national de la recherche scientifique/CN=ccstashcacheli01.in2p3.fr
     FQDN: ccstashcacheli01.in2p3.fr
     Services:
       XRootD cache server:

diff --git a/topology/Internet2/Internet2Chicago/I2ChicagoInfrastructure.yaml b/topology/Internet2/Internet2Chicago/I2ChicagoInfrastructure.yaml
@@ -18,7 +18,7 @@ Resources:
           Name: Fabio Andrijauskas 
           ID: OSG1000162
     FQDN: osg-chicago-stashcache.nrp.internet2.edu
-    DN: /CN=osg-chicago-stashcache.nrp.internet2.edu
+    DN: /DC=org/DC=incommon/C=US/ST=Michigan/O=University Corporation For Advanced Internet Development/CN=osg-chicago-stashcache.nrp.internet2.edu
     Services:
       XRootD cache server:
         Description: Internet2 Chicago Cache

diff --git a/topology/Purdue University/Purdue CMS/Purdue_downtime.yaml b/topology/Purdue University/Purdue CMS/Purdue_downtime.yaml
@@ -2131,3 +2131,14 @@
   Services:
   - net.perfSONAR.Latency
 # ---------------------------------------------------------
+- Class: SCHEDULED
+  ID: 1819162937
+  Description: Power transfer for datacenter circuits
+  Severity: Severe
+  StartTime: May 30, 2024 11:00 +0000
+  EndTime: May 30, 2024 21:00 +0000
+  CreatedTime: May 28, 2024 17:11 +0000
+  ResourceName: Purdue-EOS-SE
+  Services:
+  - EOS
+# ---------------------------------------------------------