From 8cee80f8d50b9f454b5b1a0c816315daac2d6af4 Mon Sep 17 00:00:00 2001 From: Tim Cartwright Date: Thu, 23 May 2024 11:01:06 -0500 Subject: [PATCH 01/10] Added MI-HORUS local contacts --- .../Michigan HORUS/MI-HORUS.yaml | 55 ++++++++----------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml b/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml index 148637d81..ef5d9d311 100644 --- a/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml +++ b/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml @@ -27,48 +27,39 @@ Resources: # If you cannot find the contact above XML, please register the contact: # https://opensciencegrid.org/docs/common/registration/#registering-contacts ContactLists: - # Administrative Contacts are persons or groups of people (i.e., - # mailing lists) that are directly responsible for the - # maintenance of the resource Administrative Contact: Primary: Name: Jeffrey Michael Dost ID: 3a8eb6436a8b78ca50f7e93bb2a4d1f0141212ba - # Secondary: - # Name: - # ID: - # Tertiary: - # Name: - # ID: - # Security Contact are persons or groups of people (i.e., - # mailing lists) that are responsible for handling security - # issues related to the resource Security Contact: Primary: Name: Jeffrey Michael Dost ID: 3a8eb6436a8b78ca50f7e93bb2a4d1f0141212ba - # Secondary: - # Name: - # ID: - # Tertiary: - # Name: - # ID: - # Site contact (optional) are persons or groups of people (i.e., - # mailing lists) that are generally responsible for a site's - # relationship with the OSG (e.g., principal investigators, - # local administrator contact for OSG Hosted CEs) - # Site Contact: - # Primary: - # Name: - # ID: - # Secondary: - # Name: - # ID: - # Tertiary: - # Name: - # ID: + Local Operational Contact: + Primary: + Name: Muhammad Akhdhor + ID: OSG1000654 + Secondary: + Name: Michael Thompson + ID: f84214412f7b3dbf848808c4ba30d179c5743766 + Tertiary: + Name: Andrew Keen + ID: OSG1000660 + + Local Security Contact: + Primary: + Name: HORUS Security Mailing List + ID: d7ec710e472267ffcc69de4070491f48582ccc2f + + Local Executive Contact: + Primary: + Name: Shawn McKee + ID: a833f94ad3840fe779702409c6b778d79aaed76f + Secondary: + Name: Robert Eugene Stovall + ID: OSG1000685 # FQDN is the fully qualified domain name of the host running this resource FQDN: mi-horus-ce1.svc.opensciencegrid.org From a54acd2923b7452c53c140f74134fd590242dc5b Mon Sep 17 00:00:00 2001 From: Fabio Andrijauskas Date: Wed, 22 May 2024 18:21:18 -0700 Subject: [PATCH 02/10] Adding ANY to ESnet cache Adding ANY to ESnet cache --- topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml | 2 +- topology/Energy Sciences Network/London/ESnetLondon.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml b/topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml index b81bda81b..f2ed868fd 100644 --- a/topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml +++ b/topology/Energy Sciences Network/Amsterdam/EsnetAmsterdam.yaml @@ -22,4 +22,4 @@ Resources: XRootD cache server: Description: ESnet Amsterdam Cache AllowedVOs: - - LIGO + - ANY \ No newline at end of file diff --git a/topology/Energy Sciences Network/London/ESnetLondon.yaml b/topology/Energy Sciences Network/London/ESnetLondon.yaml index 5fe582291..934ded268 100644 --- a/topology/Energy Sciences Network/London/ESnetLondon.yaml +++ b/topology/Energy Sciences Network/London/ESnetLondon.yaml @@ -22,4 +22,4 @@ Resources: XRootD cache server: Description: Internet2 London Cache AllowedVOs: - - LIGO + - ANY From 2bb7448e1d97a494100f86daab86728b8ef2a362 Mon Sep 17 00:00:00 2001 From: Tim Cartwright Date: Fri, 24 May 2024 10:06:10 -0500 Subject: [PATCH 03/10] Added remaining HORUS contact; FD #74888 --- topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml b/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml index ef5d9d311..5ed2ade22 100644 --- a/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml +++ b/topology/University of Michigan/Michigan HORUS/MI-HORUS.yaml @@ -60,6 +60,9 @@ Resources: Secondary: Name: Robert Eugene Stovall ID: OSG1000685 + Tertiary: + Name: Robert Michael Thompson + ID: OSG1000697 # FQDN is the fully qualified domain name of the host running this resource FQDN: mi-horus-ce1.svc.opensciencegrid.org From 646eaff2e560cb9641e4db1b588352b9efa04de8 Mon Sep 17 00:00:00 2001 From: Christina K Date: Fri, 24 May 2024 13:03:56 -0500 Subject: [PATCH 04/10] Create UWMadison_OConnor.yaml --- projects/UWMadison_OConnor.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 projects/UWMadison_OConnor.yaml diff --git a/projects/UWMadison_OConnor.yaml b/projects/UWMadison_OConnor.yaml new file mode 100644 index 000000000..36a01870e --- /dev/null +++ b/projects/UWMadison_OConnor.yaml @@ -0,0 +1,8 @@ +Department: Pathology and Laboratory Medicine +Description: "The overarching goal of the O\u2019Connor lab is to contribute meaningfully\ + \ to the global response to viral infections impacting human health." +FieldOfScience: Biological and Biomedical Sciences +FieldOfScienceID: 26.0999b +InstitutionID: Unknown +Organization: University of Wisconsin-Madison +PIName: David O'Connor From 35cfcd93efe275ee6d32987cca52d3294f7134bb Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 24 May 2024 13:19:44 -0500 Subject: [PATCH 05/10] Add InstitutionID --- projects/UWMadison_OConnor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/UWMadison_OConnor.yaml b/projects/UWMadison_OConnor.yaml index 36a01870e..ec1ccba51 100644 --- a/projects/UWMadison_OConnor.yaml +++ b/projects/UWMadison_OConnor.yaml @@ -3,6 +3,6 @@ Description: "The overarching goal of the O\u2019Connor lab is to contribute mea \ to the global response to viral infections impacting human health." FieldOfScience: Biological and Biomedical Sciences FieldOfScienceID: 26.0999b -InstitutionID: Unknown +InstitutionID: 'https://osg-htc.org/iid/fq8thqsj99zh' Organization: University of Wisconsin-Madison PIName: David O'Connor From 1f3b8792c9cf376d0ce966ef19eeb085b4ebad2d Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 24 May 2024 20:46:38 -0500 Subject: [PATCH 06/10] Cache results of DN -> DN hash conversion --- src/webapp/x509.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webapp/x509.py b/src/webapp/x509.py index 5724cb60f..ba6d939dc 100644 --- a/src/webapp/x509.py +++ b/src/webapp/x509.py @@ -1,3 +1,4 @@ +import functools import hashlib import re @@ -17,6 +18,7 @@ } +@functools.lru_cache(maxsize=2048) def generate_dn_hash(dn: str) -> str: """ Given a DN one-liner as commonly encoded in the grid world From c4ae64f8d8fc731a1692c4272571fc31602a4c00 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 24 May 2024 21:08:30 -0500 Subject: [PATCH 07/10] Cache and pre-calculate authfile lines to speed up generation --- src/stashcache.py | 14 +++++------ src/webapp/data_federation.py | 46 +++++++++++++++++------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/stashcache.py b/src/stashcache.py index f2db0de99..a9ac3a5a7 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -172,10 +172,10 @@ def fetch_ligo_authz_list_if_needed(): for authz in extended_authz_list: if authz.used_in_authfile: - self.id_to_paths[authz.get_authfile_id()].add(path) - self.id_to_str[authz.get_authfile_id()] = str(authz) + self.id_to_paths[authz.authfile_id].add(path) + self.id_to_str[authz.authfile_id] = str(authz) if authz.used_in_grid_mapfile: - self.grid_mapfile_lines.add(authz.get_grid_mapfile_line()) + self.grid_mapfile_lines.add(authz.grid_mapfile_line) return self @@ -220,10 +220,10 @@ def for_origin(cls, topology: Topology, vos_data: VOsData, for authz in authz_list: if authz.used_in_authfile: - self.id_to_paths[authz.get_authfile_id()].add(path) - self.id_to_str[authz.get_authfile_id()] = str(authz) + self.id_to_paths[authz.authfile_id].add(path) + self.id_to_str[authz.authfile_id] = str(authz) if authz.used_in_grid_mapfile: - self.grid_mapfile_lines.add(authz.get_grid_mapfile_line()) + self.grid_mapfile_lines.add(authz.grid_mapfile_line) return self @@ -534,7 +534,7 @@ def get_credential_generation_dict_for_namespace(ns: Namespace) -> Optional[Dict def get_scitokens_list_for_namespace(ns: Namespace) -> List[Dict]: """Return the list of scitokens issuer info for the .namespaces[*].scitokens attribute in the namespaces JSON""" return list( - filter(None, (a.get_namespaces_scitokens_block() for a in ns.authz_list)) + filter(None, (a.namespaces_scitokens_block for a in ns.authz_list)) ) diff --git a/src/webapp/data_federation.py b/src/webapp/data_federation.py index 5f6ab796a..f7ae234ef 100644 --- a/src/webapp/data_federation.py +++ b/src/webapp/data_federation.py @@ -1,3 +1,4 @@ +import functools import re import urllib import urllib.parse @@ -12,22 +13,20 @@ class AuthMethod: + __slots__ = ("authfile_id", "grid_mapfile_line", "namespaces_scitokens_block") is_public = False used_in_authfile = False used_in_scitokens_conf = False used_in_grid_mapfile = False - def get_authfile_id(self): - return "" + def __init__(self): + self.authfile_id = "" + self.grid_mapfile_line = "" + self.namespaces_scitokens_block = None def get_scitokens_conf_block(self, service_name: str): return "" - def get_grid_mapfile_line(self): - return "" - - def get_namespaces_scitokens_block(self): - return None class NullAuth(AuthMethod): pass @@ -37,59 +36,60 @@ class PublicAuth(AuthMethod): is_public = True used_in_authfile = True + def __init__(self): + super().__init__() + self.authfile_id = "u *" + def __str__(self): return "PUBLIC" - def get_authfile_id(self): - return "u *" - class DNAuth(AuthMethod): + __slots__ = ("dn", "dn_hash") used_in_authfile = True used_in_grid_mapfile = True def __init__(self, dn: str): + super().__init__() self.dn = dn + self.dn_hash = generate_dn_hash(dn) + self.authfile_id = f"u {self.dn_hash}" + self.grid_mapfile_line = f'"{self.dn}" {self.dn_hash}' def __str__(self): return "DN: " + self.dn - def get_dn_hash(self): - return generate_dn_hash(self.dn) - - def get_authfile_id(self): - return f"u {self.get_dn_hash()}" - - def get_grid_mapfile_line(self): - return f'"{self.dn}" {self.get_dn_hash()}' - class FQANAuth(AuthMethod): + __slots__ = ("fqan",) used_in_authfile = True def __init__(self, fqan: str): + super().__init__() self.fqan = fqan + self.authfile_id = f"g {self.fqan}" def __str__(self): return "FQAN: " + self.fqan - def get_authfile_id(self): - return f"g {self.fqan}" - class SciTokenAuth(AuthMethod): + __slots__ = ("issuer", "base_path", "restricted_path", "map_subject") used_in_scitokens_conf = True def __init__(self, issuer: str, base_path: str, restricted_path: Optional[str], map_subject: bool): + super().__init__() self.issuer = issuer self.base_path = base_path self.restricted_path = restricted_path self.map_subject = map_subject + self.namespaces_scitokens_block = self._get_namespaces_scitokens_block() def __str__(self): return f"SciToken: issuer={self.issuer} base_path={self.base_path} restricted_path={self.restricted_path} " \ f"map_subject={self.map_subject}" + @functools.lru_cache(4) def get_scitokens_conf_block(self, service_name: str): if service_name not in [XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER]: raise ValueError(f"service_name must be '{XROOTD_CACHE_SERVER}' or '{XROOTD_ORIGIN_SERVER}'") @@ -103,7 +103,7 @@ def get_scitokens_conf_block(self, service_name: str): return block - def get_namespaces_scitokens_block(self): + def _get_namespaces_scitokens_block(self): base_path = re.split(r"\s*,\s*", self.base_path) restricted_path = re.split(r"\s*,\s*", self.restricted_path) if self.restricted_path else [] return { From 33b56f7dfa7cdb039d814e9f3d7c163d4b76cdc3 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 24 May 2024 21:45:12 -0500 Subject: [PATCH 08/10] Don't cache get_scitokens_conf_block() care must be taken when caching methods (see https://docs.python.org/3/faq/programming.html#how-do-i-cache-method-calls) and it's not called frequently enough to be worth it. --- src/webapp/data_federation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webapp/data_federation.py b/src/webapp/data_federation.py index f7ae234ef..8542f527f 100644 --- a/src/webapp/data_federation.py +++ b/src/webapp/data_federation.py @@ -1,4 +1,3 @@ -import functools import re import urllib import urllib.parse @@ -89,7 +88,6 @@ def __str__(self): return f"SciToken: issuer={self.issuer} base_path={self.base_path} restricted_path={self.restricted_path} " \ f"map_subject={self.map_subject}" - @functools.lru_cache(4) def get_scitokens_conf_block(self, service_name: str): if service_name not in [XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER]: raise ValueError(f"service_name must be '{XROOTD_CACHE_SERVER}' or '{XROOTD_ORIGIN_SERVER}'") From 8d918b8511e4ed1e11f402d11adc283fdf56a87b Mon Sep 17 00:00:00 2001 From: smithnp <60900055+smithnp@users.noreply.github.com> Date: Tue, 28 May 2024 13:13:03 -0400 Subject: [PATCH 09/10] Update Purdue_downtime.yaml --- .../Purdue University/Purdue CMS/Purdue_downtime.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/topology/Purdue University/Purdue CMS/Purdue_downtime.yaml b/topology/Purdue University/Purdue CMS/Purdue_downtime.yaml index 270898dcb..5ba31f0a8 100644 --- a/topology/Purdue University/Purdue CMS/Purdue_downtime.yaml +++ b/topology/Purdue University/Purdue CMS/Purdue_downtime.yaml @@ -2131,3 +2131,14 @@ Services: - net.perfSONAR.Latency # --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819162937 + Description: Power transfer for datacenter circuits + Severity: Severe + StartTime: May 30, 2024 11:00 +0000 + EndTime: May 30, 2024 21:00 +0000 + CreatedTime: May 28, 2024 17:11 +0000 + ResourceName: Purdue-EOS-SE + Services: + - EOS +# --------------------------------------------------------- From a8ad9c03db0e7face3bda8e0fa425d350215a9a1 Mon Sep 17 00:00:00 2001 From: jlstephen Date: Tue, 28 May 2024 15:40:19 -0500 Subject: [PATCH 10/10] Add downtime for MWT2 for EL9 upgrades --- .../MWT2 ATLAS UC/MWT2_downtime.yaml | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/topology/University of Chicago/MWT2 ATLAS UC/MWT2_downtime.yaml b/topology/University of Chicago/MWT2 ATLAS UC/MWT2_downtime.yaml index 96a13392e..2da2fcf9c 100644 --- a/topology/University of Chicago/MWT2 ATLAS UC/MWT2_downtime.yaml +++ b/topology/University of Chicago/MWT2 ATLAS UC/MWT2_downtime.yaml @@ -1574,3 +1574,102 @@ Services: - XRootD component # --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819285909 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:36 +0000 + ResourceName: MWT2_CE_IU + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286180 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:36 +0000 + ResourceName: MWT2_CE_IU2 + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286287 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:37 +0000 + ResourceName: MWT2_CE_UC + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286377 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:37 +0000 + ResourceName: MWT2_CE_UC2 + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286612 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:37 +0000 + ResourceName: MWT2_CE_UIUC + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286698 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:37 +0000 + ResourceName: MWT2_CE_UIUC2 + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286864 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:38 +0000 + ResourceName: MWT2_UC_SE + Services: + - SRMv2 +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819286951 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:38 +0000 + ResourceName: MWT2_UC_WebDAV + Services: + - WebDAV +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1819287109 + Description: EL9 Upgrades + Severity: Outage + StartTime: Jun 10, 2024 14:00 +0000 + EndTime: Jun 10, 2024 22:00 +0000 + CreatedTime: May 28, 2024 20:38 +0000 + ResourceName: MWT2_UC_XRootD_door + Services: + - XRootD component +# ---------------------------------------------------------