From b93cf47efc52106991565cd10144711e35186888 Mon Sep 17 00:00:00 2001 From: Aiden Cohen Date: Fri, 28 Oct 2022 15:02:57 -0500 Subject: [PATCH 01/15] initial changes --- src/topology_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index 28cc6da24..441fc9d3a 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -6,6 +6,7 @@ import os import sys import urllib +import urllib3 import fnmatch import urllib.parse as urlparse @@ -45,19 +46,21 @@ def get_auth_session(args): if args.key: key = args.key - session = requests.Session() - + session = {} + session['cert_reqs'] = 'CERT_REQUIRED' if os.path.exists(cert): - session.cert = cert + session["cert_file"] = cert else: raise InvalidPathError("Error: could not find cert at %s" % cert) if os.path.exists(key): - session.cert = (cert, key) + session["cert_key"] = key else: raise InvalidPathError("Error: could not find key at %s" % key) + dpass = input("Decryption Pass") + session["key_password"] = dpass - return session + return urllib3.PoolManager(*session) def update_url_hostname(url, args): From c0110a560b6502641766c84ec0a14c12bacc2af1 Mon Sep 17 00:00:00 2001 From: Fabio Andrijauskas Date: Thu, 20 Oct 2022 11:23:03 -0700 Subject: [PATCH 02/15] Update I2ManhattanInfrastructure.yaml --- .../Internet2Manhattan/I2ManhattanInfrastructure.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/topology/Internet2/Internet2Manhattan/I2ManhattanInfrastructure.yaml b/topology/Internet2/Internet2Manhattan/I2ManhattanInfrastructure.yaml index fdc195c9d..0c1bcbef4 100644 --- a/topology/Internet2/Internet2Manhattan/I2ManhattanInfrastructure.yaml +++ b/topology/Internet2/Internet2Manhattan/I2ManhattanInfrastructure.yaml @@ -22,8 +22,5 @@ Resources: Services: XRootD cache server: Description: Internet2 Manhattan Cache - Details: - #endpoint_override: osg-new-york-stashcache.nrp.internet2.edu:8000 - auth_endpoint_override: osg-new-york-stashcache.nrp.internet2.edu:8444 AllowedVOs: - ANY From f2dc5a57ab3c86059d69e79ea2729220ffd3ceb0 Mon Sep 17 00:00:00 2001 From: Aiden Cohen Date: Fri, 28 Oct 2022 15:10:51 -0500 Subject: [PATCH 03/15] aiden --- src/topology_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index 28cc6da24..a551b8e3b 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -45,19 +45,20 @@ def get_auth_session(args): if args.key: key = args.key - session = requests.Session() + session = {} if os.path.exists(cert): - session.cert = cert + session["cert_file"] = cert else: raise InvalidPathError("Error: could not find cert at %s" % cert) if os.path.exists(key): - session.cert = (cert, key) + session["key_file"] = key else: raise InvalidPathError("Error: could not find key at %s" % key) - - return session + session['cert_reqs'] = 'CERT_REQUIRED' + session['key_password'] = input("decryption password: ") + return urlib3.PoolManager(*session) def update_url_hostname(url, args): From a8ec70fda073aac5c66a46f942f8fbab83551d0d Mon Sep 17 00:00:00 2001 From: Aiden Cohen Date: Tue, 29 Nov 2022 14:18:11 -0600 Subject: [PATCH 04/15] finished last touches --- src/topology_utils.py | 80 +++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index a551b8e3b..b4ded1d6b 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -5,10 +5,10 @@ import os import sys -import urllib +import urllib3 import fnmatch import urllib.parse as urlparse - +from getpass import getpass import xml.etree.ElementTree as ET import requests @@ -57,8 +57,7 @@ def get_auth_session(args): else: raise InvalidPathError("Error: could not find key at %s" % key) session['cert_reqs'] = 'CERT_REQUIRED' - session['key_password'] = input("decryption password: ") - return urlib3.PoolManager(*session) + return urllib3.PoolManager(**session, key_password=getpass("decryption password: ")) def update_url_hostname(url, args): @@ -132,7 +131,7 @@ def get_contact_list_info(contact_list): return contact_list_info -def get_vo_map(args, session=None): +def get_vo_map(args, pm=None): """ Generate a dictionary mapping from the VO name (key) to the VO ID (value). @@ -142,11 +141,11 @@ def get_vo_map(args, session=None): url = update_url_hostname("https://topology.opensciencegrid.org/vosummary" "/xml?all_vos=on&active_value=1", args) - if session is None: - with get_auth_session(args) as session: - response = session.get(url) + if pm is None: + pm = get_auth_session(args) + response = pm.request('GET',url) else: - response = session.get(url) + response = pm.request('GET',url) if old_no_proxy is not None: os.environ['no_proxy'] = old_no_proxy @@ -171,7 +170,7 @@ def get_vo_map(args, session=None): if 'ID' in vo_info and 'Name' in vo_info: vo_map[vo_info['Name'].lower()] = vo_info['ID'] - return vo_map + return vo_map,pm SERVICE_IDS = {'ce': 1, @@ -182,7 +181,7 @@ def get_vo_map(args, session=None): 'perfsonar-latency': 130, 'gums': 101, } -def mangle_url(url, args, session=None): +def mangle_url(url, args, pm=None): """ Given a MyOSG URL, switch to using the hostname specified in the arguments @@ -207,7 +206,7 @@ def mangle_url(url, args, session=None): qs_list.append(("service_sel[]", str(service_id))) if getattr(args, 'owner_vo', None): - vo_map = get_vo_map(args, session) + vo_map,pm = get_vo_map(args, pm) if 'voown' not in qs_dict: qs_list.append(("voown", "on")) for vo in args.owner_vo.split(","): @@ -220,10 +219,10 @@ def mangle_url(url, args, session=None): url_list[3] = urlparse.urlencode(qs_list, doseq=True) - return urlparse.urlunsplit(url_list) + return urlparse.urlunsplit(url_list),pm -def get_contacts(args, urltype, roottype): +def get_contacts(args, urltype, roottype, session=None): """ Get one type of contacts for OSG. """ @@ -232,43 +231,46 @@ def get_contacts(args, urltype, roottype): base_url = "https://topology.opensciencegrid.org/" + urltype + "summary/xml?" \ "&active=on&active_value=1&disable=on&disable_value=0" - with get_auth_session(args) as session: - url = mangle_url(base_url, args, session) + if(session == None): + session = get_auth_session(args) + url,session = mangle_url(base_url, args, session) + try: + response = session.request('GET',url) + except requests.exceptions.ConnectionError as exc: + print(exc) try: - response = session.get(url) - except requests.exceptions.ConnectionError as exc: - try: - if exc.args[0].args[1].errno == 22: - raise IncorrectPasswordError("Incorrect password, please try again") - else: - raise exc - except (TypeError, AttributeError, IndexError): - raise exc + if exc.args[0].args[1].errno == 22: + raise IncorrectPasswordError("Incorrect password, please try again") + else: + raise exc + except (TypeError, AttributeError, IndexError): + print(exc) + raise exc if old_no_proxy is not None: os.environ['no_proxy'] = old_no_proxy else: del os.environ['no_proxy'] - if response.status_code != requests.codes.ok: + if response.status != requests.codes.ok: print("MyOSG request failed (status %d): %s" % \ (response.status_code, response.text[:2048]), file=sys.stderr) return None - root = ET.fromstring(response.content) + root = ET.fromstring(response.data) if root.tag != roottype + 'Summary': print("MyOSG returned invalid XML with root tag %s" % root.tag, file=sys.stderr) return None - return root + return root, session -def get_vo_contacts(args): +def get_vo_contacts(args, pm = None): """ Get resource contacts for OSG. Return results. """ - root = get_contacts(args, 'vo', 'VO') + root,pm = get_contacts(args, 'vo', 'VO', pm) if root is None: return 1 @@ -291,17 +293,17 @@ def get_vo_contacts(args): if name and contact_list_info: results[name] = contact_list_info - return results + return results, pm -def get_resource_contacts_by_name_and_fqdn(args): +def get_resource_contacts_by_name_and_fqdn(args, pm = None): """ Get resource contacts for OSG. Return results. Returns two dictionaries, one keyed on the resource name and one keyed on the resource FQDN. """ - root = get_contacts(args, 'rg', 'Resource') + root, pm = get_contacts(args, 'rg', 'Resource', pm) if root is None: return {}, {} @@ -336,15 +338,17 @@ def get_resource_contacts_by_name_and_fqdn(args): if resource_fqdn: results_by_fqdn[resource_fqdn] = contact_list_info - return results_by_name, results_by_fqdn + return results_by_name, results_by_fqdn, pm -def get_resource_contacts(args): - return get_resource_contacts_by_name_and_fqdn(args)[0] +def get_resource_contacts(args, pm = None): + res = get_resource_contacts_by_name_and_fqdn(args, pm) + return res[0], res[2] -def get_resource_contacts_by_fqdn(args): - return get_resource_contacts_by_name_and_fqdn(args)[1] +def get_resource_contacts_by_fqdn(args, pm=None): + res = get_resource_contacts_by_name_and_fqdn(args, pm) + return res[1], res[2] def filter_contacts(args, results): From 08b7804973c48952dedf0bc07ffee02eea2d0736 Mon Sep 17 00:00:00 2001 From: Aiden Cohen Date: Tue, 29 Nov 2022 14:19:37 -0600 Subject: [PATCH 05/15] finished last touches --- bin/osg-notify | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/osg-notify b/bin/osg-notify index be2c043a3..a77065c3e 100755 --- a/bin/osg-notify +++ b/bin/osg-notify @@ -173,13 +173,13 @@ def has_non_printable_ascii_characters(contents): def main(): args = parseargs() - + pm = None recipients = set(args.recipients.split()) if args.oim_recipients and 'vos' in args.oim_recipients: attempts = 3 while attempts > 0: try: - results = topology_utils.get_vo_contacts(args) + results,pm = topology_utils.get_vo_contacts(args, pm) except topology_utils.InvalidPathError as exc: print(exc) exit(1) @@ -202,9 +202,9 @@ def main(): while attempts > 0: try: if args.fqdn_filter: - results = topology_utils.get_resource_contacts_by_fqdn(args) + results, pm = topology_utils.get_resource_contacts_by_fqdn(args, pm) else: - results = topology_utils.get_resource_contacts(args) + results, pm = topology_utils.get_resource_contacts(args, pm) except topology_utils.InvalidPathError as exc: exit(str(exc)) except topology_utils.IncorrectPasswordError as exc: From 4a32d2122f3fdf8dbabee47bbaea24acc8aa3a09 Mon Sep 17 00:00:00 2001 From: Aiden Cohen Date: Tue, 13 Dec 2022 15:34:04 -0600 Subject: [PATCH 06/15] Revamped structure --- bin/osg-notify | 13 +- src/topology_utils.py | 749 +++++++++++++++++++++--------------------- 2 files changed, 375 insertions(+), 387 deletions(-) diff --git a/bin/osg-notify b/bin/osg-notify index a77065c3e..9f982a9ab 100755 --- a/bin/osg-notify +++ b/bin/osg-notify @@ -21,6 +21,7 @@ if __name__ == "__main__" and __package__ is None: sys.path.append(_parent + "/src") import topology_utils +from topology_utils import OSGPoolManager import net_name_addr_utils # Parts of this implementation are from the following StackOverflow answer: @@ -173,13 +174,13 @@ def has_non_printable_ascii_characters(contents): def main(): args = parseargs() - pm = None + pm = OSGPoolManager() recipients = set(args.recipients.split()) if args.oim_recipients and 'vos' in args.oim_recipients: attempts = 3 while attempts > 0: try: - results,pm = topology_utils.get_vo_contacts(args, pm) + results = pm.get_vo_contacts(args) except topology_utils.InvalidPathError as exc: print(exc) exit(1) @@ -190,7 +191,7 @@ def main(): exit(1) else: print(exc) - results = topology_utils.filter_contacts(args, results) + results = OSGPoolManager.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: @@ -202,9 +203,9 @@ def main(): while attempts > 0: try: if args.fqdn_filter: - results, pm = topology_utils.get_resource_contacts_by_fqdn(args, pm) + results = pm.get_resource_contacts_by_fqdn(args) else: - results, pm = topology_utils.get_resource_contacts(args, pm) + results = pm.get_resource_contacts(args) except topology_utils.InvalidPathError as exc: exit(str(exc)) except topology_utils.IncorrectPasswordError as exc: @@ -213,7 +214,7 @@ def main(): exit("Too many incorrect password attempts, exiting") else: print(exc) - results = topology_utils.filter_contacts(args, results) + results = OSGPoolManager.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: diff --git a/src/topology_utils.py b/src/topology_utils.py index 5009211d9..6e8f14706 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -5,10 +5,6 @@ import os import sys -<<<<<<< HEAD -======= -import urllib ->>>>>>> master import urllib3 import fnmatch import urllib.parse as urlparse @@ -29,384 +25,375 @@ class InvalidPathError(AuthError): class IncorrectPasswordError(AuthError): pass -def get_auth_session(args): - """ - Return a requests session ready for an XML query. - """ - euid = os.geteuid() - if euid == 0: - cert = '/etc/grid-security/hostcert.pem' - key = '/etc/grid-security/hostkey.pem' - else: - cert = '/tmp/x509up_u%d' % euid - key = '/tmp/x509up_u%d' % euid - - cert = os.environ.get('X509_USER_PROXY', cert) - key = os.environ.get('X509_USER_PROXY', key) - - if args.cert: - cert = args.cert - if args.key: - key = args.key - - session = {} -<<<<<<< HEAD - -======= - session['cert_reqs'] = 'CERT_REQUIRED' ->>>>>>> master - if os.path.exists(cert): - session["cert_file"] = cert - else: - raise InvalidPathError("Error: could not find cert at %s" % cert) - - if os.path.exists(key): -<<<<<<< HEAD - session["key_file"] = key - else: - raise InvalidPathError("Error: could not find key at %s" % key) - session['cert_reqs'] = 'CERT_REQUIRED' - return urllib3.PoolManager(**session, key_password=getpass("decryption password: ")) -======= - session["cert_key"] = key - else: - raise InvalidPathError("Error: could not find key at %s" % key) - dpass = input("Decryption Pass") - session["key_password"] = dpass - - return urllib3.PoolManager(*session) ->>>>>>> master - - -def update_url_hostname(url, args): - """ - Given a URL and an argument object, update the URL's hostname - according to args.host and return the newly-formed URL. - """ - if not args.host: - return url - url_list = list(urlparse.urlsplit(url)) - url_list[1] = args.host - return urlparse.urlunsplit(url_list) - - -def get_contact_list_info(contact_list): - """ - Get contact list info out of contact list - - In rgsummary, this looks like: - - - Administrative Contact - - - Matyas Selmeci +class OSGPoolManager(urllib3.PoolManager): + + def __init__(self): + self.session = False + super().__init__() + + def get_auth_session(self,args): + """ + Return a requests session ready for an XML query. + """ + euid = os.geteuid() + if euid == 0: + cert = '/etc/grid-security/hostcert.pem' + key = '/etc/grid-security/hostkey.pem' + else: + cert = '/tmp/x509up_u%d' % euid + key = '/tmp/x509up_u%d' % euid + + cert = os.environ.get('X509_USER_PROXY', cert) + key = os.environ.get('X509_USER_PROXY', key) + + if args.cert: + cert = args.cert + if args.key: + key = args.key + + session = {} + if os.path.exists(cert): + session["cert_file"] = cert + else: + raise InvalidPathError("Error: could not find cert at %s" % cert) + + if os.path.exists(key): + session["key_file"] = key + else: + raise InvalidPathError("Error: could not find key at %s" % key) + session['cert_reqs'] = 'CERT_REQUIRED' + session['key_password'] = getpass("decryption password: ") + super().__dict__.update(**session) + return True + + @staticmethod + def update_url_hostname(url, args): + """ + Given a URL and an argument object, update the URL's hostname + according to args.host and return the newly-formed URL. + """ + if not args.host: + return url + url_list = list(urlparse.urlsplit(url)) + url_list[1] = args.host + return urlparse.urlunsplit(url_list) + + @staticmethod + def get_contact_list_info(contact_list): + """ + Get contact list info out of contact list + + In rgsummary, this looks like: + + + Administrative Contact + + + Matyas Selmeci + ... + + + + ... + + + and the arg `contact_list` is the contents of a single + + If vosummary, this looks like: + + + Miscellaneous Contact + + + ... + ... + ... - - - - ... - - - and the arg `contact_list` is the contents of a single - - If vosummary, this looks like: - - - Miscellaneous Contact - - - ... - ... - - ... - - - ... - - - and the arg `contact_list` is the contents of - - - Returns: a list of dicts that each look like: - { 'ContactType': 'Administrative Contact', - 'Name': 'Matyas Selmeci', - 'Email': '...', - ... - } - """ - contact_list_info = [] - for contact in contact_list: - if contact.tag == 'ContactType' or contact.tag == 'Type': - contact_list_type = contact.text.lower() - if contact.tag == 'Contacts': - for con in contact: - contact_info = { 'ContactType' : contact_list_type } - for contact_contents in con: - contact_info[contact_contents.tag] = contact_contents.text - contact_list_info.append(contact_info) - - return contact_list_info - - -def get_vo_map(args, pm=None): - """ - Generate a dictionary mapping from the VO name (key) to the - VO ID (value). - """ - old_no_proxy = os.environ.pop('no_proxy', None) - os.environ['no_proxy'] = '.opensciencegrid.org' - - url = update_url_hostname("https://topology.opensciencegrid.org/vosummary" - "/xml?all_vos=on&active_value=1", args) - if pm is None: - pm = get_auth_session(args) - response = pm.request('GET',url) - else: - response = pm.request('GET',url) - - if old_no_proxy is not None: - os.environ['no_proxy'] = old_no_proxy - else: - del os.environ['no_proxy'] - - if response.status_code != requests.codes.ok: - raise Exception("MyOSG request failed (status %d): %s" % \ - (response.status_code, response.text[:2048])) - - root = ET.fromstring(response.content) - if root.tag != 'VOSummary': - raise Exception("MyOSG returned invalid XML with root tag %s" % root.tag) - vo_map = {} - for child_vo in root: - if child_vo.tag != "VO": - raise Exception("MyOSG returned a non-VO (%s) inside VO summary." % \ - root.tag) - vo_info = {} - for child_info in child_vo: - vo_info[child_info.tag] = child_info.text - if 'ID' in vo_info and 'Name' in vo_info: - vo_map[vo_info['Name'].lower()] = vo_info['ID'] - - return vo_map,pm - - -SERVICE_IDS = {'ce': 1, - 'srmv2': 3, - 'gridftp': 5, - 'xrootd': 142, - 'perfsonar-bandwidth': 130, - 'perfsonar-latency': 130, - 'gums': 101, - } -def mangle_url(url, args, pm=None): - """ - Given a MyOSG URL, switch to using the hostname specified in the - arguments - """ - if not args.host: - return url - url_list = list(urlparse.urlsplit(url)) - url_list[1] = args.host - - qs_dict = urlparse.parse_qs(url_list[3]) - qs_list = urlparse.parse_qsl(url_list[3]) - - if getattr(args, 'provides_service', None): - if 'service' not in qs_dict: - qs_list.append(("service", "on")) - for service in args.provides_service.split(","): - service = service.strip().lower() - service_id = SERVICE_IDS.get(service) - if not service_id: - raise Exception("Requested service %s not known; known service" - " names: %s" % (service, ", ".join(SERVICE_IDS))) - qs_list.append(("service_sel[]", str(service_id))) - - if getattr(args, 'owner_vo', None): - vo_map,pm = get_vo_map(args, pm) - if 'voown' not in qs_dict: - qs_list.append(("voown", "on")) - for vo in args.owner_vo.split(","): - vo = vo.strip().lower() - vo_id = vo_map.get(vo) - if not vo_id: - raise Exception("Requested owner VO %s not known; known VOs: %s" \ - % (vo, ", ".join(vo_map))) - qs_list.append(("voown_sel[]", str(vo_id))) - - url_list[3] = urlparse.urlencode(qs_list, doseq=True) - - return urlparse.urlunsplit(url_list),pm - - -def get_contacts(args, urltype, roottype, session=None): - """ - Get one type of contacts for OSG. - """ - old_no_proxy = os.environ.pop('no_proxy', None) - os.environ['no_proxy'] = '.opensciencegrid.org' - - base_url = "https://topology.opensciencegrid.org/" + urltype + "summary/xml?" \ - "&active=on&active_value=1&disable=on&disable_value=0" - if(session == None): - session = get_auth_session(args) - url,session = mangle_url(base_url, args, session) - try: - response = session.request('GET',url) - except requests.exceptions.ConnectionError as exc: - print(exc) + + + ... + + + and the arg `contact_list` is the contents of + + + Returns: a list of dicts that each look like: + { 'ContactType': 'Administrative Contact', + 'Name': 'Matyas Selmeci', + 'Email': '...', + ... + } + """ + contact_list_info = [] + for contact in contact_list: + if contact.tag == 'ContactType' or contact.tag == 'Type': + contact_list_type = contact.text.lower() + if contact.tag == 'Contacts': + for con in contact: + contact_info = { 'ContactType' : contact_list_type } + for contact_contents in con: + contact_info[contact_contents.tag] = contact_contents.text + contact_list_info.append(contact_info) + + return contact_list_info + + + def get_vo_map(self,args): + """ + Generate a dictionary mapping from the VO name (key) to the + VO ID (value). + """ + old_no_proxy = os.environ.pop('no_proxy', None) + os.environ['no_proxy'] = '.opensciencegrid.org' + + url = update_url_hostname("https://topology.opensciencegrid.org/vosummary" + "/xml?all_vos=on&active_value=1", args) + if not self.session: + self.session = self.get_auth_session(args) + response = self.request('GET',url) + else: + response = self.request('GET',url) + + if old_no_proxy is not None: + os.environ['no_proxy'] = old_no_proxy + else: + del os.environ['no_proxy'] + + if response.status_code != requests.codes.ok: + raise Exception("MyOSG request failed (status %d): %s" % \ + (response.status_code, response.text[:2048])) + + root = ET.fromstring(response.content) + if root.tag != 'VOSummary': + raise Exception("MyOSG returned invalid XML with root tag %s" % root.tag) + vo_map = {} + for child_vo in root: + if child_vo.tag != "VO": + raise Exception("MyOSG returned a non-VO (%s) inside VO summary." % \ + root.tag) + vo_info = {} + for child_info in child_vo: + vo_info[child_info.tag] = child_info.text + if 'ID' in vo_info and 'Name' in vo_info: + vo_map[vo_info['Name'].lower()] = vo_info['ID'] + + return vo_map + + + SERVICE_IDS = {'ce': 1, + 'srmv2': 3, + 'gridftp': 5, + 'xrootd': 142, + 'perfsonar-bandwidth': 130, + 'perfsonar-latency': 130, + 'gums': 101, + } + def mangle_url(self,url, args): + """ + Given a MyOSG URL, switch to using the hostname specified in the + arguments + """ + if not args.host: + return url + url_list = list(urlparse.urlsplit(url)) + url_list[1] = args.host + + qs_dict = urlparse.parse_qs(url_list[3]) + qs_list = urlparse.parse_qsl(url_list[3]) + + if getattr(args, 'provides_service', None): + if 'service' not in qs_dict: + qs_list.append(("service", "on")) + for service in args.provides_service.split(","): + service = service.strip().lower() + service_id = SERVICE_IDS.get(service) + if not service_id: + raise Exception("Requested service %s not known; known service" + " names: %s" % (service, ", ".join(SERVICE_IDS))) + qs_list.append(("service_sel[]", str(service_id))) + + if getattr(args, 'owner_vo', None): + vo_map = self.get_vo_map(args) + if 'voown' not in qs_dict: + qs_list.append(("voown", "on")) + for vo in args.owner_vo.split(","): + vo = vo.strip().lower() + vo_id = vo_map.get(vo) + if not vo_id: + raise Exception("Requested owner VO %s not known; known VOs: %s" \ + % (vo, ", ".join(vo_map))) + qs_list.append(("voown_sel[]", str(vo_id))) + + url_list[3] = urlparse.urlencode(qs_list, doseq=True) + + return urlparse.urlunsplit(url_list) + + + def get_contacts(self, args, urltype, roottype): + """ + Get one type of contacts for OSG. + """ + old_no_proxy = os.environ.pop('no_proxy', None) + os.environ['no_proxy'] = '.opensciencegrid.org' + + base_url = "https://topology.opensciencegrid.org/" + urltype + "summary/xml?" \ + "&active=on&active_value=1&disable=on&disable_value=0" + if(not self.session): + self.session = self.get_auth_session(args) + url = self.mangle_url(base_url, args) try: - if exc.args[0].args[1].errno == 22: - raise IncorrectPasswordError("Incorrect password, please try again") - else: - raise exc - except (TypeError, AttributeError, IndexError): - print(exc) - raise exc - - if old_no_proxy is not None: - os.environ['no_proxy'] = old_no_proxy - else: - del os.environ['no_proxy'] - - if response.status != requests.codes.ok: - print("MyOSG request failed (status %d): %s" % \ - (response.status_code, response.text[:2048]), file=sys.stderr) - return None - - root = ET.fromstring(response.data) - if root.tag != roottype + 'Summary': - print("MyOSG returned invalid XML with root tag %s" % root.tag, - file=sys.stderr) - return None - - return root, session - - -def get_vo_contacts(args, pm = None): - """ - Get resource contacts for OSG. Return results. - """ - root,pm = get_contacts(args, 'vo', 'VO', pm) - if root is None: - return 1 - - results = {} - for child_vo in root: - if child_vo.tag != "VO": - print("MyOSG returned a non-VO (%s) inside summary." % \ - root.tag, file=sys.stderr) + response = self.request('GET',url) + except requests.exceptions.ConnectionError as exc: + print(exc) + try: + if exc.args[0].args[1].errno == 22: + raise IncorrectPasswordError("Incorrect password, please try again") + else: + raise exc + except (TypeError, AttributeError, IndexError): + print(exc) + raise exc + + if old_no_proxy is not None: + os.environ['no_proxy'] = old_no_proxy + else: + del os.environ['no_proxy'] + + if response.status != requests.codes.ok: + print("MyOSG request failed (status %d): %s" % \ + (response.status_code, response.text[:2048]), file=sys.stderr) + return None + + root = ET.fromstring(response.data) + if root.tag != roottype + 'Summary': + print("MyOSG returned invalid XML with root tag %s" % root.tag, + file=sys.stderr) + return None + + return root + + + def get_vo_contacts(self,args): + """ + Get resource contacts for OSG. Return results. + """ + root = self.get_contacts(args, 'vo', 'VO') + if root is None: return 1 - name = None - contact_list_info = [] - for item in child_vo: - if item.tag == 'Name': - name = item.text - if item.tag == "ContactTypes": - for contact_type in item: - contact_list_info.extend( \ - get_contact_list_info(contact_type)) - - if name and contact_list_info: - results[name] = contact_list_info - - return results, pm - - -def get_resource_contacts_by_name_and_fqdn(args, pm = None): - """ - Get resource contacts for OSG. Return results. - - Returns two dictionaries, one keyed on the resource name and one keyed on - the resource FQDN. - """ - root, pm = get_contacts(args, 'rg', 'Resource', pm) - if root is None: - return {}, {} - - results_by_name = {} - results_by_fqdn = {} - for child_rg in root: - if child_rg.tag != "ResourceGroup": - print("MyOSG returned a non-resource group (%s) inside summary." % \ - root.tag, file=sys.stderr) + + results = {} + for child_vo in root: + if child_vo.tag != "VO": + print("MyOSG returned a non-VO (%s) inside summary." % \ + root.tag, file=sys.stderr) + return 1 + name = None + contact_list_info = [] + for item in child_vo: + if item.tag == 'Name': + name = item.text + if item.tag == "ContactTypes": + for contact_type in item: + contact_list_info.extend( \ + get_contact_list_info(contact_type)) + + if name and contact_list_info: + results[name] = contact_list_info + + return results + + + def get_resource_contacts_by_name_and_fqdn(self,args): + """ + Get resource contacts for OSG. Return results. + + Returns two dictionaries, one keyed on the resource name and one keyed on + the resource FQDN. + """ + root = self.get_contacts(args, 'rg', 'Resource') + if root is None: return {}, {} - for child_res in child_rg: - if child_res.tag != "Resources": - continue - for resource in child_res: - resource_name = None - resource_fqdn = None - contact_list_info = [] - for resource_tag in resource: - if resource_tag.tag == 'Name': - resource_name = resource_tag.text - if resource_tag.tag == 'FQDN': - resource_fqdn = resource_tag.text - if resource_tag.tag == 'ContactLists': - for contact_list in resource_tag: - if contact_list.tag == 'ContactList': - contact_list_info.extend( \ - get_contact_list_info(contact_list)) - - if contact_list_info: - if resource_name: - results_by_name[resource_name] = contact_list_info - if resource_fqdn: - results_by_fqdn[resource_fqdn] = contact_list_info - - return results_by_name, results_by_fqdn, pm - - -def get_resource_contacts(args, pm = None): - res = get_resource_contacts_by_name_and_fqdn(args, pm) - return res[0], res[2] - - -def get_resource_contacts_by_fqdn(args, pm=None): - res = get_resource_contacts_by_name_and_fqdn(args, pm) - return res[1], res[2] - - -def filter_contacts(args, results): - """ - Given a set of result contacts, filter them according to given arguments - """ - results = dict(results) # make a copy so we don't modify the original - - if getattr(args, 'name_filter', None): - # filter out undesired names - for name in list(results): - if not fnmatch.fnmatch(name, args.name_filter) and \ - args.name_filter not in name: - del results[name] - elif getattr(args, 'fqdn_filter', None): - # filter out undesired FQDNs - for fqdn in list(results): - if not fnmatch.fnmatch(fqdn, args.fqdn_filter) and \ - args.fqdn_filter not in fqdn: - del results[fqdn] - - if args.contact_type != 'all': - # filter out undesired contact types - for name in list(results): - contact_list = [] - for contact in results[name]: - contact_type = contact['ContactType'] - if contact_type.startswith(args.contact_type): - contact_list.append(contact) - if contact_list == []: - del results[name] - else: - results[name] = contact_list - - if getattr(args, 'contact_emails', None): - for name in list(results): - contact_list = [contact for contact in results[name] if contact['Email'] in args.contact_emails] - if not contact_list: - del results[name] - else: - results[name] = contact_list - - return results + + results_by_name = {} + results_by_fqdn = {} + for child_rg in root: + if child_rg.tag != "ResourceGroup": + print("MyOSG returned a non-resource group (%s) inside summary." % \ + root.tag, file=sys.stderr) + return {}, {} + for child_res in child_rg: + if child_res.tag != "Resources": + continue + for resource in child_res: + resource_name = None + resource_fqdn = None + contact_list_info = [] + for resource_tag in resource: + if resource_tag.tag == 'Name': + resource_name = resource_tag.text + if resource_tag.tag == 'FQDN': + resource_fqdn = resource_tag.text + if resource_tag.tag == 'ContactLists': + for contact_list in resource_tag: + if contact_list.tag == 'ContactList': + contact_list_info.extend( \ + get_contact_list_info(contact_list)) + + if contact_list_info: + if resource_name: + results_by_name[resource_name] = contact_list_info + if resource_fqdn: + results_by_fqdn[resource_fqdn] = contact_list_info + + return results_by_name, results_by_fqdn + + + def get_resource_contacts(self,args): + return self.get_resource_contacts_by_name_and_fqdn(args)[0] + + + def get_resource_contacts_by_fqdn(self, args): + return self.get_resource_contacts_by_name_and_fqdn(args)[1] + + @staticmethod + def filter_contacts(args, results): + """ + Given a set of result contacts, filter them according to given arguments + """ + results = dict(results) # make a copy so we don't modify the original + + if getattr(args, 'name_filter', None): + # filter out undesired names + for name in list(results): + if not fnmatch.fnmatch(name, args.name_filter) and \ + args.name_filter not in name: + del results[name] + elif getattr(args, 'fqdn_filter', None): + # filter out undesired FQDNs + for fqdn in list(results): + if not fnmatch.fnmatch(fqdn, args.fqdn_filter) and \ + args.fqdn_filter not in fqdn: + del results[fqdn] + + if args.contact_type != 'all': + # filter out undesired contact types + for name in list(results): + contact_list = [] + for contact in results[name]: + contact_type = contact['ContactType'] + if contact_type.startswith(args.contact_type): + contact_list.append(contact) + if contact_list == []: + del results[name] + else: + results[name] = contact_list + + if getattr(args, 'contact_emails', None): + for name in list(results): + contact_list = [contact for contact in results[name] if contact['Email'] in args.contact_emails] + if not contact_list: + del results[name] + else: + results[name] = contact_list + + return results From 1f337fbf59adf661dc4b91d3aca34d1eec574d14 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Wed, 21 Dec 2022 17:40:36 -0600 Subject: [PATCH 07/15] Rename class to avoid future typos --- bin/osg-notify | 8 ++++---- src/topology_utils.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/osg-notify b/bin/osg-notify index 9f982a9ab..545587f2a 100755 --- a/bin/osg-notify +++ b/bin/osg-notify @@ -21,7 +21,7 @@ if __name__ == "__main__" and __package__ is None: sys.path.append(_parent + "/src") import topology_utils -from topology_utils import OSGPoolManager +from topology_utils import TopologyPoolManager import net_name_addr_utils # Parts of this implementation are from the following StackOverflow answer: @@ -174,7 +174,7 @@ def has_non_printable_ascii_characters(contents): def main(): args = parseargs() - pm = OSGPoolManager() + pm = TopologyPoolManager() recipients = set(args.recipients.split()) if args.oim_recipients and 'vos' in args.oim_recipients: attempts = 3 @@ -191,7 +191,7 @@ def main(): exit(1) else: print(exc) - results = OSGPoolManager.filter_contacts(args, results) + results = TopologyPoolManager.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: @@ -214,7 +214,7 @@ def main(): exit("Too many incorrect password attempts, exiting") else: print(exc) - results = OSGPoolManager.filter_contacts(args, results) + results = TopologyPoolManager.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: diff --git a/src/topology_utils.py b/src/topology_utils.py index 6e8f14706..79ca2ec74 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -25,7 +25,7 @@ class InvalidPathError(AuthError): class IncorrectPasswordError(AuthError): pass -class OSGPoolManager(urllib3.PoolManager): +class TopologyPoolManager(urllib3.PoolManager): def __init__(self): self.session = False From 971c946c49311e233110ea28640d15d69d4b9d6a Mon Sep 17 00:00:00 2001 From: College Date: Fri, 16 Feb 2024 13:20:32 -0600 Subject: [PATCH 08/15] removed static methods in TopologyPoolManager --- bin/osg-notify | 4 ++-- src/topology_utils.py | 13 +++++-------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/osg-notify b/bin/osg-notify index 545587f2a..c326f4dd9 100755 --- a/bin/osg-notify +++ b/bin/osg-notify @@ -191,7 +191,7 @@ def main(): exit(1) else: print(exc) - results = TopologyPoolManager.filter_contacts(args, results) + results = pm.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: @@ -214,7 +214,7 @@ def main(): exit("Too many incorrect password attempts, exiting") else: print(exc) - results = TopologyPoolManager.filter_contacts(args, results) + results = pm.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: diff --git a/src/topology_utils.py b/src/topology_utils.py index 79ca2ec74..7172653e3 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -66,7 +66,6 @@ def get_auth_session(self,args): super().__dict__.update(**session) return True - @staticmethod def update_url_hostname(url, args): """ Given a URL and an argument object, update the URL's hostname @@ -78,7 +77,6 @@ def update_url_hostname(url, args): url_list[1] = args.host return urlparse.urlunsplit(url_list) - @staticmethod def get_contact_list_info(contact_list): """ Get contact list info out of contact list @@ -146,7 +144,7 @@ def get_vo_map(self,args): old_no_proxy = os.environ.pop('no_proxy', None) os.environ['no_proxy'] = '.opensciencegrid.org' - url = update_url_hostname("https://topology.opensciencegrid.org/vosummary" + url = self.update_url_hostname("https://topology.opensciencegrid.org/vosummary" "/xml?all_vos=on&active_value=1", args) if not self.session: self.session = self.get_auth_session(args) @@ -206,10 +204,10 @@ def mangle_url(self,url, args): qs_list.append(("service", "on")) for service in args.provides_service.split(","): service = service.strip().lower() - service_id = SERVICE_IDS.get(service) + service_id = self.SERVICE_IDS.get(service) if not service_id: raise Exception("Requested service %s not known; known service" - " names: %s" % (service, ", ".join(SERVICE_IDS))) + " names: %s" % (service, ", ".join(self.SERVICE_IDS))) qs_list.append(("service_sel[]", str(service_id))) if getattr(args, 'owner_vo', None): @@ -295,7 +293,7 @@ def get_vo_contacts(self,args): if item.tag == "ContactTypes": for contact_type in item: contact_list_info.extend( \ - get_contact_list_info(contact_type)) + self.get_contact_list_info(contact_type)) if name and contact_list_info: results[name] = contact_list_info @@ -337,7 +335,7 @@ def get_resource_contacts_by_name_and_fqdn(self,args): for contact_list in resource_tag: if contact_list.tag == 'ContactList': contact_list_info.extend( \ - get_contact_list_info(contact_list)) + self.get_contact_list_info(contact_list)) if contact_list_info: if resource_name: @@ -355,7 +353,6 @@ def get_resource_contacts(self,args): def get_resource_contacts_by_fqdn(self, args): return self.get_resource_contacts_by_name_and_fqdn(args)[1] - @staticmethod def filter_contacts(args, results): """ Given a set of result contacts, filter them according to given arguments From 0b1e4832a63b8d8de3ff6e01aee7d7c132297779 Mon Sep 17 00:00:00 2001 From: College Date: Mon, 19 Feb 2024 11:17:40 -0600 Subject: [PATCH 09/15] fixing self references from static class removal --- src/topology_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index 7172653e3..a37c41d45 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -66,7 +66,7 @@ def get_auth_session(self,args): super().__dict__.update(**session) return True - def update_url_hostname(url, args): + def update_url_hostname(self, url, args): """ Given a URL and an argument object, update the URL's hostname according to args.host and return the newly-formed URL. @@ -77,7 +77,7 @@ def update_url_hostname(url, args): url_list[1] = args.host return urlparse.urlunsplit(url_list) - def get_contact_list_info(contact_list): + def get_contact_list_info(self, contact_list): """ Get contact list info out of contact list @@ -353,7 +353,7 @@ def get_resource_contacts(self,args): def get_resource_contacts_by_fqdn(self, args): return self.get_resource_contacts_by_name_and_fqdn(args)[1] - def filter_contacts(args, results): + def filter_contacts(self, args, results): """ Given a set of result contacts, filter them according to given arguments """ From fbf23448823415d3358bd2d28e46207918486563 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 26 Feb 2024 17:22:17 -0600 Subject: [PATCH 10/15] Remove bad merges --- src/topology_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index a37c41d45..34e7cbe43 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -56,7 +56,7 @@ def get_auth_session(self,args): session["cert_file"] = cert else: raise InvalidPathError("Error: could not find cert at %s" % cert) - + if os.path.exists(key): session["key_file"] = key else: @@ -372,14 +372,15 @@ def filter_contacts(self, args, results): args.fqdn_filter not in fqdn: del results[fqdn] - if args.contact_type != 'all': + if 'all' not in args.contact_type: # filter out undesired contact types for name in list(results): contact_list = [] for contact in results[name]: contact_type = contact['ContactType'] - if contact_type.startswith(args.contact_type): - contact_list.append(contact) + for args_contact_type in args.contact_type: + if contact_type.startswith(args_contact_type): + contact_list.append(contact) if contact_list == []: del results[name] else: From 7a6b772c2e73d5ccb511837de257795dfdd3146d Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 26 Feb 2024 17:38:52 -0600 Subject: [PATCH 11/15] Move static methods to module level to retain API consistency --- bin/osg-notify | 4 +- src/topology_utils.py | 234 +++++++++++++++++++++--------------------- 2 files changed, 121 insertions(+), 117 deletions(-) diff --git a/bin/osg-notify b/bin/osg-notify index c326f4dd9..d6aa0f19b 100755 --- a/bin/osg-notify +++ b/bin/osg-notify @@ -191,7 +191,7 @@ def main(): exit(1) else: print(exc) - results = pm.filter_contacts(args, results) + results = topology_utils.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: @@ -214,7 +214,7 @@ def main(): exit("Too many incorrect password attempts, exiting") else: print(exc) - results = pm.filter_contacts(args, results) + results = topology_utils.filter_contacts(args, results) emails = set() for name in results.keys(): for contact in results[name]: diff --git a/src/topology_utils.py b/src/topology_utils.py index 34e7cbe43..5fe99c42f 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -25,6 +25,122 @@ class InvalidPathError(AuthError): class IncorrectPasswordError(AuthError): pass + +def update_url_hostname(url, args): + """ + Given a URL and an argument object, update the URL's hostname + according to args.host and return the newly-formed URL. + """ + if not args.host: + return url + url_list = list(urlparse.urlsplit(url)) + url_list[1] = args.host + return urlparse.urlunsplit(url_list) + + +def get_contact_list_info(contact_list): + """ + Get contact list info out of contact list + + In rgsummary, this looks like: + + + Administrative Contact + + + Matyas Selmeci + ... + + + + ... + + + and the arg `contact_list` is the contents of a single + + If vosummary, this looks like: + + + Miscellaneous Contact + + + ... + ... + + ... + + + ... + + + and the arg `contact_list` is the contents of + + + Returns: a list of dicts that each look like: + { 'ContactType': 'Administrative Contact', + 'Name': 'Matyas Selmeci', + 'Email': '...', + ... + } + """ + contact_list_info = [] + for contact in contact_list: + if contact.tag == 'ContactType' or contact.tag == 'Type': + contact_list_type = contact.text.lower() + if contact.tag == 'Contacts': + for con in contact: + contact_info = { 'ContactType' : contact_list_type } + for contact_contents in con: + contact_info[contact_contents.tag] = contact_contents.text + contact_list_info.append(contact_info) + + return contact_list_info + + +def filter_contacts(args, results): + """ + Given a set of result contacts, filter them according to given arguments + """ + results = dict(results) # make a copy so we don't modify the original + + if getattr(args, 'name_filter', None): + # filter out undesired names + for name in list(results): + if not fnmatch.fnmatch(name, args.name_filter) and \ + args.name_filter not in name: + del results[name] + elif getattr(args, 'fqdn_filter', None): + # filter out undesired FQDNs + for fqdn in list(results): + if not fnmatch.fnmatch(fqdn, args.fqdn_filter) and \ + args.fqdn_filter not in fqdn: + del results[fqdn] + + if 'all' not in args.contact_type: + # filter out undesired contact types + for name in list(results): + contact_list = [] + for contact in results[name]: + contact_type = contact['ContactType'] + for args_contact_type in args.contact_type: + if contact_type.startswith(args_contact_type): + contact_list.append(contact) + if contact_list == []: + del results[name] + else: + results[name] = contact_list + + if getattr(args, 'contact_emails', None): + for name in list(results): + contact_list = [contact for contact in results[name] if contact['Email'] in args.contact_emails] + if not contact_list: + del results[name] + else: + results[name] = contact_list + + return results + + class TopologyPoolManager(urllib3.PoolManager): def __init__(self): @@ -66,75 +182,6 @@ def get_auth_session(self,args): super().__dict__.update(**session) return True - def update_url_hostname(self, url, args): - """ - Given a URL and an argument object, update the URL's hostname - according to args.host and return the newly-formed URL. - """ - if not args.host: - return url - url_list = list(urlparse.urlsplit(url)) - url_list[1] = args.host - return urlparse.urlunsplit(url_list) - - def get_contact_list_info(self, contact_list): - """ - Get contact list info out of contact list - - In rgsummary, this looks like: - - - Administrative Contact - - - Matyas Selmeci - ... - - - - ... - - - and the arg `contact_list` is the contents of a single - - If vosummary, this looks like: - - - Miscellaneous Contact - - - ... - ... - - ... - - - ... - - - and the arg `contact_list` is the contents of - - - Returns: a list of dicts that each look like: - { 'ContactType': 'Administrative Contact', - 'Name': 'Matyas Selmeci', - 'Email': '...', - ... - } - """ - contact_list_info = [] - for contact in contact_list: - if contact.tag == 'ContactType' or contact.tag == 'Type': - contact_list_type = contact.text.lower() - if contact.tag == 'Contacts': - for con in contact: - contact_info = { 'ContactType' : contact_list_type } - for contact_contents in con: - contact_info[contact_contents.tag] = contact_contents.text - contact_list_info.append(contact_info) - - return contact_list_info - def get_vo_map(self,args): """ @@ -144,7 +191,7 @@ def get_vo_map(self,args): old_no_proxy = os.environ.pop('no_proxy', None) os.environ['no_proxy'] = '.opensciencegrid.org' - url = self.update_url_hostname("https://topology.opensciencegrid.org/vosummary" + url = update_url_hostname("https://topology.opensciencegrid.org/vosummary" "/xml?all_vos=on&active_value=1", args) if not self.session: self.session = self.get_auth_session(args) @@ -293,7 +340,7 @@ def get_vo_contacts(self,args): if item.tag == "ContactTypes": for contact_type in item: contact_list_info.extend( \ - self.get_contact_list_info(contact_type)) + get_contact_list_info(contact_type)) if name and contact_list_info: results[name] = contact_list_info @@ -335,7 +382,7 @@ def get_resource_contacts_by_name_and_fqdn(self,args): for contact_list in resource_tag: if contact_list.tag == 'ContactList': contact_list_info.extend( \ - self.get_contact_list_info(contact_list)) + get_contact_list_info(contact_list)) if contact_list_info: if resource_name: @@ -352,46 +399,3 @@ def get_resource_contacts(self,args): def get_resource_contacts_by_fqdn(self, args): return self.get_resource_contacts_by_name_and_fqdn(args)[1] - - def filter_contacts(self, args, results): - """ - Given a set of result contacts, filter them according to given arguments - """ - results = dict(results) # make a copy so we don't modify the original - - if getattr(args, 'name_filter', None): - # filter out undesired names - for name in list(results): - if not fnmatch.fnmatch(name, args.name_filter) and \ - args.name_filter not in name: - del results[name] - elif getattr(args, 'fqdn_filter', None): - # filter out undesired FQDNs - for fqdn in list(results): - if not fnmatch.fnmatch(fqdn, args.fqdn_filter) and \ - args.fqdn_filter not in fqdn: - del results[fqdn] - - if 'all' not in args.contact_type: - # filter out undesired contact types - for name in list(results): - contact_list = [] - for contact in results[name]: - contact_type = contact['ContactType'] - for args_contact_type in args.contact_type: - if contact_type.startswith(args_contact_type): - contact_list.append(contact) - if contact_list == []: - del results[name] - else: - results[name] = contact_list - - if getattr(args, 'contact_emails', None): - for name in list(results): - contact_list = [contact for contact in results[name] if contact['Email'] in args.contact_emails] - if not contact_list: - del results[name] - else: - results[name] = contact_list - - return results From dc5f9ec99ca0ef7039ac007a64314bcc9411214f Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 26 Feb 2024 17:40:59 -0600 Subject: [PATCH 12/15] Fix pool manager keyword updates --- src/topology_utils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index 5fe99c42f..8de940ddd 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -167,21 +167,18 @@ def get_auth_session(self,args): if args.key: key = args.key - session = {} if os.path.exists(cert): - session["cert_file"] = cert + self.connection_pool_kw["cert_file"] = cert else: raise InvalidPathError("Error: could not find cert at %s" % cert) if os.path.exists(key): - session["key_file"] = key + self.connection_pool_kw["key_file"] = key else: raise InvalidPathError("Error: could not find key at %s" % key) - session['cert_reqs'] = 'CERT_REQUIRED' - session['key_password'] = getpass("decryption password: ") - super().__dict__.update(**session) - return True + self.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED' + self.connection_pool_kw['key_password'] = getpass("decryption password: ") def get_vo_map(self,args): """ From 105b84d4d2d1fdd69cf68bcaf8ee654b728a8c53 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 26 Feb 2024 17:42:53 -0600 Subject: [PATCH 13/15] Use f-strings, fix some pylint warnings --- src/topology_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index 8de940ddd..f145e5c6d 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -147,7 +147,7 @@ def __init__(self): self.session = False super().__init__() - def get_auth_session(self,args): + def get_auth_session(self, args): """ Return a requests session ready for an XML query. """ @@ -156,8 +156,8 @@ def get_auth_session(self,args): cert = '/etc/grid-security/hostcert.pem' key = '/etc/grid-security/hostkey.pem' else: - cert = '/tmp/x509up_u%d' % euid - key = '/tmp/x509up_u%d' % euid + cert = f'/tmp/x509up_u{euid}' + key = f'/tmp/x509up_u{euid}' cert = os.environ.get('X509_USER_PROXY', cert) key = os.environ.get('X509_USER_PROXY', key) @@ -170,12 +170,12 @@ def get_auth_session(self,args): if os.path.exists(cert): self.connection_pool_kw["cert_file"] = cert else: - raise InvalidPathError("Error: could not find cert at %s" % cert) + raise InvalidPathError(f"Error: could not find cert at {cert}") if os.path.exists(key): self.connection_pool_kw["key_file"] = key else: - raise InvalidPathError("Error: could not find key at %s" % key) + raise InvalidPathError(f"Error: could not find key at {key}") self.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED' self.connection_pool_kw['key_password'] = getpass("decryption password: ") From b848e6f18eceb33d90a076ed290bf8070b137684 Mon Sep 17 00:00:00 2001 From: Emily Yao Date: Fri, 8 Mar 2024 10:22:49 -0600 Subject: [PATCH 14/15] re-added missing break statement --- bin/osg-notify | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/osg-notify b/bin/osg-notify index d6aa0f19b..9d33f33be 100755 --- a/bin/osg-notify +++ b/bin/osg-notify @@ -206,6 +206,7 @@ def main(): results = pm.get_resource_contacts_by_fqdn(args) else: results = pm.get_resource_contacts(args) + break except topology_utils.InvalidPathError as exc: exit(str(exc)) except topology_utils.IncorrectPasswordError as exc: From 19c1d15cf3423e063b0c21ff9491da909741b41b Mon Sep 17 00:00:00 2001 From: Emily Yao Date: Fri, 8 Mar 2024 10:25:16 -0600 Subject: [PATCH 15/15] reverted pool manager keywords --- src/topology_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/topology_utils.py b/src/topology_utils.py index f145e5c6d..ff3525413 100644 --- a/src/topology_utils.py +++ b/src/topology_utils.py @@ -167,18 +167,21 @@ def get_auth_session(self, args): if args.key: key = args.key + session = {} if os.path.exists(cert): - self.connection_pool_kw["cert_file"] = cert + session["cert_file"] = cert else: raise InvalidPathError(f"Error: could not find cert at {cert}") if os.path.exists(key): - self.connection_pool_kw["key_file"] = key + session["key_file"] = key else: raise InvalidPathError(f"Error: could not find key at {key}") - self.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED' - self.connection_pool_kw['key_password'] = getpass("decryption password: ") + session['cert_reqs'] = 'CERT_REQUIRED' + session['key_password'] = getpass("decryption password: ") + super().__dict__.update(**session) + return True def get_vo_map(self,args): """