From 438afd1c6e5a396eb3f26e2bf35e9e0a40cdf743 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Wed, 4 Oct 2023 17:25:28 -0400 Subject: [PATCH 1/5] fixed requirements --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1a464430..556a8dcd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ colorama msgpack netifaces ntplib -pymongo<4 pyzmq regex setproctitle From 9bfb78a642d57bb87bc4789cbccb3ab6e91e191e Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 5 Oct 2023 13:39:17 +0200 Subject: [PATCH 2/5] remove spurious mongodb helpers --- src/radical/utils/misc.py | 115 -------------------------------------- 1 file changed, 115 deletions(-) diff --git a/src/radical/utils/misc.py b/src/radical/utils/misc.py index ea4644456..1ac5077d9 100644 --- a/src/radical/utils/misc.py +++ b/src/radical/utils/misc.py @@ -33,121 +33,6 @@ _RU_exit = None -# ------------------------------------------------------------------------------ -# -def split_dburl(dburl, default_dburl=None): - ''' - we split the url into the base mongodb URL, and the path element, whose - first element is the database name, and the remainder is interpreted as - collection id. - ''' - - # if the given URL does not contain schema nor host, the default URL is used - # as base, and the given URL string is appended to the path element. - - url = ruu.Url(dburl) - - if not url.schema and not url.host: - url = ruu.Url(default_dburl) - url.path = dburl - - # NOTE: add other data base schemes here... - if 'mongodb' not in url.schema.split('+'): - raise ValueError("expected 'mongodb[+ssl]://' url, not '%s'" % dburl) - - host = url.host - port = url.port - path = url.path - user = url.username - pwd = unquote_plus(url.password) - - query_options = {'ssl': False} - - if 'ssl' in url.schema.split('+'): - query_options['ssl'] = True - url.schema = 'mongodb' - - if not host: - host = 'localhost' - - if url.query: - from urllib.parse import parse_qsl - # get dict query from url.query (str) - q = dict(parse_qsl(url.query)) - # control of which options are transferred - query_options['tlsAllowInvalidCertificates'] = bool( - q.get('tlsAllowInvalidCertificates', '0').lower() in ['true', '1']) - - if path.startswith('/'): - path = path[1:] - path_elems = path.split('/') - - dbname = None - cname = None - pname = None - - if len(path_elems) > 0: - dbname = path_elems[0] - - if len(path_elems) > 1: - dbname = path_elems[0] - cname = path_elems[1] - - if len(path_elems) > 2: - dbname = path_elems[0] - cname = path_elems[1] - pname = '.'.join(path_elems[2:]) - - if dbname == '.': - dbname = None - - return [host, port, dbname, cname, pname, user, pwd, query_options] - - -# ------------------------------------------------------------------------------ -# -def mongodb_connect(dburl, default_dburl=None): - ''' - connect to the given mongodb, perform auth for the database (if a database - was given). - ''' - - try: - import pymongo - - except ImportError as e: - msg = " \n\npymongo is not available -- install RU with: \n\n" - msg += " (1) pip install --upgrade -e '.[pymongo]'\n" - msg += " (2) pip install --upgrade 'radical.utils[pymongo]'\n\n" - msg += "to resolve that dependency (or install pymongo manually).\n" - msg += "The first version will work for local installation, \n" - msg += "the second one for installation from pypi.\n\n" - raise ImportError(msg) from e - - [host, port, dbname, cname, pname, - user, pwd, options] = split_dburl(dburl, default_dburl) - - mongo = pymongo.MongoClient(host=host, port=port, **options) - db = None - - if dbname: - db = mongo[dbname] - - if user and pwd: - db.authenticate(user, pwd) - - else: - - # if no DB is given, we try to auth against all databases. - for dbname in mongo.database_names(): - try: - mongo[dbname].authenticate(user, pwd) - except: - pass - - return mongo, db, dbname, cname, pname - - # ------------------------------------------------------------------------------ # def parse_file_staging_directives(directives): From 63cc1d762a7c9c62f4a274765e0c48355dc9ff31 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Thu, 5 Oct 2023 12:53:27 -0400 Subject: [PATCH 3/5] fixed linting --- src/radical/utils/misc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/radical/utils/misc.py b/src/radical/utils/misc.py index 1ac5077d9..23b88034f 100644 --- a/src/radical/utils/misc.py +++ b/src/radical/utils/misc.py @@ -10,8 +10,6 @@ from typing import List, Tuple, Union, Any -from urllib.parse import unquote_plus - from . import url as ruu from .ru_regex import ReString From c00e13ea78a8b3144f22a1a0d4ce7c91f6b85810 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Thu, 5 Oct 2023 12:54:09 -0400 Subject: [PATCH 4/5] removed obsolete mongo-related module --- bin/radical-utils-mongodb.py | 221 ----------------------------------- setup.py | 1 - 2 files changed, 222 deletions(-) delete mode 100755 bin/radical-utils-mongodb.py diff --git a/bin/radical-utils-mongodb.py b/bin/radical-utils-mongodb.py deleted file mode 100755 index eb18921c8..000000000 --- a/bin/radical-utils-mongodb.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 - - -import os -import sys -import pprint -import radical.utils as ru - - -_DEFAULT_MODE = 'list' -_DEFAULT_DBURL = 'mongodb://localhost:27017/' -_DEFAULT_DBURL = 'mongodb://ec2-184-72-89-141.compute-1.amazonaws.com:27017/' - -if 'RADICAL_PILOT_DBURL' in os.environ : - _DEFAULT_DBURL = os.environ['RADICAL_PILOT_DBURL'] - - -# ------------------------------------------------------------------------------ -# -def usage (msg=None, noexit=False) : - - if msg : - print("\n\t%s\n" % msg) - - print(""" - - usage : %s -m mode [-d url] - example : %s mongodb://localhost/synapse_profiles/profiles/ - - The URL is interpreted as: - [schema]://[host]:[port]/[database]/[collection]/[document_id] - - modes are: - - help: show this message - tree: show a tree of the hierarchy, but only document IDs, no content - dump: show a tree of the hierarchy, including document contents - list: list entries in the subtree, but do not traverse - remove: remove the specified subtree - - The default command is 'tree'. - The default MongoDB is """ + "'%s'\n\n" % _DEFAULT_DBURL) - - if msg : - sys.exit (1) - - if not noexit : - sys.exit (0) - - -# ------------------------------------------------------------------------------ -# -def dump (url, mode) : - """ - Connect to mongodb at the given location, and traverse the data bases - """ - - mongo, _, dbname, cname, pname = ru.mongodb_connect (url, _DEFAULT_DBURL) - - print(dbname) - - if dbname : dbnames = [dbname] - else : dbnames = mongo.database_names() - - for name in dbnames : - - if mode == 'list' and not dbname : - print(" +-- db %s" % name) - - elif mode == 'remove' : - - if (not dbname) or (name == dbname) : - try : - mongo.drop_database (name) - print(" removed database %s" % name) - except : - pass # ignore system tables - - else : - handle_db (mongo, mode, name, cname, pname) - - mongo.disconnect () - - -# ------------------------------------------------------------------------------ -def handle_db (mongo, mode, dbname, cname, pname) : - """ - For the given db, traverse collections - """ - - database = mongo[dbname] - print(" +-- db %s" % dbname) - - - if cname : cnames = [cname] - else : cnames = database.collection_names () - - for name in cnames : - - if mode == 'list' and not cname : - print(" | +-- coll %s" % name) - - elif mode == 'remove' and not pname : - try : - database.drop_collection (name) - print(" removed collection %s" % name) - except : - pass # ignore errors - - else : - handle_coll (database, mode, name, pname) - - -# ------------------------------------------------------------------------------ -def handle_coll (database, mode, cname, pname) : - """ - For a given collection, traverse all documents - """ - - if 'indexes' in cname : - return - - collection = database[cname] - print(" | +-- coll %s" % cname) - - docs = collection.find () - - for doc in docs : - - name = doc['_id'] - - if mode == 'list' and not pname : - print(" | | +-- doc %s" % name) - - elif mode == 'remove' : - if (not pname) or (str(name) == str(pname)) : - try : - collection.remove (name) - print(" removed document %s" % name) - except Exception: - pass # ignore errors - - else : - if (not pname) or (str(name) == str(pname)) : - handle_doc (mode, doc) - - -# ------------------------------------------------------------------------------ -def handle_doc (mode, doc) : - """ - And, surprise, for a given document, show it according to 'mode' - """ - - name = doc['_id'] - - if mode == 'list' : - - for key in doc : - print(" | | | +-- %s" % (key)) - - elif mode == 'tree' : - print(" | | +-- doc %s" % (name)) - for key in doc : - print(" | | | +-- %s" % (key)) - - elif mode == 'dump' : - print(" | | +-- doc %s" % (name)) - for key in doc : - txt_in = pprint.pformat (doc[key]) - txt_out = "" - lnum = 1 - for line in txt_in.split ('\n') : - if lnum != 1 : - txt_out += ' | | | | ' - txt_out += line - txt_out += '\n' - lnum += 1 - - # remove last \n - print(" | | | +-- %-10s : %s" % (key, txt_out[:-1])) - - -# ------------------------------------------------------------------------------ -# -if __name__ == '__main__' : - - import optparse - parser = optparse.OptionParser (add_help_option=False) - - parser.add_option('-m', '--mode', dest='mode') - parser.add_option('-d', '--dburl', dest='dburl') - parser.add_option('-h', '--help', dest='help', action="store_true") - - options, args = parser.parse_args () - - if args : usage ("Too many arguments (%s)" % args) - if options.help : usage () - - modes = options.mode - dburl = options.dburl - - if not modes : modes = _DEFAULT_MODE - if not dburl : dburl = _DEFAULT_DBURL - - print("modes : %s" % modes) - print("db url : %s" % dburl) - - for m in modes.split (',') : - - if m not in ['list', 'dump', 'tree', 'remove', 'help'] : - usage ("Unsupported mode '%s'" % m) - - elif m == 'tree' : dump (dburl, m) - elif m == 'dump' : dump (dburl, m) - elif m == 'list' : dump (dburl, m) - elif m == 'remove' : dump (dburl, m) - elif m == 'help' : usage (noexit=True) - else : usage ("unknown mode '%s'" % m) - - -# ------------------------------------------------------------------------------ diff --git a/setup.py b/setup.py index e62e3e3e1..0ba0800c3 100755 --- a/setup.py +++ b/setup.py @@ -235,7 +235,6 @@ def run(self): 'packages' : find_namespace_packages('src', include=['radical.*']), 'package_dir' : {'': 'src'}, 'scripts' : ['bin/radical-utils-fix-headers.pl', - 'bin/radical-utils-mongodb.py', 'bin/radical-utils-version', 'bin/radical-utils-pwatch', 'bin/radical-utils-pylint.sh', From a793d5417070b34e6b2952d13c02b8d32378d5bb Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Thu, 5 Oct 2023 20:27:28 -0400 Subject: [PATCH 5/5] added ifaddress to prioritize net interface for Frontier --- src/radical/utils/host.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/radical/utils/host.py b/src/radical/utils/host.py index 0b4c7377f..26f3f29dd 100644 --- a/src/radical/utils/host.py +++ b/src/radical/utils/host.py @@ -55,6 +55,7 @@ def get_hostip(req=None, log=None): req = as_list(req) white_list = [ + 'hsn0', # Frontier (HPE Cray EX) 'ipogif0', # Cray's 'br0', # SuperMIC 'eth0', # desktops etc.