From 6bad6fa43b4bb3a3ccabd064b1149dddbcbe0d34 Mon Sep 17 00:00:00 2001 From: Aniket Singh Rawat Date: Tue, 7 Nov 2023 23:01:32 +0530 Subject: [PATCH 1/2] Replaced imp module with importlib.util --- retriever/lib/repository.py | 10 ++++++---- retriever/lib/scripts.py | 22 ++++++++++++++-------- test/test_regression.py | 23 ++++++++++++----------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/retriever/lib/repository.py b/retriever/lib/repository.py index cdacc166d..38f1cbf25 100644 --- a/retriever/lib/repository.py +++ b/retriever/lib/repository.py @@ -1,7 +1,7 @@ """Checks the repository for updates.""" import os import requests -import imp +import importlib.util from tqdm import tqdm from pkg_resources import parse_version from retriever.lib.defaults import (REPOSITORY, RETRIEVER_REPOSITORY, SCRIPT_WRITE_PATH, @@ -59,9 +59,11 @@ def check_for_updates(repo=REPOSITORY): _download_from_repository("scripts/" + script_name, download_path, repo) need_to_download = False try: - file_object, pathname, desc = imp.find_module( - ''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH]) - new_module = imp.load_module(script_name, file_object, pathname, desc) + _script_name = ''.join(script_name.split('.')[:-1]) + script_path = f"{SCRIPT_WRITE_PATH}/{_script_name}.py" + spec = importlib.util.spec_from_file_location(_script_name, script_path) + new_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(new_module) m = str(new_module.SCRIPT.version) need_to_download = parse_version(str(script_version)) > parse_version(m) except: diff --git a/retriever/lib/scripts.py b/retriever/lib/scripts.py index 5367eb5ab..6a85bf823 100644 --- a/retriever/lib/scripts.py +++ b/retriever/lib/scripts.py @@ -1,5 +1,5 @@ import csv -import imp +import importlib.util import io import os import re @@ -29,7 +29,7 @@ def check_retriever_minimum_version(module): m = module.name if hasattr(module, "retriever_minimum_version"): - if not parse_version(VERSION) >= parse_version("{}".format(mod_ver)): + if mod_ver != '' and not parse_version(VERSION) >= parse_version("{}".format(mod_ver)): print("{} is supported by Retriever version " "{}".format(m, mod_ver)) print("Current version is {}".format(VERSION)) @@ -80,9 +80,11 @@ def reload_scripts(): script_name = ".".join(script.split(".")[:-1]) if script_name not in loaded_files: loaded_files.append(script_name) - file, pathname, desc = imp.find_module(script_name, [search_path]) + script_path = f"{search_path}/{script_name}.py" + spec = importlib.util.spec_from_file_location(script_name, script_path) try: - new_module = imp.load_module(script_name, file, pathname, desc) + new_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(new_module) if not hasattr(new_module, "SCRIPT"): continue if hasattr(new_module.SCRIPT, "retriever_minimum_version"): @@ -274,8 +276,10 @@ def get_script_upstream(dataset, repo=REPOSITORY): setattr(read_script, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name)) setattr(read_script, "_name", script) return read_script - file, pathname, desc = imp.find_module(script, [SCRIPT_WRITE_PATH]) - new_module = imp.load_module(script, file, pathname, desc) + script_path = f"{SCRIPT_WRITE_PATH}/{script_name}.py" + spec = importlib.util.spec_from_file_location(script_name, script_path) + new_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(new_module) setattr(new_module.SCRIPT, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name)) setattr(new_module.SCRIPT, "_name", script) return new_module.SCRIPT @@ -461,9 +465,11 @@ def read_json_version(json_file): def read_py_version(script_name, search_path): """Read the version of a script from a python file""" - file, pathname, desc = imp.find_module(script_name, [search_path]) + script_path = f"{search_path}/{script_name}.py" + spec = importlib.util.spec_from_file_location(script_name, script_path) try: - new_module = imp.load_module(script_name, file, pathname, desc) + new_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(new_module) if hasattr(new_module.SCRIPT, "version"): return new_module.SCRIPT.version except: diff --git a/test/test_regression.py b/test/test_regression.py index 115697a30..75adc9046 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -import imp +import importlib import os import shlex import shutil @@ -8,7 +8,6 @@ import sys import time from distutils.dir_util import copy_tree -from imp import reload import pytest import retriever as rt @@ -35,7 +34,7 @@ mysqldb_host = "mysqldb_retriever" mysql_engine, postgres_engine, sqlite_engine, msaccess_engine, \ -csv_engine, download_engine, json_engine, xml_engine, _ = engine_list + csv_engine, download_engine, json_engine, xml_engine, _ = engine_list file_location = os.path.dirname(os.path.realpath(__file__)) retriever_root_dir = os.path.abspath(os.path.join(file_location, os.pardir)) working_script_dir = os.path.abspath(os.path.join(retriever_root_dir, "scripts")) @@ -58,11 +57,11 @@ ] spatial_db_md5 = [ - ("test-eco-level-four", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'd1c01d8046143e9700f5cf92cbd6be3d',[]), + ("test-eco-level-four", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'd1c01d8046143e9700f5cf92cbd6be3d', []), # ("test-raster-bio1clip", ["rid","filename"], '29f702992ae42cc221d6ca9149635024',[60, 50, 100, 0]), - ("test-raster-bio1", ["rid", "filename"], '27e0472ddc2da9fe807bfb48b786a251',[]), - ("test-raster-bio2", ["rid", "filename"], '2983a9f7e099355db2ce2fa312a94cc6',[]), - ("test-us-eco", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'eaab9fa30c745557ff6ba7c116910b45',[]), + ("test-raster-bio1", ["rid", "filename"], '27e0472ddc2da9fe807bfb48b786a251', []), + ("test-raster-bio2", ["rid", "filename"], '2983a9f7e099355db2ce2fa312a94cc6', []), + ("test-us-eco", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'eaab9fa30c745557ff6ba7c116910b45', []), # h5py has compatibility issues in linux-Travis # Tests pass locally # ("sample-hdf", ["*"], '31e61867e9990138788a946542c4b1bf') @@ -99,7 +98,7 @@ 'consumerstageid', 'resourcestageid', 'linknumber', 'linktype', 'linkevidence', 'linkevidencenotes', 'linkfrequency', 'linkn', 'dietfraction', 'consumptionrate', 'vectorfrom', 'preyfrom'] - ] + ] }]) ] @@ -143,9 +142,11 @@ def teardown_module(): def get_script_module(script_name): """Load a script module""" if script_name in python_files: - file, pathname, desc = imp.find_module(script_name, - [working_script_dir]) - return imp.load_module(script_name + '.py', file, pathname, desc) + script_path = f"{working_script_dir}/{script_name}.py" + spec = importlib.util.spec_from_file_location(script_name, script_path) + new_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(new_module) + return new_module return read_json(os.path.join(retriever_root_dir, 'scripts', script_name)) From 3b6bb448e5bdc04acfec12428682506741ea1667 Mon Sep 17 00:00:00 2001 From: Aniket Singh Rawat Date: Tue, 7 Nov 2023 23:05:15 +0530 Subject: [PATCH 2/2] removed version condition --- retriever/lib/scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retriever/lib/scripts.py b/retriever/lib/scripts.py index 6a85bf823..61283f7c9 100644 --- a/retriever/lib/scripts.py +++ b/retriever/lib/scripts.py @@ -29,7 +29,7 @@ def check_retriever_minimum_version(module): m = module.name if hasattr(module, "retriever_minimum_version"): - if mod_ver != '' and not parse_version(VERSION) >= parse_version("{}".format(mod_ver)): + if not parse_version(VERSION) >= parse_version("{}".format(mod_ver)): print("{} is supported by Retriever version " "{}".format(m, mod_ver)) print("Current version is {}".format(VERSION))