From d0d061bf7e9dcb857231dde40b0008c97e9de638 Mon Sep 17 00:00:00 2001 From: Adam Souzis Date: Wed, 13 Nov 2024 07:33:33 -0800 Subject: [PATCH] loader: avoid sys.modules --- tests/examples/dsl_configurator.py | 2 +- tosca-package/tosca/_tosca.py | 7 ++++--- tosca-package/tosca/loader.py | 26 ++++++++++++++------------ tosca-package/tosca/python2yaml.py | 4 ++-- unfurl/planrequests.py | 6 ++++-- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tests/examples/dsl_configurator.py b/tests/examples/dsl_configurator.py index 84dc38a0..95fe0af3 100644 --- a/tests/examples/dsl_configurator.py +++ b/tests/examples/dsl_configurator.py @@ -59,7 +59,7 @@ class MyDataType(tosca.OpenDataEntity): assert not data_list_type_info.instance_check(1), "data_list_type_info.instance_check(1)" assert data_list_type_info.instance_check([]), "data_list_type_info.instance_check([])" assert not data_list_type_info.instance_check([1]), "not data_list_type_info.instance_check([1])" -assert data_list_type_info.instance_check([MyDataType()]) +assert data_list_type_info.instance_check([MyDataType()]), "data_list_type_info.instance_check([MyDataType()])" generic = unfurl.nodes.Generic("generic") assert generic._name == "generic" diff --git a/tosca-package/tosca/_tosca.py b/tosca-package/tosca/_tosca.py index a9948217..cc3b837a 100644 --- a/tosca-package/tosca/_tosca.py +++ b/tosca-package/tosca/_tosca.py @@ -2248,10 +2248,11 @@ def _lookup_class(cls, qname: str): globals = cls._globals else: globals = {} - if cls.__module__ in sys.modules and cls.__module__ != "builtins": - mod_globals = sys.modules[cls.__module__].__dict__ - elif cls.__module__ in global_state.modules: + # global_state.modules get priority + if cls.__module__ in global_state.modules: mod_globals = global_state.modules[cls.__module__].__dict__ + elif cls.__module__ in sys.modules and cls.__module__ != "builtins": + mod_globals = sys.modules[cls.__module__].__dict__ else: mod_globals = {} locals = cls._namespace or {} diff --git a/tosca-package/tosca/loader.py b/tosca-package/tosca/loader.py index 794e009f..eafb3e1a 100644 --- a/tosca-package/tosca/loader.py +++ b/tosca-package/tosca/loader.py @@ -233,12 +233,8 @@ def __getattribute__(self, __name: str) -> Any: def load_private_module(base_dir: str, modules: Dict[str, ModuleType], name: str): parent, sep, last = name.rpartition(".") - if parent: - if parent not in modules: - if parent in sys.modules: - modules[parent] = sys.modules[parent] - else: - load_private_module(base_dir, modules, parent) + if parent and parent not in modules: + load_private_module(base_dir, modules, parent) if name in modules: # cf. "Crazy side-effects!" in _bootstrap.py (e.g. parent could have imported child) return modules[name] @@ -601,8 +597,8 @@ def install(import_resolver_: Optional[ImportResolver], base_dir=None) -> str: old_basedir = service_template_basedir if base_dir: service_template_basedir = base_dir - if base_dir != old_basedir: - _clear_special_modules() # these are bad + # if base_dir != old_basedir: + # _clear_special_modules() # these are bad else: service_template_basedir = os.getcwd() global installed @@ -746,14 +742,20 @@ def restricted_exec( if result.errors: raise SyntaxError("\n".join(result.errors)) temp_module = None - if full_name not in sys.modules: - # dataclass._process_class() might assume the current module is in sys.modules - # so to make it happy add a dummy one if its missing + if full_name not in modules: temp_module = ModuleType(full_name) temp_module.__dict__.update(namespace) if full_name == "service_template": temp_module.__path__ = [service_template_basedir] + modules[full_name] = temp_module + else: + temp_module = modules[full_name] + remove_temp_module = False + if full_name not in sys.modules: + # dataclass._process_class() might assume the current module is in sys.modules + # so to make it happy add a dummy one if its missing sys.modules[full_name] = temp_module + remove_temp_module = True previous_safe_mode = global_state.safe_mode previous_mode = global_state.mode try: @@ -768,6 +770,6 @@ def restricted_exec( finally: global_state.safe_mode = previous_safe_mode global_state.mode = previous_mode - if temp_module: + if remove_temp_module: del sys.modules[full_name] return result diff --git a/tosca-package/tosca/python2yaml.py b/tosca-package/tosca/python2yaml.py index f90a3209..01b6553e 100644 --- a/tosca-package/tosca/python2yaml.py +++ b/tosca-package/tosca/python2yaml.py @@ -70,7 +70,7 @@ def find_yaml_import( self, module_name: str ) -> Tuple[Optional[ModuleType], Optional[Path]]: "Find the given Python module and corresponding yaml file path" - module = self.modules.get(module_name) or sys.modules.get(module_name) + module = self.modules.get(module_name) if not module: return None, None path = module.__file__ @@ -93,7 +93,7 @@ def _set_repository_for_module( else: root_package = parts[0] repo_name = parts[0] - root_module = self.modules.get(root_package, sys.modules.get(root_package)) + root_module = self.modules.get(root_package) if not root_module: return "", None root_path = root_module.__file__ diff --git a/unfurl/planrequests.py b/unfurl/planrequests.py index b7dc9310..7d730a32 100644 --- a/unfurl/planrequests.py +++ b/unfurl/planrequests.py @@ -131,12 +131,14 @@ def create(self) -> "Configurator": from .dsl import DslMethodConfigurator module_name, qualname, action = className.split(":") - logger.warning("loading dsl configurator %s %s", module_name, tosca.safe_mode()) if tosca.loader.import_resolver: assert not tosca.loader.import_resolver.get_safe_mode(), module_name else: assert not tosca.safe_mode(), module_name - module = importlib.import_module(module_name) + if module_name not in sys.modules: + module = importlib.import_module(module_name) + else: + module = sys.modules[module_name] cls_name, sep, func_name = qualname.rpartition(".") cls = getattr(module, cls_name) return DslMethodConfigurator(cls, getattr(cls, func_name), action)