From b9e2735108b2959d6fdce2fb1ffd49e5cb809604 Mon Sep 17 00:00:00 2001 From: Adam Souzis Date: Wed, 13 Nov 2024 11:19:30 -0800 Subject: [PATCH] dsl: loader: preload allowed modules in safe mode --- smoketest.sh | 2 +- tests/test_constraints.py | 2 - tests/test_docs.py | 3 +- tests/test_dsl.py | 6 ++- tosca-package/tosca/loader.py | 75 +++++++++++++++++++----------- tosca-package/tosca/python2yaml.py | 4 +- unfurl/dsl.py | 2 +- 7 files changed, 58 insertions(+), 36 deletions(-) diff --git a/smoketest.sh b/smoketest.sh index 9833512c..69e7ff1a 100755 --- a/smoketest.sh +++ b/smoketest.sh @@ -1,3 +1,3 @@ set -e .tox/${1:-py310}/bin/mypy unfurl --install-types --non-interactive -UNFURL_TEST_SKIP=docker+slow+k8s+helm+$UNFURL_TEST_SKIP tox --skip-pkg-install -e ${1:-py310} -- -v --no-cov -n auto --dist loadfile $2 $3 $4 $5 $6 $7 +UNFURL_TEST_SKIP_BUILD_RUST=1 UNFURL_TEST_SKIP=docker+slow+k8s+helm+$UNFURL_TEST_SKIP tox --skip-pkg-install -e ${1:-py310} -- -v --no-cov -n auto --dist loadfile $2 $3 $4 $5 $6 $7 diff --git a/tests/test_constraints.py b/tests/test_constraints.py index cd5d20c0..3739c439 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -328,8 +328,6 @@ def test_computed_properties(): "skipped": 0, "changed": 1, } - # XXX we need to delete this module because mytypes gets re-evaluated, breaking class identity - # is this a scenario we need to worry about outside unit tests? result, job, summary = run_job_cmd( cli_runner, ["-vvv", "undeploy"], print_result=True ) diff --git a/tests/test_docs.py b/tests/test_docs.py index e4a2fab4..eb3484cd 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -107,4 +107,5 @@ def test_quickstart(): with open("service_template.py", "a") as f: f.write(deployment_blueprint) run_cmd(runner, "plan production") - run_cmd(runner, "deploy --dryrun --approve development") + if "slow" not in os.getenv("UNFURL_TEST_SKIP", ""): + run_cmd(runner, "deploy --dryrun --approve development") diff --git a/tests/test_dsl.py b/tests/test_dsl.py index 0c00b38d..64b25d56 100644 --- a/tests/test_dsl.py +++ b/tests/test_dsl.py @@ -1212,14 +1212,16 @@ def test_sandbox(capsys): """foo = dict(); foo[1] = 2; bar = list(); bar.append(1); baz = tuple()""", """import math; math.floor(1.0)""", """from unfurl.configurators.templates.dns import unfurl_relationships_DNSRecords""", - # """from unfurl.tosca_plugins import k8s; k8s.kube_artifacts""", + """from unfurl import artifacts""", + """import unfurl; unfurl.artifacts""", + """from unfurl.tosca_plugins import k8s; k8s.kube_artifacts""", """import tosca node = tosca.nodes.Root() node._name = "test" """, ] for src in allowed: - print("allowed", src) + # print("allowed?", src) assert _to_yaml(src, True) diff --git a/tosca-package/tosca/loader.py b/tosca-package/tosca/loader.py index 51379f67..425bbdca 100644 --- a/tosca-package/tosca/loader.py +++ b/tosca-package/tosca/loader.py @@ -25,6 +25,33 @@ logger = logging.getLogger("tosca") +# python standard library modules matches those added to utility_builtins +ALLOWED_MODULES = ( + "typing", + "typing_extensions", + "tosca", + "random", + "math", + "string", + "DateTime", + "unfurl", + "urllib.parse", +) + +# XXX have the unfurl package set these: +ALLOWED_PRIVATE_PACKAGES = [ + "unfurl.tosca_plugins", + "unfurl.configurators.templates", +] + + +def get_allowed_modules() -> Dict[str, "ImmutableModule"]: + allowed = {} + for name in ALLOWED_MODULES: + if name in sys.modules: + allowed[name] = ImmutableModule(name, sys.modules[name]) + return allowed + def get_module_path(module) -> str: if getattr(module, "__spec__", None) and module.__spec__.origin: @@ -68,7 +95,10 @@ def find_spec(cls, fullname: str, path=None, target=None, modules=None): if os.path.exists(init_path): loader = ToscaYamlLoader(fullname, init_path, modules) return spec_from_file_location( - fullname, init_path, loader=loader, submodule_search_locations=[repo_path] + fullname, + init_path, + loader=loader, + submodule_search_locations=[repo_path], ) else: return ModuleSpec( @@ -88,7 +118,10 @@ def find_spec(cls, fullname: str, path=None, target=None, modules=None): if os.path.exists(init_path): loader = ToscaYamlLoader(fullname, init_path, modules) return spec_from_file_location( - fullname, init_path, loader=loader, submodule_search_locations=[origin_path] + fullname, + init_path, + loader=loader, + submodule_search_locations=[origin_path], ) else: return ModuleSpec( @@ -162,7 +195,11 @@ def exec_module(self, module): python_filepath = str(path) with open(path) as f: src = f.read() - safe_mode = import_resolver.get_safe_mode() if import_resolver else global_state.safe_mode + safe_mode = ( + import_resolver.get_safe_mode() + if import_resolver + else global_state.safe_mode + ) module.__dict__["__file__"] = python_filepath for i in range(self.full_name.count(".")): path = path.parent @@ -183,13 +220,14 @@ class ImmutableModule(ModuleType): def __init__(self, name, module): super().__init__(name) - object.__getattribute__(self, "__dict__")["__protected_module__"] = module + object.__getattribute__(self, "__dict__")["__protected_module__"] = module def __getattribute__(self, __name: str) -> Any: module = super().__getattribute__("__protected_module__") if ( __name not in ImmutableModule.__always_safe__ - and __name not in getattr(module, "__safe__", getattr(module, "__all__", ())) + and __name + not in getattr(module, "__safe__", getattr(module, "__all__", ())) and module.__name__ != "math" ): # special case "math", it doesn't have __all__ @@ -342,7 +380,7 @@ def __safe_import__( for from_name in fromlist: if not hasattr(module, from_name): # e.g. from tosca_repositories.repo import module - load_private_module(base_dir, modules, name+"."+from_name) + load_private_module(base_dir, modules, name + "." + from_name) if name in ALLOWED_MODULES: _check_fromlist(module, fromlist) elif "*" in fromlist: @@ -392,6 +430,7 @@ def __safe_import__( ) return module + def _validate_star(module): # ok if there's no __all__ because default * will exclude names that start with "_" safe = getattr(module, "__safe__", None) @@ -426,26 +465,6 @@ def get_descriptions(body): return doc_strings -# python standard library modules matches those added to utility_builtins -ALLOWED_MODULES = ( - "typing", - "typing_extensions", - "tosca", - "random", - "math", - "string", - "DateTime", - "unfurl", - "urllib.parse", -) - -# XXX have the unfurl package set these: -ALLOWED_PRIVATE_PACKAGES = [ - "unfurl.tosca_plugins", - "unfurl.configurators.templates", -] - - def default_guarded_getattr(ob, name): return getattr(ob, name) @@ -593,11 +612,13 @@ def check_import_names(self, node): import_resolver: Optional[ImportResolver] = None service_template_basedir = "" + def _clear_special_modules(): for name in list(sys.modules): if name.startswith("service_template") or name.startswith("tosca_repositories"): del sys.modules[name] + def install(import_resolver_: Optional[ImportResolver], base_dir=None) -> str: # insert the path hook ahead of other path hooks global import_resolver @@ -735,7 +756,7 @@ def restricted_exec( # try to guess file path from module name if full_name.startswith("service_template."): # assume service_template is just our dummy package - module_name = full_name[len("service_template."):] + module_name = full_name[len("service_template.") :] else: module_name = full_name namespace["__file__"] = ( diff --git a/tosca-package/tosca/python2yaml.py b/tosca-package/tosca/python2yaml.py index 079fe8fa..cfd3181c 100644 --- a/tosca-package/tosca/python2yaml.py +++ b/tosca-package/tosca/python2yaml.py @@ -32,7 +32,7 @@ EvalData, Namespace, ) -from .loader import restricted_exec, get_module_path +from .loader import restricted_exec, get_module_path, get_allowed_modules class PythonToYaml: @@ -441,7 +441,7 @@ def python_src_to_yaml_obj( import_resolver=None, ) -> dict: if modules is None: - modules = {} + modules = get_allowed_modules() global_state.modules = modules if namespace is None: namespace = {} diff --git a/unfurl/dsl.py b/unfurl/dsl.py index bfbb82fe..ffd6b2bf 100644 --- a/unfurl/dsl.py +++ b/unfurl/dsl.py @@ -123,7 +123,7 @@ def convert_to_yaml( if safe_mode_override: safe_mode = safe_mode_override != "never" if import_resolver.manifest.modules is None: - import_resolver.manifest.modules = {} + import_resolver.manifest.modules = tosca.loader.get_allowed_modules() yaml_src = python_src_to_yaml_obj( contents, namespace,