dsl: loader: preload allowed modules in safe mode

smoketest.sh

@@ -1,3 +1,3 @@
 set -e
 .tox/${1:-py310}/bin/mypy unfurl --install-types --non-interactive
-UNFURL_TEST_SKIP=docker+slow+k8s+helm+$UNFURL_TEST_SKIP tox --skip-pkg-install -e ${1:-py310} -- -v --no-cov -n auto --dist loadfile $2 $3 $4 $5 $6 $7
+UNFURL_TEST_SKIP_BUILD_RUST=1 UNFURL_TEST_SKIP=docker+slow+k8s+helm+$UNFURL_TEST_SKIP tox --skip-pkg-install -e ${1:-py310} -- -v --no-cov -n auto --dist loadfile $2 $3 $4 $5 $6 $7

tests/test_constraints.py

@@ -328,8 +328,6 @@ def test_computed_properties():
             "skipped": 0,
             "changed": 1,
         }
-        # XXX we need to delete this module because mytypes gets re-evaluated, breaking class identity
-        # is this a scenario we need to worry about outside unit tests?
         result, job, summary = run_job_cmd(
             cli_runner, ["-vvv", "undeploy"], print_result=True
         )

tests/test_docs.py

@@ -107,4 +107,5 @@ def test_quickstart():
         with open("service_template.py", "a") as f:
             f.write(deployment_blueprint)
         run_cmd(runner, "plan production")
-        run_cmd(runner, "deploy --dryrun --approve development")
+        if "slow" not in os.getenv("UNFURL_TEST_SKIP", ""):
+            run_cmd(runner, "deploy --dryrun --approve development")

tests/test_dsl.py

@@ -1212,14 +1212,16 @@ def test_sandbox(capsys):
         """foo = dict(); foo[1] = 2; bar = list(); bar.append(1); baz = tuple()""",
         """import math; math.floor(1.0)""",
         """from unfurl.configurators.templates.dns import unfurl_relationships_DNSRecords""",
-         # """from unfurl.tosca_plugins import k8s; k8s.kube_artifacts""",
+        """from unfurl import artifacts""",
+        """import unfurl; unfurl.artifacts""",
+        """from unfurl.tosca_plugins import k8s; k8s.kube_artifacts""",
         """import tosca
 node = tosca.nodes.Root()
 node._name = "test"
         """,
     ]
     for src in allowed:
-        print("allowed", src)
+        # print("allowed?", src)
         assert _to_yaml(src, True)
 
 

tosca-package/tosca/loader.py

@@ -25,6 +25,33 @@
 
 logger = logging.getLogger("tosca")
 
+# python standard library modules matches those added to utility_builtins
+ALLOWED_MODULES = (
+    "typing",
+    "typing_extensions",
+    "tosca",
+    "random",
+    "math",
+    "string",
+    "DateTime",
+    "unfurl",
+    "urllib.parse",
+)
+
+# XXX have the unfurl package set these:
+ALLOWED_PRIVATE_PACKAGES = [
+    "unfurl.tosca_plugins",
+    "unfurl.configurators.templates",
+]
+
+
+def get_allowed_modules() -> Dict[str, "ImmutableModule"]:
+    allowed = {}
+    for name in ALLOWED_MODULES:
+        if name in sys.modules:
+            allowed[name] = ImmutableModule(name, sys.modules[name])
+    return allowed
+
 
 def get_module_path(module) -> str:
     if getattr(module, "__spec__", None) and module.__spec__.origin:
@@ -68,7 +95,10 @@ def find_spec(cls, fullname: str, path=None, target=None, modules=None):
                         if os.path.exists(init_path):
                             loader = ToscaYamlLoader(fullname, init_path, modules)
                             return spec_from_file_location(
-                                fullname, init_path, loader=loader, submodule_search_locations=[repo_path]
+                                fullname,
+                                init_path,
+                                loader=loader,
+                                submodule_search_locations=[repo_path],
                             )
                         else:
                             return ModuleSpec(
@@ -88,7 +118,10 @@ def find_spec(cls, fullname: str, path=None, target=None, modules=None):
                         if os.path.exists(init_path):
                             loader = ToscaYamlLoader(fullname, init_path, modules)
                             return spec_from_file_location(
-                                fullname, init_path, loader=loader, submodule_search_locations=[origin_path]
+                                fullname,
+                                init_path,
+                                loader=loader,
+                                submodule_search_locations=[origin_path],
                             )
                         else:
                             return ModuleSpec(
@@ -162,7 +195,11 @@ def exec_module(self, module):
             python_filepath = str(path)
             with open(path) as f:
                 src = f.read()
-        safe_mode = import_resolver.get_safe_mode() if import_resolver else global_state.safe_mode
+        safe_mode = (
+            import_resolver.get_safe_mode()
+            if import_resolver
+            else global_state.safe_mode
+        )
         module.__dict__["__file__"] = python_filepath
         for i in range(self.full_name.count(".")):
             path = path.parent
@@ -183,13 +220,14 @@ class ImmutableModule(ModuleType):
 
     def __init__(self, name, module):
         super().__init__(name)
-        object.__getattribute__(self, "__dict__")["__protected_module__"] =  module
+        object.__getattribute__(self, "__dict__")["__protected_module__"] = module
 
     def __getattribute__(self, __name: str) -> Any:
         module = super().__getattribute__("__protected_module__")
         if (
             __name not in ImmutableModule.__always_safe__
-            and __name not in getattr(module, "__safe__", getattr(module, "__all__", ()))
+            and __name
+            not in getattr(module, "__safe__", getattr(module, "__all__", ()))
             and module.__name__ != "math"
         ):
             # special case "math", it doesn't have __all__
@@ -342,7 +380,7 @@ def __safe_import__(
                 for from_name in fromlist:
                     if not hasattr(module, from_name):
                         # e.g. from tosca_repositories.repo import module
-                        load_private_module(base_dir, modules, name+"."+from_name)
+                        load_private_module(base_dir, modules, name + "." + from_name)
             if name in ALLOWED_MODULES:
                 _check_fromlist(module, fromlist)
             elif "*" in fromlist:
@@ -392,6 +430,7 @@ def __safe_import__(
         )
     return module
 
+
 def _validate_star(module):
     # ok if there's no __all__ because default * will exclude names that start with "_"
     safe = getattr(module, "__safe__", None)
@@ -426,26 +465,6 @@ def get_descriptions(body):
     return doc_strings
 
 
-# python standard library modules matches those added to utility_builtins
-ALLOWED_MODULES = (
-    "typing",
-    "typing_extensions",
-    "tosca",
-    "random",
-    "math",
-    "string",
-    "DateTime",
-    "unfurl",
-    "urllib.parse",
-)
-
-# XXX have the unfurl package set these:
-ALLOWED_PRIVATE_PACKAGES = [
-    "unfurl.tosca_plugins",
-    "unfurl.configurators.templates",
-]
-
-
 def default_guarded_getattr(ob, name):
     return getattr(ob, name)
 
@@ -593,11 +612,13 @@ def check_import_names(self, node):
 import_resolver: Optional[ImportResolver] = None
 service_template_basedir = ""
 
+
 def _clear_special_modules():
     for name in list(sys.modules):
         if name.startswith("service_template") or name.startswith("tosca_repositories"):
             del sys.modules[name]
 
+
 def install(import_resolver_: Optional[ImportResolver], base_dir=None) -> str:
     # insert the path hook ahead of other path hooks
     global import_resolver
@@ -735,7 +756,7 @@ def restricted_exec(
         # try to guess file path from module name
         if full_name.startswith("service_template."):
             # assume service_template is just our dummy package
-            module_name = full_name[len("service_template."):]
+            module_name = full_name[len("service_template.") :]
         else:
             module_name = full_name
         namespace["__file__"] = (

tosca-package/tosca/python2yaml.py

@@ -32,7 +32,7 @@
     EvalData,
     Namespace,
 )
-from .loader import restricted_exec, get_module_path
+from .loader import restricted_exec, get_module_path, get_allowed_modules
 
 
 class PythonToYaml:
@@ -441,7 +441,7 @@ def python_src_to_yaml_obj(
     import_resolver=None,
 ) -> dict:
     if modules is None:
-        modules = {}
+        modules = get_allowed_modules()
     global_state.modules = modules
     if namespace is None:
         namespace = {}

unfurl/dsl.py

@@ -123,7 +123,7 @@ def convert_to_yaml(
     if safe_mode_override:
         safe_mode = safe_mode_override != "never"
     if import_resolver.manifest.modules is None:
-        import_resolver.manifest.modules = {}
+        import_resolver.manifest.modules = tosca.loader.get_allowed_modules()
     yaml_src = python_src_to_yaml_obj(
         contents,
         namespace,