Merge pull request #252 from caracal-pipeline/issue-251

Issue 251

scabha/cargo.py

@@ -411,7 +411,7 @@ def _resolve_implicit_parameters(self, params, subst: Optional[SubstitutionNS]=N
                     current[name] = schema.implicit
 
 
-    def prevalidate(self, params: Optional[Dict[str, Any]], subst: Optional[SubstitutionNS]=None, root=False):
+    def prevalidate(self, params: Optional[Dict[str, Any]], subst: Optional[SubstitutionNS]=None, backend=None, root=False):
         """Does pre-validation.
         No parameter substitution is done, but will check for missing params and such.
         A dynamic schema, if defined, is applied at this point."""

stimela/backends/__init__.py

@@ -47,7 +47,7 @@ class StimelaBackendOptions(object):
     # overrides registries -- useful if you have a pull-through cache set up
     override_registries: Dict[str, str] = EmptyDictDefault()
 
-    select: Any = ""   # should be Union[str, List[str]], but OmegaConf doesn't support it, so handle in __post_init__ for now
+    select: Any = "singularity,native"   # should be Union[str, List[str]], but OmegaConf doesn't support it, so handle in __post_init__ for now
 
     singularity: Optional[SingularityBackendOptions] = None
     kube: Optional[KubeBackendOptions] = None

stimela/backends/runner.py

@@ -48,7 +48,7 @@ def validate_backend_settings(backend_opts: Dict[str, Any], log: logging.Logger)
         backend_opts = OmegaConf.to_object(backend_opts)
 
     backend_name = backend = None
-    selected = backend_opts.select or ['native']
+    selected = backend_opts.select or ['singularity', 'native']
     # select containerization engine, if any
     for name in selected: 
         # check that backend has not been disabled

stimela/config.py

@@ -1,4 +1,4 @@
-import glob
+import importlib
 import os, os.path, time, sys, platform, traceback
 from typing import Any, List, Dict, Optional, Union
 from enum import Enum
@@ -12,8 +12,6 @@
 from stimela.exceptions import *
 from stimela import log_exception
 
-CONFIG_FILE = os.path.expanduser("~/.config/stimela.conf")
-
 from scabha import configuratt
 from scabha.cargo import ListOrString, EmptyDictDefault, EmptyListDefault
 from stimela.backends import StimelaBackendOptions
@@ -64,12 +62,19 @@ def DefaultDirs():
 _CONFIG_BASENAME = "stimela.conf"
 _STIMELA_CONFDIR = os.path.os.path.expanduser("~/.stimela")
 
+# check for cultcargo module
+try:
+    ccmod = importlib.import_module("cultcargo")
+except ImportError:
+    ccmod = None
+
 # dict of config file locations to check, in order of preference
 CONFIG_LOCATIONS = OrderedDict(
     package = os.path.join(os.path.dirname(__file__), _CONFIG_BASENAME),
     local   = _CONFIG_BASENAME,
     venv    = os.environ.get('VIRTUAL_ENV', None) and os.path.join(os.environ['VIRTUAL_ENV'], _CONFIG_BASENAME),
     stimela = os.path.isdir(_STIMELA_CONFDIR) and os.path.join(_STIMELA_CONFDIR, _CONFIG_BASENAME),
+    cultcargo = ccmod and os.path.join(os.path.dirname(ccmod.__file__), _CONFIG_BASENAME),
     user    = os.path.join(os.path.expanduser("~/.config"), _CONFIG_BASENAME),
 )
 

stimela/kitchen/recipe.py

@@ -739,11 +739,13 @@ def _preprocess_parameters(self, params: Dict[str, Any]):
         return own_params, unset_params
 
 
-    def prevalidate(self, params: Dict[str, Any], subst: Optional[SubstitutionNS]=None, root=False):
-        self.finalize()
+    def prevalidate(self, params: Dict[str, Any], subst: Optional[SubstitutionNS]=None, backend=None, root=False):
+        self.finalize(backend=backend)
         self.log.debug("prevalidating recipe")
         errors = []
 
+        backend = OmegaConf.merge(backend or self.config.opts.backend, self.backend or {})
+
         # split parameters into our own, and per-step, and UNSET directives
         params,  unset_params = self._preprocess_parameters(params)
 
@@ -785,7 +787,7 @@ def prevalidate(self, params: Dict[str, Any], subst: Optional[SubstitutionNS]=No
         # we call this twice, potentially, so define as a function
         def prevalidate_self(params):
             try:
-                params1 = Cargo.prevalidate(self, params, subst=subst_outer)
+                params1 = Cargo.prevalidate(self, params, subst=subst_outer, backend=backend)
                 # mark params that have become unset 
                 unset_params.update(set(params) - set(params1))
                 params = params1

stimela/kitchen/step.py

@@ -266,8 +266,8 @@ def finalize(self, config=None, log=None, fqname=None, backend=None, nesting=0):
             runner.validate_backend_settings(backend_opts, log=log)
 
 
-    def prevalidate(self, subst: Optional[SubstitutionNS]=None, root=False):
-        self.finalize()
+    def prevalidate(self, subst: Optional[SubstitutionNS]=None, root=False, backend=None):
+        self.finalize(backend=backend)
         self.cargo.apply_dynamic_schemas(self.params, subst)
         # validate cab or recipe
         params = self.validated_params = self.cargo.prevalidate(self.params, subst, root=root)
@@ -320,11 +320,11 @@ def assign_value(self, key: str, value: Any, override: bool = False):
                 raise AssignmentError(f"{self.name}: invalid assignment {key}={value}", exc)
 
 
-    def build(self, backend={}, rebuild=False, build_skips=False, log: Optional[logging.Logger] = None):
+    def build(self, backend=None, rebuild=False, build_skips=False, log: Optional[logging.Logger] = None):
         # skipping step? ignore the build
         if self.skip is True and not build_skips:
             return
-        backend = OmegaConf.merge(backend, self.cargo.backend or {}, self.backend or {})
+        backend = OmegaConf.merge(backend or {}, self.cargo.backend or {}, self.backend or {})
         log = log or self.log
         # recurse into sub-recipe 
         from .recipe import Recipe
@@ -334,7 +334,7 @@ def build(self, backend={}, rebuild=False, build_skips=False, log: Optional[logg
         else:
             # validate backend settings and call the build function
             try:
-                backend_opts = OmegaConf.merge(backend, self.cargo.backend or {}, self.backend or {})
+                backend_opts = OmegaConf.merge(self.config.opts.backend, backend)
                 if getattr(backend_opts, 'verbose', 0):
                     opts_yaml = OmegaConf.to_yaml(backend_opts)
                     log_rich_payload(self.log, "effective backend settings are", opts_yaml, syntax="yaml") 
@@ -348,7 +348,7 @@ def build(self, backend={}, rebuild=False, build_skips=False, log: Optional[logg
                 return backend_runner.build(self.cargo, log=log, rebuild=rebuild)
 
 
-    def run(self, backend: Optional[Dict] = {}, subst: Optional[Dict[str, Any]] = None, 
+    def run(self, backend: Optional[Dict] = None, subst: Optional[Dict[str, Any]] = None, 
             is_outer_step: bool=False,
             parent_log: Optional[logging.Logger] = None) -> Dict[str, Any]:
         """executes the step
@@ -376,11 +376,11 @@ def run(self, backend: Optional[Dict] = {}, subst: Optional[Dict[str, Any]] = No
         if parent_log is None:
             parent_log = self.log
 
-        backend = OmegaConf.merge(backend, self.cargo.backend or {}, self.backend or {})
+        backend = OmegaConf.merge(backend or {}, self.cargo.backend or {}, self.backend or {})
 
         # validate backend settings
         try:
-            backend_opts = OmegaConf.merge(stimela.CONFIG.opts.backend, backend)
+            backend_opts = OmegaConf.merge(self.config.opts.backend, backend)
             backend_opts = evaluate_and_substitute_object(backend_opts, subst, recursion_level=-1, location=[self.fqname, "backend"])
             if not is_outer_step and backend_opts.verbose:
                 opts_yaml = OmegaConf.to_yaml(backend_opts)

stimela/stimela.conf

@@ -1,10 +1,13 @@
+# Defines default versions of images for python and casa-flavour cabs
+# Note that if cult-cargo is installed, its own version of stimela.conf will
+# be read afterwards, and so can overwite these settings.
 images:
     default-python: 
         registry: quay.io/stimela2
         name: python-astro
-        version: cc0.0.2
+        version: cc0.1.2
     default-casa: 
         registry: quay.io/stimela2
         name: casa
-        version: cc0.0.2
+        version: cc0.1.2
 

tests/scabha_tests/testconf.yaml

@@ -16,6 +16,13 @@ hierarchical:
 relative:
   _include: (.)test_include2.yaml
 
+structured:
+  _include:
+    (.):
+      test_include2.yaml
+    .:
+      test_include2.yaml
+
 flat:
   _use: hierarchical
   _flatten: true