From 7e41f978e8f9fa804787a670b6aa32b58dc62d17 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 2 Mar 2024 17:40:14 +0200 Subject: [PATCH 1/2] added line to test --- tests/scabha_tests/testconf.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/scabha_tests/testconf.yaml b/tests/scabha_tests/testconf.yaml index b0d34280..6941ef3d 100644 --- a/tests/scabha_tests/testconf.yaml +++ b/tests/scabha_tests/testconf.yaml @@ -16,6 +16,13 @@ hierarchical: relative: _include: (.)test_include2.yaml +structured: + _include: + (.): + test_include2.yaml + .: + test_include2.yaml + flat: _use: hierarchical _flatten: true From 1d49023331c79eb2b6a367e0038051afee536dbf Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sun, 3 Mar 2024 16:55:32 +0200 Subject: [PATCH 2/2] fixes #251. Minor cleanups to backend passing. Add lookup of stimela.conf in cult-cargo, if installed --- scabha/cargo.py | 2 +- stimela/backends/__init__.py | 2 +- stimela/backends/runner.py | 2 +- stimela/config.py | 11 ++++++++--- stimela/kitchen/recipe.py | 8 +++++--- stimela/kitchen/step.py | 16 ++++++++-------- stimela/stimela.conf | 7 +++++-- 7 files changed, 29 insertions(+), 19 deletions(-) diff --git a/scabha/cargo.py b/scabha/cargo.py index ee69595b..f0157f4d 100644 --- a/scabha/cargo.py +++ b/scabha/cargo.py @@ -411,7 +411,7 @@ def _resolve_implicit_parameters(self, params, subst: Optional[SubstitutionNS]=N current[name] = schema.implicit - def prevalidate(self, params: Optional[Dict[str, Any]], subst: Optional[SubstitutionNS]=None, root=False): + def prevalidate(self, params: Optional[Dict[str, Any]], subst: Optional[SubstitutionNS]=None, backend=None, root=False): """Does pre-validation. No parameter substitution is done, but will check for missing params and such. A dynamic schema, if defined, is applied at this point.""" diff --git a/stimela/backends/__init__.py b/stimela/backends/__init__.py index bbebd5c2..834528eb 100644 --- a/stimela/backends/__init__.py +++ b/stimela/backends/__init__.py @@ -47,7 +47,7 @@ class StimelaBackendOptions(object): # overrides registries -- useful if you have a pull-through cache set up override_registries: Dict[str, str] = EmptyDictDefault() - select: Any = "" # should be Union[str, List[str]], but OmegaConf doesn't support it, so handle in __post_init__ for now + select: Any = "singularity,native" # should be Union[str, List[str]], but OmegaConf doesn't support it, so handle in __post_init__ for now singularity: Optional[SingularityBackendOptions] = None kube: Optional[KubeBackendOptions] = None diff --git a/stimela/backends/runner.py b/stimela/backends/runner.py index ad17e9f7..c8c7807d 100644 --- a/stimela/backends/runner.py +++ b/stimela/backends/runner.py @@ -48,7 +48,7 @@ def validate_backend_settings(backend_opts: Dict[str, Any], log: logging.Logger) backend_opts = OmegaConf.to_object(backend_opts) backend_name = backend = None - selected = backend_opts.select or ['native'] + selected = backend_opts.select or ['singularity', 'native'] # select containerization engine, if any for name in selected: # check that backend has not been disabled diff --git a/stimela/config.py b/stimela/config.py index 18aa9bfd..454683d8 100644 --- a/stimela/config.py +++ b/stimela/config.py @@ -1,4 +1,4 @@ -import glob +import importlib import os, os.path, time, sys, platform, traceback from typing import Any, List, Dict, Optional, Union from enum import Enum @@ -12,8 +12,6 @@ from stimela.exceptions import * from stimela import log_exception -CONFIG_FILE = os.path.expanduser("~/.config/stimela.conf") - from scabha import configuratt from scabha.cargo import ListOrString, EmptyDictDefault, EmptyListDefault from stimela.backends import StimelaBackendOptions @@ -64,12 +62,19 @@ def DefaultDirs(): _CONFIG_BASENAME = "stimela.conf" _STIMELA_CONFDIR = os.path.os.path.expanduser("~/.stimela") +# check for cultcargo module +try: + ccmod = importlib.import_module("cultcargo") +except ImportError: + ccmod = None + # dict of config file locations to check, in order of preference CONFIG_LOCATIONS = OrderedDict( package = os.path.join(os.path.dirname(__file__), _CONFIG_BASENAME), local = _CONFIG_BASENAME, venv = os.environ.get('VIRTUAL_ENV', None) and os.path.join(os.environ['VIRTUAL_ENV'], _CONFIG_BASENAME), stimela = os.path.isdir(_STIMELA_CONFDIR) and os.path.join(_STIMELA_CONFDIR, _CONFIG_BASENAME), + cultcargo = ccmod and os.path.join(os.path.dirname(ccmod.__file__), _CONFIG_BASENAME), user = os.path.join(os.path.expanduser("~/.config"), _CONFIG_BASENAME), ) diff --git a/stimela/kitchen/recipe.py b/stimela/kitchen/recipe.py index 3f98faef..91ce77d6 100644 --- a/stimela/kitchen/recipe.py +++ b/stimela/kitchen/recipe.py @@ -739,11 +739,13 @@ def _preprocess_parameters(self, params: Dict[str, Any]): return own_params, unset_params - def prevalidate(self, params: Dict[str, Any], subst: Optional[SubstitutionNS]=None, root=False): - self.finalize() + def prevalidate(self, params: Dict[str, Any], subst: Optional[SubstitutionNS]=None, backend=None, root=False): + self.finalize(backend=backend) self.log.debug("prevalidating recipe") errors = [] + backend = OmegaConf.merge(backend or self.config.opts.backend, self.backend or {}) + # split parameters into our own, and per-step, and UNSET directives params, unset_params = self._preprocess_parameters(params) @@ -785,7 +787,7 @@ def prevalidate(self, params: Dict[str, Any], subst: Optional[SubstitutionNS]=No # we call this twice, potentially, so define as a function def prevalidate_self(params): try: - params1 = Cargo.prevalidate(self, params, subst=subst_outer) + params1 = Cargo.prevalidate(self, params, subst=subst_outer, backend=backend) # mark params that have become unset unset_params.update(set(params) - set(params1)) params = params1 diff --git a/stimela/kitchen/step.py b/stimela/kitchen/step.py index ade9554d..5bfd8976 100644 --- a/stimela/kitchen/step.py +++ b/stimela/kitchen/step.py @@ -266,8 +266,8 @@ def finalize(self, config=None, log=None, fqname=None, backend=None, nesting=0): runner.validate_backend_settings(backend_opts, log=log) - def prevalidate(self, subst: Optional[SubstitutionNS]=None, root=False): - self.finalize() + def prevalidate(self, subst: Optional[SubstitutionNS]=None, root=False, backend=None): + self.finalize(backend=backend) self.cargo.apply_dynamic_schemas(self.params, subst) # validate cab or recipe params = self.validated_params = self.cargo.prevalidate(self.params, subst, root=root) @@ -320,11 +320,11 @@ def assign_value(self, key: str, value: Any, override: bool = False): raise AssignmentError(f"{self.name}: invalid assignment {key}={value}", exc) - def build(self, backend={}, rebuild=False, build_skips=False, log: Optional[logging.Logger] = None): + def build(self, backend=None, rebuild=False, build_skips=False, log: Optional[logging.Logger] = None): # skipping step? ignore the build if self.skip is True and not build_skips: return - backend = OmegaConf.merge(backend, self.cargo.backend or {}, self.backend or {}) + backend = OmegaConf.merge(backend or {}, self.cargo.backend or {}, self.backend or {}) log = log or self.log # recurse into sub-recipe from .recipe import Recipe @@ -334,7 +334,7 @@ def build(self, backend={}, rebuild=False, build_skips=False, log: Optional[logg else: # validate backend settings and call the build function try: - backend_opts = OmegaConf.merge(backend, self.cargo.backend or {}, self.backend or {}) + backend_opts = OmegaConf.merge(self.config.opts.backend, backend) if getattr(backend_opts, 'verbose', 0): opts_yaml = OmegaConf.to_yaml(backend_opts) log_rich_payload(self.log, "effective backend settings are", opts_yaml, syntax="yaml") @@ -348,7 +348,7 @@ def build(self, backend={}, rebuild=False, build_skips=False, log: Optional[logg return backend_runner.build(self.cargo, log=log, rebuild=rebuild) - def run(self, backend: Optional[Dict] = {}, subst: Optional[Dict[str, Any]] = None, + def run(self, backend: Optional[Dict] = None, subst: Optional[Dict[str, Any]] = None, is_outer_step: bool=False, parent_log: Optional[logging.Logger] = None) -> Dict[str, Any]: """executes the step @@ -376,11 +376,11 @@ def run(self, backend: Optional[Dict] = {}, subst: Optional[Dict[str, Any]] = No if parent_log is None: parent_log = self.log - backend = OmegaConf.merge(backend, self.cargo.backend or {}, self.backend or {}) + backend = OmegaConf.merge(backend or {}, self.cargo.backend or {}, self.backend or {}) # validate backend settings try: - backend_opts = OmegaConf.merge(stimela.CONFIG.opts.backend, backend) + backend_opts = OmegaConf.merge(self.config.opts.backend, backend) backend_opts = evaluate_and_substitute_object(backend_opts, subst, recursion_level=-1, location=[self.fqname, "backend"]) if not is_outer_step and backend_opts.verbose: opts_yaml = OmegaConf.to_yaml(backend_opts) diff --git a/stimela/stimela.conf b/stimela/stimela.conf index 7ca83b4d..5121f01d 100644 --- a/stimela/stimela.conf +++ b/stimela/stimela.conf @@ -1,10 +1,13 @@ +# Defines default versions of images for python and casa-flavour cabs +# Note that if cult-cargo is installed, its own version of stimela.conf will +# be read afterwards, and so can overwite these settings. images: default-python: registry: quay.io/stimela2 name: python-astro - version: cc0.0.2 + version: cc0.1.2 default-casa: registry: quay.io/stimela2 name: casa - version: cc0.0.2 + version: cc0.1.2 \ No newline at end of file