Merge pull request #259 from caracal-pipeline/issue-257

Issue 257 & 258

stimela/backends/singularity.py

@@ -252,7 +252,10 @@ def run(cab: 'stimela.kitchen.cab.Cab', params: Dict[str, Any], fqname: str,
     mounts = {cwd: True}
     # get extra required filesystem bindings
     resolve_required_mounts(mounts, params, cab.inputs, cab.outputs)
-    for path, rw in mounts.items():
+
+    # sort mount paths before iterating -- this ensures that parent directories come first
+    # (singularity doesn't like it if you specify a bind of a subdir before a bind of a parent) 
+    for path, rw in sorted(mounts.items()):
         args += ["--bind", f"{path}:{path}:{'rw' if rw else 'ro'}"]
 
     args += [simg_path]

stimela/backends/utils.py

@@ -20,10 +20,12 @@ def resolve_required_mounts(mounts: Dict[str, bool],
     def add_target(param_name, path, must_exist, readwrite):
         if not os.path.isdir(path):
             path = os.path.dirname(path)
+        # if file doesn't exit, bind parent or throw error
         if not os.path.exists(path):
             if must_exist:
                 raise SchemaError(f"parameter '{param_name}': path '{path}' does not exist")
-            return
+            path = os.path.dirname(path)
+        path = path.rstrip("/")
         mounts[path] = mounts.get(path) or readwrite
 
     # go through parameters and accumulate target paths
@@ -50,30 +52,24 @@ def add_target(param_name, path, must_exist, readwrite):
             path = os.path.abspath(path).rstrip("/")
             realpath = os.path.abspath(os.path.realpath(path))
             add_target(name, realpath, must_exist=must_exist, readwrite=readwrite)
+            add_target(name, path, must_exist=must_exist, readwrite=readwrite)
             # check if parent directory access is required
             if schema.access_parent_dir or schema.write_parent_dir:
                 add_target(name, os.path.dirname(path), must_exist=True, readwrite=schema.write_parent_dir)
                 add_target(name, os.path.dirname(realpath), must_exist=True, readwrite=schema.write_parent_dir)
-            # initial set of things that need to be mounted, this includes the path for a start
+
+    # now, for any mount that has a symlink in the path, add the symlink target to mounts
+    for path, readwrite in list(mounts.items()):
+        while path != "/":
             if os.path.islink(path):
                 chain = [path]
                 while os.path.islink(path):
-                    # Resolve the symlink one step.
                     path = os.readlink(path)
                     # Check if the path is absolute; if not, resolve it relative to the directory of the previous link.
                     if not os.path.isabs(path):
-                        path = os.path.join(os.path.dirname(chain[-1]), path)
+                        path = os.path.abspath(os.path.join(os.path.dirname(chain[-1]), path))
                     chain.append(path)
-                    add_target(name, os.path.dirname(path), must_exist=True, readwrite=readwrite)
-            else:
-                add_target(name, path, must_exist=must_exist, readwrite=readwrite)
-
-    # now, for any mount that has a symlink in the path, add the symlink target to mounts
-    for path, readwrite in list(mounts.items()):
-        while path != "/":
-            if os.path.islink(path):
-                realpath = os.path.realpath(path)
-                mounts[realpath] = mounts.get(realpath) or readwrite
+                    add_target(name, path, must_exist=True, readwrite=readwrite)
             path = os.path.dirname(path)
 
     # now eliminate unnecessary mounts (those that have a parent mount with no lower read/write privileges)
@@ -91,7 +87,6 @@ def add_target(param_name, path, must_exist, readwrite):
     for path in skip_targets:
         mounts.pop(path)
 
-
 def resolve_remote_mounts(params: Dict[str, Any], 
                             inputs: Dict[str, Parameter], 
                             outputs: Dict[str, Parameter],

stimela/commands/run.py

@@ -334,15 +334,18 @@ def log_available_runnables():
     if cab_name is not None:
         # create step config by merging in settings (var=value pairs from the command line) 
         outer_step = Step(cab=cab_name, params=params)
-        outer_step.name = outer_step.fqname = cab_name
+        outer_step.name = cab_name
         # provide basic substitutions for running the step below
         subst = SubstitutionNS()
         info = SubstitutionNS(fqname=cab_name, label=cab_name, label_parts=[], suffix='', taskname=cab_name)
         subst._add_('info', info, nosubst=True)
         subst._add_('config', stimela.CONFIG, nosubst=True) 
         subst._add_('current', SubstitutionNS(**params))
-
+        # create step logger manually, since we won't be doing the normal recipe-level log management
+        step_logger = stimela.logger().getChild(cab_name)
+        step_logger.propagate = True
         try:
+            outer_step.finalize(fqname=cab_name, log=step_logger)
             outer_step.prevalidate(root=True, subst=subst)
         except ScabhaBaseException as exc:
             log_exception(exc)

stimela/kitchen/step.py

@@ -242,7 +242,7 @@ def finalize(self, config=None, log=None, fqname=None, backend=None, nesting=0):
             # if logger is not provided, then init one
             if log is None:
                 log = stimela.logger().getChild(self.fqname)
-                log.propagate = True
+                log.propagate = False
 
             # finalize the cargo
             self.cargo.finalize(config, log=log, fqname=self.fqname, backend=backend, nesting=nesting)