Merge branch 'caracal-pipeline:master' into clickfy_documentation

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "stimela"
-version = "2.0.1"
+version = "2.0.1.1"
 description = "Framework for system agnostic pipelines for (not just) radio interferometry"
 authors = ["Oleg Smirnov and RATT <[email protected]>", "Sphesihle Makhathini <[email protected]>"]
 readme = "README.rst"

stimela/main.py

@@ -52,7 +52,7 @@ def resolve_command(self, ctx, args):
                 help="Add directory to _include paths. Can be given multiple times.")
 @click.option('--clear-cache', '-C', is_flag=True, 
                 help="Reset the configuration cache. First thing to try in case of strange configuration errors.")
-@click.option('--boring', '-b', is_flag=True, 
+@click.option('--boring', '-B', is_flag=True, 
                 help="Disables progress bar and any other fancy console outputs.")
 @click.option('--verbose', '-v', is_flag=True, 
               help='Be extra verbose in output.')

stimela/task_stats.py

@@ -237,7 +237,7 @@ def update_stats(now: datetime, sample: TaskStatsDatum):
         _taskstats[key][0] = (now - start).total_seconds()
 
 
-def update_process_status(stimela_process=None, child_processes=None):
+def update_process_status():
     # current subtask info
     ti = _task_stack[-1] if _task_stack else None
 
@@ -247,27 +247,11 @@ def update_process_status(stimela_process=None, child_processes=None):
 
     # form up sample datum
     s = TaskStatsDatum(num_samples=1)
-
-    # If we have the process objects, we don't need to block.
-    interval = 0 if child_processes else 1
-
-    # Grab the stimela process and its children (recursively).
-    stimela_process = stimela_process or psutil.Process()
-    stimela_children = child_processes or stimela_process.children(recursive=True)
-
-    # Assume that all child processes belong to the same task.
-    # TODO: Handling of children is rudimentary at present.
-    # How would this work for scattered/parallel steps?
-    if stimela_children and ti:
-        processes = stimela_children
-    else:
-        processes = [stimela_process]
-
     # CPU and memory
-    s.cpu = sum(p.cpu_percent(interval=interval) for p in processes)
-    system_memory = psutil.virtual_memory().total
-    s.mem_used = round(sum(p.memory_info().rss for p in processes) / 2**30)
-    s.mem_total = round(system_memory / 2**30)
+    s.cpu = psutil.cpu_percent()
+    mem = psutil.virtual_memory()
+    s.mem_used = round(mem.total*mem.percent/100 / 2**30)
+    s.mem_total = round(mem.total / 2**30)
     # load
     s.load, _, _ = psutil.getloadavg()
 
@@ -335,16 +319,8 @@ def update_process_status(stimela_process=None, child_processes=None):
 async def run_process_status_update():
     if progress_bar:
         with contextlib.suppress(asyncio.CancelledError):
-            stimela_process = psutil.Process()
-            child_processes = set()
             while True:
-                new_children = {c for c in stimela_process.children(recursive=True) if c not in child_processes}
-                old_children = {c for c in child_processes if c not in stimela_process.children(recursive=True)}
-                child_processes = child_processes.union(new_children) - old_children
-                try:
-                    update_process_status(stimela_process, child_processes)
-                except psutil.NoSuchProcess:
-                    continue
+                update_process_status()
                 await asyncio.sleep(1)
 
 _printed_stats = dict(

tests/stimela_tests/test_callables.py

@@ -11,7 +11,7 @@ def callable_function_dict(a: int, b: str):
 
 def test_wrangler_replace_suppress():
     print("===== expecting no errors =====")
-    retcode, output = run("stimela -v run test_callables.yml")
+    retcode, output = run("stimela -v -b native run test_callables.yml")
     assert retcode == 0
     print(output)
     assert verify_output(output, 'y = 46barbar')

tests/stimela_tests/test_recipe.py

@@ -36,14 +36,14 @@ def verify_output(output, *regexes):
 
 def test_test_aliasing():
     print("===== expecting an error since required parameters are missing =====")
-    retcode, _ = run(f"stimela -v exec test_aliasing.yml")
+    retcode, _ = run(f"stimela -v -b native exec test_aliasing.yml")
     assert retcode != 0 
 
     print("===== expecting no errors now =====")
     retcode, output = run("stimela -v doc test_aliasing.yml")
 
     print("===== expecting no errors now =====")
-    retcode, output = run("stimela -v exec test_aliasing.yml a=1 s3.a=1 s4.a=1 e=e f=f")
+    retcode, output = run("stimela -v -b native exec test_aliasing.yml a=1 s3.a=1 s4.a=1 e=e f=f")
     assert retcode == 0
     print(output)
     assert verify_output(output, 
@@ -53,58 +53,58 @@ def test_test_aliasing():
 
 def test_test_nesting():
     print("===== expecting no errors =====")
-    retcode, output = run("stimela -v exec test_nesting.yml demo_recipe")
+    retcode, output = run("stimela -v -b native exec test_nesting.yml demo_recipe")
     assert retcode == 0
     print(output)
 
 
 def test_test_recipe():
     print("===== expecting an error since 'msname' parameter is missing =====")
-    retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar")
+    retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar")
     assert retcode != 0 
 
     print("===== expecting an error due to elem-xyz choices wrong =====")
-    retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=t elemlist-xyz=[x,y]")
+    retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=t elemlist-xyz=[x,y]")
     assert retcode != 0
 
     print("===== expecting an error due to elem-xyz choices wrong =====")
-    retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,t]")
+    retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,t]")
     assert retcode != 0
 
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,y]")
+    retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,y]")
     assert retcode == 0
 
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=x")
+    retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=x")
     assert retcode == 0
 
 def test_test_loop_recipe():
     print("===== expecting an error since 'ms' parameter is missing =====")
-    retcode = os.system("stimela -v exec test_loop_recipe.yml cubical_image_loop")
+    retcode = os.system("stimela -v -b native exec test_loop_recipe.yml cubical_image_loop")
     assert retcode != 0
 
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela -v exec test_loop_recipe.yml cubical_image_loop ms=foo")
+    retcode = os.system("stimela -v -b native exec test_loop_recipe.yml cubical_image_loop ms=foo")
     assert retcode == 0
 
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela exec test_loop_recipe.yml same_as_cubical_image_loop ms=foo")
+    retcode = os.system("stimela -v -b native exec test_loop_recipe.yml same_as_cubical_image_loop ms=foo")
     assert retcode == 0
 
     for name in "abc":
         msname = f"test-{name}.ms"
         if not os.path.exists(msname):
             os.mkdir(msname)
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela exec test_loop_recipe.yml loop_recipe")
+    retcode = os.system("stimela -v -b native exec test_loop_recipe.yml loop_recipe")
     assert retcode == 0
 
 def test_scatter():
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela exec test_scatter.yml basic_loop")
+    retcode = os.system("stimela -v -b native exec test_scatter.yml basic_loop")
     assert retcode == 0
 
     print("===== expecting no errors now =====")
-    retcode = os.system("stimela exec test_scatter.yml nested_loop")
+    retcode = os.system("stimela -v -b native exec test_scatter.yml nested_loop")
     assert retcode == 0

tests/stimela_tests/test_wranglers.py

@@ -3,7 +3,7 @@
 
 def test_wrangler_replace_suppress():
     print("===== expecting no errors =====")
-    retcode, output = run("stimela run test_wranglers.yml test_replace_suppress")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_replace_suppress")
     assert retcode == 0
     print(output)
     assert verify_output(output, "Michael J. Fox", "don't need roads!")
@@ -12,40 +12,40 @@ def test_wrangler_replace_suppress():
 
 def test_wrangler_force_success():
     print("===== expecting no errors =====")
-    retcode, output = run("stimela run test_wranglers.yml test_force_success")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_force_success")
     assert retcode == 0
     print(output)
     assert verify_output(output, "deliberately declared")
 
 
 def test_wrangler_force_failure():
     print("===== expecting an error =====")
-    retcode, output = run("stimela run test_wranglers.yml test_force_failure")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_force_failure")
     assert retcode != 0
     print(output)
     assert verify_output(output, "cab marked as failed")
 
     print("===== expecting an error =====")
-    retcode, output = run("stimela run test_wranglers.yml test_force_failure2")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_force_failure2")
     assert retcode != 0
     print(output)
     assert verify_output(output, "Nobody expected the fox!")
 
 def test_wrangler_parse():
     print("===== expecting no errors =====")
-    retcode, output = run("stimela run test_wranglers.yml test_parse")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_parse")
     assert retcode == 0
     print(output)
     assert verify_output(output, "The bloody cheetah ate 22 dogs!")
 
     print("===== expecting no errors =====")
-    retcode, output = run("stimela run test_wranglers.yml test_parse2")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_parse2")
     assert retcode == 0
     print(output)
     assert verify_output(output, "The bloody cheetah ate 22 dogs!")
 
     print("===== expecting no errors =====")
-    retcode, output = run("stimela run test_wranglers.yml test_parse3")
+    retcode, output = run("stimela -b native run test_wranglers.yml test_parse3")
     assert retcode == 0
     print(output)
     assert verify_output(output, "The bloody cheetah ate 22 dogs!")