diff --git a/pyproject.toml b/pyproject.toml index 957d230..0be688d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "stimela" -version = "2.0.1" +version = "2.0.1.1" description = "Framework for system agnostic pipelines for (not just) radio interferometry" authors = ["Oleg Smirnov and RATT ", "Sphesihle Makhathini "] readme = "README.rst" diff --git a/stimela/main.py b/stimela/main.py index 875aa5a..ace9bb3 100644 --- a/stimela/main.py +++ b/stimela/main.py @@ -52,7 +52,7 @@ def resolve_command(self, ctx, args): help="Add directory to _include paths. Can be given multiple times.") @click.option('--clear-cache', '-C', is_flag=True, help="Reset the configuration cache. First thing to try in case of strange configuration errors.") -@click.option('--boring', '-b', is_flag=True, +@click.option('--boring', '-B', is_flag=True, help="Disables progress bar and any other fancy console outputs.") @click.option('--verbose', '-v', is_flag=True, help='Be extra verbose in output.') diff --git a/stimela/task_stats.py b/stimela/task_stats.py index 0c27fef..0d865ac 100644 --- a/stimela/task_stats.py +++ b/stimela/task_stats.py @@ -237,7 +237,7 @@ def update_stats(now: datetime, sample: TaskStatsDatum): _taskstats[key][0] = (now - start).total_seconds() -def update_process_status(stimela_process=None, child_processes=None): +def update_process_status(): # current subtask info ti = _task_stack[-1] if _task_stack else None @@ -247,27 +247,11 @@ def update_process_status(stimela_process=None, child_processes=None): # form up sample datum s = TaskStatsDatum(num_samples=1) - - # If we have the process objects, we don't need to block. - interval = 0 if child_processes else 1 - - # Grab the stimela process and its children (recursively). - stimela_process = stimela_process or psutil.Process() - stimela_children = child_processes or stimela_process.children(recursive=True) - - # Assume that all child processes belong to the same task. - # TODO: Handling of children is rudimentary at present. - # How would this work for scattered/parallel steps? - if stimela_children and ti: - processes = stimela_children - else: - processes = [stimela_process] - # CPU and memory - s.cpu = sum(p.cpu_percent(interval=interval) for p in processes) - system_memory = psutil.virtual_memory().total - s.mem_used = round(sum(p.memory_info().rss for p in processes) / 2**30) - s.mem_total = round(system_memory / 2**30) + s.cpu = psutil.cpu_percent() + mem = psutil.virtual_memory() + s.mem_used = round(mem.total*mem.percent/100 / 2**30) + s.mem_total = round(mem.total / 2**30) # load s.load, _, _ = psutil.getloadavg() @@ -335,16 +319,8 @@ def update_process_status(stimela_process=None, child_processes=None): async def run_process_status_update(): if progress_bar: with contextlib.suppress(asyncio.CancelledError): - stimela_process = psutil.Process() - child_processes = set() while True: - new_children = {c for c in stimela_process.children(recursive=True) if c not in child_processes} - old_children = {c for c in child_processes if c not in stimela_process.children(recursive=True)} - child_processes = child_processes.union(new_children) - old_children - try: - update_process_status(stimela_process, child_processes) - except psutil.NoSuchProcess: - continue + update_process_status() await asyncio.sleep(1) _printed_stats = dict( diff --git a/tests/stimela_tests/test_callables.py b/tests/stimela_tests/test_callables.py index b1a33cf..06fc9b8 100644 --- a/tests/stimela_tests/test_callables.py +++ b/tests/stimela_tests/test_callables.py @@ -11,7 +11,7 @@ def callable_function_dict(a: int, b: str): def test_wrangler_replace_suppress(): print("===== expecting no errors =====") - retcode, output = run("stimela -v run test_callables.yml") + retcode, output = run("stimela -v -b native run test_callables.yml") assert retcode == 0 print(output) assert verify_output(output, 'y = 46barbar') diff --git a/tests/stimela_tests/test_recipe.py b/tests/stimela_tests/test_recipe.py index 6c71d80..feb1b1d 100644 --- a/tests/stimela_tests/test_recipe.py +++ b/tests/stimela_tests/test_recipe.py @@ -36,14 +36,14 @@ def verify_output(output, *regexes): def test_test_aliasing(): print("===== expecting an error since required parameters are missing =====") - retcode, _ = run(f"stimela -v exec test_aliasing.yml") + retcode, _ = run(f"stimela -v -b native exec test_aliasing.yml") assert retcode != 0 print("===== expecting no errors now =====") retcode, output = run("stimela -v doc test_aliasing.yml") print("===== expecting no errors now =====") - retcode, output = run("stimela -v exec test_aliasing.yml a=1 s3.a=1 s4.a=1 e=e f=f") + retcode, output = run("stimela -v -b native exec test_aliasing.yml a=1 s3.a=1 s4.a=1 e=e f=f") assert retcode == 0 print(output) assert verify_output(output, @@ -53,43 +53,43 @@ def test_test_aliasing(): def test_test_nesting(): print("===== expecting no errors =====") - retcode, output = run("stimela -v exec test_nesting.yml demo_recipe") + retcode, output = run("stimela -v -b native exec test_nesting.yml demo_recipe") assert retcode == 0 print(output) def test_test_recipe(): print("===== expecting an error since 'msname' parameter is missing =====") - retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar") + retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar") assert retcode != 0 print("===== expecting an error due to elem-xyz choices wrong =====") - retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=t elemlist-xyz=[x,y]") + retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=t elemlist-xyz=[x,y]") assert retcode != 0 print("===== expecting an error due to elem-xyz choices wrong =====") - retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,t]") + retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,t]") assert retcode != 0 print("===== expecting no errors now =====") - retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,y]") + retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=[x,y]") assert retcode == 0 print("===== expecting no errors now =====") - retcode = os.system("stimela -v exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=x") + retcode = os.system("stimela -v -b native exec test_recipe.yml selfcal.image_name=bar msname=foo elem-xyz=x elemlist-xyz=x") assert retcode == 0 def test_test_loop_recipe(): print("===== expecting an error since 'ms' parameter is missing =====") - retcode = os.system("stimela -v exec test_loop_recipe.yml cubical_image_loop") + retcode = os.system("stimela -v -b native exec test_loop_recipe.yml cubical_image_loop") assert retcode != 0 print("===== expecting no errors now =====") - retcode = os.system("stimela -v exec test_loop_recipe.yml cubical_image_loop ms=foo") + retcode = os.system("stimela -v -b native exec test_loop_recipe.yml cubical_image_loop ms=foo") assert retcode == 0 print("===== expecting no errors now =====") - retcode = os.system("stimela exec test_loop_recipe.yml same_as_cubical_image_loop ms=foo") + retcode = os.system("stimela -v -b native exec test_loop_recipe.yml same_as_cubical_image_loop ms=foo") assert retcode == 0 for name in "abc": @@ -97,14 +97,14 @@ def test_test_loop_recipe(): if not os.path.exists(msname): os.mkdir(msname) print("===== expecting no errors now =====") - retcode = os.system("stimela exec test_loop_recipe.yml loop_recipe") + retcode = os.system("stimela -v -b native exec test_loop_recipe.yml loop_recipe") assert retcode == 0 def test_scatter(): print("===== expecting no errors now =====") - retcode = os.system("stimela exec test_scatter.yml basic_loop") + retcode = os.system("stimela -v -b native exec test_scatter.yml basic_loop") assert retcode == 0 print("===== expecting no errors now =====") - retcode = os.system("stimela exec test_scatter.yml nested_loop") + retcode = os.system("stimela -v -b native exec test_scatter.yml nested_loop") assert retcode == 0 diff --git a/tests/stimela_tests/test_wranglers.py b/tests/stimela_tests/test_wranglers.py index 50cfe90..cc605de 100644 --- a/tests/stimela_tests/test_wranglers.py +++ b/tests/stimela_tests/test_wranglers.py @@ -3,7 +3,7 @@ def test_wrangler_replace_suppress(): print("===== expecting no errors =====") - retcode, output = run("stimela run test_wranglers.yml test_replace_suppress") + retcode, output = run("stimela -b native run test_wranglers.yml test_replace_suppress") assert retcode == 0 print(output) assert verify_output(output, "Michael J. Fox", "don't need roads!") @@ -12,7 +12,7 @@ def test_wrangler_replace_suppress(): def test_wrangler_force_success(): print("===== expecting no errors =====") - retcode, output = run("stimela run test_wranglers.yml test_force_success") + retcode, output = run("stimela -b native run test_wranglers.yml test_force_success") assert retcode == 0 print(output) assert verify_output(output, "deliberately declared") @@ -20,32 +20,32 @@ def test_wrangler_force_success(): def test_wrangler_force_failure(): print("===== expecting an error =====") - retcode, output = run("stimela run test_wranglers.yml test_force_failure") + retcode, output = run("stimela -b native run test_wranglers.yml test_force_failure") assert retcode != 0 print(output) assert verify_output(output, "cab marked as failed") print("===== expecting an error =====") - retcode, output = run("stimela run test_wranglers.yml test_force_failure2") + retcode, output = run("stimela -b native run test_wranglers.yml test_force_failure2") assert retcode != 0 print(output) assert verify_output(output, "Nobody expected the fox!") def test_wrangler_parse(): print("===== expecting no errors =====") - retcode, output = run("stimela run test_wranglers.yml test_parse") + retcode, output = run("stimela -b native run test_wranglers.yml test_parse") assert retcode == 0 print(output) assert verify_output(output, "The bloody cheetah ate 22 dogs!") print("===== expecting no errors =====") - retcode, output = run("stimela run test_wranglers.yml test_parse2") + retcode, output = run("stimela -b native run test_wranglers.yml test_parse2") assert retcode == 0 print(output) assert verify_output(output, "The bloody cheetah ate 22 dogs!") print("===== expecting no errors =====") - retcode, output = run("stimela run test_wranglers.yml test_parse3") + retcode, output = run("stimela -b native run test_wranglers.yml test_parse3") assert retcode == 0 print(output) assert verify_output(output, "The bloody cheetah ate 22 dogs!")