Merge branch 'master' into benc-k8s-kind-ci

parsl/app/bash.py

@@ -1,5 +1,5 @@
 import logging
-from functools import partial, update_wrapper
+from functools import partial
 from inspect import Parameter, signature
 
 from parsl.app.app import AppBase
@@ -123,11 +123,10 @@ def __init__(self, func, data_flow_kernel=None, cache=False, executors='all', ig
             if sig.parameters[s].default is not Parameter.empty:
                 self.kwargs[s] = sig.parameters[s].default
 
-        # update_wrapper allows remote_side_bash_executor to masquerade as self.func
         # partial is used to attach the first arg the "func" to the remote_side_bash_executor
         # this is done to avoid passing a function type in the args which parsl.serializer
         # doesn't support
-        remote_fn = partial(update_wrapper(remote_side_bash_executor, self.func), self.func)
+        remote_fn = partial(remote_side_bash_executor, self.func)
         remote_fn.__name__ = self.func.__name__
         self.wrapped_remote_function = wrap_error(remote_fn)
 

parsl/channels/local/local.py

@@ -55,6 +55,7 @@ def execute_wait(self, cmd, walltime=None, envs={}):
         current_env.update(envs)
 
         try:
+            logger.debug("Creating process with command '%s'", cmd)
             proc = subprocess.Popen(
                 cmd,
                 stdout=subprocess.PIPE,
@@ -64,12 +65,16 @@ def execute_wait(self, cmd, walltime=None, envs={}):
                 shell=True,
                 preexec_fn=os.setpgrp
             )
+            logger.debug("Created process with pid %s. Performing communicate", proc.pid)
             (stdout, stderr) = proc.communicate(timeout=walltime)
             retcode = proc.returncode
+            logger.debug("Process %s returned %s", proc.pid, proc.returncode)
 
-        except Exception as e:
-            logger.warning("Execution of command '{}' failed due to \n{}".format(cmd, e))
+        except Exception:
+            logger.exception(f"Execution of command failed:\n{cmd}")
             raise
+        else:
+            logger.debug("Execution of command in process %s completed normally", proc.pid)
 
         return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))
 

parsl/configs/ASPIRE1.py

@@ -4,6 +4,7 @@
 from parsl.launchers import MpiRunLauncher
 from parsl.monitoring.monitoring import MonitoringHub
 from parsl.providers import PBSProProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
         executors=[
@@ -39,5 +40,6 @@
         strategy='simple',
         retries=3,
         app_cache=True,
-        checkpoint_mode='task_exit'
+        checkpoint_mode='task_exit',
+        usage_tracking=LEVEL_1,
 )

parsl/configs/Azure.py

@@ -8,6 +8,7 @@
 from parsl.data_provider.rsync import RSyncStaging
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AzureProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 vm_reference = {
     # All fields below are required
@@ -33,5 +34,6 @@
                             FTPInTaskStaging(),
                             RSyncStaging(getpass.getuser() + "@" + address_by_query())],
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/ad_hoc.py

@@ -4,6 +4,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AdHocProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 user_opts: Dict[str, Dict[str, Any]]
 user_opts = {'adhoc':
@@ -33,4 +34,5 @@
     ],
     #  AdHoc Clusters should not be setup with scaling strategy.
     strategy='none',
+    usage_tracking=LEVEL_1,
 )

parsl/configs/bridges.py

@@ -3,6 +3,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Bridges at PSC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -34,5 +35,6 @@
                 cmd_timeout=120,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/cc_in2p3.py

@@ -2,6 +2,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import GridEngineProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -19,4 +20,5 @@
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/ec2.py

@@ -1,6 +1,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AWSProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -25,4 +26,5 @@
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/expanse.py

@@ -2,6 +2,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -24,5 +25,6 @@
                 nodes_per_block=2,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/frontera.py

@@ -3,6 +3,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -32,4 +33,5 @@
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/htex_local.py

@@ -2,6 +2,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -15,4 +16,5 @@
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/illinoiscluster.py

@@ -2,6 +2,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of the Campus Cluster at UIUC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -25,4 +26,5 @@
                ),
           )
      ],
+     usage_tracking=LEVEL_1,
 )

parsl/configs/kubernetes.py

@@ -2,6 +2,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import KubernetesProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -36,5 +37,6 @@
                 max_blocks=10,
             ),
         ),
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/local_threads.py

@@ -1,4 +1,8 @@
 from parsl.config import Config
 from parsl.executors.threads import ThreadPoolExecutor
+from parsl.usage_tracking.levels import LEVEL_1
 
-config = Config(executors=[ThreadPoolExecutor()])
+config = Config(
+    executors=[ThreadPoolExecutor()],
+    usage_tracking=LEVEL_1,
+)

parsl/configs/midway.py

@@ -3,6 +3,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -28,4 +29,5 @@
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/osg.py

@@ -1,6 +1,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import CondorProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -26,5 +27,6 @@
             worker_logdir_root='$OSG_WN_TMP',
             worker_ports=(31000, 31001)
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/polaris.py

@@ -3,6 +3,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import MpiExecLauncher
 from parsl.providers import PBSProProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 # There are three user parameters to change for the PBSProProvider:
 #  YOUR_ACCOUNT: Account to charge usage
@@ -34,5 +35,6 @@
                 cpus_per_node=64,
             ),
         ),
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/stampede2.py

@@ -4,6 +4,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -34,4 +35,5 @@
         )
 
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/summit.py

@@ -3,6 +3,7 @@
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import JsrunLauncher
 from parsl.providers import LSFProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -26,4 +27,5 @@
         )
 
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/toss3_llnl.py

@@ -2,6 +2,7 @@
 from parsl.executors import FluxExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -24,5 +25,6 @@
                 cmd_timeout=120,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/vineex_local.py

@@ -2,6 +2,7 @@
 
 from parsl.config import Config
 from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -15,5 +16,6 @@
             # To disable status reporting, comment out the project_name.
             manager_config=TaskVineManagerConfig(project_name="parsl-vine-" + str(uuid.uuid4())),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/wqex_local.py

@@ -2,6 +2,7 @@
 
 from parsl.config import Config
 from parsl.executors import WorkQueueExecutor
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -21,5 +22,6 @@
             # A shared filesystem is not needed when using Work Queue.
             shared_fs=False
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )