Merge branch 'master' into trackingFileProvenance

Parsl · Dec 10, 2024 · 9444f42 · 9444f42
2 parents 53f323d + d646aaa
commit 9444f42
Show file tree

Hide file tree

Showing 46 changed files with 8 additions and 125 deletions.
diff --git a/README.rst b/README.rst
@@ -1,6 +1,6 @@
 Parsl - Parallel Scripting Library
 ==================================
-|licence| |docs| |NSF-1550588| |NSF-1550476| |NSF-1550562| |NSF-1550528| |CZI-EOSS|
+|licence| |docs| |NSF-1550588| |NSF-1550476| |NSF-1550562| |NSF-1550528| |NumFOCUS| |CZI-EOSS| 
 
 Parsl extends parallelism in Python beyond a single computer.
 
@@ -64,6 +64,9 @@ then explore the `parallel computing patterns <https://parsl.readthedocs.io/en/s
 .. |CZI-EOSS| image:: https://chanzuckerberg.github.io/open-science/badges/CZI-EOSS.svg
    :target: https://czi.co/EOSS
    :alt: CZI's Essential Open Source Software for Science
+.. |NumFOCUS| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A
+    :target: https://numfocus.org
+    :alt: Powered by NumFOCUS
 
 
 Quickstart

diff --git a/parsl/channels/__init__.py b/parsl/channels/__init__.py
diff --git a/parsl/channels/base.py b/parsl/channels/base.py
diff --git a/parsl/channels/local/__init__.py b/parsl/channels/local/__init__.py
diff --git a/parsl/channels/local/local.py b/parsl/channels/local/local.py
diff --git a/parsl/configs/cc_in2p3.py b/parsl/configs/cc_in2p3.py
@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import GridEngineProvider
@@ -10,7 +9,6 @@
             label='cc_in2p3_htex',
             max_workers_per_node=2,
             provider=GridEngineProvider(
-                channel=LocalChannel(),
                 nodes_per_block=1,
                 init_blocks=2,
                 max_blocks=2,

diff --git a/parsl/configs/frontera.py b/parsl/configs/frontera.py
@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
@@ -15,7 +14,6 @@
             max_workers_per_node=1,          # Set number of workers per node
             provider=SlurmProvider(
                 cmd_timeout=60,     # Add extra time for slow scheduler responses
-                channel=LocalChannel(),
                 nodes_per_block=2,
                 init_blocks=1,
                 min_blocks=1,

diff --git a/parsl/configs/htex_local.py b/parsl/configs/htex_local.py
@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
@@ -10,7 +9,6 @@
             label="htex_local",
             cores_per_worker=1,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
             ),

diff --git a/parsl/providers/cluster_provider.py b/parsl/providers/cluster_provider.py
@@ -18,8 +18,6 @@ class ClusterProvider(ExecutionProvider):
     ----------
       label : str
         Label for this provider.
-      channel : Channel
-        Channel for accessing this provider.
       walltime : str
         Walltime requested per block in HH:MM:SS.
       launcher : Launcher
@@ -45,7 +43,6 @@ class ClusterProvider(ExecutionProvider):
 
     def __init__(self,
                  label,
-                 channel,
                  nodes_per_block,
                  init_blocks,
                  min_blocks,
@@ -56,7 +53,6 @@ def __init__(self,
                  cmd_timeout=10):
 
         self._label = label
-        self.channel = channel
         self.nodes_per_block = nodes_per_block
         self.init_blocks = init_blocks
         self.min_blocks = min_blocks

diff --git a/parsl/providers/condor/condor.py b/parsl/providers/condor/condor.py
@@ -5,7 +5,6 @@
 
 import typeguard
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.launchers.base import Launcher
@@ -18,8 +17,6 @@
 
 from typing import Dict, List, Optional
 
-from parsl.channels.base import Channel
-
 # See http://pages.cs.wisc.edu/~adesmet/status.html
 translate_table = {
     '1': JobState.PENDING,
@@ -36,8 +33,6 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     cores_per_slot : int
@@ -79,7 +74,6 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
     """
     @typeguard.typechecked
     def __init__(self,
-                 channel: Channel = LocalChannel(),
                  nodes_per_block: int = 1,
                  cores_per_slot: Optional[int] = None,
                  mem_per_slot: Optional[float] = None,
@@ -100,7 +94,6 @@ def __init__(self,
 
         label = 'condor'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,

diff --git a/parsl/providers/grid_engine/grid_engine.py b/parsl/providers/grid_engine/grid_engine.py
@@ -2,7 +2,6 @@
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.cluster_provider import ClusterProvider
@@ -36,8 +35,6 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     min_blocks : int
@@ -62,7 +59,6 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
     """
 
     def __init__(self,
-                 channel=LocalChannel(),
                  nodes_per_block=1,
                  init_blocks=1,
                  min_blocks=0,
@@ -76,7 +72,6 @@ def __init__(self,
                  queue=None):
         label = 'grid_engine'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,

diff --git a/parsl/providers/local/local.py b/parsl/providers/local/local.py
@@ -2,7 +2,6 @@
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.base import ExecutionProvider
@@ -37,7 +36,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
     """
 
     def __init__(self,
-                 channel=LocalChannel(),
                  nodes_per_block=1,
                  launcher=SingleNodeLauncher(),
                  init_blocks=1,
@@ -46,7 +44,6 @@ def __init__(self,
                  worker_init='',
                  cmd_timeout=30,
                  parallelism=1):
-        self.channel = channel
         self._label = 'local'
         self.nodes_per_block = nodes_per_block
         self.launcher = launcher

diff --git a/parsl/providers/lsf/lsf.py b/parsl/providers/lsf/lsf.py
@@ -3,7 +3,6 @@
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.cluster_provider import ClusterProvider
@@ -32,8 +31,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
         When request_by_nodes is False, it is computed by cores_per_block / cores_per_node.
@@ -77,7 +74,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
     """
 
     def __init__(self,
-                 channel=LocalChannel(),
                  nodes_per_block=1,
                  cores_per_block=None,
                  cores_per_node=None,
@@ -96,7 +92,6 @@ def __init__(self,
                  launcher=SingleNodeLauncher()):
         label = 'LSF'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,

diff --git a/parsl/providers/pbspro/pbspro.py b/parsl/providers/pbspro/pbspro.py
@@ -3,7 +3,6 @@
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.pbspro.template import template_string
@@ -17,8 +16,6 @@ class PBSProProvider(TorqueProvider):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     account : str
         Account the job will be charged against.
     queue : str
@@ -51,7 +48,6 @@ class PBSProProvider(TorqueProvider):
         :class:`~parsl.launchers.SingleNodeLauncher`.
     """
     def __init__(self,
-                 channel=LocalChannel(),
                  account=None,
                  queue=None,
                  scheduler_options='',
@@ -66,8 +62,7 @@ def __init__(self,
                  launcher=SingleNodeLauncher(),
                  walltime="00:20:00",
                  cmd_timeout=120):
-        super().__init__(channel,
-                         account,
+        super().__init__(account,
                          queue,
                          scheduler_options,
                          worker_init,

diff --git a/parsl/providers/slurm/slurm.py b/parsl/providers/slurm/slurm.py
@@ -7,8 +7,6 @@
 
 import typeguard
 
-from parsl.channels import LocalChannel
-from parsl.channels.base import Channel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.launchers.base import Launcher
@@ -73,8 +71,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
     clusters : str
         Slurm cluster name, or comma seperated cluster list, used to choose between different clusters in a federated Slurm instance.
         If unspecified or ``None``, no slurm directive for clusters will be added.
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     cores_per_node : int
@@ -119,7 +115,6 @@ def __init__(self,
                  qos: Optional[str] = None,
                  constraint: Optional[str] = None,
                  clusters: Optional[str] = None,
-                 channel: Channel = LocalChannel(),
                  nodes_per_block: int = 1,
                  cores_per_node: Optional[int] = None,
                  mem_per_node: Optional[int] = None,
@@ -136,7 +131,6 @@ def __init__(self,
                  launcher: Launcher = SingleNodeLauncher()):
         label = 'slurm'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,

diff --git a/parsl/providers/torque/torque.py b/parsl/providers/torque/torque.py
@@ -2,7 +2,6 @@
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import AprunLauncher
 from parsl.providers.cluster_provider import ClusterProvider
@@ -33,8 +32,6 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     account : str
         Account the job will be charged against.
     queue : str
@@ -65,7 +62,6 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
 
     """
     def __init__(self,
-                 channel=LocalChannel(),
                  account=None,
                  queue=None,
                  scheduler_options='',
@@ -80,7 +76,6 @@ def __init__(self,
                  cmd_timeout=120):
         label = 'torque'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,
@@ -170,7 +165,6 @@ def submit(self, command, tasks_per_node, job_name="parsl.torque"):
                      tasks_per_node)
 
         job_config = {}
-        # TODO : script_path might need to change to accommodate script dir set via channels
         job_config["submit_script_dir"] = self.script_dir
         job_config["nodes"] = self.nodes_per_block
         job_config["task_blocks"] = self.nodes_per_block * tasks_per_node

diff --git a/parsl/tests/configs/cc_in2p3.py b/parsl/tests/configs/cc_in2p3.py
@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import GridEngineProvider
@@ -14,7 +13,6 @@ def fresh_config():
                 max_workers_per_node=1,
                 encrypted=True,
                 provider=GridEngineProvider(
-                    channel=LocalChannel(),
                     nodes_per_block=2,
                     init_blocks=2,
                     max_blocks=2,

diff --git a/parsl/tests/configs/frontera.py b/parsl/tests/configs/frontera.py
@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
@@ -20,7 +19,6 @@ def fresh_config():
                 encrypted=True,
                 provider=SlurmProvider(
                     cmd_timeout=60,     # Add extra time for slow scheduler responses
-                    channel=LocalChannel(),
                     nodes_per_block=2,
                     init_blocks=1,
                     min_blocks=1,

diff --git a/parsl/tests/configs/htex_local.py b/parsl/tests/configs/htex_local.py
@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -15,7 +14,6 @@ def fresh_config():
                 cores_per_worker=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                     launcher=SimpleLauncher(),