From 3fa1599477f83214e85adc1127a7aa953304afa4 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Fri, 8 Nov 2024 14:45:44 +0000 Subject: [PATCH 1/3] Remove unused channel directory handling code (#3689) This is part of implementing #3515 channel removal code. These methods became unused in PR #3688. ## Type of change - Code maintenance/cleanup --- parsl/channels/base.py | 28 ---------------------------- parsl/channels/local/local.py | 28 ---------------------------- 2 files changed, 56 deletions(-) diff --git a/parsl/channels/base.py b/parsl/channels/base.py index e8acfc1088..ee0097f0d0 100644 --- a/parsl/channels/base.py +++ b/parsl/channels/base.py @@ -80,31 +80,3 @@ def pull_file(self, remote_source: str, local_dir: str) -> str: destination_path (string) ''' pass - - @abstractmethod - def makedirs(self, path: str, mode: int = 0o511, exist_ok: bool = False) -> None: - """Create a directory. - - If intermediate directories do not exist, they will be created. - - Parameters - ---------- - path : str - Path of directory to create. - mode : int - Permissions (posix-style) for the newly-created directory. - exist_ok : bool - If False, raise an OSError if the target directory already exists. - """ - pass - - @abstractmethod - def isdir(self, path: str) -> bool: - """Return true if the path refers to an existing directory. - - Parameters - ---------- - path : str - Path of directory to check. - """ - pass diff --git a/parsl/channels/local/local.py b/parsl/channels/local/local.py index 40b7eac34f..1d7a15dfdf 100644 --- a/parsl/channels/local/local.py +++ b/parsl/channels/local/local.py @@ -91,34 +91,6 @@ def push_file(self, source, dest_dir): def pull_file(self, remote_source, local_dir): return self.push_file(remote_source, local_dir) - def isdir(self, path): - """Return true if the path refers to an existing directory. - - Parameters - ---------- - path : str - Path of directory to check. - """ - - return os.path.isdir(path) - - def makedirs(self, path, mode=0o700, exist_ok=False): - """Create a directory. - - If intermediate directories do not exist, they will be created. - - Parameters - ---------- - path : str - Path of directory to create. - mode : int - Permissions (posix-style) for the newly-created directory. - exist_ok : bool - If False, raise an OSError if the target directory already exists. - """ - - return os.makedirs(path, mode, exist_ok) - @property def script_dir(self): return self._script_dir From 166cb03ba3b651f088bfb0403d66baee22973241 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Fri, 8 Nov 2024 15:53:51 +0000 Subject: [PATCH 2/3] Fix filesystem radio type annotation: the wrong kind of Queue (#3691) Prior to this PR, filesystem_receiver was annotated as taking queue.Queue. This was incorrect (but not detected or enforced by any tooling - because type annotations are not carried across the invoking Process constructor) The receiver actually takes a multiprocessing.Queue, which is used without a namespace, like the rest of this source file. ## Type of change - Code maintenance/cleanup --- parsl/monitoring/monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsl/monitoring/monitoring.py b/parsl/monitoring/monitoring.py index c63b45611f..a1b20f2705 100644 --- a/parsl/monitoring/monitoring.py +++ b/parsl/monitoring/monitoring.py @@ -258,7 +258,7 @@ def close(self) -> None: @wrap_with_logs -def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]", run_dir: str) -> None: +def filesystem_receiver(logdir: str, q: Queue[TaggedMonitoringMessage], run_dir: str) -> None: logger = set_file_logger("{}/monitoring_filesystem_radio.log".format(logdir), name="monitoring_filesystem_radio", level=logging.INFO) From 3151ca84e083524645e7989c337573f8f773d4cd Mon Sep 17 00:00:00 2001 From: Colin Thomas <33940547+colinthomas-z80@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:31:26 -0500 Subject: [PATCH 3/3] TaskVine Executor: add vine tune to manager config (#3692) # Description Make vine_tune config parameters accessible through the manager config while using the TaskVine executor ## Type of change - New feature --- parsl/executors/taskvine/manager.py | 6 ++++++ parsl/executors/taskvine/manager_config.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/parsl/executors/taskvine/manager.py b/parsl/executors/taskvine/manager.py index e5a4986062..94cfb2e391 100644 --- a/parsl/executors/taskvine/manager.py +++ b/parsl/executors/taskvine/manager.py @@ -44,11 +44,17 @@ def _set_manager_attributes(m, config): # Enable peer transfer feature between workers if specified if config.enable_peer_transfers: m.enable_peer_transfers() + else: + m.disable_peer_transfers() # Set catalog report to parsl if project name exists if m.name: m.set_property("framework", "parsl") + if config.tune_parameters is not None: + for k, v in config.tune_parameters.items(): + m.tune(k, v) + def _prepare_environment_serverless(manager_config, env_cache_dir, poncho_create_script): # Return path to a packaged poncho environment diff --git a/parsl/executors/taskvine/manager_config.py b/parsl/executors/taskvine/manager_config.py index 18e58a0b90..0ef112fda4 100644 --- a/parsl/executors/taskvine/manager_config.py +++ b/parsl/executors/taskvine/manager_config.py @@ -156,6 +156,10 @@ class TaskVineManagerConfig: Directory to store TaskVine logging facilities. Default is None, in which all TaskVine logs will be contained in the Parsl logging directory. + + tune_parameters: Optional[dict] + Extended vine_tune parameters, expressed in a dictionary + by { 'tune-parameter' : value }. """ # Connection and communication settings @@ -181,6 +185,7 @@ class TaskVineManagerConfig: autocategory: bool = True enable_peer_transfers: bool = True wait_for_workers: Optional[int] = None + tune_parameters: Optional[dict] = None # Logging settings vine_log_dir: Optional[str] = None