diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py index d6cb0597fc..83ea2e31cf 100644 --- a/parsl/dataflow/dflow.py +++ b/parsl/dataflow/dflow.py @@ -6,7 +6,6 @@ import inspect import logging import os -import pathlib import pickle import random import sys @@ -25,7 +24,6 @@ import parsl from parsl.app.errors import RemoteExceptionWrapper from parsl.app.futures import DataFuture -from parsl.channels import Channel from parsl.config import Config from parsl.data_provider.data_manager import DataManager from parsl.data_provider.files import File @@ -49,7 +47,6 @@ from parsl.monitoring.message_type import MessageType from parsl.monitoring.remote import monitor_wrapper from parsl.process_loggers import wrap_with_logs -from parsl.providers.base import ExecutionProvider from parsl.usage_tracking.usage import UsageTracker from parsl.utils import Timer, get_all_checkpoints, get_std_fname_mode, get_version @@ -1143,36 +1140,6 @@ def log_task_states(self) -> None: logger.info("End of summary") - def _create_remote_dirs_over_channel(self, provider: ExecutionProvider, channel: Channel) -> None: - """Create script directories across a channel - - Parameters - ---------- - provider: Provider obj - Provider for which scripts dirs are being created - channel: Channel obj - Channel over which the remote dirs are to be created - """ - run_dir = self.run_dir - if channel.script_dir is None: - - # This case will be detected as unreachable by mypy, because of - # the type of script_dir, which is str, not Optional[str]. - # The type system doesn't represent the initialized/uninitialized - # state of a channel so cannot represent that a channel needs - # its script directory set or not. - - channel.script_dir = os.path.join(run_dir, 'submit_scripts') # type: ignore[unreachable] - - # Only create dirs if we aren't on a shared-fs - if not channel.isdir(run_dir): - parent, child = pathlib.Path(run_dir).parts[-2:] - remote_run_dir = os.path.join(parent, child) - channel.script_dir = os.path.join(remote_run_dir, 'remote_submit_scripts') - provider.script_dir = os.path.join(run_dir, 'local_submit_scripts') - - channel.makedirs(channel.script_dir, exist_ok=True) - def add_executors(self, executors: Sequence[ParslExecutor]) -> None: for executor in executors: executor.run_id = self.run_id @@ -1186,7 +1153,7 @@ def add_executors(self, executors: Sequence[ParslExecutor]) -> None: executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts') os.makedirs(executor.provider.script_dir, exist_ok=True) - self._create_remote_dirs_over_channel(executor.provider, executor.provider.channel) + executor.provider.channel.script_dir = executor.provider.script_dir self.executors[executor.label] = executor executor.start()