From b75a3ae7acfd417f35fc07319ffcccafc642716d Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Mon, 19 Aug 2024 02:36:03 -0400 Subject: [PATCH 1/4] function data in temp --- parsl/executors/taskvine/executor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parsl/executors/taskvine/executor.py b/parsl/executors/taskvine/executor.py index 2e1efb211f..891899f6fc 100644 --- a/parsl/executors/taskvine/executor.py +++ b/parsl/executors/taskvine/executor.py @@ -18,6 +18,7 @@ import threading import uuid from concurrent.futures import Future +from datetime import datetime from typing import List, Literal, Optional, Union # Import other libraries @@ -215,7 +216,8 @@ def __create_data_and_logging_dirs(self): # Create directories for data and results log_dir = os.path.join(run_dir, self.label) - self._function_data_dir = os.path.join(run_dir, self.label, "function_data") + tmp_dir = os.path.join('/tmp/', f'{self.label}-{os.getlogin()}') + self._function_data_dir = os.path.join(tmp_dir, datetime.now().strftime('%Y%m%d%H%M%S%f'), "function_data") os.makedirs(log_dir) os.makedirs(self._function_data_dir) From 2c18d6cadc433e7fd959321cab6578120a9f7642 Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Mon, 19 Aug 2024 10:42:48 -0400 Subject: [PATCH 2/4] use getpass for username --- parsl/executors/taskvine/executor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parsl/executors/taskvine/executor.py b/parsl/executors/taskvine/executor.py index 891899f6fc..b242c51dac 100644 --- a/parsl/executors/taskvine/executor.py +++ b/parsl/executors/taskvine/executor.py @@ -3,6 +3,7 @@ high-throughput system for delegating Parsl tasks to thousands of remote machines """ +import getpass import hashlib import inspect import itertools @@ -216,7 +217,7 @@ def __create_data_and_logging_dirs(self): # Create directories for data and results log_dir = os.path.join(run_dir, self.label) - tmp_dir = os.path.join('/tmp/', f'{self.label}-{os.getlogin()}') + tmp_dir = os.path.join('/tmp/', f'{self.label}-{getpass.getuser()}') self._function_data_dir = os.path.join(tmp_dir, datetime.now().strftime('%Y%m%d%H%M%S%f'), "function_data") os.makedirs(log_dir) os.makedirs(self._function_data_dir) From c201ec1a68fcba827a001070bf5831ea7f354db1 Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Tue, 20 Aug 2024 10:28:46 -0400 Subject: [PATCH 3/4] use tempfile module --- parsl/executors/taskvine/executor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parsl/executors/taskvine/executor.py b/parsl/executors/taskvine/executor.py index b242c51dac..73e80086ab 100644 --- a/parsl/executors/taskvine/executor.py +++ b/parsl/executors/taskvine/executor.py @@ -217,10 +217,10 @@ def __create_data_and_logging_dirs(self): # Create directories for data and results log_dir = os.path.join(run_dir, self.label) - tmp_dir = os.path.join('/tmp/', f'{self.label}-{getpass.getuser()}') - self._function_data_dir = os.path.join(tmp_dir, datetime.now().strftime('%Y%m%d%H%M%S%f'), "function_data") os.makedirs(log_dir) - os.makedirs(self._function_data_dir) + + tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-' + self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix) # put TaskVine logs outside of a Parsl run as TaskVine caches between runs while # Parsl does not. @@ -230,7 +230,7 @@ def __create_data_and_logging_dirs(self): # factory logs go with manager logs regardless self.factory_config.scratch_dir = self.manager_config.vine_log_dir - logger.debug(f"Function data directory: {self._function_data_dir}, log directory: {log_dir}") + logger.debug(f"Function data directory: {self._function_data_dir.name}, log directory: {log_dir}") logger.debug( f"TaskVine manager log directory: {self.manager_config.vine_log_dir}, " f"factory log directory: {self.factory_config.scratch_dir}") @@ -296,7 +296,7 @@ def _path_in_task(self, executor_task_id, *path_components): 'map': Pickled file with a dict between local parsl names, and remote taskvine names. """ task_dir = "{:04d}".format(executor_task_id) - return os.path.join(self._function_data_dir, task_dir, *path_components) + return os.path.join(self._function_data_dir.name, task_dir, *path_components) def submit(self, func, resource_specification, *args, **kwargs): """Processes the Parsl app by its arguments and submits the function From 9f6b0377d17218e28e7f848cf4e9cc7018fb85c8 Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Tue, 20 Aug 2024 10:34:14 -0400 Subject: [PATCH 4/4] flake etc --- parsl/executors/taskvine/executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/parsl/executors/taskvine/executor.py b/parsl/executors/taskvine/executor.py index 73e80086ab..a15a444d2c 100644 --- a/parsl/executors/taskvine/executor.py +++ b/parsl/executors/taskvine/executor.py @@ -218,7 +218,6 @@ def __create_data_and_logging_dirs(self): # Create directories for data and results log_dir = os.path.join(run_dir, self.label) os.makedirs(log_dir) - tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-' self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix)