diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e74b6bb6f8..33af0b8ec7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -99,6 +99,7 @@ jobs: tag: ${{ github.event.inputs.tag }} build-conda: + needs: lint runs-on: ubuntu-latest timeout-minutes: 60 env: diff --git a/taskvine/src/bindings/python3/ndcctools/taskvine/task.py b/taskvine/src/bindings/python3/ndcctools/taskvine/task.py index cbb478b9a5..c1c9d5a41d 100644 --- a/taskvine/src/bindings/python3/ndcctools/taskvine/task.py +++ b/taskvine/src/bindings/python3/ndcctools/taskvine/task.py @@ -124,7 +124,7 @@ def _free(self): self._task = None @staticmethod - def _determine_mount_flags(watch=False, failure_only=False, success_only=False, strict_input=False): + def _determine_mount_flags(watch=False, failure_only=False, success_only=False, strict_input=False, mount_symlink=False): flags = cvine.VINE_TRANSFER_ALWAYS if watch: flags |= cvine.VINE_WATCH @@ -134,6 +134,8 @@ def _determine_mount_flags(watch=False, failure_only=False, success_only=False, flags |= cvine.VINE_SUCCESS_ONLY if strict_input: flags |= cvine.VINE_FIXED_LOCATION + if mount_symlink: + flags |= cvine.VINE_MOUNT_SYMLINK return flags @staticmethod @@ -256,12 +258,12 @@ def add_feature(self, name): # >>> f = m.declare_untar(url) # >>> task.add_input(f,"data") # @endcode - def add_input(self, file, remote_name, strict_input=False): + def add_input(self, file, remote_name, strict_input=False, mount_symlink=False): # SWIG expects strings if not isinstance(remote_name, str): raise TypeError(f"remote_name {remote_name} is not a str") - flags = Task._determine_mount_flags(strict_input=strict_input) + flags = Task._determine_mount_flags(strict_input=strict_input, mount_symlink=mount_symlink) if cvine.vine_task_add_input(self._task, file._file, remote_name, flags)==0: raise ValueError("invalid file description") diff --git a/taskvine/src/manager/taskvine.h b/taskvine/src/manager/taskvine.h index fbfabfd736..91c3280681 100644 --- a/taskvine/src/manager/taskvine.h +++ b/taskvine/src/manager/taskvine.h @@ -46,7 +46,8 @@ typedef enum { VINE_WATCH = 2, /**< Watch the output file and send back changes as the task runs. */ VINE_FAILURE_ONLY = 4, /**< Only return this output file if the task failed. (Useful for returning large log files.) */ VINE_SUCCESS_ONLY = 8, /**< Only return this output file if the task succeeded. */ - VINE_RETRACT_ON_RESET = 16 /**< Remove this file from the mount lists if the task is reset. (TaskVine internal use only.) */ + VINE_RETRACT_ON_RESET = 16, /**< Remove this file from the mount lists if the task is reset. (TaskVine internal use only.) */ + VINE_MOUNT_SYMLINK = 32 /**< Permit this directory to be mounted via symlink instead of hardlink. */ } vine_mount_flags_t; /** Control caching and sharing behavior of file objects. diff --git a/taskvine/src/worker/vine_sandbox.c b/taskvine/src/worker/vine_sandbox.c index 59e3ef5848..56b8478e15 100644 --- a/taskvine/src/worker/vine_sandbox.c +++ b/taskvine/src/worker/vine_sandbox.c @@ -81,7 +81,16 @@ static int stage_input_file(struct vine_process *p, struct vine_mount *m, struct if (status == VINE_CACHE_STATUS_READY) { create_dir_parents(sandbox_path, 0777); debug(D_VINE, "input: link %s -> %s", cache_path, sandbox_path); - result = file_link_recursive(cache_path, sandbox_path, vine_worker_symlinks_enabled); + if (m->flags & VINE_MOUNT_SYMLINK) { + /* If the user has requested a symlink, just do that b/c it is faster for large dirs. */ + result = symlink(cache_path, sandbox_path); + /* Change sense of Unix result to true/false. */ + result = !result; + } else { + /* Otherwise recursively hard-link the object into the sandbox. */ + result = file_link_recursive(cache_path, sandbox_path, 1); + } + if (!result) debug(D_VINE, "couldn't link %s into sandbox as %s: %s", diff --git a/taskvine/src/worker/vine_worker.c b/taskvine/src/worker/vine_worker.c index 5aa7e180b4..b56ab6a524 100644 --- a/taskvine/src/worker/vine_worker.c +++ b/taskvine/src/worker/vine_worker.c @@ -117,9 +117,6 @@ static int sigchld_received_flag = 0; // Password shared between manager and worker. char *vine_worker_password = 0; -// Allow worker to use symlinks when link() fails. Enabled by default. -int vine_worker_symlinks_enabled = 1; - int mini_task_id = 0; // Worker id. A unique id for this worker instance. @@ -2123,7 +2120,6 @@ enum { LONG_OPT_DISK, LONG_OPT_DISK_PERCENT, LONG_OPT_GPUS, - LONG_OPT_DISABLE_SYMLINKS, LONG_OPT_IDLE_TIMEOUT, LONG_OPT_CONNECT_TIMEOUT, LONG_OPT_SINGLE_SHOT, @@ -2153,7 +2149,6 @@ static const struct option long_options[] = {{"advertise", no_argument, 0, 'a'}, {"min-backoff", required_argument, 0, 'i'}, {"max-backoff", required_argument, 0, 'b'}, {"single-shot", no_argument, 0, LONG_OPT_SINGLE_SHOT}, - {"disable-symlinks", no_argument, 0, LONG_OPT_DISABLE_SYMLINKS}, {"disk-threshold", required_argument, 0, 'z'}, {"memory-threshold", required_argument, 0, LONG_OPT_MEMORY_THRESHOLD}, {"arch", required_argument, 0, 'A'}, @@ -2332,9 +2327,6 @@ int main(int argc, char *argv[]) warn(D_NOTICE, "Ignoring --wall-time, a positive integer is expected."); } break; - case LONG_OPT_DISABLE_SYMLINKS: - vine_worker_symlinks_enabled = 0; - break; case LONG_OPT_SINGLE_SHOT: single_shot_mode = 1; break; diff --git a/taskvine/src/worker/vine_worker.h b/taskvine/src/worker/vine_worker.h index 26511a22af..845b9cd7be 100644 --- a/taskvine/src/worker/vine_worker.h +++ b/taskvine/src/worker/vine_worker.h @@ -9,7 +9,6 @@ void vine_worker_send_cache_update( struct link *manager, const char *cachename, int64_t size, timestamp_t transfer_time, timestamp_t transfer_start ); void vine_worker_send_cache_invalid( struct link *manager, const char *cachename, const char *message ); -extern int vine_worker_symlinks_enabled; extern char *vine_worker_password; #endif diff --git a/taskvine/test/vine_python.py b/taskvine/test/vine_python.py index 0b6a32537d..a53ff5edca 100755 --- a/taskvine/test/vine_python.py +++ b/taskvine/test/vine_python.py @@ -115,6 +115,19 @@ def next_output_name(): t = q.wait(wait_time) report_task(t, "success", 0, [path.join(test_dir, output_name)]) + # same thing, but this time symlink the input directory. + output_name = next_output_name() + t = vine.Task(f"cd my_dir && ./{exec_name} {input_name} 2>&1 > {output_name}") + in_dir = q.declare_file(test_dir, cache=True) + t.add_input(exec_file, exec_name) + t.add_input(in_dir, "my_dir", mount_symlink=True) + output_file = q.declare_file(path.join(test_dir, output_name), cache=False) + t.add_output(output_file, path.join("my_dir", output_name)) + + q.submit(t) + t = q.wait(wait_time) + report_task(t, "success", 0, [path.join(test_dir, output_name)]) + # we bring back the outputs from a directory: output_name = next_output_name() t = vine.Task(f"mkdir outs && ./{exec_name} {input_name} 2>&1 > outs/{output_name}")