From e0f68dd175a43cab1b492b2b343740ea734dffc1 Mon Sep 17 00:00:00 2001 From: Florian Rupprecht Date: Thu, 23 May 2024 11:23:49 -0400 Subject: [PATCH] Move runners to external package --- src/styx/runners/__init__.py | 1 - src/styx/runners/docker.py | 112 ------------------ tests/test_carg_building.py | 20 ++-- tests/test_default_values.py | 4 +- tests/test_groups.py | 8 +- tests/test_numeric_ranges.py | 12 +- tests/test_output_files.py | 8 +- tests/utils/__init__.py | 1 + .../dummy.py => tests/utils/dummy_runner.py | 0 9 files changed, 27 insertions(+), 139 deletions(-) delete mode 100644 src/styx/runners/__init__.py delete mode 100644 src/styx/runners/docker.py create mode 100644 tests/utils/__init__.py rename src/styx/runners/dummy.py => tests/utils/dummy_runner.py (100%) diff --git a/src/styx/runners/__init__.py b/src/styx/runners/__init__.py deleted file mode 100644 index 490e419..0000000 --- a/src/styx/runners/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Example runners.""" diff --git a/src/styx/runners/docker.py b/src/styx/runners/docker.py deleted file mode 100644 index 71feb72..0000000 --- a/src/styx/runners/docker.py +++ /dev/null @@ -1,112 +0,0 @@ -import pathlib as pl -import re -from collections import deque -from concurrent.futures import ThreadPoolExecutor -from functools import partial -from subprocess import PIPE, CalledProcessError, Popen -from typing import Callable - -from styxdefs import Execution, Metadata, Runner - - -def _docker_mount(host_path: str, container_path: str, readonly: bool) -> str: - host_path = host_path.replace('"', r"\"") - container_path = container_path.replace('"', r"\"") - host_path = host_path.replace("\\", "\\\\") - container_path = container_path.replace("\\", "\\\\") - return f"type=bind,source={host_path},target={container_path}{',readonly' if readonly else ''}" - - -class DockerExecution(Execution): - def __init__(self, metadata: Metadata, output_dir: pl.Path) -> None: - self.metadata = metadata - self.input_files: list[tuple[pl.Path, str]] = [] - self.input_file_next_id = 0 - self.output_files: list[tuple[pl.Path, str]] = [] - self.output_file_next_id = 0 - self.output_dir = output_dir - - def input_file(self, host_file: pl.Path | str) -> str: - _host_file = pl.Path(host_file) - local_file = f"/styx_input/{self.input_file_next_id}/{_host_file.name}" - self.input_file_next_id += 1 - self.input_files.append((_host_file, local_file)) - return local_file - - def output_file(self, local_file: str, optional: bool = False) -> pl.Path: - return self.output_dir / local_file - - def run(self, cargs: list[str]) -> None: - mounts: list[str] = [] - - for i, (host_file, local_file) in enumerate(self.input_files): - mounts.append("--mount") - mounts.append(_docker_mount(host_file.absolute().as_posix(), local_file, readonly=True)) - - # Output directory - self.output_dir.mkdir(parents=True, exist_ok=True) - - mounts.append("--mount") - mounts.append(_docker_mount(self.output_dir.absolute().as_posix(), "/styx_output", readonly=False)) - - docker_extra_args: list[str] = [] - container = self.metadata.container_image_tag - - if container is None: - raise ValueError("No container image tag specified in metadata") - - docker_command = [ - "docker", - "run", - "--rm", - "-w", - "/styx_output", - *mounts, - "--entrypoint", - "/bin/bash", - *docker_extra_args, - container, - "-l", - "-c", - " ".join(cargs), - ] - - print(f"Executing docker command: '{docker_command}'") - - def stdout_handler(line: str) -> None: - print(line) - - def stderr_handler(line: str) -> None: - print(line) - - with Popen(docker_command, text=True, stdout=PIPE, stderr=PIPE) as process: - with ThreadPoolExecutor(2) as pool: # two threads to handle the streams - exhaust = partial(pool.submit, partial(deque, maxlen=0)) - exhaust(stdout_handler(line[:-1]) for line in process.stdout) # type: ignore - exhaust(stderr_handler(line[:-1]) for line in process.stderr) # type: ignore - return_code = process.poll() - if return_code: - raise CalledProcessError(return_code, process.args) - - -def _default_execution_output_dir(metadata: Metadata) -> pl.Path: - filesafe_name = re.sub(r"\W+", "_", metadata.name) - return pl.Path(f"output_{filesafe_name}") - - -class DockerRunner(Runner): - def __init__(self, execution_output_dir: Callable[[Metadata], pl.Path] | None = None) -> None: - """Create a new DockerRunner. - - Args: - execution_output_dir: A function that returns the output directory for a given Metadata. - If None, a folder named 'output_' will be used. - This function is called once before every execution. - """ - self.execution_output_dir: Callable[[Metadata], pl.Path] = ( - _default_execution_output_dir if execution_output_dir is None else execution_output_dir - ) - - def start_execution(self, metadata: Metadata) -> Execution: - output_dir = self.execution_output_dir(metadata) - return DockerExecution(metadata, output_dir) diff --git a/tests/test_carg_building.py b/tests/test_carg_building.py index 6edeba3..870c855 100644 --- a/tests/test_carg_building.py +++ b/tests/test_carg_building.py @@ -2,7 +2,7 @@ import styx.compiler.core import styx.compiler.settings -import styx.runners.dummy +import tests.utils.dummy_runner from tests.utils.dynmodule import ( BT_TYPE_FILE, BT_TYPE_FLAG, @@ -30,7 +30,7 @@ def test_positional_string_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x="my_string") assert dummy_runner.last_cargs is not None @@ -54,7 +54,7 @@ def test_positional_number_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x="123") assert dummy_runner.last_cargs is not None @@ -78,7 +78,7 @@ def test_positional_file_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x="/my/file.txt") assert dummy_runner.last_cargs is not None @@ -103,7 +103,7 @@ def test_flag_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x="my_string") assert dummy_runner.last_cargs is not None @@ -128,7 +128,7 @@ def test_named_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x="my_string") assert dummy_runner.last_cargs is not None @@ -162,7 +162,7 @@ def test_list_of_strings_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x=["my_string1", "my_string2"], y=["my_string3", "my_string4"]) assert dummy_runner.last_cargs is not None @@ -196,7 +196,7 @@ def test_list_of_numbers_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x=[1, 2], y=[3, 4]) assert dummy_runner.last_cargs is not None @@ -221,7 +221,7 @@ def test_static_args() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner, x="my_string") assert dummy_runner.last_cargs is not None @@ -266,7 +266,7 @@ def test_arg_order() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy("aaa", "bbb", runner=dummy_runner) assert dummy_runner.last_cargs is not None diff --git a/tests/test_default_values.py b/tests/test_default_values.py index 132d961..b7702de 100644 --- a/tests/test_default_values.py +++ b/tests/test_default_values.py @@ -2,7 +2,7 @@ import styx.compiler.core import styx.compiler.settings -import styx.runners.dummy +import tests.utils.dummy_runner from tests.utils.dynmodule import ( BT_TYPE_STRING, boutiques_dummy, @@ -28,7 +28,7 @@ def test_default_string_arg() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() test_module.dummy(runner=dummy_runner) assert dummy_runner.last_cargs is not None diff --git a/tests/test_groups.py b/tests/test_groups.py index 9a6a5c1..44d4e23 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -4,7 +4,7 @@ import styx.compiler.core import styx.compiler.settings -import styx.runners.dummy +import tests.utils.dummy_runner from tests.utils.dynmodule import ( BT_TYPE_NUMBER, boutiques_dummy, @@ -57,7 +57,7 @@ def test_mutually_exclusive() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=1, y=2) @@ -89,7 +89,7 @@ def test_all_or_none() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=1, y=2) with pytest.raises(ValueError): @@ -118,7 +118,7 @@ def test_one_required() -> None: print(compiled_module) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner) diff --git a/tests/test_numeric_ranges.py b/tests/test_numeric_ranges.py index e6ea115..ea95e6a 100644 --- a/tests/test_numeric_ranges.py +++ b/tests/test_numeric_ranges.py @@ -4,7 +4,7 @@ import styx.compiler.core import styx.compiler.settings -import styx.runners.dummy +import tests.utils.dummy_runner from tests.utils.dynmodule import ( BT_TYPE_NUMBER, boutiques_dummy, @@ -31,7 +31,7 @@ def test_below_range_minimum_inclusive() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=4) @@ -55,7 +55,7 @@ def test_above_range_maximum_inclusive() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=6) @@ -80,7 +80,7 @@ def test_above_range_maximum_exclusive() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=5) @@ -105,7 +105,7 @@ def test_below_range_minimum_exclusive() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=5) @@ -130,7 +130,7 @@ def test_outside_range() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() with pytest.raises(ValueError): test_module.dummy(runner=dummy_runner, x=11) diff --git a/tests/test_output_files.py b/tests/test_output_files.py index 0c9c644..0c73432 100644 --- a/tests/test_output_files.py +++ b/tests/test_output_files.py @@ -2,7 +2,7 @@ import styx.compiler.core import styx.compiler.settings -import styx.runners.dummy +import tests.utils.dummy_runner from tests.utils.dynmodule import ( BT_TYPE_FILE, BT_TYPE_NUMBER, @@ -35,7 +35,7 @@ def test_output_file() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() out = test_module.dummy(runner=dummy_runner, x=5) assert dummy_runner.last_cargs is not None @@ -68,7 +68,7 @@ def test_output_file_with_template() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() out = test_module.dummy(runner=dummy_runner, x=5) assert dummy_runner.last_cargs is not None @@ -102,7 +102,7 @@ def test_output_file_with_template_and_stripped_extensions() -> None: compiled_module = styx.compiler.core.compile_boutiques_dict(model) test_module = dynamic_module(compiled_module, "test_module") - dummy_runner = styx.runners.dummy.DummyRunner() + dummy_runner = tests.utils.dummy_runner.DummyRunner() out = test_module.dummy(runner=dummy_runner, x="in.txt") assert dummy_runner.last_cargs is not None diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..6742ae6 --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1 @@ +"""Test utilities.""" diff --git a/src/styx/runners/dummy.py b/tests/utils/dummy_runner.py similarity index 100% rename from src/styx/runners/dummy.py rename to tests/utils/dummy_runner.py