diff --git a/.gitignore b/.gitignore index 90bb4dc..be5dccc 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,15 @@ cython_debug/ # ignore all .out files generated by jsi runs **/*.out + +# Generated by Cargo +# will have compiled files and executables +**/debug/ +**/target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/jsi-client-rs/Cargo.toml b/jsi-client-rs/Cargo.toml new file mode 100644 index 0000000..8a34637 --- /dev/null +++ b/jsi-client-rs/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "jsif" +version = "0.1.0" +edition = "2021" +description = "just solve it fast - rust client for the jsi daemon" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/jsi-client-rs/src/main.rs b/jsi-client-rs/src/main.rs new file mode 100644 index 0000000..fd7de11 --- /dev/null +++ b/jsi-client-rs/src/main.rs @@ -0,0 +1,49 @@ +use std::env; +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::process; +use std::time::Instant; + +fn get_server_home() -> Option { + env::var_os("HOME").map(|home| { + let mut path = PathBuf::from(home); + path.push(".jsi"); + path.push("daemon"); + path + }) +} + +fn send_command(command: &str) -> Result<(), Box> { + let socket_path = get_server_home().unwrap().join("server.sock"); + let mut stream = UnixStream::connect(socket_path)?; + + // Send the command + stream.write_all(command.as_bytes())?; + stream.flush()?; + + // Read the response + let start = Instant::now(); + let mut response = String::new(); + stream.read_to_string(&mut response)?; + + println!("{}", response); + println!("; response time: {:?}", start.elapsed()); + Ok(()) +} + +fn main() { + let args: Vec = env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: {} ", args[0]); + process::exit(1); + } + + let command = args[1..].join(" "); + + match send_command(&command) { + Ok(_) => (), + Err(e) => eprintln!("Error: {}", e), + } +} diff --git a/pyproject.toml b/pyproject.toml index c7b8040..506f714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ authors = [ dependencies = [ "pyright>=1.1.378", "rich>=13.8.1", + "python-daemon>=3.0.1", ] readme = "README.md" requires-python = ">= 3.11" diff --git a/requirements-dev.lock b/requirements-dev.lock index bf381c4..7641834 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -14,8 +14,12 @@ click==8.1.7 colorama==0.4.6 ; sys_platform == 'win32' or platform_system == 'Windows' # via click # via pytest +docutils==0.21.2 + # via python-daemon iniconfig==2.0.0 # via pytest +lockfile==0.12.2 + # via python-daemon markdown-it-py==3.0.0 # via rich mdurl==0.1.2 @@ -31,5 +35,9 @@ pygments==2.18.0 pyright==1.1.378 # via jsi pytest==8.3.2 +python-daemon==3.0.1 + # via jsi rich==13.8.1 # via jsi +setuptools==75.1.0 + # via python-daemon diff --git a/requirements.lock b/requirements.lock index 6854d9c..be5ed9d 100644 --- a/requirements.lock +++ b/requirements.lock @@ -10,6 +10,10 @@ # universal: false -e file:. +docutils==0.21.2 + # via python-daemon +lockfile==0.12.2 + # via python-daemon markdown-it-py==3.0.0 # via rich mdurl==0.1.2 @@ -20,7 +24,11 @@ pygments==2.18.0 # via rich pyright==1.1.382.post0 # via jsi +python-daemon==3.0.1 + # via jsi rich==13.8.1 # via jsi +setuptools==75.1.0 + # via python-daemon typing-extensions==4.12.2 # via pyright diff --git a/src/jsi/cli.py b/src/jsi/cli.py index 71366cd..c5f72b8 100644 --- a/src/jsi/cli.py +++ b/src/jsi/cli.py @@ -55,13 +55,15 @@ import threading from functools import partial -from jsi.config.loader import Config, SolverDefinition, load_definitions +from jsi.config.loader import Config, find_available_solvers, load_definitions from jsi.core import ( Command, ProcessController, Task, TaskResult, TaskStatus, + base_commands, + set_input_output, ) from jsi.utils import ( LogLevel, @@ -99,53 +101,6 @@ def get_status(): return NoopStatus() -def find_available_solvers( - solver_definitions: dict[str, SolverDefinition], -) -> dict[str, str]: - if os.path.exists(solver_paths): - stderr.print(f"loading solver paths from cache ({solver_paths})") - import json - - with open(solver_paths) as f: - try: - paths = json.load(f) - except json.JSONDecodeError as err: - logger.error(f"error loading solver cache: {err}") - paths = {} - - if paths: - return paths - - stderr.print("looking for solvers available on PATH:") - paths: dict[str, str] = {} - - import shutil - - for solver_name, solver_def in solver_definitions.items(): - path = shutil.which(solver_def.executable) # type: ignore - - if path is None: - stderr.print(f"{solver_name:>12} not found") - continue - - paths[solver_name] = path - stderr.print(f"{solver_name:>12} [green]OK[/green]") - - stderr.print() - - # save the paths to the solver_paths file - if paths: - import json - - if not os.path.exists(jsi_home): - os.makedirs(jsi_home) - - with open(solver_paths, "w") as f: - json.dump(paths, f) - - return paths - - def setup_signal_handlers(controller: ProcessController): event = threading.Event() @@ -221,6 +176,8 @@ def parse_args(args: list[str]) -> Config: config.csv = True case "--supervisor": config.supervisor = True + case "--daemon": + config.daemon = True case "--timeout": config.timeout_seconds = parse_time(args[i]) i += 1 @@ -239,14 +196,15 @@ def parse_args(args: list[str]) -> Config: config.input_file = arg - if not config.input_file: - raise BadParameterError("no input file provided") + if not config.daemon: + if not config.input_file: + raise BadParameterError("no input file provided") - if not os.path.exists(config.input_file): - raise BadParameterError(f"input file does not exist: {config.input_file}") + if not os.path.exists(config.input_file): + raise BadParameterError(f"input file does not exist: {config.input_file}") - if not os.path.isfile(config.input_file): - raise BadParameterError(f"input file is not a file: {config.input_file}") + if not os.path.isfile(config.input_file): + raise BadParameterError(f"input file is not a file: {config.input_file}") if config.output_dir and not os.path.exists(config.output_dir): raise BadParameterError(f"output directory does not exist: {config.output_dir}") @@ -261,7 +219,7 @@ def parse_args(args: list[str]) -> Config: raise BadParameterError(f"invalid interval value: {config.interval_seconds}") # output directory defaults to the parent of the input file - if config.output_dir is None: + if config.output_dir is None and config.input_file: config.output_dir = os.path.dirname(config.input_file) return config @@ -301,8 +259,16 @@ def main(args: list[str] | None = None) -> int: if config.debug: logger.enable(console=stderr, level=LogLevel.DEBUG) + if config.daemon: + import jsi.server + + # this detaches the server from the current shell, + # this returns immediately, leaving the server running in the background + jsi.server.daemonize(config) + return 0 + with timer("load_config"): - solver_definitions = load_definitions() + solver_definitions = load_definitions(config) if not solver_definitions: stderr.print("error: no solver definitions found", style="red") @@ -310,62 +276,31 @@ def main(args: list[str] | None = None) -> int: with timer("find_available_solvers"): # maps solver name to executable path - available_solvers: dict[str, str] = find_available_solvers(solver_definitions) + available_solvers = find_available_solvers(solver_definitions, config) if not available_solvers: stderr.print("error: no solvers found on PATH", style="red") return 1 # build the commands to run the solvers - file = config.input_file - output = config.output_dir - - assert file is not None - assert output is not None - - commands: list[Command] = [] - basename = os.path.basename(file) - # run the solvers in the specified sequence, or fallback to the default order - for solver_name in config.sequence or available_solvers: - solver_def: SolverDefinition | None = solver_definitions.get(solver_name) - if not solver_def: - stderr.print(f"error: unknown solver: {solver_name}", style="red") - return 1 - - executable_path = available_solvers[solver_name] - args = [executable_path] - - # append the model option if requested - if config.model: - if model_arg := solver_def.model: - args.append(model_arg) - else: - stderr.print(f"warn: solver {solver_name} has no model option") - - # append solver-specific extra arguments - args.extend(solver_def.args) - - command = Command( - name=solver_name, - args=args, - input_file=file, - ) - - stdout_file = os.path.join(output, f"{basename}.{solver_name}.out") - stderr_file = os.path.join(output, f"{basename}.{solver_name}.err") - command.stdout = open(stdout_file, "w") # noqa: SIM115 - command.stderr = open(stderr_file, "w") # noqa: SIM115 - commands.append(command) + commands: list[Command] = base_commands( + config.sequence or list(available_solvers.keys()), + solver_definitions, + available_solvers, + config, + ) + + set_input_output(commands, config) # initialize the controller - task = Task(name=str(file)) + task = Task(name=str(config.input_file)) controller = ProcessController(task, commands, config, get_exit_callback()) setup_signal_handlers(controller) stderr.print(f"starting {len(commands)} solvers") - stderr.print(f"output will be written to: {output}{os.sep}") + stderr.print(f"output will be written to: {config.output_dir}{os.sep}") status = get_status() try: # all systems go @@ -414,7 +349,12 @@ def main(args: list[str] | None = None) -> int: from jsi.output.basic import get_results_csv csv = get_results_csv(controller) - csv_file = os.path.join(output, f"{basename}.csv") + + assert config.input_file is not None + assert config.output_dir is not None + + basename = os.path.basename(config.input_file) + csv_file = os.path.join(config.output_dir, f"{basename}.csv") stderr.print(f"writing results to: {csv_file}") with open(csv_file, "w") as f: f.write(csv) diff --git a/src/jsi/client.py b/src/jsi/client.py new file mode 100644 index 0000000..d063930 --- /dev/null +++ b/src/jsi/client.py @@ -0,0 +1,26 @@ +# client.py + +import socket +import sys + +from jsi.server import SOCKET_PATH + + +def send_command(command: str): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + try: + client.connect(SOCKET_PATH.as_posix()) + client.sendall(command.encode()) + response = client.recv(4096).decode() + print(response) + except OSError as e: + print(f"Error: {e}") + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python client.py ") + sys.exit(1) + + command = " ".join(sys.argv[1:]) + send_command(command) diff --git a/src/jsi/config/definitions.json b/src/jsi/config/definitions.json index 949c07a..20b7cc2 100644 --- a/src/jsi/config/definitions.json +++ b/src/jsi/config/definitions.json @@ -33,7 +33,10 @@ "executable": "z3", "model": "--model", "args": [] + }, + "always-sat": { + "executable": "echo", + "model": null, + "args": ["sat", "\n; input:"] } } - - diff --git a/src/jsi/config/loader.py b/src/jsi/config/loader.py index 87d91af..9d7e201 100644 --- a/src/jsi/config/loader.py +++ b/src/jsi/config/loader.py @@ -1,12 +1,18 @@ import json import os from collections.abc import Sequence +from importlib.abc import Traversable from importlib.resources import files from jsi.utils import Printable, get_consoles, logger, simple_stderr, simple_stdout class Config: + jsi_home: str + definitions_file: str + solver_paths: str + definitions_default_path: Traversable + stdout: Printable stderr: Printable @@ -22,6 +28,7 @@ def __init__( sequence: Sequence[str] | None = None, model: bool = False, csv: bool = False, + daemon: bool = False, ): self.early_exit = early_exit self.timeout_seconds = timeout_seconds @@ -33,9 +40,16 @@ def __init__( self.sequence = sequence self.model = model self.csv = csv + self.daemon = daemon self.stdout = simple_stdout self.stderr = simple_stderr + # global defaults + self.jsi_home = os.path.expanduser("~/.jsi") + self.solver_paths = os.path.join(self.jsi_home, "solvers.json") + self.definitions_file = os.path.join(self.jsi_home, "definitions.json") + self.definitions_default_path = files("jsi.config").joinpath("definitions.json") + def setup_consoles(self): self.stdout, self.stderr = get_consoles() @@ -71,15 +85,69 @@ def parse_definitions(data: dict[str, object]) -> dict[str, SolverDefinition]: } -def load_definitions() -> dict[str, SolverDefinition]: +def load_definitions(config: Config) -> dict[str, SolverDefinition]: _, stderr = get_consoles() - custom_path = os.path.expanduser("~/.jsi/definitions.json") + custom_path = config.definitions_file if os.path.exists(custom_path): logger.debug(f"Loading definitions from {custom_path}") with open(custom_path) as f: return parse_definitions(json.load(f)) - stderr.print(f"no custom definitions file found ({custom_path}), loading default") - data = files("jsi.config").joinpath("definitions.json").read_text() + default_path = config.definitions_default_path + stderr.print(f"no custom definitions file found ('{custom_path}')") + stderr.print(f"loading defaults ('{default_path}')") + + data = default_path.read_text() return parse_definitions(json.loads(data)) + + +def find_available_solvers( + solver_definitions: dict[str, SolverDefinition], + config: Config, +) -> dict[str, str]: + stderr = config.stderr + + solver_paths = config.solver_paths + if os.path.exists(solver_paths): + stderr.print(f"loading solver paths from cache ('{solver_paths}')") + import json + + with open(solver_paths) as f: + try: + paths = json.load(f) + except json.JSONDecodeError as err: + logger.error(f"error loading solver cache: {err}") + paths = {} + + if paths: + return paths + + stderr.print("looking for solvers available on PATH:") + paths: dict[str, str] = {} + + import shutil + + for solver_name, solver_def in solver_definitions.items(): + path = shutil.which(solver_def.executable) # type: ignore + + if path is None: + stderr.print(f"{solver_name:>12} not found") + continue + + paths[solver_name] = path + stderr.print(f"{solver_name:>12} [green]OK[/green]") + + stderr.print() + + # save the paths to the solver_paths file + if paths: + import json + + if not os.path.exists(config.jsi_home): + os.makedirs(config.jsi_home) + + with open(solver_paths, "w") as f: + json.dump(paths, f) + + return paths diff --git a/src/jsi/core.py b/src/jsi/core.py index acbb4e1..6796a95 100644 --- a/src/jsi/core.py +++ b/src/jsi/core.py @@ -10,7 +10,7 @@ from enum import Enum from subprocess import PIPE, Popen, TimeoutExpired -from jsi.config.loader import Config +from jsi.config.loader import Config, SolverDefinition from jsi.utils import logger, timer sat, unsat, error, unknown, timeout, killed = ( @@ -384,6 +384,57 @@ def pid(self): return self._process.pid +def base_commands( + solver_names: Sequence[str], + solver_definitions: dict[str, SolverDefinition], + available_solvers: dict[str, str], + config: Config, +) -> list[Command]: + """Command "templates" corresponding to a particular configuration, + but without input and output filenames.""" + commands: list[Command] = [] + + for solver_name in solver_names: + solver_def: SolverDefinition | None = solver_definitions.get(solver_name) + if not solver_def: + raise RuntimeError(f"unknown solver: {solver_name}") + + executable_path = available_solvers[solver_name] + args = [executable_path] + + # append the model option if requested + if config.model and (model_arg := solver_def.model): + args.append(model_arg) + + # append solver-specific extra arguments + args.extend(solver_def.args) + commands.append( + Command( + name=solver_name, + args=args, + ) + ) + + return commands + + +def set_input_output(commands: list[Command], config: Config): + file = config.input_file + output = config.output_dir + + assert file is not None + assert output is not None + + basename = os.path.basename(file) + + for command in commands: + command.input_file = file + stdout_file = os.path.join(output, f"{basename}.{command.name}.out") + stderr_file = os.path.join(output, f"{basename}.{command.name}.err") + command.stdout = open(stdout_file, "w") # noqa: SIM115 + command.stderr = open(stderr_file, "w") # noqa: SIM115 + + class Task: """Mutable class that keeps track of a high level task (query to be solved), involving potentially multiple solver subprocesses. diff --git a/src/jsi/server.py b/src/jsi/server.py new file mode 100644 index 0000000..cd66120 --- /dev/null +++ b/src/jsi/server.py @@ -0,0 +1,226 @@ +""" +A daemon that listens for requests on a unix socket. + +Can be started with: + + # with a command line interface to parse the config: + jsi [options] --daemon + + # or with a default config: + python -m jsi.server + +These commands return immediately, as a detached daemon runs in the background. + +The daemon: +- checks if there is an existing daemon (as indicated by ~/.jsi/daemon/server.pid) +- it kills the existing daemon if found +- it writes its own pid to ~/.jsi/daemon/server.pid +- it outputs logs to ~/.jsi/daemon/server.{err,out} +- it listens for requests on a unix domain socket (by default ~/.jsi/daemon/server.sock) +- each request is a single line of text, the path to a file to solve +- for each request, it runs the sequence of solvers defined in the config +- it returns the output of the solvers, based on the config +- it runs until terminated by the user or another daemon +""" + +import asyncio +import contextlib +import os +import signal +import threading +from pathlib import Path + +import daemon # type: ignore + +from jsi.config.loader import ( + Config, + SolverDefinition, + find_available_solvers, + load_definitions, +) +from jsi.core import ( + Command, + ProcessController, + Task, + base_commands, + set_input_output, +) +from jsi.utils import get_consoles, pid_exists, unexpand_home + +SERVER_HOME = Path.home() / ".jsi" / "daemon" +SOCKET_PATH = SERVER_HOME / "server.sock" +STDOUT_PATH = SERVER_HOME / "server.out" +STDERR_PATH = SERVER_HOME / "server.err" +PID_PATH = SERVER_HOME / "server.pid" +CONN_BUFFER_SIZE = 1024 + + +unexpanded_pid = unexpand_home(PID_PATH) +server_usage = f"""[bold white]starting daemon...[/] + +- tail logs: + [green]tail -f {unexpand_home(STDERR_PATH)[:-4]}.{{err,out}}[/] + +- view pid of running daemon: + [green]cat {unexpanded_pid}[/] + +- display useful info about current daemon: + [green]ps -o pid,etime,command -p $(cat {unexpanded_pid})[/] + +- terminate daemon (gently, with SIGTERM): + [green]kill $(cat {unexpanded_pid})[/] + +- terminate daemon (forcefully, with SIGKILL): + [green]kill -9 $(cat {unexpanded_pid})[/] + +(use the commands above to monitor the daemon, this process will exit immediately)""" + + +class ResultListener: + def __init__(self): + self.event = threading.Event() + self._result: str | None = None + + def exit_callback(self, command: Command, task: Task): + if self.event.is_set(): + return + + if command.done() and command.ok() and (stdout_text := command.stdout_text): + self.event.set() + self._result = f"{stdout_text.strip()}\n; (result from {command.name})" + name, result = command.name, command.result() + print(f"{name} returned {result} in {command.elapsed():.03f}s") + + @property + def result(self) -> str: + self.event.wait() + + assert self._result is not None + return self._result + + +class PIDFile: + def __init__(self, path: Path): + self.path = path + # don't get the pid here, as it may not be the current pid anymore + # by the time we enter the context manager + + def __enter__(self): + try: + with open(self.path) as fd: + print(f"pid file already exists: {self.path}") + other_pid = fd.read() + + if pid_exists(int(other_pid)): + print(f"killing existing daemon ({other_pid=})") + os.kill(int(other_pid), signal.SIGKILL) + + else: + print(f"pid file points to dead daemon ({other_pid=})") + except FileNotFoundError: + # pid file doesn't exist, we're good to go + pass + + # overwrite the file if it already exists + pid = os.getpid() + with open(self.path, "w") as fd: + fd.write(str(pid)) + + print(f"created pid file: {self.path} ({pid=})") + return self.path + + def __exit__(self, exc_type, exc_value, traceback): # type: ignore + print(f"removing pid file: {self.path}") + + # ignore if the file was already removed + with contextlib.suppress(FileNotFoundError): + os.remove(self.path) + + +class Server: + solver_definitions: dict[str, SolverDefinition] + available_solvers: dict[str, str] + config: Config + + def __init__(self, config: Config): + self.config = config + self.solver_definitions = load_definitions(config) + self.available_solvers = find_available_solvers(self.solver_definitions, config) + + async def start(self): + server = await asyncio.start_unix_server( + self.handle_client, path=str(SOCKET_PATH) + ) + + async with server: + print(f"server started on {unexpand_home(SOCKET_PATH)}") + await server.serve_forever() + + async def handle_client( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ): + try: + data: bytes = await reader.read(1024) + if data: + message: str = data.decode() + result = await self.solve(message) + writer.write(result.encode()) + await writer.drain() + except Exception as e: + print(f"Error handling client: {e}") + finally: + writer.close() + await writer.wait_closed() + + async def solve(self, file: str) -> str: + # Assuming solve is CPU-bound, we use run_in_executor + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(None, self.sync_solve, file) + return result + + def sync_solve(self, file: str) -> str: + # initialize the controller + task = Task(name=str(file)) + + # FIXME: don't mutate the config + self.config.input_file = file + self.config.output_dir = os.path.dirname(file) + + commands = base_commands( + self.config.sequence or list(self.available_solvers.keys()), + self.solver_definitions, + self.available_solvers, + self.config, + ) + set_input_output(commands, self.config) + + listener = ResultListener() + controller = ProcessController( + task, commands, self.config, exit_callback=listener.exit_callback + ) + controller.start() + + return listener.result + + +def daemonize(config: Config): + stdout, _ = get_consoles() + stdout.print(server_usage) + + async def run_server(): + server = Server(config) + await server.start() + + stdout_file = open(STDOUT_PATH, "w+") # noqa: SIM115 + stderr_file = open(STDERR_PATH, "w+") # noqa: SIM115 + + with daemon.DaemonContext( + stdout=stdout_file, + stderr=stderr_file, + pidfile=PIDFile(PID_PATH), + ): + asyncio.run(run_server()) + + +if __name__ == "__main__": + daemonize(Config()) diff --git a/src/jsi/utils.py b/src/jsi/utils.py index b1bc7e1..95c2012 100644 --- a/src/jsi/utils.py +++ b/src/jsi/utils.py @@ -6,6 +6,7 @@ import time from datetime import datetime from enum import Enum +from pathlib import Path class Closeable: @@ -32,6 +33,10 @@ def is_terminal(self) -> bool: return False +def unexpand_home(path: str | Path) -> str: + return str(path).replace(str(Path.home()), "~") + + def is_terminal(file: object) -> bool: return hasattr(file, "isatty") and file.isatty() # type: ignore diff --git a/tests/test_cli.py b/tests/test_cli.py index ba8b9c2..0664f89 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,7 @@ import pytest from jsi.cli import BadParameterError, parse_args -from jsi.config.loader import load_definitions +from jsi.config.loader import Config, load_definitions def capture_stdout( @@ -48,7 +48,7 @@ def test_cli_version(): def test_load_definitions(): - definitions = load_definitions() + definitions = load_definitions(Config()) assert "z3" in definitions assert "bitwuzla" in definitions assert "cvc4" in definitions