diff --git a/README.md b/README.md index f2719ca..c22e3da 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,33 @@ # Spinner -![Logo](spinner.svg) +drawing + +## Contributting +Read the [Contribution Guidelines](docs/contribute.md). ## Setting up environment ```sh +python3 -m ensurepip +python3 -m pip3 install virtualenv python3 -m virtualenv .venv -python3 -m pip install pip --upgrade -python -m pip install ".[dev]" source .venv/bin/activate +python3 -m pip install pip --upgrade +python -m pip install . ``` ## Running -```sh -python3 spinner/main.py -b F -r T -c spinner/bench_settings.yaml -e F -python3 spinner/main.py -g T -s 1000000000 -o random_numbers.bin -r T -c spinner/bench_settings.yaml -e T -h "sdumont[6174-6177]" -``` +Check examples in the docs folder. Check plotting examples in the notebook `spinner/exporter/reporter.ipynb`. This is the notebook that the command `--export` uses. -# Mandatory development policy -`.git/hooks/pre-commit` ```sh -#!/bin/bash - -if git rev-parse --verify HEAD >/dev/null 2>&1; then - against=HEAD -else - # Initial commit: diff against an empty tree object - against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 -fi - -# Function to format all C/C++ files using clang-format -format_c_cpp_files() { - if command -v clang-format >/dev/null 2>&1; then - for file in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.(c|h|cpp|hpp)$'); do - clang-format -i "$file" - git add "$file" - done - else - echo "clang-format not found, skipping C/C++ file formatting." - fi -} - -# Function to format all Python files using black -format_python_files() { - if command -v black >/dev/null 2>&1; then - for file in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.py$'); do - black "$file" - isort "$file" - git add "$file" - done - else - echo "black not found, skipping Python file formatting." - fi - if command -v black >/dev/null 2>&1; then - for file in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.ipynb$'); do - black "$file" - isort "$file" - git add "$file" - done - else - echo "black for notebook not found, skipping Python file formatting." - fi -} - -# Function to clear output in Jupyter notebooks -clear_notebook_output() { - if command -v jupyter >/dev/null 2>&1; then - for notebook in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.ipynb$'); do - jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace "$notebook" - git add "$notebook" - done - else - echo "jupyter not found, skipping Jupyter notebook output clearing." - fi -} - -# Run the formatting functions -format_c_cpp_files -format_python_files -clear_notebook_output - -# Check for changes -exec git diff-index --check --cached "$against" -- +spinner --help +Usage: spinner [OPTIONS] + +Options: + -c, --config TEXT Benchmark configuration file + -r, --run BOOLEAN Run all benchmarks + -e, --export BOOLEAN Export results to report.html + -h, --hosts TEXT Hosts list + --help Show this message and exit. ``` diff --git a/docs/contribute.md b/docs/contribute.md new file mode 100644 index 0000000..766e111 --- /dev/null +++ b/docs/contribute.md @@ -0,0 +1,80 @@ +# Contribute to Spinner + +## Setting up the environment as a developer + +```sh +python3 -m ensurepip +python3 -m pip3 install virtualenv +python3 -m virtualenv .venv +source .venv/bin/activate +python3 -m pip install pip --upgrade +python -m pip install -e ".[dev]" +``` + +## Mandatory lint policy + +We recomend the following commit hook (`.git/hooks/pre-commit`) +```sh +#!/bin/bash + +if git rev-parse --verify HEAD >/dev/null 2>&1; then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 +fi + +# Function to format all C/C++ files using clang-format +format_c_cpp_files() { + if command -v clang-format >/dev/null 2>&1; then + for file in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.(c|h|cpp|hpp)$'); do + clang-format -i "$file" + git add "$file" + done + else + echo "clang-format not found, skipping C/C++ file formatting." + fi +} + +# Function to format all Python files using black +format_python_files() { + if command -v black >/dev/null 2>&1; then + for file in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.py$'); do + black "$file" + isort "$file" + git add "$file" + done + else + echo "black not found, skipping Python file formatting." + fi + if command -v black >/dev/null 2>&1; then + for file in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.ipynb$'); do + black "$file" + isort "$file" + git add "$file" + done + else + echo "black for notebook not found, skipping Python file formatting." + fi +} + +# Function to clear output in Jupyter notebooks +clear_notebook_output() { + if command -v jupyter >/dev/null 2>&1; then + for notebook in $(git diff --cached --name-only --diff-filter=ACM "$against" | grep -E '\.ipynb$'); do + jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace "$notebook" + git add "$notebook" + done + else + echo "jupyter not found, skipping Jupyter notebook output clearing." + fi +} + +# Run the formatting functions +format_c_cpp_files +format_python_files +clear_notebook_output + +# Check for changes +exec git diff-index --check --cached "$against" -- +``` \ No newline at end of file diff --git a/docs/examples/capture_output.yaml b/docs/examples/capture_output.yaml new file mode 100644 index 0000000..8c0e5cb --- /dev/null +++ b/docs/examples/capture_output.yaml @@ -0,0 +1,41 @@ +metadata: + description: shell time capture output + version: "1.0" + runs: 5 + timeout: 7 + retry: True + retry_limit: 0 + + sleep_test: + command: + template: > + bash -c "time sleep {{sleep_ammount}}" + + output: + - type: contains + pattern: "real" + to_float: + name: real_time + lambda: > + lambda x: ( + float(x.split("m")[0].split("\t")[1]) * 60 + + + float(x.split("m")[1].split("s")[0]) + ) + + + plot_axis: + - title: Real Time vs Sleep arg + x_axis: sleep_ammount + y_axis: real_time + group_by: sleep_ammount + + - title: Measure Time vs sleep arg + x_axis: sleep_ammount + y_axis: time + group_by: sleep_ammount + +sleep_test: + sleep_ammount: + - 0.31 + - 0.28 diff --git a/docs/examples/sleep_benchmark.yaml b/docs/examples/sleep_benchmark.yaml new file mode 100644 index 0000000..cf648f8 --- /dev/null +++ b/docs/examples/sleep_benchmark.yaml @@ -0,0 +1,19 @@ +metadata: + description: Sleep check + version: "1.0" + runs: 2 + timeout: 5 + retry: True + retry_limit: 0 + + sleep_test: + command: + template: > + sleep {{sleep_ammount}} && printf " + I slept {{sleep_ammount}} + " + +sleep_test: + sleep_ammount: + - 1 + - 2 diff --git a/docs/examples/timeout_test.yaml b/docs/examples/timeout_test.yaml new file mode 100644 index 0000000..3a7095a --- /dev/null +++ b/docs/examples/timeout_test.yaml @@ -0,0 +1,20 @@ +metadata: + description: Timeout test + version: "1.0" + runs: 1 + timeout: 5 + retry: False + retry_limit: 0 + + sleep_test: + command: + template: > + sleep {{sleep_ammount}} && printf " + I slept {{sleep_ammount}} + # this may not print due to timeout! + " + +sleep_test: + sleep_ammount: + - 1 + - 200 diff --git a/docs/future.MD b/docs/future.MD deleted file mode 100644 index 460bd5a..0000000 --- a/docs/future.MD +++ /dev/null @@ -1,175 +0,0 @@ -# Future - -This contains our view of the Spinner application, when ready. - -## Possible usages - -```sh -spinner --use-slurm sorgan.json --group-jobs-by nodes,repetition -c benchmark.json -r T -``` - -This will launch one separeted Slurm job per input configuration, but inputs that are repetition (for each nrun) and inputs with specific number of nodes will be aggregated on the same slurm job. - -```sh -spinner --use-slurm sorgan.json --group-jobs-by nodes,repetition -c benchmark.json -r T --dry-run --assert-environmnet -``` - -This will assert that the enrivonment is corret: LD_LIBRARY_PATH, if binaries are there, if we have the permissions on the partitions. There is a fixed list, common to all benchmarks, and a list that inherited classes can provide, additionaly. The dry-run flag make it so nothing is actually ran, only reported. - - -## Features - -- Asserts - - Benchmark-wise: check if commads are feasible before start (i.e. check for libraries and build binaries) - - System-wise: check if current number of nodes are avaliable - -- Runner - - Collect metrics and save to dataframe - - Track progress - - Automatically launch Slurm, or mpirun directly if run from allocation (e.g. check for env vars in slurm envinroments) - -- Resiliency - - Keep going of excepetion happens at one execution (and notify user) - - Save status after each run to PKL file - -- Profile - - Automatic plot of total time - - Automatic instrumentation stacked plot (using JSON expression match) - - Metadata tracker: instrument enviroment and save to PKL with each dataframe of all executions - - Statistics: use best-practices for confidence interval (not using mean, not using std dev - use Bootstrap intead) - - -## Configurations (JSON) - -Cluster config: account, partitions and limits. Templates for launching applications: `srun`, or `mpirun` (or neither). They should prefix the application binary if the application specifies a launcher in their JSON. - -Application config: sweep parameters, their usage in the invocation (template, JINJA for example). Profiling and instrumentation: if on, a REGEXP on how to grep the output of the application to plot stacked bars. - -Example: - -### Application Configuration JSON - -```json -{ - "metadata": { - "description": "I/O benchmarks", - "version": "1.0", - "runs": 30, - "timeout": 600, - "input_file": "random_numbers.bin", - "command_template": "srun --nodes={{ nodes }} --ntasks={{ tasks }} --time={{ timeout }} {{ executable }}", - "use_allocated_node": false, - "cluster_config": "cluster_config.json", - "profiling": { - "enabled": true, - "patterns": { - "execution_time": "Execution time: (\\d+\\.\\d+) seconds", - "memory_usage": "Memory usage: (\\d+\\.\\d+) MB" - } - }, - "omp-tasks": { - "build": { - "cmake_flags": ["-DCMAKE_BUILD_TYPE=Release"], - "make_flags": ["-j"], - "env": {"CC": "gcc", "CXX": "g++"}, - "path": "benchs/omp-tasks" - } - }, - "mpi-io": { - "build": { - "cmake_flags": ["-DCMAKE_BUILD_TYPE=Release"], - "make_flags": ["-j"], - "env": {"CC": "mpicc", "CXX": "mpicxx"}, - "path": "benchs/mpi-io" - } - } - }, - "omp-tasks": { - "tasks": [1, 2, 32, 64], - "read_step": [100, 100000, 1000000] - }, - "mpi-io": { - "nodes": [1, 2], - "procs": [1, 8, 32], - "read_step": [100, 100000, 1000000] - } -} -``` - -### Cluster Configuration JSON - -```json -{ - "srun_account": "default_account", - "partition": "default_partition", - "time_limit": "01:00:00" -} -``` - -## CLI - -Command line options: - -- `--config`, `-c`: Path to the benchmark configuration file. Default is `bench_settings.json`. -- `--build`, `-b`: Boolean flag to build all benchmarks. Default is `False`. -- `--output`, `-o`: Path to the output file. -- `--run`, `-r`: Boolean flag to run all benchmarks. Default is `False`. -- `--export`, `-e`: Boolean flag to export results to `report.html`. Default is `True`. -- `--hosts`, `-h`: Comma-separated list of hosts. -- `--dry-run`, `-d`: Boolean flag to perform a dry run without executing the commands. Default is `False`. - -## Modules architecture and signatures - -### Modules - -1. **runner/config_loader.py** - - Load and validate configuration files. - -2. **runner/command_executor.py** - - Execute commands and handle dry runs. - -3. **runner/profiler.py** - - Parse profiling information from command output. - -4. **runner/benchmark_builder.py** - - Build benchmarks. - -5. **runner/benchmark_runner.py** - - Run benchmarks and handle results. - -### Function Signatures - -#### config_loader.py - -```python -def load_config(config_path: str) -> dict: - """Load and validate the configuration file.""" -``` - -#### command_executor.py - -```python -def execute_command(template: str, context: dict, dry_run: bool) -> tuple: - """Execute a command using a Jinja template and context.""" -``` - -#### profiler.py - -```python -def parse_profiling_info(output: str, patterns: dict) -> dict: - """Parse profiling information from command output using regex patterns.""" -``` - -#### benchmark_builder.py - -```python -def build_all(config: dict): - """Build all benchmarks as specified in the configuration.""" -``` - -#### benchmark_runner.py - -```python -def run_benchmarks(config: dict, hosts: str): - """Run all benchmarks as specified in the configuration.""" -``` \ No newline at end of file diff --git a/docs/slurm.MD b/docs/slurm.MD new file mode 100644 index 0000000..5550ffd --- /dev/null +++ b/docs/slurm.MD @@ -0,0 +1,29 @@ +# Using with SLURM + +Spinner is designed to work well with Slurm, but for simplicity it does not directly launch or interact with jobs. + +```sh +#!/bin/bash +#SBATCH --nodes=4 +#SBATCH --partition=gpu-partition +#SBATCH --time=04:00:00 +#SBATCH --job-name=cool-benchmark +#SBATCH --output=job_log_%j.out + +set -e + +# Load required modules +module purge +module load cmake/3.27.9 +module load mpich/4.0.2-cuda-12.4.0-ucx +module load singularity/3.7.1 + +# Load Spinner env +source spinner/.venv/bin/activate + +# Get MPI host lsit from Slurm and format it for MPI +host_list=$(scontrol show hostname $(echo "$SLURM_JOB_NODELIST" | head -n 4 | tr '\n' ',' | sed 's/,$//')) + +spinner -c bench_settings.yaml -r T -e T --hosts $host_list + +``` \ No newline at end of file diff --git a/spinner.png b/spinner.png new file mode 100644 index 0000000..0005c55 Binary files /dev/null and b/spinner.png differ diff --git a/spinner.svg b/spinner.svg deleted file mode 100644 index 4c856cc..0000000 --- a/spinner.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/spinner/bench_settings.yaml b/spinner/bench_settings.yaml deleted file mode 100644 index 0bbe52b..0000000 --- a/spinner/bench_settings.yaml +++ /dev/null @@ -1,51 +0,0 @@ -metadata: - description: I/O benchmarks - version: "1.0" - runs: 2 - timeout: 600 - input_file: random_numbers.bin - omp-tasks: - bench_path: benchs/omp-tasks - env: - CC: gcc - CXX: g++ - build_instructions: - - command: mkdir -p build - cwd: "{{bench_path}}" - - command: cmake .. -DCMAKE_BUILD_TYPE=Release - cwd: "{{bench_path}}/build" - - command: make -j - cwd: "{{bench_path}}/build" - command: - template: "{{bench_path}}/build/lustre1 -t {{tasks}} -r {{read_step}} -f {{input_file}}" - mpi-io: - bench_path: benchs/mpi-io - env: - CC: mpicc - CXX: mpicxx - build_instructions: - - command: mkdir -p build - cwd: "{{bench_path}}" - - command: cmake .. -DCMAKE_BUILD_TYPE=Release - cwd: "{{bench_path}}/build" - - command: make -j - cwd: "{{bench_path}}/build" - command: - template: "mpirun -np {{procs}} -ppn {{ppn}} -hosts {{hosts}} {{bench_path}}/build/mpi_io_count {{input_file}} {{read_step}}" - -omp-tasks: - tasks: - - 1 - - 2 - read_step: - - 100 - - 101 - -mpi-io: - nodes: - - 1 - procs: - - 1 - - 2 - read_step: - - 100