diff --git a/.gitignore b/.gitignore index a955ea3dd..84b1804f3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ deb/ **.DS_Store out.txt +out.json # Default docker volume data/ diff --git a/changelog.md b/changelog.md index 338196861..ec4ae0318 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,8 @@ title: "Changelog" heading: "Bencher Changelog" sortOrder: 4 --- +## Pending `v0.3.13` +- Add ability for `bencher run` to read output from a file without running a command argument ## `v0.3.12` - Change Metric `lower_bound` and `upper_bound` to `lower_value` and `upper_value` respectively diff --git a/services/cli/src/bencher/sub/project/run/error.rs b/services/cli/src/bencher/sub/project/run/error.rs index fdc8150ed..18c2c392c 100644 --- a/services/cli/src/bencher/sub/project/run/error.rs +++ b/services/cli/src/bencher/sub/project/run/error.rs @@ -33,8 +33,6 @@ pub enum RunError { #[error("Failed to run command due to a non-zero exit code: {0}")] ExitStatus(crate::bencher::sub::Output), - #[error("Failed to open output file: {0}")] - OutputFileOpen(std::io::Error), #[error("Failed to read from output file: {0}")] OutputFileRead(std::io::Error), diff --git a/services/cli/src/bencher/sub/project/run/runner/mod.rs b/services/cli/src/bencher/sub/project/run/runner/mod.rs index 366bd335b..3ce5acd9b 100644 --- a/services/cli/src/bencher/sub/project/run/runner/mod.rs +++ b/services/cli/src/bencher/sub/project/run/runner/mod.rs @@ -1,9 +1,4 @@ -use std::{ - convert::{TryFrom, TryInto}, - fs::File, - io::{BufRead, BufReader}, - path::PathBuf, -}; +use std::{convert::TryFrom, path::PathBuf}; use crate::parser::project::run::CliRunCommand; @@ -24,16 +19,23 @@ pub enum Runner { Pipe(Pipe), Command(Command), CommandToFile(Command, PathBuf), + File(PathBuf), } impl TryFrom for Runner { type Error = RunError; - fn try_from(mut command: CliRunCommand) -> Result { - if let Some(cmd) = command.cmd.take() { - (command, cmd).try_into() - } else if let Ok(cmd) = std::env::var(BENCHER_CMD) { - (command, cmd).try_into() + fn try_from(command: CliRunCommand) -> Result { + let cmd_str = command.cmd.or_else(|| std::env::var(BENCHER_CMD).ok()); + if let Some(cmd_str) = cmd_str { + let cmd = Command::try_from((command.shell, cmd_str))?; + Ok(if let Some(file) = command.file { + Self::CommandToFile(cmd, file) + } else { + Self::Command(cmd) + }) + } else if let Some(file) = command.file { + Ok(Self::File(file)) } else if let Some(pipe) = Pipe::new() { Ok(Self::Pipe(pipe)) } else { @@ -42,41 +44,26 @@ impl TryFrom for Runner { } } -impl TryFrom<(CliRunCommand, String)> for Runner { - type Error = RunError; - - fn try_from((command, cmd): (CliRunCommand, String)) -> Result { - let cmd = Command::try_from((command.shell, cmd))?; - Ok(if let Some(file) = command.file { - Self::CommandToFile(cmd, file) - } else { - Self::Command(cmd) - }) - } -} - impl Runner { pub async fn run(&self) -> Result { Ok(match self { Self::Pipe(pipe) => pipe.output(), Self::Command(command) => command.run().await?, Self::CommandToFile(command, file_path) => { - let mut output: Output = command.run().await?; - let capacity = std::fs::metadata(file_path) - .ok() - .and_then(|metadata| usize::try_from(metadata.len()).ok()) - .unwrap_or_default(); - let mut result = String::with_capacity(capacity); - - let output_file = File::open(file_path).map_err(RunError::OutputFileOpen)?; - let buffered = BufReader::new(output_file); - for line in buffered.lines() { - result.push_str(&line.map_err(RunError::OutputFileRead)?); - } - + let mut output = command.run().await?; + let result = + std::fs::read_to_string(file_path).map_err(RunError::OutputFileRead)?; output.result = Some(result); output }, + Self::File(file_path) => { + let result = + std::fs::read_to_string(file_path).map_err(RunError::OutputFileRead)?; + Output { + result: Some(result), + ..Default::default() + } + }, }) } } diff --git a/services/cli/src/parser/project/run.rs b/services/cli/src/parser/project/run.rs index 32934e643..389ad3bfa 100644 --- a/services/cli/src/parser/project/run.rs +++ b/services/cli/src/parser/project/run.rs @@ -98,7 +98,7 @@ pub struct CliRunBranch { #[derive(Args, Debug)] pub struct CliRunCommand { /// Benchmark command output file path - #[clap(long, requires = "cmd")] + #[clap(long)] pub file: Option, #[clap(flatten)] diff --git a/services/console/src/content/explanation/bencher-run.mdx b/services/console/src/content/explanation/bencher-run.mdx index 1c7cd0d85..b45513669 100644 --- a/services/console/src/content/explanation/bencher-run.mdx +++ b/services/console/src/content/explanation/bencher-run.mdx @@ -13,6 +13,7 @@ This page will explain the options, flags, and arguments that can be passed to ` The first and only argument to `bencher run` is the optional benchmark command. This is the command that will be executed, invoking your benchmark harness. +It can also be set using the `BENCHER_CMD` environment variable. The command is executed in a shell, which can be configured with the `--shell` and `--flag` options. Its output is parsed by a benchmark harness adapter, which can be set using the `--adapter` option. However, if the benchmark harness outputs to a file then the `--file` option must also be used to specify the output file path. @@ -27,8 +28,9 @@ The benchmark command can be run multiple times using the `--iter` option, and those results can be folded into a single result using the `--fold` option. If any of the iterations fail, then the entire command is considered to have failed unless the `--allow-failure` flag is set. -If the benchmark command is not specified, then `bencher run` will read from `stdin` instead. -This allows you to pipe the output of another command into `bencher run`. +If the benchmark command is not specified but the `--file` option is, then `bencher run` will read from output file path instead. +If both the benchmark command and `--file` option are not specified, then `bencher run` will read from `stdin` instead. +This allows you to save the output of another command to a file or pipe it into `bencher run`, respectively. ## Options diff --git a/services/console/src/content/how_to/github-actions.mdx b/services/console/src/content/how_to/github-actions.mdx index 00c506e25..55152df92 100644 --- a/services/console/src/content/how_to/github-actions.mdx +++ b/services/console/src/content/how_to/github-actions.mdx @@ -223,13 +223,14 @@ jobs: - uses: bencherdev/bencher@main - name: Track Benchmarks with Bencher run: | - cat $BENCHMARK_RESULTS | bencher run \\ + bencher run \\ --if-branch "${{ env.PR_HEAD }}" \\ --else-if-branch "${{ env.PR_BASE }}" \\ --else-if-branch main \\ --err \\ --github-actions \${{ secrets.GITHUB_TOKEN }} \\ - --ci-number ${{ env.PR_NUMBER }} + --ci-number ${{ env.PR_NUMBER }} \\ + --file $BENCHMARK_RESULTS ```