diff --git a/.gitignore b/.gitignore index a1e31910..82fb403d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ **swp /Cargo.lock /src/docs/*/book +/.idea diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a81835..9943fc68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - [#568](https://github.com/tag1consulting/goose/pull/568) don't panic when truncating non utf-8 string - [#574](https://github.com/tag1consulting/goose/pull/574) update [`http`](https://docs.rs/http), [`itertools`](https://docs.rs/itertools) [`nix`](https://docs.rs/nix), [`rustls`](https://docs.rs/rustls/), and [`serial_test`](https://docs.rs/serial_test) - [#575](https://github.com/tag1consulting/goose/pull/575) add test coverage for sessions and cookies, revert [#557](https://github.com/tag1consulting/goose/pull/557) to avoid sharing the CookieJar between all users + - [#600](https://github.com/tag1consulting/goose/pull/600) Refactor reports/metrics, add JSON and markdown report ## 0.17.2 August 28, 2023 - [#557](https://github.com/tag1consulting/goose/pull/557) speed up user initialization on Linux diff --git a/Cargo.toml b/Cargo.toml index 06e18e73..a8f0f831 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ url = "2" [features] default = ["reqwest/default-tls"] rustls-tls = ["reqwest/rustls-tls", "tokio-tungstenite/rustls"] +gaggle = [] [dev-dependencies] httpmock = "0.6" diff --git a/src/config.rs b/src/config.rs index 5a17186f..59a5920e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -100,9 +100,9 @@ pub struct GooseConfiguration { /// Doesn't display an error summary #[options(no_short)] pub no_error_summary: bool, - /// Create an html-formatted report + /// Create reports, can be used multiple times (supports .html, .htm, .md, .json) #[options(no_short, meta = "NAME")] - pub report_file: String, + pub report_file: Vec, /// Disable granular graphs in report file #[options(no_short)] pub no_granular_report: bool, @@ -282,7 +282,7 @@ pub(crate) struct GooseDefaults { /// An optional default for not displaying an error summary. pub no_error_summary: Option, /// An optional default for the html-formatted report file name. - pub report_file: Option, + pub report_file: Option>, /// An optional default for the flag that disables granular data in HTML report graphs. pub no_granular_report: Option, /// An optional default for the requests log file name. @@ -569,7 +569,7 @@ impl GooseDefaultType<&str> for GooseAttack { Some(value.to_string()) } } - GooseDefault::ReportFile => self.defaults.report_file = Some(value.to_string()), + GooseDefault::ReportFile => self.defaults.report_file = Some(vec![value.to_string()]), GooseDefault::RequestLog => self.defaults.request_log = Some(value.to_string()), GooseDefault::ScenarioLog => self.defaults.scenario_log = Some(value.to_string()), GooseDefault::Scenarios => { @@ -1161,6 +1161,24 @@ impl GooseConfigure for GooseConfiguration { None } } +impl GooseConfigure> for GooseConfiguration { + /// Use [`GooseValue`] to set a [`String`] value. + fn get_value(&self, values: Vec>>) -> Option> { + for value in values { + if let Some(v) = value.value { + if value.filter { + continue; + } else { + if !value.message.is_empty() { + info!("{} = {:?}", value.message, v) + } + return Some(v); + } + } + } + None + } +} impl GooseConfigure for GooseConfiguration { /// Use [`GooseValue`] to set a [`bool`] value. fn get_value(&self, values: Vec>) -> Option { @@ -1563,23 +1581,22 @@ impl GooseConfiguration { .unwrap_or(false); // Configure `report_file`. - self.report_file = match self.get_value(vec![ - // Use --report-file if set. - GooseValue { - value: Some(self.report_file.to_string()), - filter: self.report_file.is_empty(), - message: "report_file", - }, - // Otherwise use GooseDefault if set. - GooseValue { - value: defaults.report_file.clone(), - filter: defaults.report_file.is_none(), - message: "report_file", - }, - ]) { - Some(v) => v, - None => "".to_string(), - }; + self.report_file = self + .get_value(vec![ + // Use --report-file if set. + GooseValue { + value: Some(self.report_file.clone()), + filter: self.report_file.is_empty(), + message: "report_file", + }, + // Otherwise use GooseDefault if set. + GooseValue { + value: defaults.report_file.clone(), + filter: defaults.report_file.is_none(), + message: "report_file", + }, + ]) + .unwrap_or_default(); // Configure `no_granular_report`. self.no_debug_body = self @@ -2013,7 +2030,7 @@ impl GooseConfiguration { } else if !self.report_file.is_empty() { return Err(GooseError::InvalidOption { option: "`configuration.report_file`".to_string(), - value: self.report_file.to_string(), + value: format!("{:?}", self.report_file), detail: "`configuration.report_file` can not be set with `configuration.no_metrics`." .to_string(), @@ -2273,7 +2290,7 @@ mod test { assert!(goose_attack.defaults.no_autostart == Some(true)); assert!(goose_attack.defaults.timeout == Some(timeout)); assert!(goose_attack.defaults.no_gzip == Some(true)); - assert!(goose_attack.defaults.report_file == Some(report_file)); + assert!(goose_attack.defaults.report_file == Some(vec![report_file])); assert!(goose_attack.defaults.request_log == Some(request_log)); assert!(goose_attack.defaults.request_format == Some(GooseLogFormat::Raw)); assert!(goose_attack.defaults.error_log == Some(error_log)); diff --git a/src/controller.rs b/src/controller.rs index 46edbe7e..8bf4dbea 100644 --- a/src/controller.rs +++ b/src/controller.rs @@ -33,13 +33,13 @@ use tokio_tungstenite::tungstenite::Message; /// - Commands will be displayed in the help screen in the order defined here, so /// they should be logically grouped. /// 2. Add the new command to `ControllerCommand::details` and populate all -/// `ControllerCommandDetails`, using other commands as an implementation reference. -/// - The `regex` is used to identify the command, and optionally to extract a -/// value (for example see `Hatchrate` and `Users`) -/// - If additional validation is required beyond the regular expression, add -/// the necessary logic to `ControllerCommand::validate_value`. +/// `ControllerCommandDetails`, using other commands as an implementation reference. +/// - The `regex` is used to identify the command, and optionally to extract a +/// value (for example see `Hatchrate` and `Users`) +/// - If additional validation is required beyond the regular expression, add +/// the necessary logic to `ControllerCommand::validate_value`. /// 3. Add any necessary parent process logic for the command to -/// `GooseAttack::handle_controller_requests` (also in this file). +/// `GooseAttack::handle_controller_requests` (also in this file). /// 4. Add a test for the new command in tests/controller.rs. #[derive(Clone, Debug, EnumIter, PartialEq, Eq)] pub enum ControllerCommand { @@ -642,10 +642,8 @@ impl GooseAttack { AttackPhase::Idle => { let current_users = if !self.test_plan.steps.is_empty() { self.test_plan.steps[self.test_plan.current].0 - } else if let Some(users) = self.configuration.users { - users } else { - 0 + self.configuration.users.unwrap_or_default() }; info!( "changing users from {:?} to {}", @@ -1410,13 +1408,7 @@ impl Controller for ControllerState { raw_value: ControllerTelnetMessage, ) -> Result { let command_string = match str::from_utf8(&raw_value) { - Ok(m) => { - if let Some(c) = m.lines().next() { - c - } else { - "" - } - } + Ok(m) => m.lines().next().unwrap_or_default(), Err(e) => { let error = format!("ignoring unexpected input from telnet controller: {}", e); info!("{}", error); diff --git a/src/docs/goose-book/src/config/defaults.md b/src/docs/goose-book/src/config/defaults.md index fc244ec0..d55d952f 100644 --- a/src/docs/goose-book/src/config/defaults.md +++ b/src/docs/goose-book/src/config/defaults.md @@ -18,7 +18,7 @@ The following defaults can be configured with a `&str`: - host: `GooseDefault::Host` - set a per-request timeout: `GooseDefault::Timeout` - users to start per second: `GooseDefault::HatchRate` - - html-formatted report file name: `GooseDefault::ReportFile` + - report file names: `GooseDefault::ReportFile` - goose log file name: `GooseDefault::GooseLog` - request log file name: `GooseDefault::RequestLog` - transaction log file name: `GooseDefault::TransactionLog` @@ -62,7 +62,7 @@ The following defaults can be configured with a `bool`: - enable Manager mode: `GooseDefault::Manager` - enable Worker mode: `GooseDefault::Worker` - ignore load test checksum: `GooseDefault::NoHashCheck` - - do not collect granular data in the HTML report: `GooseDefault::NoGranularData` + - do not collect granular data in the reports: `GooseDefault::NoGranularData` The following defaults can be configured with a `GooseLogFormat`: - request log file format: `GooseDefault::RequestFormat` diff --git a/src/docs/goose-book/src/getting-started/common.md b/src/docs/goose-book/src/getting-started/common.md index aea11fd2..64a5de36 100644 --- a/src/docs/goose-book/src/getting-started/common.md +++ b/src/docs/goose-book/src/getting-started/common.md @@ -82,9 +82,13 @@ cargo run --release -- --iterations 5 ## Writing An HTML-formatted Report -By default, Goose displays [text-formatted metrics](metrics.md) when a load test finishes. It can also optionally write an HTML-formatted report if you enable the `--report-file ` run-time option, where `` is an absolute or relative path to the report file to generate. Any file that already exists at the specified path will be overwritten. +By default, Goose displays [text-formatted metrics](metrics.md) when a load test finishes. -The HTML report includes some graphs that rely on the [eCharts JavaScript library](https://echarts.apache.org). The HTML report loads the library via CDN, which means that the graphs won't be loaded correctly if the CDN is not accessible. +It can also optionally write one or more reports in HTML, Markdown, or JSON format. For that, you need to provide one or more `--report-file ` run-time options. All requested reports will be written. + +The value of `` is an absolute or relative path to the report file to generate. The file extension will evaluate the type of report to write. Any file that already exists at the specified path will be overwritten. + +For more information, see [Metrics Reports](metrics.md#metrics-reports). ![Requests per second graph](rps.png) @@ -94,3 +98,10 @@ _Write an HTML-formatted report to `report.html` when the load test finishes._ ```bash cargo run --release -- --report-file report.html ``` + +### HTML & Markdown report example +_Write a Markdown and an HTML-formatted report when the load test finishes._ + +```bash +cargo run --release -- --report-file report.md --report-file report.html +``` diff --git a/src/docs/goose-book/src/getting-started/metrics.md b/src/docs/goose-book/src/getting-started/metrics.md index 18576e85..8b77e244 100644 --- a/src/docs/goose-book/src/getting-started/metrics.md +++ b/src/docs/goose-book/src/getting-started/metrics.md @@ -289,30 +289,38 @@ All 9 users hatched. ------------------------------------------------------------------------------ ``` -## HTML metrics -In addition to the above metrics displayed on the CLI, we've also told Goose to create an HTML report. +## Metrics reports +In addition to the above metrics displayed on the CLI, we've also told Goose to create reports on other formats, like Markdown, JSON, or HTML. -### Overview +It is possible to create one or more reports at the same time, using one or more `--report-file` arguments. The type of report is chosen by the file extension. An unsupported file extension will lead to an error. + +The following subsections describe the reports on more detail. + +### HTML report + +#### Overview The HTML report starts with a brief overview table, offering the same information found in the [ASCII overview](#ascii-metrics) above: ![Metrics overview](metrics-overview.jpg) -### Requests +**NOTE:** The HTML report includes some graphs that rely on the [eCharts JavaScript library](https://echarts.apache.org). The HTML report loads the library via CDN, which means that the graphs won't be loaded correctly if the CDN is not accessible. + +#### Requests Next the report includes a graph of all requests made during the duration of the load test. By default, the graph includes an aggregated average, as well as per-request details. It's possible to click on the request names at the top of the graph to hide/show specific requests on the graphs. In this case, the graph shows that most requests made by the load test were for static assets. Below the graph is a table that shows per-request details, only partially included in this screenshot: ![Request metrics](metrics-requests.jpg) -### Response times +#### Response times The next graph shows the response times measured for each request made. In the following graph, it's apparent that POST requests had the slowest responses, which is logical as they are not cached. As before, it's possible to click on the request names at the top of the graph to hide/show details about specific requests. Below the graph is a table that shows per-request details: ![Response time metrics](metrics-response-time.jpg) -### Status codes +#### Status codes All status codes returned by the server are displayed in a table, per-request and in aggregate. In our simple test, we received only `200 OK` responses. ![Status code metrics](metrics-status-codes.jpg) -### Transactions +#### Transactions The next graph summarizes all Transactions run during the load test. One or more requests are grouped logically inside Transactions. For example, the Transaction named `0.0 anon /` includes an anonymous (not-logged-in) request for the front page, as well as requests for all static assets found on the front page. Whereas a Request automatically fails based on the web server response code, the code that defines a Transaction must manually return an error for a Task to be considered failed. For example, the logic may be written to fail the Transaction of the html request fails, but not if one or more static asset requests fail. @@ -320,7 +328,7 @@ Whereas a Request automatically fails based on the web server response code, the This graph is also followed by a table showing details on all Transactions, partially shown here: ![Transaction metrics](metrics-transactions.jpg) -### Scenarios +#### Scenarios The next graph summarizes all Scenarios run during the load test. One or more Transactions are grouped logically inside Scenarios. For example, the Scenario named `Anonymous English user` includes the above `anon /` Transaction, the `anon /en/basicpage`, and all the rest of the Transactions requesting pages in English. @@ -330,9 +338,17 @@ It is followed by a table, shown in entirety here because this load test only ha As our example only ran for 60 seconds, and the `Admin user` Scenario took >30 seconds to run once, the load test only ran completely through this scenario one time, also reflected in the following table: ![Scenario metrics](metrics-scenarios.jpg) -### Users +#### Users The final graph shows how many users were running at the various stages of the load test. As configured, Goose quickly ramped up to 9 users, then sustained that level of traffic for a minute before shutting down: ![User metrics](metrics-users.jpg) +### Markdown report + +The Markdown report follows the structure of the [HTML report](#html-report). However, it does not include the chart elements. + +### JSON report + +The JSON report is a dump of the internal metrics collection. It is a JSON serialization of the `ReportData` structure. Mainly having a field named `raw_metrics`, carrying the content of [`GooseMetrics`](https://docs.rs/goose/latest/goose/metrics/struct.GooseMetrics.html). + ### Developer documentation Additional details about how metrics are collected, stored, and displayed can be found [in the developer documentation](https://docs.rs/goose/*/goose/metrics/index.html). diff --git a/src/docs/goose-book/src/getting-started/running.md b/src/docs/goose-book/src/getting-started/running.md index e19f7207..666f10f0 100644 --- a/src/docs/goose-book/src/getting-started/running.md +++ b/src/docs/goose-book/src/getting-started/running.md @@ -14,7 +14,7 @@ Error: InvalidOption { option: "--host", value: "", detail: "A host must be defi The load test fails with an error as it hasn't been told the host you want to load test. -So, let's try again, this time passing in the `--host` flag. We will also add the `--report-file` flag, [which will generate a HTML report](common.html#writing-an-html-formatted-report), and `--no-reset-metrics` to preserve all information including the load test startup. The same information will also [be printed to the command line](metrics.md) (without graphs). After running for a few seconds, press `ctrl-c` one time to gracefully stop the load test: +So, let's try again, this time passing in the `--host` flag. We will also add the `--report-file` flag with a `.html` file extension, [which will generate an HTML report](common.html#writing-an-html-formatted-report), and `--no-reset-metrics` to preserve all information including the load test startup. The same information will also [be printed to the command line](metrics.md) (without graphs). After running for a few seconds, press `ctrl-c` one time to gracefully stop the load test: ```bash % cargo run --release -- --host http://umami.ddev.site --report-file=report.html --no-reset-metrics diff --git a/src/docs/goose-book/src/getting-started/runtime-options.md b/src/docs/goose-book/src/getting-started/runtime-options.md index bfa2e375..d36c5b5a 100644 --- a/src/docs/goose-book/src/getting-started/runtime-options.md +++ b/src/docs/goose-book/src/getting-started/runtime-options.md @@ -33,7 +33,7 @@ Metrics: --no-scenario-metrics Doesn't track scenario metrics --no-print-metrics Doesn't display metrics at end of load test --no-error-summary Doesn't display an error summary - --report-file NAME Create an html-formatted report + --report-file NAME Create reports, can be used multiple times (supports .html, .htm, .md, .json) --no-granular-report Disable granular graphs in report file -R, --request-log NAME Sets request log file name --request-format FORMAT Sets request log format (csv, json, raw, pretty) @@ -69,4 +69,4 @@ Advanced: --accept-invalid-certs Disables validation of https certificates ``` -All of the above configuration options are [defined in the developer documentation](https://docs.rs/goose/*/goose/config/struct.GooseConfiguration.html). \ No newline at end of file +All of the above configuration options are [defined in the developer documentation](https://docs.rs/goose/*/goose/config/struct.GooseConfiguration.html). diff --git a/src/goose.rs b/src/goose.rs index 68c0450f..e785b1a4 100644 --- a/src/goose.rs +++ b/src/goose.rs @@ -522,7 +522,7 @@ impl Scenario { Scenario { name: name.to_string(), machine_name: Scenario::get_machine_name(name), - scenarios_index: usize::max_value(), + scenarios_index: usize::MAX, weight: 1, transaction_wait: None, transactions: Vec::new(), @@ -961,7 +961,7 @@ impl GooseUser { metrics_channel: None, shutdown_channel: None, // A value of max_value() indicates this user isn't fully initialized yet. - weighted_users_index: usize::max_value(), + weighted_users_index: usize::MAX, load_test_hash, request_cadence: GooseRequestCadence::new(), slept: 0, @@ -1144,9 +1144,9 @@ impl GooseUser { /// of precedence: /// 1. `--host` (host specified on the command line when running load test) /// 2. [`Scenario`](./struct.Scenario.html)`.host` (default host defined for the - /// current scenario) + /// current scenario) /// 3. [`GooseDefault::Host`](../config/enum.GooseDefault.html#variant.Host) (default host - /// defined for the current load test) + /// defined for the current load test) pub fn build_url(&self, path: &str) -> Result> { // If URL includes a host, simply use it. if let Ok(parsed_path) = Url::parse(path) { @@ -2177,7 +2177,7 @@ impl GooseUser { /// - A manually built client is specific to a single Goose thread -- if you are /// generating a large load test with many users, each will need to manually build their /// own client (typically you'd do this in a Transaction that is registered with - /// [`Transaction::set_on_start()`] in each Scenario requiring a custom client; + /// [`Transaction::set_on_start()`] in each Scenario requiring a custom client; /// - Manually building a client will completely replace the automatically built client /// with a brand new one, so any configuration, cookies or headers set in the previously /// built client will be gone; @@ -2811,7 +2811,7 @@ impl Transaction { pub fn new(function: TransactionFunction) -> Self { trace!("new transaction"); Transaction { - transactions_index: usize::max_value(), + transactions_index: usize::MAX, name: "".to_string(), weight: 1, sequence: 0, @@ -3081,7 +3081,7 @@ mod tests { let mut scenario = scenario!("foo"); assert_eq!(scenario.name, "foo"); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); assert_eq!(scenario.weight, 1); assert_eq!(scenario.transaction_wait, None); assert!(scenario.host.is_none()); @@ -3094,7 +3094,7 @@ mod tests { scenario = scenario.register_transaction(transaction!(test_function_a)); assert_eq!(scenario.transactions.len(), 1); assert_eq!(scenario.weighted_transactions.len(), 0); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); assert_eq!(scenario.weight, 1); assert_eq!(scenario.transaction_wait, None); assert!(scenario.host.is_none()); @@ -3103,7 +3103,7 @@ mod tests { scenario = scenario.register_transaction(transaction!(test_function_b)); assert_eq!(scenario.transactions.len(), 2); assert_eq!(scenario.weighted_transactions.len(), 0); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); assert_eq!(scenario.weight, 1); assert_eq!(scenario.transaction_wait, None); assert!(scenario.host.is_none()); @@ -3112,7 +3112,7 @@ mod tests { scenario = scenario.register_transaction(transaction!(test_function_a)); assert_eq!(scenario.transactions.len(), 3); assert_eq!(scenario.weighted_transactions.len(), 0); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); assert_eq!(scenario.weight, 1); assert_eq!(scenario.transaction_wait, None); assert!(scenario.host.is_none()); @@ -3122,7 +3122,7 @@ mod tests { assert_eq!(scenario.weight, 50); assert_eq!(scenario.transactions.len(), 3); assert_eq!(scenario.weighted_transactions.len(), 0); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); assert_eq!(scenario.transaction_wait, None); assert!(scenario.host.is_none()); @@ -3136,7 +3136,7 @@ mod tests { assert_eq!(scenario.weight, 5); assert_eq!(scenario.transactions.len(), 3); assert_eq!(scenario.weighted_transactions.len(), 0); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); assert_eq!(scenario.transaction_wait, None); // Host field can be changed. @@ -3155,7 +3155,7 @@ mod tests { assert_eq!(scenario.weight, 5); assert_eq!(scenario.transactions.len(), 3); assert_eq!(scenario.weighted_transactions.len(), 0); - assert_eq!(scenario.scenarios_index, usize::max_value()); + assert_eq!(scenario.scenarios_index, usize::MAX); // Wait time can be changed. scenario = scenario @@ -3178,7 +3178,7 @@ mod tests { // Initialize scenario. let mut transaction = transaction!(test_function_a); - assert_eq!(transaction.transactions_index, usize::max_value()); + assert_eq!(transaction.transactions_index, usize::MAX); assert_eq!(transaction.name, "".to_string()); assert_eq!(transaction.weight, 1); assert_eq!(transaction.sequence, 0); @@ -3254,7 +3254,7 @@ mod tests { let base_url = get_base_url(Some(HOST.to_string()), None, None).unwrap(); let user = GooseUser::new(0, "".to_string(), base_url, &configuration, 0, None).unwrap(); assert_eq!(user.scenarios_index, 0); - assert_eq!(user.weighted_users_index, usize::max_value()); + assert_eq!(user.weighted_users_index, usize::MAX); // Confirm the URLs are correctly built using the default_host. let url = user.build_url("/foo").unwrap(); diff --git a/src/lib.rs b/src/lib.rs index ace680d8..bb4363de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,7 +61,6 @@ use std::hash::{Hash, Hasher}; use std::sync::{atomic::AtomicUsize, Arc, RwLock}; use std::time::{self, Duration}; use std::{fmt, io}; -use tokio::fs::File; use crate::config::{GooseConfiguration, GooseDefaults}; use crate::controller::{ControllerProtocol, ControllerRequest}; @@ -102,6 +101,8 @@ pub enum GooseError { Reqwest(reqwest::Error), /// Wraps a ['tokio::task::JoinError'](https://tokio-rs.github.io/tokio/doc/tokio/task/struct.JoinError.html). TokioJoin(tokio::task::JoinError), + /// Wraps a ['serde_json::Error'](https://docs.rs/serde_json/*/serde_json/struct.Error.html). + Serde(serde_json::Error), /// Failed attempt to use code that requires a compile-time feature be enabled. FeatureNotEnabled { /// The missing compile-time feature. @@ -160,6 +161,7 @@ impl GooseError { match *self { GooseError::Io(_) => "io::Error", GooseError::Reqwest(_) => "reqwest::Error", + GooseError::Serde(_) => "serde_json::Error", GooseError::TokioJoin(_) => "tokio::task::JoinError", GooseError::FeatureNotEnabled { .. } => "required compile-time feature not enabled", GooseError::InvalidHost { .. } => "failed to parse hostname", @@ -228,6 +230,13 @@ impl From for GooseError { } } +/// Auto-convert serde_json errors. +impl From for GooseError { + fn from(err: serde_json::Error) -> GooseError { + GooseError::Serde(err) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] /// A [`GooseAttack`](./struct.GooseAttack.html) load test operates in one (and only one) /// of the following modes. @@ -803,17 +812,6 @@ impl GooseAttack { self.attack_phase = phase; } - // If enabled, returns the path of the report_file, otherwise returns None. - fn get_report_file_path(&mut self) -> Option { - // Return if enabled. - if !self.configuration.report_file.is_empty() { - Some(self.configuration.report_file.to_string()) - // Otherwise there is no report file. - } else { - None - } - } - // Display all scenarios (sorted by machine name). fn print_scenarios(&self) { let mut scenarios = BTreeMap::new(); @@ -951,8 +949,8 @@ impl GooseAttack { ); print!("{}", self.metrics); - // Write an html report, if enabled. - self.write_html_report().await?; + // Write reports + self.write_reports().await?; } Ok(self.metrics) @@ -1164,15 +1162,6 @@ impl GooseAttack { Some(controller_request_rx) } - // Prepare an asynchronous file writer for `report_file` (if enabled). - async fn prepare_report_file(&mut self) -> Result, GooseError> { - if let Some(report_file_path) = self.get_report_file_path() { - Ok(Some(File::create(&report_file_path).await?)) - } else { - Ok(None) - } - } - // Invoke `test_start` transactions if existing. async fn run_test_start(&self) -> Result<(), GooseError> { // First run global test_start_transaction, if defined. @@ -1548,8 +1537,8 @@ impl GooseAttack { if !self.configuration.no_metrics { println!("{}", self.metrics); } - // Write an html report, if enabled. - self.write_html_report().await?; + // Write reports, if enabled. + self.write_reports().await?; // Return to an Idle state. self.set_attack_phase(goose_attack_run_state, AttackPhase::Idle); } @@ -1733,17 +1722,8 @@ impl GooseAttack { goose_attack_run_state.throttle_threads_tx = throttle_threads_tx; goose_attack_run_state.parent_to_throttle_tx = parent_to_throttle_tx; - // If enabled, try to create the report file to confirm access. - let _report_file = match self.prepare_report_file().await { - Ok(f) => f, - Err(e) => { - return Err(GooseError::InvalidOption { - option: "--report-file".to_string(), - value: self.get_report_file_path().unwrap(), - detail: format!("Failed to create report file: {}", e), - }) - } - }; + // Try to create the requested report files, to confirm access. + self.create_reports().await?; // Record when the GooseAttack officially started. self.started = Some(time::Instant::now()); diff --git a/src/metrics.rs b/src/metrics.rs index 86cd5e6a..fe8313a5 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -8,9 +8,14 @@ //! contained [`TransactionMetrics`], [`GooseRequestMetrics`], and //! [`GooseErrorMetrics`] are displayed in tables. +mod common; + +pub(crate) use common::ReportData; + use crate::config::GooseDefaults; use crate::goose::{get_base_url, GooseMethod, Scenario}; use crate::logger::GooseLog; +use crate::metrics::common::ReportOptions; use crate::report; use crate::test_plan::{TestPlanHistory, TestPlanStepAction}; use crate::util; @@ -24,9 +29,13 @@ use serde::ser::SerializeStruct; use serde::{Deserialize, Serialize, Serializer}; use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::ffi::OsStr; use std::fmt::Write; +use std::io::BufWriter; +use std::path::PathBuf; use std::str::FromStr; use std::{f32, fmt}; +use tokio::fs::File; use tokio::io::AsyncWriteExt; /// Used to send metrics from [`GooseUser`](../goose/struct.GooseUser.html) threads @@ -2419,7 +2428,7 @@ impl GooseMetrics { } // Determine the seconds, minutes and hours between two chrono:DateTimes. - fn get_seconds_minutes_hours( + pub(crate) fn get_seconds_minutes_hours( &self, start: &chrono::DateTime, end: &chrono::DateTime, @@ -2994,46 +3003,134 @@ impl GooseAttack { }; } - // Write an HTML-formatted report, if enabled. - pub(crate) async fn write_html_report(&mut self) -> Result<(), GooseError> { - // If enabled, try to create the report file to confirm access. - let report_file = match self.prepare_report_file().await { - Ok(f) => f, - Err(e) => { - return Err(GooseError::InvalidOption { + /// Process all requested reports. + /// + /// If `write` is true, then also write the data. Otherwise just create the files to ensure + /// we have access. + async fn process_reports(&self, write: bool) -> Result<(), GooseError> { + let create = |path: PathBuf| async move { + File::create(&path) + .await + .map_err(|err| GooseError::InvalidOption { option: "--report-file".to_string(), - value: self.get_report_file_path().unwrap(), - detail: format!("Failed to create report file: {}", e), + value: path.to_string_lossy().to_string(), + detail: format!("Failed to create report file: {}", err), }) - } }; + for report in &self.configuration.report_file { + let path = PathBuf::from(report); + match path.extension().map(OsStr::to_string_lossy).as_deref() { + Some("html" | "htm") => { + let file = create(path).await?; + if write { + self.write_html_report(file, report).await?; + } + } + Some("json") => { + let file = create(path).await?; + if write { + self.write_json_report(file).await?; + } + } + Some("md") => { + let file = create(path).await?; + if write { + self.write_markdown_report(file).await?; + } + } + None => { + return Err(GooseError::InvalidOption { + option: "--report-file".to_string(), + value: report.clone(), + detail: "Missing file extension for report".to_string(), + }) + } + Some(ext) => { + return Err(GooseError::InvalidOption { + option: "--report-file".to_string(), + value: report.clone(), + detail: format!("Unknown report file type: {ext}"), + }) + } + } + } + + Ok(()) + } + + /// Create all requested reports, to ensure we have access. + pub(crate) async fn create_reports(&self) -> Result<(), GooseError> { + self.process_reports(false).await + } + + /// Write all requested reports. + pub(crate) async fn write_reports(&self) -> Result<(), GooseError> { + self.process_reports(true).await + } + + /// Write a JSON report. + pub(crate) async fn write_json_report(&self, report_file: File) -> Result<(), GooseError> { + let data = common::prepare_data( + ReportOptions { + no_transaction_metrics: self.configuration.no_transaction_metrics, + no_scenario_metrics: self.configuration.no_scenario_metrics, + no_status_codes: self.configuration.no_status_codes, + }, + &self.metrics, + ); + + serde_json::to_writer_pretty(BufWriter::new(report_file.into_std().await), &data)?; + + Ok(()) + } + + /// Write a Markdown report. + pub(crate) async fn write_markdown_report(&self, report_file: File) -> Result<(), GooseError> { + let data = common::prepare_data( + ReportOptions { + no_transaction_metrics: self.configuration.no_transaction_metrics, + no_scenario_metrics: self.configuration.no_scenario_metrics, + no_status_codes: self.configuration.no_status_codes, + }, + &self.metrics, + ); + + report::write_markdown_report(&mut BufWriter::new(report_file.into_std().await), data) + } + + // Write an HTML-formatted report. + pub(crate) async fn write_html_report( + &self, + mut report_file: File, + path: &str, + ) -> Result<(), GooseError> { // Only write the report if enabled. - if let Some(mut report_file) = report_file { - let test_start_time = self.metrics.history.first().unwrap().timestamp; - - // Prepare report summary variables. - let users = self.metrics.maximum_users.to_string(); - - let mut steps_overview = String::new(); - for step in self.metrics.history.windows(2) { - let (seconds, minutes, hours) = self - .metrics - .get_seconds_minutes_hours(&step[0].timestamp, &step[1].timestamp); - let started = Local - .timestamp_opt(step[0].timestamp.timestamp(), 0) - // @TODO: error handling - .unwrap() - .format("%y-%m-%d %H:%M:%S"); - let stopped = Local - .timestamp_opt(step[1].timestamp.timestamp(), 0) - // @TODO: error handling - .unwrap() - .format("%y-%m-%d %H:%M:%S"); - match &step[0].action { - // For maintaining just show the current number of users. - TestPlanStepAction::Maintaining => { - let _ = write!(steps_overview, + + let test_start_time = self.metrics.history.first().unwrap().timestamp; + + // Prepare report summary variables. + let users = self.metrics.maximum_users.to_string(); + + let mut steps_overview = String::new(); + for step in self.metrics.history.windows(2) { + let (seconds, minutes, hours) = self + .metrics + .get_seconds_minutes_hours(&step[0].timestamp, &step[1].timestamp); + let started = Local + .timestamp_opt(step[0].timestamp.timestamp(), 0) + // @TODO: error handling + .unwrap() + .format("%y-%m-%d %H:%M:%S"); + let stopped = Local + .timestamp_opt(step[1].timestamp.timestamp(), 0) + // @TODO: error handling + .unwrap() + .format("%y-%m-%d %H:%M:%S"); + match &step[0].action { + // For maintaining just show the current number of users. + TestPlanStepAction::Maintaining => { + let _ = write!(steps_overview, "{:?}{}{}{:02}:{:02}:{:02}{}", step[0].action, started, @@ -3043,10 +3140,10 @@ impl GooseAttack { seconds, step[0].users, ); - } - // For increasing show the current number of users to the new number of users. - TestPlanStepAction::Increasing => { - let _ = write!(steps_overview, + } + // For increasing show the current number of users to the new number of users. + TestPlanStepAction::Increasing => { + let _ = write!(steps_overview, "{:?}{}{}{:02}:{:02}:{:02}{} → {}", step[0].action, started, @@ -3057,10 +3154,10 @@ impl GooseAttack { step[0].users, step[1].users, ); - } - // For decreasing show the new number of users from the current number of users. - TestPlanStepAction::Decreasing | TestPlanStepAction::Canceling => { - let _ = write!(steps_overview, + } + // For decreasing show the new number of users from the current number of users. + TestPlanStepAction::Decreasing | TestPlanStepAction::Canceling => { + let _ = write!(steps_overview, "{:?}{}{}{:02}:{:02}:{:02}{} ← {}", step[0].action, started, @@ -3071,479 +3168,116 @@ impl GooseAttack { step[1].users, step[0].users, ); - } - TestPlanStepAction::Finished => { - unreachable!("there shouldn't be a step after finished"); - } } - } - - // Build a comma separated list of hosts. - let hosts = &self.metrics.hosts.clone().into_iter().join(", "); - - // Prepare requests and responses variables. - let mut raw_request_metrics = Vec::new(); - let mut co_request_metrics = Vec::new(); - let mut raw_response_metrics = Vec::new(); - let mut co_response_metrics = Vec::new(); - let mut raw_aggregate_total_count = 0; - let mut co_aggregate_total_count = 0; - let mut raw_aggregate_fail_count = 0; - let mut raw_aggregate_response_time_counter: usize = 0; - let mut raw_aggregate_response_time_minimum: usize = 0; - let mut raw_aggregate_response_time_maximum: usize = 0; - let mut raw_aggregate_response_times: BTreeMap = BTreeMap::new(); - let mut co_aggregate_response_time_counter: usize = 0; - let mut co_aggregate_response_time_maximum: usize = 0; - let mut co_aggregate_response_times: BTreeMap = BTreeMap::new(); - let mut co_data = false; - for (request_key, request) in self.metrics.requests.iter().sorted() { - // Determine whether or not to include Coordinated Omission data. - if !co_data && request.coordinated_omission_data.is_some() { - co_data = true; + TestPlanStepAction::Finished => { + unreachable!("there shouldn't be a step after finished"); } - let method = format!("{}", request.method); - // The request_key is "{method} {name}", so by stripping the "{method} " - // prefix we get the name. - let name = request_key - .strip_prefix(&format!("{} ", request.method)) - .unwrap() - .to_string(); - let total_request_count = request.success_count + request.fail_count; - let (requests_per_second, failures_per_second) = per_second_calculations( - self.metrics.duration, - total_request_count, - request.fail_count, - ); - // Prepare per-request metrics. - raw_request_metrics.push(report::RequestMetric { - method: method.to_string(), - name: name.to_string(), - number_of_requests: total_request_count, - number_of_failures: request.fail_count, - response_time_average: format!( - "{:.2}", - request.raw_data.total_time as f32 / request.raw_data.counter as f32 - ), - response_time_minimum: request.raw_data.minimum_time, - response_time_maximum: request.raw_data.maximum_time, - requests_per_second: format!("{:.2}", requests_per_second), - failures_per_second: format!("{:.2}", failures_per_second), - }); - - // Prepare per-response metrics. - raw_response_metrics.push(report::get_response_metric( - &method, - &name, - &request.raw_data.times, - request.raw_data.counter, - request.raw_data.minimum_time, - request.raw_data.maximum_time, - )); - - // Collect aggregated request and response metrics. - raw_aggregate_total_count += total_request_count; - raw_aggregate_fail_count += request.fail_count; - raw_aggregate_response_time_counter += request.raw_data.total_time; - raw_aggregate_response_time_minimum = update_min_time( - raw_aggregate_response_time_minimum, - request.raw_data.minimum_time, - ); - raw_aggregate_response_time_maximum = update_max_time( - raw_aggregate_response_time_maximum, - request.raw_data.maximum_time, - ); - raw_aggregate_response_times = - merge_times(raw_aggregate_response_times, request.raw_data.times.clone()); - } - - // Prepare aggregate per-request metrics. - let (raw_aggregate_requests_per_second, raw_aggregate_failures_per_second) = - per_second_calculations( - self.metrics.duration, - raw_aggregate_total_count, - raw_aggregate_fail_count, - ); - raw_request_metrics.push(report::RequestMetric { - method: "".to_string(), - name: "Aggregated".to_string(), - number_of_requests: raw_aggregate_total_count, - number_of_failures: raw_aggregate_fail_count, - response_time_average: format!( - "{:.2}", - raw_aggregate_response_time_counter as f32 / raw_aggregate_total_count as f32 - ), - response_time_minimum: raw_aggregate_response_time_minimum, - response_time_maximum: raw_aggregate_response_time_maximum, - requests_per_second: format!("{:.2}", raw_aggregate_requests_per_second), - failures_per_second: format!("{:.2}", raw_aggregate_failures_per_second), - }); - - // Prepare aggregate per-response metrics. - raw_response_metrics.push(report::get_response_metric( - "", - "Aggregated", - &raw_aggregate_response_times, - raw_aggregate_total_count, - raw_aggregate_response_time_minimum, - raw_aggregate_response_time_maximum, - )); - - // Compile the request metrics template. - let mut raw_requests_rows = Vec::new(); - for metric in raw_request_metrics { - raw_requests_rows.push(report::raw_request_metrics_row(metric)); } + } - // Compile the response metrics template. - let mut raw_responses_rows = Vec::new(); - for metric in raw_response_metrics { - raw_responses_rows.push(report::response_metrics_row(metric)); - } - - let co_requests_template: String; - let co_responses_template: String; - if co_data { - for (request_key, request) in self.metrics.requests.iter().sorted() { - if let Some(coordinated_omission_data) = - request.coordinated_omission_data.as_ref() - { - let method = format!("{}", request.method); - // The request_key is "{method} {name}", so by stripping the "{method} " - // prefix we get the name. - let name = request_key - .strip_prefix(&format!("{} ", request.method)) - .unwrap() - .to_string(); - let raw_average = - request.raw_data.total_time as f32 / request.raw_data.counter as f32; - let co_average = coordinated_omission_data.total_time as f32 - / coordinated_omission_data.counter as f32; - // Prepare per-request metrics. - co_request_metrics.push(report::CORequestMetric { - method: method.to_string(), - name: name.to_string(), - response_time_average: format!("{:.2}", co_average), - response_time_standard_deviation: format!( - "{:.2}", - util::standard_deviation(raw_average, co_average) - ), - response_time_maximum: coordinated_omission_data.maximum_time, - }); - - // Prepare per-response metrics. - co_response_metrics.push(report::get_response_metric( - &method, - &name, - &coordinated_omission_data.times, - coordinated_omission_data.counter, - coordinated_omission_data.minimum_time, - coordinated_omission_data.maximum_time, - )); - - // Collect aggregated request and response metrics. - co_aggregate_response_time_counter += coordinated_omission_data.total_time; - co_aggregate_response_time_maximum = update_max_time( - co_aggregate_response_time_maximum, - coordinated_omission_data.maximum_time, - ); - co_aggregate_response_times = merge_times( - co_aggregate_response_times, - coordinated_omission_data.times.clone(), - ); - } - let total_request_count = request.success_count + request.fail_count; - co_aggregate_total_count += total_request_count; - } - let co_average = - co_aggregate_response_time_counter as f32 / co_aggregate_total_count as f32; - let raw_average = - raw_aggregate_response_time_counter as f32 / raw_aggregate_total_count as f32; - co_request_metrics.push(report::CORequestMetric { - method: "".to_string(), - name: "Aggregated".to_string(), - response_time_average: format!( - "{:.2}", - co_aggregate_response_time_counter as f32 / co_aggregate_total_count as f32 - ), - response_time_standard_deviation: format!( - "{:.2}", - util::standard_deviation(raw_average, co_average), - ), - response_time_maximum: co_aggregate_response_time_maximum, - }); + // Build a comma separated list of hosts. + let hosts = &self.metrics.hosts.clone().into_iter().join(", "); + + let ReportData { + raw_metrics: _, + raw_request_metrics, + raw_response_metrics, + co_request_metrics, + co_response_metrics, + scenario_metrics, + transaction_metrics, + errors, + status_code_metrics, + } = common::prepare_data( + ReportOptions { + no_transaction_metrics: self.configuration.no_transaction_metrics, + no_scenario_metrics: self.configuration.no_scenario_metrics, + no_status_codes: self.configuration.no_status_codes, + }, + &self.metrics, + ); - // Prepare aggregate per-response metrics. - co_response_metrics.push(report::get_response_metric( - "", - "Aggregated", - &co_aggregate_response_times, - co_aggregate_total_count, - raw_aggregate_response_time_minimum, - co_aggregate_response_time_maximum, - )); + // Compile the request metrics template. + let mut raw_requests_rows = Vec::new(); + for metric in raw_request_metrics { + raw_requests_rows.push(report::raw_request_metrics_row(metric)); + } - // Compile the co_request metrics rows. - let mut co_request_rows = Vec::new(); - for metric in co_request_metrics { - co_request_rows.push(report::coordinated_omission_request_metrics_row(metric)); - } + // Compile the response metrics template. + let mut raw_responses_rows = Vec::new(); + for metric in raw_response_metrics { + raw_responses_rows.push(report::response_metrics_row(metric)); + } - // Compile the status_code metrics template. - co_requests_template = report::coordinated_omission_request_metrics_template( - &co_request_rows.join("\n"), - ); + let co_requests_template = co_request_metrics + .map(|co_request_metrics| { + let co_request_rows = co_request_metrics + .into_iter() + .map(report::coordinated_omission_request_metrics_row) + .join("\n"); - // Compile the co_request metrics rows. - let mut co_response_rows = Vec::new(); - for metric in co_response_metrics { - co_response_rows - .push(report::coordinated_omission_response_metrics_row(metric)); - } + report::coordinated_omission_request_metrics_template(&co_request_rows) + }) + .unwrap_or_default(); - // Compile the status_code metrics template. - co_responses_template = report::coordinated_omission_response_metrics_template( - &co_response_rows.join("\n"), - ); - } else { - // If --status-codes is not enabled, return an empty template. - co_requests_template = "".to_string(); - co_responses_template = "".to_string(); - } + let co_responses_template = co_response_metrics + .map(|co_response_metrics| { + let co_response_rows = co_response_metrics + .into_iter() + .map(report::coordinated_omission_response_metrics_row) + .join("\n"); - // Only build the transactions template if --no-transaction-metrics isn't enabled. - let transactions_template: String; - if !self.configuration.no_transaction_metrics { - let mut transaction_metrics = Vec::new(); - let mut aggregate_total_count = 0; - let mut aggregate_fail_count = 0; - let mut aggregate_transaction_time_counter: usize = 0; - let mut aggregate_transaction_time_minimum: usize = 0; - let mut aggregate_transaction_time_maximum: usize = 0; - let mut aggregate_transaction_times: BTreeMap = BTreeMap::new(); - for (scenario_counter, scenario) in self.metrics.transactions.iter().enumerate() { - for (transaction_counter, transaction) in scenario.iter().enumerate() { - if transaction_counter == 0 { - // Only the scenario_name is used for scenarios. - transaction_metrics.push(report::TransactionMetric { - is_scenario: true, - transaction: "".to_string(), - name: transaction.scenario_name.to_string(), - number_of_requests: 0, - number_of_failures: 0, - response_time_average: "".to_string(), - response_time_minimum: 0, - response_time_maximum: 0, - requests_per_second: "".to_string(), - failures_per_second: "".to_string(), - }); - } - let total_run_count = transaction.success_count + transaction.fail_count; - let (requests_per_second, failures_per_second) = per_second_calculations( - self.metrics.duration, - total_run_count, - transaction.fail_count, - ); - let average = match transaction.counter { - 0 => 0.00, - _ => transaction.total_time as f32 / transaction.counter as f32, - }; - transaction_metrics.push(report::TransactionMetric { - is_scenario: false, - transaction: format!("{}.{}", scenario_counter, transaction_counter), - name: transaction.transaction_name.to_string(), - number_of_requests: total_run_count, - number_of_failures: transaction.fail_count, - response_time_average: format!("{:.2}", average), - response_time_minimum: transaction.min_time, - response_time_maximum: transaction.max_time, - requests_per_second: format!("{:.2}", requests_per_second), - failures_per_second: format!("{:.2}", failures_per_second), - }); - - aggregate_total_count += total_run_count; - aggregate_fail_count += transaction.fail_count; - aggregate_transaction_times = - merge_times(aggregate_transaction_times, transaction.times.clone()); - aggregate_transaction_time_counter += &transaction.counter; - aggregate_transaction_time_minimum = update_min_time( - aggregate_transaction_time_minimum, - transaction.min_time, - ); - aggregate_transaction_time_maximum = update_max_time( - aggregate_transaction_time_maximum, - transaction.max_time, - ); - } - } + report::coordinated_omission_response_metrics_template(&co_response_rows) + }) + .unwrap_or_default(); - let (aggregate_requests_per_second, aggregate_failures_per_second) = - per_second_calculations( - self.metrics.duration, - aggregate_total_count, - aggregate_fail_count, - ); - transaction_metrics.push(report::TransactionMetric { - is_scenario: false, - transaction: "".to_string(), - name: "Aggregated".to_string(), - number_of_requests: aggregate_total_count, - number_of_failures: aggregate_fail_count, - response_time_average: format!( - "{:.2}", - raw_aggregate_response_time_counter as f32 / aggregate_total_count as f32 - ), - response_time_minimum: aggregate_transaction_time_minimum, - response_time_maximum: aggregate_transaction_time_maximum, - requests_per_second: format!("{:.2}", aggregate_requests_per_second), - failures_per_second: format!("{:.2}", aggregate_failures_per_second), - }); - let mut transactions_rows = Vec::new(); - // Compile the transaction metrics template. - for metric in transaction_metrics { - transactions_rows.push(report::transaction_metrics_row(metric)); - } + let scenarios_template = scenario_metrics + .map(|scenario_metric| { + let scenarios_rows = scenario_metric + .into_iter() + .map(report::scenario_metrics_row) + .join("\n"); - transactions_template = report::transaction_metrics_template( - &transactions_rows.join("\n"), + report::scenario_metrics_template( + &scenarios_rows, self.graph_data - .get_transactions_per_second_graph(!self.configuration.no_granular_report) + .get_scenarios_per_second_graph(!self.configuration.no_granular_report) .get_markup(&self.metrics.history, test_start_time), - ); - } else { - transactions_template = "".to_string(); - } - - // Only build the scenarios template if --no-senario-metrics isn't enabled. - let scenarios_template: String; - if !self.configuration.no_scenario_metrics { - let mut scenario_metrics = Vec::new(); - let mut aggregate_users = 0; - let mut aggregate_count = 0; - let mut aggregate_scenario_time_counter: usize = 0; - let mut aggregate_scenario_time_minimum: usize = 0; - let mut aggregate_scenario_time_maximum: usize = 0; - let mut aggregate_scenario_times: BTreeMap = BTreeMap::new(); - let mut aggregate_iterations = 0.0; - let mut aggregate_response_time_counter = 0.0; - for scenario in &self.metrics.scenarios { - let (count_per_second, _failures_per_second) = - per_second_calculations(self.metrics.duration, scenario.counter, 0); - let average = match scenario.counter { - 0 => 0.00, - _ => scenario.total_time as f32 / scenario.counter as f32, - }; - let iterations = scenario.counter as f32 / scenario.users.len() as f32; - scenario_metrics.push(report::ScenarioMetric { - name: scenario.name.to_string(), - users: scenario.users.len(), - count: scenario.counter, - response_time_average: format!("{:.2}", average), - response_time_minimum: scenario.min_time, - response_time_maximum: scenario.max_time, - count_per_second: format!("{:.2}", count_per_second), - iterations: format!("{:.2}", iterations), - }); - - aggregate_users += scenario.users.len(); - aggregate_count += scenario.counter; - aggregate_scenario_times = - merge_times(aggregate_scenario_times, scenario.times.clone()); - aggregate_scenario_time_counter += &scenario.counter; - aggregate_scenario_time_minimum = - update_min_time(aggregate_scenario_time_minimum, scenario.min_time); - aggregate_scenario_time_maximum = - update_max_time(aggregate_scenario_time_maximum, scenario.max_time); - aggregate_iterations += iterations; - aggregate_response_time_counter += scenario.total_time as f32; - } + ) + }) + .unwrap_or_default(); - let (aggregate_count_per_second, _aggregate_failures_per_second) = - per_second_calculations(self.metrics.duration, aggregate_count, 0); - scenario_metrics.push(report::ScenarioMetric { - name: "Aggregated".to_string(), - users: aggregate_users, - count: aggregate_count, - response_time_average: format!( - "{:.2}", - aggregate_response_time_counter / aggregate_count as f32 - ), - response_time_minimum: aggregate_scenario_time_minimum, - response_time_maximum: aggregate_scenario_time_maximum, - count_per_second: format!("{:.2}", aggregate_count_per_second), - iterations: format!("{:.2}", aggregate_iterations), - }); - let mut scenarios_rows = Vec::new(); - // Compile the scenario metrics template. - for metric in scenario_metrics { - scenarios_rows.push(report::scenario_metrics_row(metric)); - } + let transactions_template = transaction_metrics + .map(|transaction_metrics| { + let transactions_rows = transaction_metrics + .into_iter() + .map(report::transaction_metrics_row) + .join("\n"); - scenarios_template = report::scenario_metrics_template( - &scenarios_rows.join("\n"), + report::transaction_metrics_template( + &transactions_rows, self.graph_data - .get_scenarios_per_second_graph(!self.configuration.no_granular_report) + .get_transactions_per_second_graph(!self.configuration.no_granular_report) .get_markup(&self.metrics.history, test_start_time), - ); - } else { - scenarios_template = "".to_string(); - } + ) + }) + .unwrap_or_default(); - // Only build the transactions template if --no-transaction-metrics isn't enabled. - let errors_template: String = if !self.metrics.errors.is_empty() { - let mut error_rows = Vec::new(); - for error in self.metrics.errors.values() { - error_rows.push(report::error_row(error)); - } + let errors_template = errors + .map(|errors| { + let error_rows = errors.into_iter().map(report::error_row).join("\n"); report::errors_template( - &error_rows.join("\n"), + &error_rows, self.graph_data .get_errors_per_second_graph(!self.configuration.no_granular_report) .get_markup(&self.metrics.history, test_start_time), ) - } else { - "".to_string() - }; - - // Only build the status_code template if --no-status-codes is not enabled. - let status_code_template: String = if !self.configuration.no_status_codes { - let mut status_code_metrics = Vec::new(); - let mut aggregated_status_code_counts: HashMap = HashMap::new(); - for (request_key, request) in self.metrics.requests.iter().sorted() { - let method = format!("{}", request.method); - // The request_key is "{method} {name}", so by stripping the "{method} " - // prefix we get the name. - let name = request_key - .strip_prefix(&format!("{} ", request.method)) - .unwrap() - .to_string(); - - // Build a list of status codes, and update the aggregate record. - let codes = prepare_status_codes( - &request.status_code_counts, - &mut Some(&mut aggregated_status_code_counts), - ); - - // Add a row of data for the status code table. - status_code_metrics.push(report::StatusCodeMetric { - method, - name, - status_codes: codes, - }); - } - - // Build a list of aggregate status codes. - let aggregated_codes = - prepare_status_codes(&aggregated_status_code_counts, &mut None); - - // Add a final row of aggregate data for the status code table. - status_code_metrics.push(report::StatusCodeMetric { - method: "".to_string(), - name: "Aggregated".to_string(), - status_codes: aggregated_codes, - }); + }) + .unwrap_or_default(); + let status_code_template = status_code_metrics + .map(|status_code_metrics| { // Compile the status_code metrics rows. let mut status_code_rows = Vec::new(); for metric in status_code_metrics { @@ -3552,56 +3286,50 @@ impl GooseAttack { // Compile the status_code metrics template. report::status_code_metrics_template(&status_code_rows.join("\n")) - } else { - // If --status-codes is not enabled, return an empty template. - "".to_string() - }; - - // Compile the report template. - let report = report::build_report( - &users, - &steps_overview, - hosts, - report::GooseReportTemplates { - raw_requests_template: &raw_requests_rows.join("\n"), - raw_responses_template: &raw_responses_rows.join("\n"), - co_requests_template: &co_requests_template, - co_responses_template: &co_responses_template, - transactions_template: &transactions_template, - scenarios_template: &scenarios_template, - status_codes_template: &status_code_template, - errors_template: &errors_template, - graph_rps_template: &self - .graph_data - .get_requests_per_second_graph(!self.configuration.no_granular_report) - .get_markup(&self.metrics.history, test_start_time), - graph_average_response_time_template: &self - .graph_data - .get_average_response_time_graph(!self.configuration.no_granular_report) - .get_markup(&self.metrics.history, test_start_time), - graph_users_per_second: &self - .graph_data - .get_active_users_graph(!self.configuration.no_granular_report) - .get_markup(&self.metrics.history, test_start_time), - }, - ); + }) + .unwrap_or_default(); + + // Compile the report template. + let report = report::build_report( + &users, + &steps_overview, + hosts, + report::GooseReportTemplates { + raw_requests_template: &raw_requests_rows.join("\n"), + raw_responses_template: &raw_responses_rows.join("\n"), + co_requests_template: &co_requests_template, + co_responses_template: &co_responses_template, + transactions_template: &transactions_template, + scenarios_template: &scenarios_template, + status_codes_template: &status_code_template, + errors_template: &errors_template, + graph_rps_template: &self + .graph_data + .get_requests_per_second_graph(!self.configuration.no_granular_report) + .get_markup(&self.metrics.history, test_start_time), + graph_average_response_time_template: &self + .graph_data + .get_average_response_time_graph(!self.configuration.no_granular_report) + .get_markup(&self.metrics.history, test_start_time), + graph_users_per_second: &self + .graph_data + .get_active_users_graph(!self.configuration.no_granular_report) + .get_markup(&self.metrics.history, test_start_time), + }, + ); - // Write the report to file. - if let Err(e) = report_file.write_all(report.as_ref()).await { - return Err(GooseError::InvalidOption { - option: "--report-file".to_string(), - value: self.get_report_file_path().unwrap(), - detail: format!("Failed to create report file: {}", e), - }); - }; - // Be sure the file flushes to disk. - report_file.flush().await?; + // Write the report to file. + if let Err(e) = report_file.write_all(report.as_ref()).await { + return Err(GooseError::InvalidOption { + option: "--report-file".to_string(), + value: path.to_string(), + detail: format!("Failed to create report file: {}", e), + }); + }; + // Be sure the file flushes to disk. + report_file.flush().await?; - info!( - "html report file written to: {}", - self.get_report_file_path().unwrap() - ); - } + info!("html report file written to: {path}"); Ok(()) } @@ -3678,7 +3406,7 @@ pub(crate) fn calculate_response_time_percentile( min: usize, max: usize, percent: f32, -) -> String { +) -> usize { let percentile_request = (total_requests as f32 * percent).round() as usize; debug!( "percentile: {}, request {} of total {}", @@ -3691,15 +3419,16 @@ pub(crate) fn calculate_response_time_percentile( total_count += counter; if total_count >= percentile_request { if *value < min { - return format_number(min); + return min; } else if *value > max { - return format_number(max); + return max; } else { - return format_number(*value); + return *value; } } } - format_number(0) + + 0 } /// Helper to count and aggregate seen status codes. @@ -3782,21 +3511,42 @@ mod test { response_times.insert(2, 1); response_times.insert(3, 1); // 3 * .5 = 1.5, rounds to 2. - assert!(calculate_response_time_percentile(&response_times, 3, 1, 3, 0.5) == "2"); + assert_eq!( + calculate_response_time_percentile(&response_times, 3, 1, 3, 0.5), + 2 + ); response_times.insert(3, 2); // 4 * .5 = 2 - assert!(calculate_response_time_percentile(&response_times, 4, 1, 3, 0.5) == "2"); + assert_eq!( + calculate_response_time_percentile(&response_times, 4, 1, 3, 0.5), + 2 + ); // 4 * .25 = 1 - assert!(calculate_response_time_percentile(&response_times, 4, 1, 3, 0.25) == "1"); + assert_eq!( + calculate_response_time_percentile(&response_times, 4, 1, 3, 0.25), + 1 + ); // 4 * .75 = 3 - assert!(calculate_response_time_percentile(&response_times, 4, 1, 3, 0.75) == "3"); + assert_eq!( + calculate_response_time_percentile(&response_times, 4, 1, 3, 0.75), + 3 + ); // 4 * 1 = 4 (and the 4th response time is also 3) - assert!(calculate_response_time_percentile(&response_times, 4, 1, 3, 1.0) == "3"); + assert_eq!( + calculate_response_time_percentile(&response_times, 4, 1, 3, 1.0), + 3 + ); // 4 * .5 = 2, but uses specified minimum of 2 - assert!(calculate_response_time_percentile(&response_times, 4, 2, 3, 0.25) == "2"); + assert_eq!( + calculate_response_time_percentile(&response_times, 4, 2, 3, 0.25), + 2 + ); // 4 * .75 = 3, but uses specified maximum of 2 - assert!(calculate_response_time_percentile(&response_times, 4, 1, 2, 0.75) == "2"); + assert_eq!( + calculate_response_time_percentile(&response_times, 4, 1, 2, 0.75), + 2 + ); response_times.insert(10, 25); response_times.insert(20, 25); @@ -3804,9 +3554,18 @@ mod test { response_times.insert(50, 25); response_times.insert(100, 10); response_times.insert(200, 1); - assert!(calculate_response_time_percentile(&response_times, 115, 1, 200, 0.9) == "50"); - assert!(calculate_response_time_percentile(&response_times, 115, 1, 200, 0.99) == "100"); - assert!(calculate_response_time_percentile(&response_times, 115, 1, 200, 0.999) == "200"); + assert_eq!( + calculate_response_time_percentile(&response_times, 115, 1, 200, 0.9), + 50 + ); + assert_eq!( + calculate_response_time_percentile(&response_times, 115, 1, 200, 0.99), + 100 + ); + assert_eq!( + calculate_response_time_percentile(&response_times, 115, 1, 200, 0.999), + 200 + ); } #[test] diff --git a/src/metrics/common.rs b/src/metrics/common.rs new file mode 100644 index 00000000..b337c111 --- /dev/null +++ b/src/metrics/common.rs @@ -0,0 +1,422 @@ +use super::{ + merge_times, per_second_calculations, prepare_status_codes, report, update_max_time, + update_min_time, GooseErrorMetricAggregate, GooseMetrics, +}; +use crate::{ + report::{ + CORequestMetric, RequestMetric, ResponseMetric, ScenarioMetric, StatusCodeMetric, + TransactionMetric, + }, + util, +}; +use itertools::Itertools; +use std::collections::{BTreeMap, HashMap}; + +#[derive(Debug, serde::Serialize)] +pub(crate) struct ReportData<'m> { + pub raw_metrics: &'m GooseMetrics, + + pub raw_request_metrics: Vec, + pub raw_response_metrics: Vec, + + pub co_request_metrics: Option>, + pub co_response_metrics: Option>, + + pub scenario_metrics: Option>, + pub transaction_metrics: Option>, + + pub status_code_metrics: Option>, + + pub errors: Option>, +} + +pub struct ReportOptions { + pub no_transaction_metrics: bool, + pub no_scenario_metrics: bool, + pub no_status_codes: bool, +} + +pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportData { + // Prepare requests and responses variables. + let mut raw_request_metrics = Vec::new(); + let mut co_request_metrics = Vec::new(); + let mut raw_response_metrics = Vec::new(); + let mut co_response_metrics = Vec::new(); + let mut raw_aggregate_total_count = 0; + let mut co_aggregate_total_count = 0; + let mut raw_aggregate_fail_count = 0; + let mut raw_aggregate_response_time_counter: usize = 0; + let mut raw_aggregate_response_time_minimum: usize = 0; + let mut raw_aggregate_response_time_maximum: usize = 0; + let mut raw_aggregate_response_times: BTreeMap = BTreeMap::new(); + let mut co_aggregate_response_time_counter: usize = 0; + let mut co_aggregate_response_time_maximum: usize = 0; + let mut co_aggregate_response_times: BTreeMap = BTreeMap::new(); + let mut co_data = false; + + for (request_key, request) in metrics.requests.iter().sorted() { + // Determine whether or not to include Coordinated Omission data. + if !co_data && request.coordinated_omission_data.is_some() { + co_data = true; + } + let method = format!("{}", request.method); + // The request_key is "{method} {name}", so by stripping the "{method} " + // prefix we get the name. + let name = request_key + .strip_prefix(&format!("{} ", request.method)) + .unwrap() + .to_string(); + let total_request_count = request.success_count + request.fail_count; + let (requests_per_second, failures_per_second) = + per_second_calculations(metrics.duration, total_request_count, request.fail_count); + // Prepare per-request metrics. + raw_request_metrics.push(report::RequestMetric { + method: method.to_string(), + name: name.to_string(), + number_of_requests: total_request_count, + number_of_failures: request.fail_count, + response_time_average: request.raw_data.total_time as f32 + / request.raw_data.counter as f32, + response_time_minimum: request.raw_data.minimum_time, + response_time_maximum: request.raw_data.maximum_time, + requests_per_second, + failures_per_second, + }); + + // Prepare per-response metrics. + raw_response_metrics.push(report::get_response_metric( + &method, + &name, + &request.raw_data.times, + request.raw_data.counter, + request.raw_data.minimum_time, + request.raw_data.maximum_time, + )); + + // Collect aggregated request and response metrics. + raw_aggregate_total_count += total_request_count; + raw_aggregate_fail_count += request.fail_count; + raw_aggregate_response_time_counter += request.raw_data.total_time; + raw_aggregate_response_time_minimum = update_min_time( + raw_aggregate_response_time_minimum, + request.raw_data.minimum_time, + ); + raw_aggregate_response_time_maximum = update_max_time( + raw_aggregate_response_time_maximum, + request.raw_data.maximum_time, + ); + raw_aggregate_response_times = + merge_times(raw_aggregate_response_times, request.raw_data.times.clone()); + } + + // Prepare aggregate per-request metrics. + let (raw_aggregate_requests_per_second, raw_aggregate_failures_per_second) = + per_second_calculations( + metrics.duration, + raw_aggregate_total_count, + raw_aggregate_fail_count, + ); + raw_request_metrics.push(report::RequestMetric { + method: "".to_string(), + name: "Aggregated".to_string(), + number_of_requests: raw_aggregate_total_count, + number_of_failures: raw_aggregate_fail_count, + response_time_average: raw_aggregate_response_time_counter as f32 + / raw_aggregate_total_count as f32, + response_time_minimum: raw_aggregate_response_time_minimum, + response_time_maximum: raw_aggregate_response_time_maximum, + requests_per_second: raw_aggregate_requests_per_second, + failures_per_second: raw_aggregate_failures_per_second, + }); + + // Prepare aggregate per-response metrics. + raw_response_metrics.push(report::get_response_metric( + "", + "Aggregated", + &raw_aggregate_response_times, + raw_aggregate_total_count, + raw_aggregate_response_time_minimum, + raw_aggregate_response_time_maximum, + )); + + let (co_request_metrics, co_response_metrics) = if co_data { + for (request_key, request) in metrics.requests.iter().sorted() { + if let Some(coordinated_omission_data) = request.coordinated_omission_data.as_ref() { + let method = format!("{}", request.method); + // The request_key is "{method} {name}", so by stripping the "{method} " + // prefix we get the name. + let name = request_key + .strip_prefix(&format!("{} ", request.method)) + .unwrap() + .to_string(); + let raw_average = + request.raw_data.total_time as f32 / request.raw_data.counter as f32; + let co_average = coordinated_omission_data.total_time as f32 + / coordinated_omission_data.counter as f32; + // Prepare per-request metrics. + co_request_metrics.push(report::CORequestMetric { + method: method.to_string(), + name: name.to_string(), + response_time_average: co_average, + response_time_standard_deviation: util::standard_deviation( + raw_average, + co_average, + ), + response_time_maximum: coordinated_omission_data.maximum_time, + }); + + // Prepare per-response metrics. + co_response_metrics.push(report::get_response_metric( + &method, + &name, + &coordinated_omission_data.times, + coordinated_omission_data.counter, + coordinated_omission_data.minimum_time, + coordinated_omission_data.maximum_time, + )); + + // Collect aggregated request and response metrics. + co_aggregate_response_time_counter += coordinated_omission_data.total_time; + co_aggregate_response_time_maximum = update_max_time( + co_aggregate_response_time_maximum, + coordinated_omission_data.maximum_time, + ); + co_aggregate_response_times = merge_times( + co_aggregate_response_times, + coordinated_omission_data.times.clone(), + ); + } + let total_request_count = request.success_count + request.fail_count; + co_aggregate_total_count += total_request_count; + } + let co_average = + co_aggregate_response_time_counter as f32 / co_aggregate_total_count as f32; + let raw_average = + raw_aggregate_response_time_counter as f32 / raw_aggregate_total_count as f32; + co_request_metrics.push(report::CORequestMetric { + method: "".to_string(), + name: "Aggregated".to_string(), + response_time_average: co_aggregate_response_time_counter as f32 + / co_aggregate_total_count as f32, + response_time_standard_deviation: util::standard_deviation(raw_average, co_average), + response_time_maximum: co_aggregate_response_time_maximum, + }); + + // Prepare aggregate per-response metrics. + co_response_metrics.push(report::get_response_metric( + "", + "Aggregated", + &co_aggregate_response_times, + co_aggregate_total_count, + raw_aggregate_response_time_minimum, + co_aggregate_response_time_maximum, + )); + + (Some(co_request_metrics), Some(co_response_metrics)) + } else { + (None, None) + }; + + // Only build the transactions template if --no-transaction-metrics isn't enabled. + let transaction_metrics = if !options.no_transaction_metrics { + let mut transaction_metrics = Vec::new(); + let mut aggregate_total_count = 0; + let mut aggregate_fail_count = 0; + let mut aggregate_transaction_time_counter: usize = 0; + let mut aggregate_transaction_time_minimum: usize = 0; + let mut aggregate_transaction_time_maximum: usize = 0; + let mut aggregate_transaction_times: BTreeMap = BTreeMap::new(); + for (scenario_counter, scenario) in metrics.transactions.iter().enumerate() { + for (transaction_counter, transaction) in scenario.iter().enumerate() { + if transaction_counter == 0 { + // Only the scenario_name is used for scenarios. + transaction_metrics.push(report::TransactionMetric { + is_scenario: true, + transaction: "".to_string(), + name: transaction.scenario_name.to_string(), + number_of_requests: 0, + number_of_failures: 0, + response_time_average: None, + response_time_minimum: 0, + response_time_maximum: 0, + requests_per_second: None, + failures_per_second: None, + }); + } + let total_run_count = transaction.success_count + transaction.fail_count; + let (requests_per_second, failures_per_second) = per_second_calculations( + metrics.duration, + total_run_count, + transaction.fail_count, + ); + let average = match transaction.counter { + 0 => 0.00, + _ => transaction.total_time as f32 / transaction.counter as f32, + }; + transaction_metrics.push(report::TransactionMetric { + is_scenario: false, + transaction: format!("{}.{}", scenario_counter, transaction_counter), + name: transaction.transaction_name.to_string(), + number_of_requests: total_run_count, + number_of_failures: transaction.fail_count, + response_time_average: Some(average), + response_time_minimum: transaction.min_time, + response_time_maximum: transaction.max_time, + requests_per_second: Some(requests_per_second), + failures_per_second: Some(failures_per_second), + }); + + aggregate_total_count += total_run_count; + aggregate_fail_count += transaction.fail_count; + aggregate_transaction_times = + merge_times(aggregate_transaction_times, transaction.times.clone()); + aggregate_transaction_time_counter += &transaction.counter; + aggregate_transaction_time_minimum = + update_min_time(aggregate_transaction_time_minimum, transaction.min_time); + aggregate_transaction_time_maximum = + update_max_time(aggregate_transaction_time_maximum, transaction.max_time); + } + } + + let (aggregate_requests_per_second, aggregate_failures_per_second) = + per_second_calculations( + metrics.duration, + aggregate_total_count, + aggregate_fail_count, + ); + transaction_metrics.push(report::TransactionMetric { + is_scenario: false, + transaction: "".to_string(), + name: "Aggregated".to_string(), + number_of_requests: aggregate_total_count, + number_of_failures: aggregate_fail_count, + response_time_average: Some( + raw_aggregate_response_time_counter as f32 / aggregate_total_count as f32, + ), + response_time_minimum: aggregate_transaction_time_minimum, + response_time_maximum: aggregate_transaction_time_maximum, + requests_per_second: Some(aggregate_requests_per_second), + failures_per_second: Some(aggregate_failures_per_second), + }); + Some(transaction_metrics) + } else { + None + }; + + // Only build the scenarios template if --no-senario-metrics isn't enabled. + let scenario_metrics = if !options.no_scenario_metrics { + let mut scenario_metrics = Vec::new(); + let mut aggregate_users = 0; + let mut aggregate_count = 0; + let mut aggregate_scenario_time_counter: usize = 0; + let mut aggregate_scenario_time_minimum: usize = 0; + let mut aggregate_scenario_time_maximum: usize = 0; + let mut aggregate_scenario_times: BTreeMap = BTreeMap::new(); + let mut aggregate_iterations = 0.0; + let mut aggregate_response_time_counter = 0.0; + for scenario in &metrics.scenarios { + let (count_per_second, _failures_per_second) = + per_second_calculations(metrics.duration, scenario.counter, 0); + let average = match scenario.counter { + 0 => 0.00, + _ => scenario.total_time as f32 / scenario.counter as f32, + }; + let iterations = scenario.counter as f32 / scenario.users.len() as f32; + scenario_metrics.push(report::ScenarioMetric { + name: scenario.name.to_string(), + users: scenario.users.len(), + count: scenario.counter, + response_time_average: average, + response_time_minimum: scenario.min_time, + response_time_maximum: scenario.max_time, + count_per_second, + iterations, + }); + + aggregate_users += scenario.users.len(); + aggregate_count += scenario.counter; + aggregate_scenario_times = + merge_times(aggregate_scenario_times, scenario.times.clone()); + aggregate_scenario_time_counter += &scenario.counter; + aggregate_scenario_time_minimum = + update_min_time(aggregate_scenario_time_minimum, scenario.min_time); + aggregate_scenario_time_maximum = + update_max_time(aggregate_scenario_time_maximum, scenario.max_time); + aggregate_iterations += iterations; + aggregate_response_time_counter += scenario.total_time as f32; + } + + let (aggregate_count_per_second, _aggregate_failures_per_second) = + per_second_calculations(metrics.duration, aggregate_count, 0); + scenario_metrics.push(report::ScenarioMetric { + name: "Aggregated".to_string(), + users: aggregate_users, + count: aggregate_count, + response_time_average: aggregate_response_time_counter / aggregate_count as f32, + response_time_minimum: aggregate_scenario_time_minimum, + response_time_maximum: aggregate_scenario_time_maximum, + count_per_second: aggregate_count_per_second, + iterations: aggregate_iterations, + }); + + Some(scenario_metrics) + } else { + None + }; + + let status_code_metrics = if !options.no_status_codes { + let mut status_code_metrics = Vec::new(); + let mut aggregated_status_code_counts: HashMap = HashMap::new(); + for (request_key, request) in metrics.requests.iter().sorted() { + let method = format!("{}", request.method); + // The request_key is "{method} {name}", so by stripping the "{method} " + // prefix we get the name. + let name = request_key + .strip_prefix(&format!("{} ", request.method)) + .unwrap() + .to_string(); + + // Build a list of status codes, and update the aggregate record. + let codes = prepare_status_codes( + &request.status_code_counts, + &mut Some(&mut aggregated_status_code_counts), + ); + + // Add a row of data for the status code table. + status_code_metrics.push(report::StatusCodeMetric { + method, + name, + status_codes: codes, + }); + } + + // Build a list of aggregate status codes. + let aggregated_codes = prepare_status_codes(&aggregated_status_code_counts, &mut None); + + // Add a final row of aggregate data for the status code table. + status_code_metrics.push(report::StatusCodeMetric { + method: "".to_string(), + name: "Aggregated".to_string(), + status_codes: aggregated_codes, + }); + + Some(status_code_metrics) + } else { + None + }; + + ReportData { + raw_metrics: metrics, + raw_request_metrics, + raw_response_metrics, + co_request_metrics, + co_response_metrics, + scenario_metrics, + transaction_metrics, + status_code_metrics, + errors: metrics + .errors + .is_empty() + .then(|| metrics.errors.values().collect::>()), + } +} diff --git a/src/report.rs b/src/report.rs index 4a0ea17a..9a5ceae8 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,11 +1,15 @@ //! Optionally writes an html-formatted summary report after running a load test. +mod common; +mod markdown; -use crate::metrics; - -use std::collections::BTreeMap; -use std::mem; +pub(crate) use markdown::write_markdown_report; +use crate::{ + metrics::{self, format_number}, + report::common::OrEmpty, +}; use serde::Serialize; +use std::collections::BTreeMap; /// The following templates are necessary to build an html-formatted summary report. #[derive(Debug)] @@ -30,11 +34,11 @@ pub(crate) struct RequestMetric { pub name: String, pub number_of_requests: usize, pub number_of_failures: usize, - pub response_time_average: String, + pub response_time_average: f32, pub response_time_minimum: usize, pub response_time_maximum: usize, - pub requests_per_second: String, - pub failures_per_second: String, + pub requests_per_second: f32, + pub failures_per_second: f32, } /// Defines the metrics reported about Coordinated Omission requests. @@ -42,8 +46,8 @@ pub(crate) struct RequestMetric { pub(crate) struct CORequestMetric { pub method: String, pub name: String, - pub response_time_average: String, - pub response_time_standard_deviation: String, + pub response_time_average: f32, + pub response_time_standard_deviation: f32, pub response_time_maximum: usize, } @@ -52,14 +56,14 @@ pub(crate) struct CORequestMetric { pub(crate) struct ResponseMetric { pub method: String, pub name: String, - pub percentile_50: String, - pub percentile_60: String, - pub percentile_70: String, - pub percentile_80: String, - pub percentile_90: String, - pub percentile_95: String, - pub percentile_99: String, - pub percentile_100: String, + pub percentile_50: usize, + pub percentile_60: usize, + pub percentile_70: usize, + pub percentile_80: usize, + pub percentile_90: usize, + pub percentile_95: usize, + pub percentile_99: usize, + pub percentile_100: usize, } /// Defines the metrics reported about transactions. @@ -70,11 +74,11 @@ pub(crate) struct TransactionMetric { pub name: String, pub number_of_requests: usize, pub number_of_failures: usize, - pub response_time_average: String, + pub response_time_average: Option, pub response_time_minimum: usize, pub response_time_maximum: usize, - pub requests_per_second: String, - pub failures_per_second: String, + pub requests_per_second: Option, + pub failures_per_second: Option, } /// Defines the metrics reported about scenarios. @@ -83,14 +87,15 @@ pub(crate) struct ScenarioMetric { pub name: String, pub users: usize, pub count: usize, - pub response_time_average: String, + pub response_time_average: f32, pub response_time_minimum: usize, pub response_time_maximum: usize, - pub count_per_second: String, - pub iterations: String, + pub count_per_second: f32, + pub iterations: f32, } /// Defines the metrics reported about status codes. +#[derive(Debug, serde::Serialize)] pub(crate) struct StatusCodeMetric { pub method: String, pub name: String, @@ -122,14 +127,14 @@ pub(crate) fn get_response_metric( ResponseMetric { method: method.to_string(), name: name.to_string(), - percentile_50: mem::take(&mut percentiles[0]), - percentile_60: mem::take(&mut percentiles[1]), - percentile_70: mem::take(&mut percentiles[2]), - percentile_80: mem::take(&mut percentiles[3]), - percentile_90: mem::take(&mut percentiles[4]), - percentile_95: mem::take(&mut percentiles[5]), - percentile_99: mem::take(&mut percentiles[6]), - percentile_100: mem::take(&mut percentiles[7]), + percentile_50: percentiles[0], + percentile_60: percentiles[1], + percentile_70: percentiles[2], + percentile_80: percentiles[3], + percentile_90: percentiles[4], + percentile_95: percentiles[5], + percentile_99: percentiles[6], + percentile_100: percentiles[7], } } @@ -141,11 +146,11 @@ pub(crate) fn raw_request_metrics_row(metric: RequestMetric) -> String { {name} {number_of_requests} {number_of_failures} - {response_time_average} + {response_time_average:.2} {response_time_minimum} {response_time_maximum} - {requests_per_second} - {failures_per_second} + {requests_per_second:.2} + {failures_per_second:.2} "#, method = metric.method, name = metric.name, @@ -176,14 +181,14 @@ pub(crate) fn response_metrics_row(metric: ResponseMetric) -> String { "#, method = metric.method, name = metric.name, - percentile_50 = metric.percentile_50, - percentile_60 = metric.percentile_60, - percentile_70 = metric.percentile_70, - percentile_80 = metric.percentile_80, - percentile_90 = metric.percentile_90, - percentile_95 = metric.percentile_95, - percentile_99 = metric.percentile_99, - percentile_100 = metric.percentile_100, + percentile_50 = format_number(metric.percentile_50), + percentile_60 = format_number(metric.percentile_60), + percentile_70 = format_number(metric.percentile_70), + percentile_80 = format_number(metric.percentile_80), + percentile_90 = format_number(metric.percentile_90), + percentile_95 = format_number(metric.percentile_95), + percentile_99 = format_number(metric.percentile_99), + percentile_100 = format_number(metric.percentile_100), ) } @@ -219,8 +224,8 @@ pub(crate) fn coordinated_omission_request_metrics_row(metric: CORequestMetric) r#" {method} {name} - {average}) - {standard_deviation} + {average:.2}) + {standard_deviation:.2} {maximum} "#, method = metric.method, @@ -279,14 +284,14 @@ pub(crate) fn coordinated_omission_response_metrics_row(metric: ResponseMetric) "#, method = metric.method, name = metric.name, - percentile_50 = metric.percentile_50, - percentile_60 = metric.percentile_60, - percentile_70 = metric.percentile_70, - percentile_80 = metric.percentile_80, - percentile_90 = metric.percentile_90, - percentile_95 = metric.percentile_95, - percentile_99 = metric.percentile_99, - percentile_100 = metric.percentile_100, + percentile_50 = format_number(metric.percentile_50), + percentile_60 = format_number(metric.percentile_60), + percentile_70 = format_number(metric.percentile_70), + percentile_80 = format_number(metric.percentile_80), + percentile_90 = format_number(metric.percentile_90), + percentile_95 = format_number(metric.percentile_95), + percentile_99 = format_number(metric.percentile_99), + percentile_100 = format_number(metric.percentile_100), ) } @@ -373,21 +378,21 @@ pub(crate) fn transaction_metrics_row(metric: TransactionMetric) -> String { {transaction} {name} {number_of_requests} {number_of_failures} - {response_time_average} + {response_time_average:.2} {response_time_minimum} {response_time_maximum} - {requests_per_second} - {failures_per_second} + {requests_per_second:.2} + {failures_per_second:.2} "#, transaction = metric.transaction, name = metric.name, - number_of_requests = metrics::format_number(metric.number_of_requests), - number_of_failures = metrics::format_number(metric.number_of_failures), - response_time_average = metric.response_time_average, + number_of_requests = format_number(metric.number_of_requests), + number_of_failures = format_number(metric.number_of_failures), + response_time_average = OrEmpty(metric.response_time_average), response_time_minimum = metric.response_time_minimum, response_time_maximum = metric.response_time_maximum, - requests_per_second = metric.requests_per_second, - failures_per_second = metric.failures_per_second, + requests_per_second = OrEmpty(metric.requests_per_second), + failures_per_second = OrEmpty(metric.failures_per_second), ) } } @@ -430,15 +435,15 @@ pub(crate) fn scenario_metrics_row(metric: ScenarioMetric) -> String { {name} {users} {count} - {response_time_average} + {response_time_average:.2} {response_time_minimum} {response_time_maximum} - {count_per_second} - {iterations} + {count_per_second:.2} + {iterations:.2} "#, name = metric.name, - users = metrics::format_number(metric.users), - count = metrics::format_number(metric.count), + users = format_number(metric.users), + count = format_number(metric.count), response_time_average = metric.response_time_average, response_time_minimum = metric.response_time_minimum, response_time_maximum = metric.response_time_maximum, diff --git a/src/report/common.rs b/src/report/common.rs new file mode 100644 index 00000000..ede83130 --- /dev/null +++ b/src/report/common.rs @@ -0,0 +1,25 @@ +use std::fmt::{Display, Formatter}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OrEmpty(pub Option); + +impl Display for OrEmpty { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match &self.0 { + Some(value) => value.fmt(f), + None => f.write_str(""), + } + } +} + +#[cfg(test)] +mod test { + use crate::report::common::OrEmpty; + + #[test] + pub fn format() { + assert_eq!("1.23", format!("{:.2}", OrEmpty(Some(1.23456)))); + assert_eq!("1", format!("{:.0}", OrEmpty(Some(1.23456)))); + assert_eq!("", format!("{:.2}", OrEmpty::(None))); + } +} diff --git a/src/report/markdown.rs b/src/report/markdown.rs new file mode 100644 index 00000000..9f921dee --- /dev/null +++ b/src/report/markdown.rs @@ -0,0 +1,310 @@ +use crate::{ + metrics::{format_number, GooseErrorMetricAggregate, ReportData}, + report::{ + common::OrEmpty, RequestMetric, ResponseMetric, ScenarioMetric, StatusCodeMetric, + TransactionMetric, + }, + test_plan::TestPlanStepAction, + GooseError, +}; +use chrono::{Local, TimeZone}; +use std::io::Write; + +struct Markdown<'m, 'w, W: Write> { + w: &'w mut W, + data: ReportData<'m>, +} + +pub(crate) fn write_markdown_report( + w: &mut W, + data: ReportData, +) -> Result<(), GooseError> { + Markdown { w, data }.write() +} + +impl<'m, 'w, W: Write> Markdown<'m, 'w, W> { + pub fn write(mut self) -> Result<(), GooseError> { + self.write_header()?; + self.write_plan_overview()?; + self.write_request_metrics()?; + self.write_response_metrics()?; + self.write_status_code_metrics()?; + self.write_transaction_metrics()?; + self.write_scenario_metrics()?; + self.write_error_metrics()?; + + Ok(()) + } + + fn write_header(&mut self) -> Result<(), GooseError> { + writeln!( + self.w, + r#" +# Goose Attack Report +"# + )?; + + Ok(()) + } + + fn write_plan_overview(&mut self) -> Result<(), GooseError> { + write!( + self.w, + r#" +## Plan Overview + +| Action | Started | Stopped | Elapsed | Users | +| ------ | ------- | ------- | ------- | ----: | +"# + )?; + + for step in self.data.raw_metrics.history.windows(2) { + let (seconds, minutes, hours) = self + .data + .raw_metrics + .get_seconds_minutes_hours(&step[0].timestamp, &step[1].timestamp); + let started = Local + .timestamp_opt(step[0].timestamp.timestamp(), 0) + // @TODO: error handling + .unwrap() + .format("%y-%m-%d %H:%M:%S"); + let stopped = Local + .timestamp_opt(step[1].timestamp.timestamp(), 0) + // @TODO: error handling + .unwrap() + .format("%y-%m-%d %H:%M:%S"); + + let users = match &step[0].action { + // For maintaining just show the current number of users. + TestPlanStepAction::Maintaining => { + format!("{}", step[0].users) + } + // For increasing show the current number of users to the new number of users. + TestPlanStepAction::Increasing => { + format!("{} → {}", step[0].users, step[1].users) + } + // For decreasing show the new number of users from the current number of users. + TestPlanStepAction::Decreasing | TestPlanStepAction::Canceling => { + format!("{} ← {}", step[1].users, step[0].users,) + } + TestPlanStepAction::Finished => { + unreachable!("there shouldn't be a step after finished"); + } + }; + + writeln!( + self.w, + r#"| {action:?} | {started} | {stopped} | {hours:02}:{minutes:02}:{seconds:02} | {users} |"#, + action = step[0].action, + )?; + } + + Ok(()) + } + + fn write_request_metrics(&mut self) -> Result<(), GooseError> { + write!( + self.w, + r#" +## Request Metrics + +| Method | Name | # Requests | # Fails | Average (ms) | Min (ms) | Max (ms) | RPS | Failures/s | +| ------ | ---- | ---------: | ------: | -----------: | -------: | -------: | --: | ---------: | +"# + )?; + + for RequestMetric { + method, + name, + number_of_requests, + number_of_failures, + response_time_average, + response_time_minimum, + response_time_maximum, + requests_per_second, + failures_per_second, + } in &self.data.raw_request_metrics + { + writeln!( + self.w, + r#"| {method} | {name} | {number_of_requests} | {number_of_failures } | {response_time_average:.2 } | {response_time_minimum} | {response_time_maximum} | {requests_per_second:.2} | {failures_per_second:.2} |"#, + )?; + } + + Ok(()) + } + + fn write_response_metrics(&mut self) -> Result<(), GooseError> { + write!( + self.w, + r#" +## Response Time Metrics + +| Method | Name | 50%ile (ms) | 60%ile (ms) | 70%ile (ms) | 80%ile (ms) | 90%ile (ms) | 95%ile (ms) | 99%ile (ms) | 100%ile (ms) | +| ------ | ---- | ----------: | ----------: | ----------: | ----------: | ----------: | ----------: | ----------: | -----------: | +"# + )?; + + for ResponseMetric { + method, + name, + percentile_50, + percentile_60, + percentile_70, + percentile_80, + percentile_90, + percentile_95, + percentile_99, + percentile_100, + } in &self.data.raw_response_metrics + { + writeln!( + self.w, + r#"| {method} | {name} | {percentile_50} | {percentile_60 } | {percentile_70 } | {percentile_80} | {percentile_90} | {percentile_95} | {percentile_99} | {percentile_100} |"#, + percentile_50 = format_number(*percentile_50), + percentile_60 = format_number(*percentile_60), + percentile_70 = format_number(*percentile_70), + percentile_80 = format_number(*percentile_80), + percentile_90 = format_number(*percentile_90), + percentile_95 = format_number(*percentile_95), + percentile_99 = format_number(*percentile_99), + percentile_100 = format_number(*percentile_100), + )?; + } + + Ok(()) + } + + fn write_status_code_metrics(&mut self) -> Result<(), GooseError> { + let Some(status_code_metrics) = &self.data.status_code_metrics else { + return Ok(()); + }; + + write!( + self.w, + r#" +## Status Code Metrics + +| Method | Name | Status Codes | +| ------ | ---- | ------------ | +"# + )?; + + for StatusCodeMetric { + method, + name, + status_codes, + } in status_code_metrics + { + writeln!(self.w, r#"| {method} | {name} | {status_codes} |"#)?; + } + + Ok(()) + } + + fn write_transaction_metrics(&mut self) -> Result<(), GooseError> { + let Some(transaction_metrics) = &self.data.transaction_metrics else { + return Ok(()); + }; + + write!( + self.w, + r#" +## Transaction Metrics + +| Transaction | # Times Run | # Fails | Average (ms) | Min (ms) | Max (ms) | RPS | Failures/s | +| ----------- | ----------: | ------: | -----------: | -------: | -------: | --: | ---------: | +"# + )?; + + for TransactionMetric { + is_scenario, + transaction, + name, + number_of_requests, + number_of_failures, + response_time_average, + response_time_minimum, + response_time_maximum, + requests_per_second, + failures_per_second, + } in transaction_metrics + { + match is_scenario { + true => writeln!(self.w, r#"| **{name}** |"#)?, + false => writeln!( + self.w, + r#"| {transaction} {name} | {number_of_requests} | {number_of_failures} | {response_time_average:.2} | {response_time_minimum} | {response_time_maximum} | {requests_per_second:.2} | {failures_per_second:.2} |"#, + response_time_average = OrEmpty(*response_time_average), + requests_per_second = OrEmpty(*requests_per_second), + failures_per_second = OrEmpty(*failures_per_second), + )?, + } + } + + Ok(()) + } + + fn write_scenario_metrics(&mut self) -> Result<(), GooseError> { + let Some(scenario_metrics) = &self.data.scenario_metrics else { + return Ok(()); + }; + + write!( + self.w, + r#" +## Scenario Metrics + +| Transaction | # Users | # Times Run | Average (ms) | Min (ms) | Max (ms) | Scenarios/s | Iterations | +| ----------- | ------: | ----------: | -----------: | -------: | -------: | ----------: | ---------: | +"# + )?; + + for ScenarioMetric { + name, + users, + count, + response_time_average, + response_time_minimum, + response_time_maximum, + count_per_second, + iterations, + } in scenario_metrics + { + writeln!( + self.w, + r#"| {name} | {users} | {count} | {response_time_average:.2} | {response_time_minimum} | {response_time_maximum} | {count_per_second:.2} | {iterations:.2} |"# + )?; + } + + Ok(()) + } + + fn write_error_metrics(&mut self) -> Result<(), GooseError> { + let Some(errors) = &self.data.errors else { + return Ok(()); + }; + + write!( + self.w, + r#" +## Error Metrics + +| Method | Name | # | Error | +| ------ | ---- | --: | ----- | +"# + )?; + + for GooseErrorMetricAggregate { + method, + name, + error, + occurrences, + } in errors + { + writeln!(self.w, r#"| {method} | {name} | {occurrences} | {error} |"#)?; + } + + Ok(()) + } +}