diff --git a/changelog.md b/changelog.md index c783c661d..edb4dc628 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,9 @@ heading: "Bencher Changelog" sortOrder: 4 --- +## Pending `v0.3.12` +- Change Metric `lower_bound` and `upper_bound` to `lower_value` and `upper_value` respectively + ## `v0.3.11` - Add strongly typed IDs for database entities - Remove deprecated configuration keys (`endpoint` => `console.url` and `secret_key` => `security.secret_key`) diff --git a/lib/bencher_adapter/benches/adapter.rs b/lib/bencher_adapter/benches/adapter.rs index 82d10ff58..fd98a44c8 100644 --- a/lib/bencher_adapter/benches/adapter.rs +++ b/lib/bencher_adapter/benches/adapter.rs @@ -7,29 +7,29 @@ const JSON_RESULT: &str = r#"{ "tests::benchmark_1": { "latency": { "value": 1.0, - "lower_bound": 1.0, - "upper_bound": 1.0 + "lower_value": 1.0, + "upper_value": 1.0 } }, "tests::benchmark_2": { "latency": { "value": 22.0, - "lower_bound": 22.0, - "upper_bound": 22.0 + "lower_value": 22.0, + "upper_value": 22.0 } }, "tests::benchmark_3": { "latency": { "value": 333.0, - "lower_bound": 333.0, - "upper_bound": 333.0 + "lower_value": 333.0, + "upper_value": 333.0 } }, "tests::benchmark_4": { "latency": { "value": 4444.0, - "lower_bound": 4444.0, - "upper_bound": 4444.0 + "lower_value": 4444.0, + "upper_value": 4444.0 } } }"#; diff --git a/lib/bencher_adapter/src/adapters/c_sharp/dot_net.rs b/lib/bencher_adapter/src/adapters/c_sharp/dot_net.rs index eb60277f8..ef549da43 100644 --- a/lib/bencher_adapter/src/adapters/c_sharp/dot_net.rs +++ b/lib/bencher_adapter/src/adapters/c_sharp/dot_net.rs @@ -79,11 +79,11 @@ impl DotNet { JsonAverage::Median => (median, interquartile_range), }; let value = latency_as_nanos(average, units); - let bound = latency_as_nanos(spread, units); + let spread = latency_as_nanos(spread, units); let json_metric = JsonMetric { value, - lower_bound: Some(value - bound), - upper_bound: Some(value + bound), + lower_value: Some(value - spread), + upper_value: Some(value + spread), }; benchmark_metrics.push((benchmark_name, json_metric)); diff --git a/lib/bencher_adapter/src/adapters/cpp/catch2.rs b/lib/bencher_adapter/src/adapters/cpp/catch2.rs index 37150a96a..019978a37 100644 --- a/lib/bencher_adapter/src/adapters/cpp/catch2.rs +++ b/lib/bencher_adapter/src/adapters/cpp/catch2.rs @@ -70,8 +70,8 @@ fn parse_catch2<'i>( if benchmark_name_remainder.is_empty() && mean_remainder.is_empty() { let json_metric = JsonMetric { value: mean, - lower_bound: Some(mean - std_dev), - upper_bound: Some(mean + std_dev), + lower_value: Some(mean - std_dev), + upper_value: Some(mean + std_dev), }; Ok((benchmark_name, json_metric)) diff --git a/lib/bencher_adapter/src/adapters/cpp/google.rs b/lib/bencher_adapter/src/adapters/cpp/google.rs index 4240422d2..8d31ae04d 100644 --- a/lib/bencher_adapter/src/adapters/cpp/google.rs +++ b/lib/bencher_adapter/src/adapters/cpp/google.rs @@ -57,8 +57,8 @@ impl TryFrom for Option { let value = latency_as_nanos(real_time, time_unit); let json_metric = JsonMetric { value, - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }; benchmark_metrics.push((name, json_metric)); diff --git a/lib/bencher_adapter/src/adapters/go/bench.rs b/lib/bencher_adapter/src/adapters/go/bench.rs index 64c867848..5eaeefc14 100644 --- a/lib/bencher_adapter/src/adapters/go/bench.rs +++ b/lib/bencher_adapter/src/adapters/go/bench.rs @@ -70,8 +70,8 @@ fn parse_go_bench(input: &str) -> IResult<&str, JsonMetric> { let value = latency_as_nanos(duration, units); Ok(JsonMetric { value, - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }) }, )(input) @@ -105,8 +105,8 @@ pub(crate) mod test_go_bench { "BenchmarkFib10-8".parse().unwrap(), JsonMetric { value: 325.0.into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, ), )), @@ -119,8 +119,8 @@ pub(crate) mod test_go_bench { "BenchmarkFib20".parse().unwrap(), JsonMetric { value: 40_537.123.into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, ), )), @@ -133,8 +133,8 @@ pub(crate) mod test_go_bench { "BenchmarkFib/my_tabled_benchmark_-_10-8".parse().unwrap(), JsonMetric { value: 325.0.into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, ), )), @@ -147,8 +147,8 @@ pub(crate) mod test_go_bench { "BenchmarkFib/my_tabled_benchmark_-_20".parse().unwrap(), JsonMetric { value: 40_537.123.into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, ), )), @@ -161,8 +161,8 @@ pub(crate) mod test_go_bench { "BenchmarkFib/my/tabled/benchmark_-_20".parse().unwrap(), JsonMetric { value: 40_537.456.into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, ), )), @@ -175,8 +175,8 @@ pub(crate) mod test_go_bench { "BenchmarkFib20WithAuxMetric-8".parse().unwrap(), JsonMetric { value: 25_829.0.into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, ), )), diff --git a/lib/bencher_adapter/src/adapters/java/jmh.rs b/lib/bencher_adapter/src/adapters/java/jmh.rs index 3b06f0aa5..35caed6cb 100644 --- a/lib/bencher_adapter/src/adapters/java/jmh.rs +++ b/lib/bencher_adapter/src/adapters/java/jmh.rs @@ -73,12 +73,12 @@ impl TryFrom for Option { let time_unit = unit.parse()?; let value = latency_as_nanos(score, time_unit); - let lower_bound = latency_as_nanos(score_confidence.0, time_unit); - let upper_bound = latency_as_nanos(score_confidence.1, time_unit); + let lower_value = latency_as_nanos(score_confidence.0, time_unit); + let upper_value = latency_as_nanos(score_confidence.1, time_unit); let json_metric = JsonMetric { value, - lower_bound: Some(lower_bound), - upper_bound: Some(upper_bound), + lower_value: Some(lower_value), + upper_value: Some(upper_value), }; AdapterMetricKind::Latency(json_metric) } else if let Some((ops_slash, unit)) = score_unit.split_once("ops/") { @@ -88,12 +88,12 @@ impl TryFrom for Option { let time_unit = unit.parse()?; let value = throughput_as_secs(score, time_unit); - let lower_bound = throughput_as_secs(score_confidence.0, time_unit); - let upper_bound = throughput_as_secs(score_confidence.1, time_unit); + let lower_value = throughput_as_secs(score_confidence.0, time_unit); + let upper_value = throughput_as_secs(score_confidence.1, time_unit); let json_metric = JsonMetric { value, - lower_bound: Some(lower_bound), - upper_bound: Some(upper_bound), + lower_value: Some(lower_value), + upper_value: Some(upper_value), }; AdapterMetricKind::Throughput(json_metric) } else { diff --git a/lib/bencher_adapter/src/adapters/js/benchmark.rs b/lib/bencher_adapter/src/adapters/js/benchmark.rs index 52cb799d3..a3b162c31 100644 --- a/lib/bencher_adapter/src/adapters/js/benchmark.rs +++ b/lib/bencher_adapter/src/adapters/js/benchmark.rs @@ -77,8 +77,8 @@ fn parse_benchmark_time(input: &str) -> IResult<&str, JsonMetric> { let error = value * percent_error; JsonMetric { value, - lower_bound: Some(value - error), - upper_bound: Some(value + error), + lower_value: Some(value - error), + upper_value: Some(value + error), } }, )(input) diff --git a/lib/bencher_adapter/src/adapters/js/time.rs b/lib/bencher_adapter/src/adapters/js/time.rs index 3cba33f9c..636bae7c1 100644 --- a/lib/bencher_adapter/src/adapters/js/time.rs +++ b/lib/bencher_adapter/src/adapters/js/time.rs @@ -72,8 +72,8 @@ fn parse_time_time(input: &str) -> IResult<&str, JsonMetric> { let value = latency_as_nanos(duration, units); JsonMetric { value, - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, } }, )(input) diff --git a/lib/bencher_adapter/src/adapters/mod.rs b/lib/bencher_adapter/src/adapters/mod.rs index c1cd02fdf..10db92e24 100644 --- a/lib/bencher_adapter/src/adapters/mod.rs +++ b/lib/bencher_adapter/src/adapters/mod.rs @@ -66,24 +66,24 @@ pub(crate) mod test_util { pub fn validate_latency( metrics: &AdapterMetrics, value: f64, - lower_bound: Option, - upper_bound: Option, + lower_value: Option, + upper_value: Option, ) { - validate_metric(metrics, LATENCY_SLUG_STR, value, lower_bound, upper_bound); + validate_metric(metrics, LATENCY_SLUG_STR, value, lower_value, upper_value); } pub fn validate_throughput( metrics: &AdapterMetrics, value: f64, - lower_bound: Option, - upper_bound: Option, + lower_value: Option, + upper_value: Option, ) { validate_metric( metrics, THROUGHPUT_SLUG_STR, value, - lower_bound, - upper_bound, + lower_value, + upper_value, ); } @@ -91,13 +91,13 @@ pub(crate) mod test_util { metrics: &AdapterMetrics, key: &str, value: f64, - lower_bound: Option, - upper_bound: Option, + lower_value: Option, + upper_value: Option, ) { assert_eq!(metrics.inner.len(), 1); let metric = metrics.get(key).unwrap(); assert_eq!(metric.value, OrderedFloat::from(value)); - assert_eq!(metric.lower_bound, lower_bound.map(OrderedFloat::from)); - assert_eq!(metric.upper_bound, upper_bound.map(OrderedFloat::from)); + assert_eq!(metric.lower_value, lower_value.map(OrderedFloat::from)); + assert_eq!(metric.upper_value, upper_value.map(OrderedFloat::from)); } } diff --git a/lib/bencher_adapter/src/adapters/python/asv.rs b/lib/bencher_adapter/src/adapters/python/asv.rs index a3c7496c0..c52386184 100644 --- a/lib/bencher_adapter/src/adapters/python/asv.rs +++ b/lib/bencher_adapter/src/adapters/python/asv.rs @@ -66,11 +66,11 @@ fn parse_asv_time(input: &str) -> IResult<&str, JsonMetric> { tuple((parse_f64, tag("±"), parse_f64, parse_units, eof)), |(duration, _, range, units, _)| { let value = latency_as_nanos(duration, units); - let bound = latency_as_nanos(range, units); + let range = latency_as_nanos(range, units); JsonMetric { value, - lower_bound: Some(value - bound), - upper_bound: Some(value + bound), + lower_value: Some(value - range), + upper_value: Some(value + range), } }, )(input) diff --git a/lib/bencher_adapter/src/adapters/python/pytest.rs b/lib/bencher_adapter/src/adapters/python/pytest.rs index b7674b6bf..d0383241d 100644 --- a/lib/bencher_adapter/src/adapters/python/pytest.rs +++ b/lib/bencher_adapter/src/adapters/python/pytest.rs @@ -75,11 +75,11 @@ impl Pytest { JsonAverage::Median => (median, iqr), }; let value = latency_as_nanos(average, units); - let bound = latency_as_nanos(spread, units); + let spread = latency_as_nanos(spread, units); let json_metric = JsonMetric { value, - lower_bound: Some(value - bound), - upper_bound: Some(value + bound), + lower_value: Some(value - spread), + upper_value: Some(value + spread), }; benchmark_metrics.push((benchmark_name, json_metric)); diff --git a/lib/bencher_adapter/src/adapters/ruby/benchmark.rs b/lib/bencher_adapter/src/adapters/ruby/benchmark.rs index 8311aacf9..24cb495f3 100644 --- a/lib/bencher_adapter/src/adapters/ruby/benchmark.rs +++ b/lib/bencher_adapter/src/adapters/ruby/benchmark.rs @@ -90,8 +90,8 @@ fn parse_ruby_benchmark(input: &str) -> IResult<&str, JsonMetric> { let value = latency_as_nanos(real, units); Ok(JsonMetric { value, - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }) }, )(input) diff --git a/lib/bencher_adapter/src/adapters/rust/bench.rs b/lib/bencher_adapter/src/adapters/rust/bench.rs index dbf6f87a6..d68c24bb6 100644 --- a/lib/bencher_adapter/src/adapters/rust/bench.rs +++ b/lib/bencher_adapter/src/adapters/rust/bench.rs @@ -74,8 +74,8 @@ fn parse_cargo_bench(input: &str) -> IResult<&str, JsonMetric> { let variance = Some(latency_as_nanos(variance, units)); JsonMetric { value, - lower_bound: variance.map(|v| value - v), - upper_bound: variance.map(|v| value + v), + lower_value: variance.map(|v| value - v), + upper_value: variance.map(|v| value + v), } }, )(input) @@ -145,8 +145,8 @@ pub(crate) mod test_rust_bench { "tests::is_bench_test".parse().unwrap(), JsonMetric { value: 5_280.0.into(), - lower_bound: Some(4_947.0.into()), - upper_bound: Some(5_613.0.into()), + lower_value: Some(4_947.0.into()), + upper_value: Some(5_613.0.into()), }, ), )), diff --git a/lib/bencher_adapter/src/adapters/rust/criterion.rs b/lib/bencher_adapter/src/adapters/rust/criterion.rs index eb87e018b..4e6fd185e 100644 --- a/lib/bencher_adapter/src/adapters/rust/criterion.rs +++ b/lib/bencher_adapter/src/adapters/rust/criterion.rs @@ -85,10 +85,10 @@ fn parse_criterion_metric(input: &str) -> IResult<&str, JsonMetric> { )), tag("]"), ), - |(lower_bound, _, value, _, upper_bound)| JsonMetric { + |(lower_value, _, value, _, upper_value)| JsonMetric { value, - lower_bound: Some(lower_bound), - upper_bound: Some(upper_bound), + lower_value: Some(lower_value), + upper_value: Some(upper_value), }, )(input) } @@ -128,8 +128,8 @@ pub(crate) mod test_rust_criterion { "criterion_benchmark".parse().unwrap(), JsonMetric { value: 280.0.into(), - lower_bound: Some(222.2.into()), - upper_bound: Some(333.33.into()), + lower_value: Some(222.2.into()), + upper_value: Some(333.33.into()), }, ), )), @@ -142,8 +142,8 @@ pub(crate) mod test_rust_criterion { "criterion_benchmark".parse().unwrap(), JsonMetric { value: 5.280.into(), - lower_bound: Some(0.222.into()), - upper_bound: Some(0.33333.into()), + lower_value: Some(0.222.into()), + upper_value: Some(0.33333.into()), }, ), )), @@ -156,8 +156,8 @@ pub(crate) mod test_rust_criterion { "criterion_benchmark".parse().unwrap(), JsonMetric { value: 18_019.0.into(), - lower_bound: Some(16_652.0.into()), - upper_bound: Some(19_562.0.into()), + lower_value: Some(16_652.0.into()), + upper_value: Some(19_562.0.into()), }, ), )), diff --git a/lib/bencher_adapter/src/adapters/rust/iai.rs b/lib/bencher_adapter/src/adapters/rust/iai.rs index 5fa53ee60..6509dced8 100644 --- a/lib/bencher_adapter/src/adapters/rust/iai.rs +++ b/lib/bencher_adapter/src/adapters/rust/iai.rs @@ -132,8 +132,8 @@ fn parse_iai_metric<'a>(input: &'a str, metric_kind: &'static str) -> IResult<&' )), |(_, _, _, _, metric, _)| JsonMetric { value: (metric as f64).into(), - lower_bound: None, - upper_bound: None, + lower_value: None, + upper_value: None, }, )(input) } @@ -168,8 +168,8 @@ pub(crate) mod test_rust_iai { for (key, value) in results { let metric = metrics.get(key).unwrap(); assert_eq!(metric.value, OrderedFloat::from(value)); - assert_eq!(metric.lower_bound, None); - assert_eq!(metric.upper_bound, None); + assert_eq!(metric.lower_value, None); + assert_eq!(metric.upper_value, None); } } @@ -181,8 +181,8 @@ pub(crate) mod test_rust_iai { "", JsonMetric { value: 1234.0.into(), - upper_bound: None, - lower_bound: None + upper_value: None, + lower_value: None } )) ); @@ -193,8 +193,8 @@ pub(crate) mod test_rust_iai { "", JsonMetric { value: 1234.0.into(), - upper_bound: None, - lower_bound: None + upper_value: None, + lower_value: None } )) ); @@ -205,8 +205,8 @@ pub(crate) mod test_rust_iai { "", JsonMetric { value: 1234.0.into(), - upper_bound: None, - lower_bound: None + upper_value: None, + lower_value: None } )) ); diff --git a/lib/bencher_adapter/tool_output/json/report_latency.json b/lib/bencher_adapter/tool_output/json/report_latency.json index 48ccc222b..54e6ce7cd 100644 --- a/lib/bencher_adapter/tool_output/json/report_latency.json +++ b/lib/bencher_adapter/tool_output/json/report_latency.json @@ -2,22 +2,22 @@ "tests::benchmark_a": { "latency": { "value": 3247.0, - "lower_bound": 1044.0, - "upper_bound": 1044.0 + "lower_value": 1044.0, + "upper_value": 1044.0 } }, "tests::benchmark_b": { "latency": { "value": 3443.0, - "lower_bound": 2275.0, - "upper_bound": 2275.0 + "lower_value": 2275.0, + "upper_value": 2275.0 } }, "tests::benchmark_c": { "latency": { "value": 3361.0, - "lower_bound": 1093.0, - "upper_bound": 1093.0 + "lower_value": 1093.0, + "upper_value": 1093.0 } } } \ No newline at end of file diff --git a/lib/bencher_json/src/project/metric/mod.rs b/lib/bencher_json/src/project/metric/mod.rs index 813a9ec88..17deb2302 100644 --- a/lib/bencher_json/src/project/metric/mod.rs +++ b/lib/bencher_json/src/project/metric/mod.rs @@ -16,16 +16,20 @@ pub use median::Median; #[cfg_attr(feature = "schema", derive(JsonSchema))] pub struct JsonMetric { pub value: OrderedFloat, - pub lower_bound: Option>, - pub upper_bound: Option>, + // TODO remove in due time + #[serde(alias = "lower_bound")] + pub lower_value: Option>, + // TODO remove in due time + #[serde(alias = "upper_bound")] + pub upper_value: Option>, } impl JsonMetric { - pub fn new(value: f64, lower_bound: Option, upper_bound: Option) -> Self { + pub fn new(value: f64, lower_value: Option, upper_value: Option) -> Self { Self { value: OrderedFloat(value), - lower_bound: lower_bound.map(OrderedFloat), - upper_bound: upper_bound.map(OrderedFloat), + lower_value: lower_value.map(OrderedFloat), + upper_value: upper_value.map(OrderedFloat), } } } @@ -39,8 +43,8 @@ impl fmt::Display for JsonMetric { impl PartialEq for JsonMetric { fn eq(&self, other: &Self) -> bool { self.value == other.value - && option_eq(self.lower_bound, other.lower_bound) - && option_eq(self.upper_bound, other.upper_bound) + && option_eq(self.lower_value, other.lower_value) + && option_eq(self.upper_value, other.upper_value) } } @@ -69,9 +73,9 @@ impl Ord for JsonMetric { fn cmp(&self, other: &Self) -> Ordering { let value_order = self.value.cmp(&other.value); if Ordering::Equal == value_order { - let upper_order = self.upper_bound.cmp(&other.upper_bound); + let upper_order = self.upper_value.cmp(&other.upper_value); if Ordering::Equal == upper_order { - self.lower_bound.cmp(&other.lower_bound) + self.lower_value.cmp(&other.lower_value) } else { upper_order } @@ -86,33 +90,33 @@ impl Add for JsonMetric { fn add(self, other: Self) -> Self { let value = self.value + other.value; - let lower_bound = option_add(self.lower_bound, self.value, other.lower_bound, other.value); - let upper_bound = option_add(self.upper_bound, self.value, other.upper_bound, other.value); + let lower_value = option_add(self.lower_value, self.value, other.lower_value, other.value); + let upper_value = option_add(self.upper_value, self.value, other.upper_value, other.value); Self { value, - lower_bound, - upper_bound, + lower_value, + upper_value, } } } fn option_add( - left_bound: Option, + left_end: Option, left_value: T, - right_bound: Option, + right_end: Option, right_value: T, ) -> Option where T: Add, { - if let Some(left_bound) = left_bound { - if let Some(right_bound) = right_bound { - Some(left_bound + right_bound) + if let Some(left_end) = left_end { + if let Some(right_end) = right_end { + Some(left_end + right_end) } else { - Some(left_bound + right_value) + Some(left_end + right_value) } } else { - right_bound.map(|rb| left_value + rb) + right_end.map(|re| left_value + re) } } @@ -132,8 +136,8 @@ impl std::ops::Div for JsonMetric { fn div(self, rhs: usize) -> Self::Output { Self { value: self.value / rhs as f64, - lower_bound: self.lower_bound.map(|b| b / rhs as f64), - upper_bound: self.upper_bound.map(|b| b / rhs as f64), + lower_value: self.lower_value.map(|b| b / rhs as f64), + upper_value: self.upper_value.map(|b| b / rhs as f64), } } } diff --git a/lib/bencher_plot/decimal.json b/lib/bencher_plot/decimal.json index db64cd239..9a55f7e09 100644 --- a/lib/bencher_plot/decimal.json +++ b/lib/bencher_plot/decimal.json @@ -59,8 +59,8 @@ "threshold": null, "metric": { "value": 0.000001987604723020502, - "lower_bound": 0.000001788844250718452, - "upper_bound": 0.0000021863651953225523 + "lower_value": 0.000001788844250718452, + "upper_value": 0.0000021863651953225523 }, "boundary": { "lower_limit": null, @@ -79,8 +79,8 @@ "threshold": null, "metric": { "value": 0.000003749262692764335, - "lower_bound": 0.0000033743364234879017, - "upper_bound": 0.000004124188962040768 + "lower_value": 0.0000033743364234879017, + "upper_value": 0.000004124188962040768 }, "boundary": { "lower_limit": null, @@ -99,8 +99,8 @@ "threshold": null, "metric": { "value": 0.000003238033192281371, - "lower_bound": 0.0000029142298730532337, - "upper_bound": 0.0000035618365115095084 + "lower_value": 0.0000029142298730532337, + "upper_value": 0.0000035618365115095084 }, "boundary": { "lower_limit": null, @@ -119,8 +119,8 @@ "threshold": null, "metric": { "value": 0.000007679525913445514, - "lower_bound": 0.0000069115733221009625, - "upper_bound": 0.000008447478504790066 + "lower_value": 0.0000069115733221009625, + "upper_value": 0.000008447478504790066 }, "boundary": { "lower_limit": null, @@ -139,8 +139,8 @@ "threshold": null, "metric": { "value": 0.0000021163849153774694, - "lower_bound": 0.0000019047464238397223, - "upper_bound": 0.0000023280234069152165 + "lower_value": 0.0000019047464238397223, + "upper_value": 0.0000023280234069152165 }, "boundary": { "lower_limit": null, @@ -187,8 +187,8 @@ "threshold": null, "metric": { "value": 0.00001997991060801682, - "lower_bound": 0.00001798191954721514, - "upper_bound": 0.0000219779016688185 + "lower_value": 0.00001798191954721514, + "upper_value": 0.0000219779016688185 }, "boundary": { "lower_limit": null, @@ -207,8 +207,8 @@ "threshold": null, "metric": { "value": 0.000017274684167770026, - "lower_bound": 0.000015547215750993025, - "upper_bound": 0.000019002152584547023 + "lower_value": 0.000015547215750993025, + "upper_value": 0.000019002152584547023 }, "boundary": { "lower_limit": null, @@ -227,8 +227,8 @@ "threshold": null, "metric": { "value": 0.00001858697184760528, - "lower_bound": 0.00001672827466284475, - "upper_bound": 0.000020445669032365805 + "lower_value": 0.00001672827466284475, + "upper_value": 0.000020445669032365805 }, "boundary": { "lower_limit": null, @@ -247,8 +247,8 @@ "threshold": null, "metric": { "value": 0.000012227269064199931, - "lower_bound": 0.00001100454215777994, - "upper_bound": 0.000013449995970619926 + "lower_value": 0.00001100454215777994, + "upper_value": 0.000013449995970619926 }, "boundary": { "lower_limit": null, @@ -267,8 +267,8 @@ "threshold": null, "metric": { "value": 0.000010735099761843256, - "lower_bound": 0.00000966158978565893, - "upper_bound": 0.00001180860973802758 + "lower_value": 0.00000966158978565893, + "upper_value": 0.00001180860973802758 }, "boundary": { "lower_limit": null, @@ -315,8 +315,8 @@ "threshold": null, "metric": { "value": 0.00002170317305227053, - "lower_bound": 0.000019532855747043477, - "upper_bound": 0.000023873490357497584 + "lower_value": 0.000019532855747043477, + "upper_value": 0.000023873490357497584 }, "boundary": { "lower_limit": null, @@ -335,8 +335,8 @@ "threshold": null, "metric": { "value": 0.00002893495333655924, - "lower_bound": 0.000026041458002903314, - "upper_bound": 0.000031828448670215165 + "lower_value": 0.000026041458002903314, + "upper_value": 0.000031828448670215165 }, "boundary": { "lower_limit": null, @@ -355,8 +355,8 @@ "threshold": null, "metric": { "value": 0.000029010578983697, - "lower_bound": 0.0000261095210853273, - "upper_bound": 0.0000319116368820667 + "lower_value": 0.0000261095210853273, + "upper_value": 0.0000319116368820667 }, "boundary": { "lower_limit": null, @@ -375,8 +375,8 @@ "threshold": null, "metric": { "value": 0.000029384319299781763, - "lower_bound": 0.00002644588736980359, - "upper_bound": 0.00003232275122975994 + "lower_value": 0.00002644588736980359, + "upper_value": 0.00003232275122975994 }, "boundary": { "lower_limit": null, @@ -395,8 +395,8 @@ "threshold": null, "metric": { "value": 0.000029697462649724795, - "lower_bound": 0.00002672771638475231, - "upper_bound": 0.00003266720891469727 + "lower_value": 0.00002672771638475231, + "upper_value": 0.00003266720891469727 }, "boundary": { "lower_limit": null, diff --git a/lib/bencher_plot/perf.json b/lib/bencher_plot/perf.json index 1cf153abd..a63b84f7b 100644 --- a/lib/bencher_plot/perf.json +++ b/lib/bencher_plot/perf.json @@ -59,8 +59,8 @@ "threshold": null, "metric": { "value": 7.070632155490754, - "lower_bound": 6.363568939941679, - "upper_bound": 7.77769537103983 + "lower_value": 6.363568939941679, + "upper_value": 7.77769537103983 }, "boundary": { "lower_limit": null, @@ -79,8 +79,8 @@ "threshold": null, "metric": { "value": 0.38989849192403, - "lower_bound": 0.35090864273162703, - "upper_bound": 0.42888834111643304 + "lower_value": 0.35090864273162703, + "upper_value": 0.42888834111643304 }, "boundary": { "lower_limit": null, @@ -99,8 +99,8 @@ "threshold": null, "metric": { "value": 8.618085683005877, - "lower_bound": 7.756277114705289, - "upper_bound": 9.479894251306463 + "lower_value": 7.756277114705289, + "upper_value": 9.479894251306463 }, "boundary": { "lower_limit": null, @@ -119,8 +119,8 @@ "threshold": null, "metric": { "value": 2.02750859607544, - "lower_bound": 1.824757736467896, - "upper_bound": 2.230259455682984 + "lower_value": 1.824757736467896, + "upper_value": 2.230259455682984 }, "boundary": { "lower_limit": null, @@ -139,8 +139,8 @@ "threshold": null, "metric": { "value": 1.5980960602930727, - "lower_bound": 1.4382864542637654, - "upper_bound": 1.75790566632238 + "lower_value": 1.4382864542637654, + "upper_value": 1.75790566632238 }, "boundary": { "lower_limit": null, @@ -159,8 +159,8 @@ "threshold": null, "metric": { "value": 7.199331114196812, - "lower_bound": 6.479398002777131, - "upper_bound": 7.919264225616493 + "lower_value": 6.479398002777131, + "upper_value": 7.919264225616493 }, "boundary": { "lower_limit": null, @@ -179,8 +179,8 @@ "threshold": null, "metric": { "value": 6.11864860295499, - "lower_bound": 5.506783742659492, - "upper_bound": 6.730513463250489 + "lower_value": 5.506783742659492, + "upper_value": 6.730513463250489 }, "boundary": { "lower_limit": null, @@ -199,8 +199,8 @@ "threshold": null, "metric": { "value": 1.55004991113737, - "lower_bound": 1.395044920023633, - "upper_bound": 1.705054902251107 + "lower_value": 1.395044920023633, + "upper_value": 1.705054902251107 }, "boundary": { "lower_limit": null, @@ -247,8 +247,8 @@ "threshold": null, "metric": { "value": 10.717940411494348, - "lower_bound": 9.646146370344914, - "upper_bound": 11.789734452643785 + "lower_value": 9.646146370344914, + "upper_value": 11.789734452643785 }, "boundary": { "lower_limit": null, @@ -267,8 +267,8 @@ "threshold": null, "metric": { "value": 11.629722712218056, - "lower_bound": 10.46675044099625, - "upper_bound": 12.792694983439862 + "lower_value": 10.46675044099625, + "upper_value": 12.792694983439862 }, "boundary": { "lower_limit": null, @@ -287,8 +287,8 @@ "threshold": null, "metric": { "value": 19.234737429290924, - "lower_bound": 17.31126368636183, - "upper_bound": 21.158211172220017 + "lower_value": 17.31126368636183, + "upper_value": 21.158211172220017 }, "boundary": { "lower_limit": null, @@ -307,8 +307,8 @@ "threshold": null, "metric": { "value": 13.887107300642333, - "lower_bound": 12.4983965705781, - "upper_bound": 15.275818030706564 + "lower_value": 12.4983965705781, + "upper_value": 15.275818030706564 }, "boundary": { "lower_limit": null, @@ -327,8 +327,8 @@ "threshold": null, "metric": { "value": 12.119943678611468, - "lower_bound": 10.90794931075032, - "upper_bound": 13.331938046472612 + "lower_value": 10.90794931075032, + "upper_value": 13.331938046472612 }, "boundary": { "lower_limit": null, @@ -347,8 +347,8 @@ "threshold": null, "metric": { "value": 11.323306653976232, - "lower_bound": 10.190975988578607, - "upper_bound": 12.455637319373857 + "lower_value": 10.190975988578607, + "upper_value": 12.455637319373857 }, "boundary": { "lower_limit": null, @@ -367,8 +367,8 @@ "threshold": null, "metric": { "value": 19.205238441354258, - "lower_bound": 17.28471459721883, - "upper_bound": 21.12576228548968 + "lower_value": 17.28471459721883, + "upper_value": 21.12576228548968 }, "boundary": { "lower_limit": null, @@ -387,8 +387,8 @@ "threshold": null, "metric": { "value": 14.791563106848365, - "lower_bound": 13.312406796163527, - "upper_bound": 16.2707194175332 + "lower_value": 13.312406796163527, + "upper_value": 16.2707194175332 }, "boundary": { "lower_limit": null, @@ -435,8 +435,8 @@ "threshold": null, "metric": { "value": 20.6971015060427, - "lower_bound": 18.62739135543843, - "upper_bound": 22.76681165664697 + "lower_value": 18.62739135543843, + "upper_value": 22.76681165664697 }, "boundary": { "lower_limit": null, @@ -455,8 +455,8 @@ "threshold": null, "metric": { "value": 24.118021157821826, - "lower_bound": 21.70621904203964, - "upper_bound": 26.52982327360401 + "lower_value": 21.70621904203964, + "upper_value": 26.52982327360401 }, "boundary": { "lower_limit": null, @@ -475,8 +475,8 @@ "threshold": null, "metric": { "value": 29.33519066616913, - "lower_bound": 26.401671599552213, - "upper_bound": 32.268709732786036 + "lower_value": 26.401671599552213, + "upper_value": 32.268709732786036 }, "boundary": { "lower_limit": null, @@ -495,8 +495,8 @@ "threshold": null, "metric": { "value": 26.513651602923844, - "lower_bound": 23.862286442631454, - "upper_bound": 29.16501676321623 + "lower_value": 23.862286442631454, + "upper_value": 29.16501676321623 }, "boundary": { "lower_limit": null, @@ -515,8 +515,8 @@ "threshold": null, "metric": { "value": 26.768095038300785, - "lower_bound": 24.091285534470703, - "upper_bound": 29.44490454213086 + "lower_value": 24.091285534470703, + "upper_value": 29.44490454213086 }, "boundary": { "lower_limit": null, @@ -535,8 +535,8 @@ "threshold": null, "metric": { "value": 21.10788996443799, - "lower_bound": 18.997100967994196, - "upper_bound": 23.218678960881793 + "lower_value": 18.997100967994196, + "upper_value": 23.218678960881793 }, "boundary": { "lower_limit": null, @@ -555,8 +555,8 @@ "threshold": null, "metric": { "value": 23.44133114519035, - "lower_bound": 21.09719803067132, - "upper_bound": 25.78546425970939 + "lower_value": 21.09719803067132, + "upper_value": 25.78546425970939 }, "boundary": { "lower_limit": null, @@ -575,8 +575,8 @@ "threshold": null, "metric": { "value": 20.49978922728023, - "lower_bound": 18.449810304552205, - "upper_bound": 22.54976815000825 + "lower_value": 18.449810304552205, + "upper_value": 22.54976815000825 }, "boundary": { "lower_limit": null, diff --git a/services/api/migrations/2023-09-25-133957_metric_value/down.sql b/services/api/migrations/2023-09-25-133957_metric_value/down.sql new file mode 100644 index 000000000..be5d6ef7f --- /dev/null +++ b/services/api/migrations/2023-09-25-133957_metric_value/down.sql @@ -0,0 +1,34 @@ +PRAGMA foreign_keys = off; +CREATE TABLE down_metric ( + id INTEGER PRIMARY KEY NOT NULL, + uuid TEXT NOT NULL UNIQUE, + perf_id INTEGER NOT NULL, + metric_kind_id INTEGER NOT NULL, + value DOUBLE NOT NULL, + lower_bound DOUBLE, + upper_bound DOUBLE, + FOREIGN KEY (perf_id) REFERENCES perf (id) ON DELETE CASCADE, + FOREIGN KEY (metric_kind_id) REFERENCES metric_kind (id), + UNIQUE(perf_id, metric_kind_id) +); +INSERT INTO down_metric( + id, + uuid, + perf_id, + metric_kind_id, + value, + lower_bound, + upper_bound + ) +SELECT id, + uuid, + perf_id, + metric_kind_id, + value, + lower_value, + upper_value +FROM metric; +DROP TABLE metric; +ALTER TABLE down_metric + RENAME TO metric; +PRAGMA foreign_keys = on; \ No newline at end of file diff --git a/services/api/migrations/2023-09-25-133957_metric_value/up.sql b/services/api/migrations/2023-09-25-133957_metric_value/up.sql new file mode 100644 index 000000000..f841de069 --- /dev/null +++ b/services/api/migrations/2023-09-25-133957_metric_value/up.sql @@ -0,0 +1,34 @@ +PRAGMA foreign_keys = off; +CREATE TABLE up_metric ( + id INTEGER PRIMARY KEY NOT NULL, + uuid TEXT NOT NULL UNIQUE, + perf_id INTEGER NOT NULL, + metric_kind_id INTEGER NOT NULL, + value DOUBLE NOT NULL, + lower_value DOUBLE, + upper_value DOUBLE, + FOREIGN KEY (perf_id) REFERENCES perf (id) ON DELETE CASCADE, + FOREIGN KEY (metric_kind_id) REFERENCES metric_kind (id), + UNIQUE(perf_id, metric_kind_id) +); +INSERT INTO up_metric( + id, + uuid, + perf_id, + metric_kind_id, + value, + lower_value, + upper_value + ) +SELECT id, + uuid, + perf_id, + metric_kind_id, + value, + lower_bound, + upper_bound +FROM metric; +DROP TABLE metric; +ALTER TABLE up_metric + RENAME TO metric; +PRAGMA foreign_keys = on; \ No newline at end of file diff --git a/services/api/src/endpoints/project/perf/mod.rs b/services/api/src/endpoints/project/perf/mod.rs index 23c032345..db865b9d1 100644 --- a/services/api/src/endpoints/project/perf/mod.rs +++ b/services/api/src/endpoints/project/perf/mod.rs @@ -348,8 +348,8 @@ fn perf_query( schema::version::hash, schema::metric::id, schema::metric::value, - schema::metric::lower_bound, - schema::metric::upper_bound, + schema::metric::lower_value, + schema::metric::upper_value, )) .load::(conn) .map_err(ApiError::from)? @@ -378,8 +378,8 @@ fn perf_metric( version_hash, metric_id, value, - lower_bound, - upper_bound, + lower_value, + upper_value, ): PerfQuery, ) -> Option { // The boundary may not exist @@ -417,8 +417,8 @@ fn perf_metric( threshold, metric: JsonMetric { value: value.into(), - lower_bound: lower_bound.map(Into::into), - upper_bound: upper_bound.map(Into::into), + lower_value: lower_value.map(Into::into), + upper_value: upper_value.map(Into::into), }, boundary: boundary.map(QueryBoundary::into_json).unwrap_or_default(), alert, diff --git a/services/api/src/model/project/benchmark.rs b/services/api/src/model/project/benchmark.rs index dbba3ef2a..569f13b50 100644 --- a/services/api/src/model/project/benchmark.rs +++ b/services/api/src/model/project/benchmark.rs @@ -138,7 +138,7 @@ impl QueryBenchmark { conn: &mut DbConnection, metric_id: MetricId, ) -> Result { - let (uuid, project_id, name, slug, created, modified, value, lower_bound, upper_bound) = + let (uuid, project_id, name, slug, created, modified, value, lower_value, upper_value) = schema::metric::table .filter(schema::metric::id.eq(metric_id)) .left_join(schema::perf::table.on(schema::perf::id.eq(schema::metric::perf_id))) @@ -154,8 +154,8 @@ impl QueryBenchmark { schema::benchmark::created, schema::benchmark::modified, schema::metric::value, - schema::metric::lower_bound, - schema::metric::upper_bound, + schema::metric::lower_value, + schema::metric::upper_value, )) .first::<( String, @@ -178,7 +178,7 @@ impl QueryBenchmark { created, modified, } = Self::get_benchmark_json(conn, &uuid, project_id, &name, &slug, created, modified)?; - let metric = QueryMetric::json(value, lower_bound, upper_bound); + let metric = QueryMetric::json(value, lower_value, upper_value); let boundary = QueryBoundary::get_json(conn, metric_id); Ok(JsonBenchmarkMetric { diff --git a/services/api/src/model/project/metric.rs b/services/api/src/model/project/metric.rs index 9a0a379d1..c84016e06 100644 --- a/services/api/src/model/project/metric.rs +++ b/services/api/src/model/project/metric.rs @@ -25,8 +25,8 @@ pub struct QueryMetric { pub perf_id: PerfId, pub metric_kind_id: MetricKindId, pub value: f64, - pub lower_bound: Option, - pub upper_bound: Option, + pub lower_value: Option, + pub upper_value: Option, } impl QueryMetric { @@ -37,22 +37,22 @@ impl QueryMetric { .map_err(ApiError::from) } - pub fn json(value: f64, lower_bound: Option, upper_bound: Option) -> JsonMetric { + pub fn json(value: f64, lower_value: Option, upper_value: Option) -> JsonMetric { JsonMetric { value: value.into(), - lower_bound: lower_bound.map(Into::into), - upper_bound: upper_bound.map(Into::into), + lower_value: lower_value.map(Into::into), + upper_value: upper_value.map(Into::into), } } pub fn into_json(self) -> JsonMetric { let Self { value, - lower_bound, - upper_bound, + lower_value, + upper_value, .. } = self; - Self::json(value, lower_bound, upper_bound) + Self::json(value, lower_value, upper_value) } } @@ -63,24 +63,24 @@ pub struct InsertMetric { pub perf_id: PerfId, pub metric_kind_id: MetricKindId, pub value: f64, - pub lower_bound: Option, - pub upper_bound: Option, + pub lower_value: Option, + pub upper_value: Option, } impl InsertMetric { pub fn from_json(perf_id: PerfId, metric_kind_id: MetricKindId, metric: JsonMetric) -> Self { let JsonMetric { value, - lower_bound, - upper_bound, + lower_value, + upper_value, } = metric; Self { perf_id, metric_kind_id, uuid: Uuid::new_v4().to_string(), value: value.into(), - lower_bound: lower_bound.map(Into::into), - upper_bound: upper_bound.map(Into::into), + lower_value: lower_value.map(Into::into), + upper_value: upper_value.map(Into::into), } } } diff --git a/services/api/src/schema.rs b/services/api/src/schema.rs index fb7350b25..409f6151b 100644 --- a/services/api/src/schema.rs +++ b/services/api/src/schema.rs @@ -62,8 +62,8 @@ diesel::table! { perf_id -> Integer, metric_kind_id -> Integer, value -> Double, - lower_bound -> Nullable, - upper_bound -> Nullable, + lower_value -> Nullable, + upper_value -> Nullable, } } diff --git a/services/cli/src/bencher/sub/mock.rs b/services/cli/src/bencher/sub/mock.rs index 6aebfee3b..f52984809 100644 --- a/services/cli/src/bencher/sub/mock.rs +++ b/services/cli/src/bencher/sub/mock.rs @@ -98,8 +98,8 @@ impl Mock { inner: hmap! { LATENCY_RESOURCE_ID.clone() => JsonMetric { value: value.into(), - lower_bound: Some((value - variance).into()), - upper_bound: Some((value + variance).into()), + lower_value: Some((value - variance).into()), + upper_value: Some((value + variance).into()), } }, }, diff --git a/services/console/src/content/api/swagger.json b/services/console/src/content/api/swagger.json index 218c05ffa..e4db6761a 100644 --- a/services/console/src/content/api/swagger.json +++ b/services/console/src/content/api/swagger.json @@ -6415,12 +6415,12 @@ "JsonMetric": { "type": "object", "properties": { - "lower_bound": { + "lower_value": { "nullable": true, "type": "number", "format": "double" }, - "upper_bound": { + "upper_value": { "nullable": true, "type": "number", "format": "double" diff --git a/services/console/src/content/explanation/adapters.mdx b/services/console/src/content/explanation/adapters.mdx index bdf3c2024..9e037554d 100644 --- a/services/console/src/content/explanation/adapters.mdx +++ b/services/console/src/content/explanation/adapters.mdx @@ -42,8 +42,8 @@ Example of BMF: "benchmark_name": { "latency": { value: 88.0, - lower_bound: 87.42, - upper_bound: 88.88 + lower_value: 87.42, + upper_value: 88.88 } } } @@ -55,15 +55,15 @@ The `benchmark_name` object contains Metric Kind slugs or UUIDs as keys. In this example, `latency` is the slug for the Latency Metric Kind. Each Project by default has a Latency (ie `latency`) and Throughput (ie `throughput`) Metric Kind, which are measured in `nanosecond (ns)` and `operations / second (ops/s)` respectively. -The Metric Kind object contains a Metric with up to three measures: `value`, `lower_bound`, and `upper_bound`. -The `lower_bound` and `upper_bound` measures are optional, +The Metric Kind object contains a Metric with up to three measures: `value`, `lower_value`, and `upper_value`. +The `lower_value` and `upper_value` measures are optional, and their calculation is benchmark harness specific. In this example, the `latency` Metric Kind object contains the following measures: - A `value` of `88.0` -- A `lower_bound` of `87.42` -- An `upper_bound` of `88.88` +- A `lower_value` of `87.42` +- An `upper_value` of `88.88` If the BMF JSON is stored in a file, then you can use the bencher run CLI subcommand with the optional `--file` argument to specify that file path @@ -83,8 +83,8 @@ The C# DotNet Adapter (`c_sharp_dot_net`) expects [BenchmarkDotNet](https://gith The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. There are two options for the Metric: -- `mean` (default): The `lower_bound` and `upper_bound` are one standard deviation below and above the mean (ie `value`) respectively. -- `median`: The `lower_bound` and `upper_bound` are one interquartile range below and above the median (ie `value`) respectively. +- `mean` (default): The `lower_value` and `upper_value` are one standard deviation below and above the mean (ie `value`) respectively. +- `median`: The `lower_value` and `upper_value` are one interquartile range below and above the median (ie `value`) respectively. This can be specified in the bencher run CLI subcommand with the optional `--average` flag. @@ -96,13 +96,13 @@ The C++ Adapter (`cpp`) is a superset of `cpp_catch2` and `cpp_google`. The C++ Catch2 Adapter (`cpp_catch2`) expects [Catch2](https://github.com/catchorg/Catch2) output. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -The `lower_bound` and `upper_bound` are one standard deviation below and above the mean (ie `value`) respectively. +The `lower_value` and `upper_value` are one standard deviation below and above the mean (ie `value`) respectively. ## ➕ C++ Google The C++ Google Adapter (`cpp_google`) expects [Google Benchmark](https://github.com/google/benchmark) output in [JSON format (ie `--benchmark_format=json`)](https://github.com/google/benchmark/blob/main/docs/user_guide.md#output-formats). The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -Only the mean (ie `value`) is available. There are no `lower_bound` and `upper_bound`. +Only the mean (ie `value`) is available. There are no `lower_value` and `upper_value`. ## 🕳 Go @@ -112,7 +112,7 @@ The Go Adapter (`go`) is a superset of `go_bench`. The Go Bench Adapter (`go_bench`) expects [go test -bench](https://pkg.go.dev/testing#hdr-Benchmarks) output. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -Only the mean (ie `value`) is available. There are no `lower_bound` and `upper_bound`. +Only the mean (ie `value`) is available. There are no `lower_value` and `upper_value`. ## ☕️ Java @@ -122,7 +122,7 @@ The Java Adapter (`java`) is a superset of `java_jmh`. The Java JMH Adapter (`java_jmh`) expects [Java Microbenchmark Harness (JMH)](https://github.com/openjdk/jmh) output in [JSON format (ie `-rf json`)](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/results/format/ResultFormatType.java). Both `latency` and `throughput` Metric Kinds (ie `nanoseconds (ns)` and `operations / second (ops/sec)`) may be gathered. -The `lower_bound` and `upper_bound` are the lower and upper confidence intervals for the mean (ie `value`) respectively. +The `lower_value` and `upper_value` are the lower and upper confidence intervals for the mean (ie `value`) respectively. ## 🕸 JavaScript @@ -132,13 +132,13 @@ The JavaScript Adapter (`js`) is a superset of `js_benchmark` and `js_time`. The JavaScript Benchmark Adapter (`js_benchmark`) expects [Benchmark.js](https://github.com/bestiejs/benchmark.js) output. The `throughput` Metric Kind (ie `operations / second (ops/sec)`) is gathered. -The `lower_bound` and `upper_bound` are the relative margin of error below and above the median (ie `value`) respectively. +The `lower_value` and `upper_value` are the relative margin of error below and above the median (ie `value`) respectively. ## 🕸 JavaScript Time The JavaScript Time Adapter (`js_time`) expects [console.time](https://developer.mozilla.org/en-US/docs/Web/API/console/time)/[console.timeEnd](https://developer.mozilla.org/en-US/docs/Web/API/console/timeEnd) output. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -Only the operation time (ie `value`) is available. There are no `lower_bound` and `upper_bound`. +Only the operation time (ie `value`) is available. There are no `lower_value` and `upper_value`. ## 🐍 Python @@ -148,7 +148,7 @@ The Python Adapter (`python`) is a superset of `python_asv` and `python_pytest`. The Python ASV Adapter (`python_asv`) expects [airspeed velocity](https://github.com/airspeed-velocity/asv) CLI [asv run](https://asv.readthedocs.io/en/stable/commands.html#asv-run) output. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -The `lower_bound` and `upper_bound` are the interquartile range below and above the median (ie `value`) respectively. +The `lower_value` and `upper_value` are the interquartile range below and above the median (ie `value`) respectively. ## 🐍 Python Pytest @@ -157,8 +157,8 @@ This JSON output is saved to a file, so you must use the `bencher run` CLI `--fi The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. There are two options for the Metric: -- `mean` (default): The `lower_bound` and `upper_bound` are one standard deviation below and above the mean (ie `value`) respectively. -- `median`: The `lower_bound` and `upper_bound` are one interquartile range below and above the median (ie `value`) respectively. +- `mean` (default): The `lower_value` and `upper_value` are one standard deviation below and above the mean (ie `value`) respectively. +- `median`: The `lower_value` and `upper_value` are one interquartile range below and above the median (ie `value`) respectively. This can be specified in the bencher run CLI subcommand with the optional `--average` argument. @@ -171,7 +171,7 @@ The Ruby Adapter (`ruby`) is a superset of `ruby_benchmark`. The Ruby Benchmark Adapter (`ruby_benchmark`) expects [Benchmark module](https://github.com/ruby/benchmark) output for the `#bm`, `#bmbm`, and `#benchmark` methods. A label is required for each benchmark. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -Only the reported value (ie `value`) is available. There are no `lower_bound` and `upper_bound`. +Only the reported value (ie `value`) is available. There are no `lower_value` and `upper_value`. ## 🦀 Rust @@ -181,19 +181,19 @@ The Rust Adapter (`rust`) is a superset of `rust_bench` and `rust_criterion`. The Rust Bench Adapter (`rust_bench`) expects [libtest bench](https://doc.rust-lang.org/rustc/tests/index.html#benchmarks) output. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -The `lower_bound` and `upper_bound` are the deviation below and above the median (ie `value`) respectively. +The `lower_value` and `upper_value` are the deviation below and above the median (ie `value`) respectively. ## 🦀 Rust Criterion The Rust Criterion Adapter (`rust_criterion`) expects [Criterion](https://github.com/bheisler/criterion.rs) output. The `latency` Metric Kind (ie `nanoseconds (ns)`) is gathered. -The `lower_bound` and `upper_bound` are the lower and upper bounds of either the slope (if available) or the mean (if not) (ie `value`) respectively. +The `lower_value` and `upper_value` are the lower and upper bounds of either the slope (if available) or the mean (if not) (ie `value`) respectively. ## 🦀 Rust Iai The Rust Iai Adapter (`rust_iai`) expects [Iai](https://github.com/bheisler/iai) output. The `instructions`, `l1_access`, `l2_access`, `ram_access`, and `estimated_cycles` Metric Kinds are gathered. -Only these measures (ie `value`) are available. There are no `lower_bound` and `upper_bound` measures. +Only these measures (ie `value`) are available. There are no `lower_value` and `upper_value` measures. The Metric Kinds for this adapter are not created by default for all projects. However, when you use this adapter, these Metric Kinds will be automatically created for your Project. diff --git a/services/console/src/content/explanation/benchmarking.mdx b/services/console/src/content/explanation/benchmarking.mdx index 592a17834..2b1aedc05 100644 --- a/services/console/src/content/explanation/benchmarking.mdx +++ b/services/console/src/content/explanation/benchmarking.mdx @@ -35,8 +35,8 @@ The only exception to the above caveat is ignoring a Benchmark. See [suppressing ### Metrics A Metric is a single, point-in-time performance regression test result. -Up to three Measures may be collected for a single Metric: `value`, `lower_bound`, and `upper_bound`. -The `value` is required for all Metrics while the `lower_bound` and `upper_bound` are independently optional. +Up to three Measures may be collected for a single Metric: `value`, `lower_value`, and `upper_value`. +The `value` is required for all Metrics while the `lower_value` and `upper_value` are independently optional. Which Measures are collected is determined by the [benchmark harness adapter](/docs/explanation/adapters). ### Metric Kind diff --git a/services/console/src/content/reference/prior-art.mdx b/services/console/src/content/reference/prior-art.mdx index aac88fa2a..da644744a 100644 --- a/services/console/src/content/reference/prior-art.mdx +++ b/services/console/src/content/reference/prior-art.mdx @@ -33,6 +33,7 @@ sortOrder: 2 - [Unity-Technologies/PerformanceBenchmarkReporter](https://github.com/Unity-Technologies/PerformanceBenchmarkReporter) - Establish benchmark samples and measurements using the Performance Testing package, then use these benchmark values to compare subsequent performance test results in an html output utilizing graphical visualizations - [BenchHub](https://github.com/benchhub/benchhub) - A service for running database benchmarks and saving the result - [bench-bot](https://github.com/icebob/bench-bot) - Benchmark runner robot. Continuous benchmarking for benchmarkify the benchmark framework for NodeJS + - [CodSpeed](https://codspeed.io/) - CodSpeed provides integrated CI tools for software engineering teams to anticipate the impacts of the next delivery on system performance. - Web Specific - [Contentsquare](https://contentsquare.com/product-tour-speed-analysis/) - Speed Analysis is not only about Synthetic Monitoring but also offers powerful Real User Monitoring capabilities, a great fit for brands leading the way on customer experience across the world. - [Iron/Out](https://www.iron-out.io) - Faster websites equals better business results: improve user experience, increase conversion rate and page experience ranking, lower the bounce rate @@ -110,6 +111,7 @@ sortOrder: 2 - [Is GitHub Actions suitable for running benchmarks?](https://labs.quansight.org/blog/2021/08/github-actions-benchmarks) - [Created GitHub Action for continuous benchmarking ](https://rhysd-hatenablog-com.translate.goog/entry/2019/11/11/131505?_x_tr_sl=ja&_x_tr_tl=en&_x_tr_hl=en&_x_tr_pto=wapp) - [automatically prevent performance regressions](https://stackoverflow.com/questions/19713319/automatically-prevent-performance-regressions) + - [Exploring the Rust compiler benchmark suite](https://kobzol.github.io/rust/rustc/2023/08/18/rustc-benchmark-suite.html) ## Benchmark Comparisons diff --git a/services/console/src/content/tutorial/quick-start.mdx b/services/console/src/content/tutorial/quick-start.mdx index 19fee97cc..319672fe5 100644 --- a/services/console/src/content/tutorial/quick-start.mdx +++ b/services/console/src/content/tutorial/quick-start.mdx @@ -191,36 +191,36 @@ $ bencher mock "bencher::mock_0": { "latency": { "value": 3.7865423396154463, - "lower_bound": 3.4078881056539014, - "upper_bound": 4.165196573576991 + "lower_value": 3.4078881056539014, + "upper_value": 4.165196573576991 } }, "bencher::mock_1": { "latency": { "value": 16.398332128878437, - "lower_bound": 14.758498915990593, - "upper_bound": 18.03816534176628 + "lower_value": 14.758498915990593, + "upper_value": 18.03816534176628 } }, "bencher::mock_2": { "latency": { "value": 20.88091359871672, - "lower_bound": 18.792822238845048, - "upper_bound": 22.969004958588393 + "lower_value": 18.792822238845048, + "upper_value": 22.969004958588393 } }, "bencher::mock_3": { "latency": { "value": 33.88103801203782, - "lower_bound": 30.492934210834036, - "upper_bound": 37.2691418132416 + "lower_value": 30.492934210834036, + "upper_value": 37.2691418132416 } }, "bencher::mock_4": { "latency": { "value": 40.90515638867921, - "lower_bound": 36.81464074981129, - "upper_bound": 44.99567202754713 + "lower_value": 36.81464074981129, + "upper_value": 44.99567202754713 } } } @@ -244,36 +244,36 @@ $ bencher run --project save-walter-white-12345 "bencher mock" "bencher::mock_0": { "latency": { "value": 0.15496641529475275, - "lower_bound": 0.13946977376527747, - "upper_bound": 0.17046305682422802 + "lower_value": 0.13946977376527747, + "upper_value": 0.17046305682422802 } }, "bencher::mock_1": { "latency": { "value": 18.648298578180437, - "lower_bound": 16.783468720362393, - "upper_bound": 20.513128435998482 + "lower_value": 16.783468720362393, + "upper_value": 20.513128435998482 } }, "bencher::mock_2": { "latency": { "value": 28.20328182167366, - "lower_bound": 25.382953639506294, - "upper_bound": 31.023610003841025 + "lower_value": 25.382953639506294, + "upper_value": 31.023610003841025 } }, "bencher::mock_3": { "latency": { "value": 34.45732560787596, - "lower_bound": 31.01159304708836, - "upper_bound": 37.903058168663556 + "lower_value": 31.01159304708836, + "upper_value": 37.903058168663556 } }, "bencher::mock_4": { "latency": { "value": 44.9237520767597, - "lower_bound": 40.43137686908373, - "upper_bound": 49.41612728443567 + "lower_value": 40.43137686908373, + "upper_value": 49.41612728443567 } } } @@ -282,7 +282,7 @@ $ bencher run --project save-walter-white-12345 "bencher mock" "branch": "master", "end_time": "2023-07-18T14:21:27.796871Z", "results": [ - "{\n \"bencher::mock_0\": {\n \"latency\": {\n \"value\": 0.15496641529475275,\n \"lower_bound\": 0.13946977376527747,\n \"upper_bound\": 0.17046305682422802\n }\n },\n \"bencher::mock_1\": {\n \"latency\": {\n \"value\": 18.648298578180437,\n \"lower_bound\": 16.783468720362393,\n \"upper_bound\": 20.513128435998482\n }\n },\n \"bencher::mock_2\": {\n \"latency\": {\n \"value\": 28.20328182167366,\n \"lower_bound\": 25.382953639506294,\n \"upper_bound\": 31.023610003841025\n }\n },\n \"bencher::mock_3\": {\n \"latency\": {\n \"value\": 34.45732560787596,\n \"lower_bound\": 31.01159304708836,\n \"upper_bound\": 37.903058168663556\n }\n },\n \"bencher::mock_4\": {\n \"latency\": {\n \"value\": 44.9237520767597,\n \"lower_bound\": 40.43137686908373,\n \"upper_bound\": 49.41612728443567\n }\n }\n}\n" + "{\n \"bencher::mock_0\": {\n \"latency\": {\n \"value\": 0.15496641529475275,\n \"lower_value\": 0.13946977376527747,\n \"upper_value\": 0.17046305682422802\n }\n },\n \"bencher::mock_1\": {\n \"latency\": {\n \"value\": 18.648298578180437,\n \"lower_value\": 16.783468720362393,\n \"upper_value\": 20.513128435998482\n }\n },\n \"bencher::mock_2\": {\n \"latency\": {\n \"value\": 28.20328182167366,\n \"lower_value\": 25.382953639506294,\n \"upper_value\": 31.023610003841025\n }\n },\n \"bencher::mock_3\": {\n \"latency\": {\n \"value\": 34.45732560787596,\n \"lower_value\": 31.01159304708836,\n \"upper_value\": 37.903058168663556\n }\n },\n \"bencher::mock_4\": {\n \"latency\": {\n \"value\": 44.9237520767597,\n \"lower_value\": 40.43137686908373,\n \"upper_value\": 49.41612728443567\n }\n }\n}\n" ], "settings": {}, "start_time": "2023-07-18T14:21:27.773930Z", diff --git a/services/console/src/types/bencher.ts b/services/console/src/types/bencher.ts index 76e1085fa..03204a179 100644 --- a/services/console/src/types/bencher.ts +++ b/services/console/src/types/bencher.ts @@ -48,8 +48,8 @@ export type BenchmarkName = string; export interface JsonMetric { value: number; - lower_bound?: number; - upper_bound?: number; + lower_value?: number; + upper_value?: number; } export interface JsonBoundary {