diff --git a/lib/plugins/compare/helper.js b/lib/plugins/compare/helper.js index c5757123e5..10735d7cfe 100644 --- a/lib/plugins/compare/helper.js +++ b/lib/plugins/compare/helper.js @@ -355,3 +355,20 @@ export function getMetrics(data) { ...getCDPPerformance(data) }; } + +export function cliffsDelta(x, y) { + const n_x = x.length; + const n_y = y.length; + let n_gt = 0; // Count of x[i] > y[j] + let n_lt = 0; // Count of x[i] < y[j] + + // Compare each pair of values (one from x, one from y) + for (let xi of x) { + for (let yi of y) { + if (xi > yi) n_gt++; + if (xi < yi) n_lt++; + } + } + + return (n_gt - n_lt) / (n_x * n_y); +} diff --git a/lib/plugins/compare/index.js b/lib/plugins/compare/index.js index bf27fb9bd8..90f7b88112 100644 --- a/lib/plugins/compare/index.js +++ b/lib/plugins/compare/index.js @@ -7,7 +7,12 @@ import intel from 'intel'; import merge from 'lodash.merge'; import dayjs from 'dayjs'; -import { getStatistics, runStatisticalTests, getMetrics } from './helper.js'; +import { + getStatistics, + runStatisticalTests, + getMetrics, + cliffsDelta +} from './helper.js'; import { getBaseline, saveBaseline } from './baseline.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -182,7 +187,8 @@ export default class ComparePlugin extends SitespeedioPlugin { median: baselineStats.median(), values: baselineStats.data }, - statisticalTestU: result['p-value'] + statisticalTestU: result['p-value'], + cliffsDelta: cliffsDelta(currentStats.data, baselineStats.data) }; } } diff --git a/lib/plugins/compare/pug/index.pug b/lib/plugins/compare/pug/index.pug index a94cee03bd..1db8402659 100644 --- a/lib/plugins/compare/pug/index.pug +++ b/lib/plugins/compare/pug/index.pug @@ -2,6 +2,11 @@ - var createGraphLink = function(metricGroup, metricName) { - return '#chart-' + metricGroup.replace(/\./g, '_') + '_' + metricName.replace(/\./g, '_'); - } +- var cliffDeltaHelper = function(delta) { +- if (delta < 0.3) return `Small effect (${h.decimals(delta)})`; +- if (delta < 0.5) return `Medium effect (${h.decimals(delta)})`; +- if (delta >= 0.5) return `Large effect (${h.decimals(delta)})`; +- } h1 Compare @@ -82,7 +87,8 @@ table if values.statisticalTestU === "N/A" td No Test Conducted else - td #{values.statisticalTestU < 0.05 ? 'Yes' : 'No'} + td #{values.statisticalTestU < 0.05 ? 'Yes - ' + cliffDeltaHelper(values.cliffsDelta) : 'No'} + h2 Graphs diff --git a/lib/plugins/compare/statistical.py b/lib/plugins/compare/statistical.py index 29ca1ce6d3..8bd423b93f 100644 --- a/lib/plugins/compare/statistical.py +++ b/lib/plugins/compare/statistical.py @@ -2,10 +2,12 @@ import json from scipy.stats import wilcoxon, mannwhitneyu + def has_variability(sample): """Check if the sample has more than one unique value.""" return len(set(sample)) > 1 + def perform_test(test_type, baseline, current, **kwargs): """Perform the statistical test based on the test type.""" if not has_variability(baseline) or not has_variability(current): @@ -18,6 +20,7 @@ def perform_test(test_type, baseline, current, **kwargs): else: raise ValueError("Invalid test type. Choose 'wilcoxon' or 'mannwhitneyu'.") + input_data = json.loads(sys.stdin.read()) options = input_data['options'] test_type = options.pop('test_type')