Use Cliffs delta to know the size of the significant change (#4024)

lib/plugins/compare/helper.js

@@ -355,3 +355,20 @@ export function getMetrics(data) {
     ...getCDPPerformance(data)
   };
 }
+
+export function cliffsDelta(x, y) {
+  const n_x = x.length;
+  const n_y = y.length;
+  let n_gt = 0; // Count of x[i] > y[j]
+  let n_lt = 0; // Count of x[i] < y[j]
+
+  // Compare each pair of values (one from x, one from y)
+  for (let xi of x) {
+    for (let yi of y) {
+      if (xi > yi) n_gt++;
+      if (xi < yi) n_lt++;
+    }
+  }
+
+  return (n_gt - n_lt) / (n_x * n_y);
+}

lib/plugins/compare/index.js

@@ -7,7 +7,12 @@ import intel from 'intel';
 import merge from 'lodash.merge';
 import dayjs from 'dayjs';
 
-import { getStatistics, runStatisticalTests, getMetrics } from './helper.js';
+import {
+  getStatistics,
+  runStatisticalTests,
+  getMetrics,
+  cliffsDelta
+} from './helper.js';
 import { getBaseline, saveBaseline } from './baseline.js';
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
@@ -182,7 +187,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
                   median: baselineStats.median(),
                   values: baselineStats.data
                 },
-                statisticalTestU: result['p-value']
+                statisticalTestU: result['p-value'],
+                cliffsDelta: cliffsDelta(currentStats.data, baselineStats.data)
               };
             }
           }

lib/plugins/compare/pug/index.pug

@@ -2,6 +2,11 @@
 - var createGraphLink = function(metricGroup, metricName) {
 -   return '#chart-' + metricGroup.replace(/\./g, '_') + '_' + metricName.replace(/\./g, '_');
 - }
+- var cliffDeltaHelper = function(delta) {
+-   if (delta < 0.3) return `Small effect (${h.decimals(delta)})`;
+-   if (delta < 0.5) return `Medium effect (${h.decimals(delta)})`;
+-   if (delta >= 0.5) return `Large effect (${h.decimals(delta)})`;
+- }
 
 h1 Compare
 
@@ -82,7 +87,8 @@ table
             if values.statisticalTestU === "N/A"
               td No Test Conducted
             else
-              td #{values.statisticalTestU < 0.05 ? 'Yes' : 'No'}
+              td #{values.statisticalTestU < 0.05 ? 'Yes - ' + cliffDeltaHelper(values.cliffsDelta) : 'No'}
+
 
 h2 Graphs
 

lib/plugins/compare/statistical.py

@@ -2,10 +2,12 @@
 import json
 from scipy.stats import wilcoxon, mannwhitneyu
 
+
 def has_variability(sample):
     """Check if the sample has more than one unique value."""
     return len(set(sample)) > 1
 
+
 def perform_test(test_type, baseline, current, **kwargs):
     """Perform the statistical test based on the test type."""
     if not has_variability(baseline) or not has_variability(current):
@@ -18,6 +20,7 @@ def perform_test(test_type, baseline, current, **kwargs):
     else:
         raise ValueError("Invalid test type. Choose 'wilcoxon' or 'mannwhitneyu'.")
 
+
 input_data = json.loads(sys.stdin.read())
 options = input_data['options']
 test_type = options.pop('test_type')