From 4860af38e11a2f4f4263ef13f035cda521efac7a Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 10:56:43 -0800 Subject: [PATCH 01/10] fix(createDashboard): use env var for host We have added the feature of being able to run `vigilo` in a Kubernetes cluster which means the host tag sent to DataDog will be 'docker-container'. Therefore, let's use an environment variable to configure the host query parameter. --- src/scripts/createDashboard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index 42b8f33..115ecef 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -15,7 +15,7 @@ type TimeseriesWidgetDefinition = Partial & { requests: v1.TimeseriesWidgetRequest[] } -const HOST = 'ubreakit.com'; +const HOST = process.env.HOST || 'docker-container'; const INSPECT_LIST = URLS; const AUDITS = lhConfig.settings.onlyAudits || []; From 5acb580c7ef10c3dacd1240c2ec95728f4f9b6c9 Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:01:50 -0800 Subject: [PATCH 02/10] feat(createDashboard): add alerts for query value widget We will be adding a Query Value widget to the dashboard so we will need to add alerts to said widget. Let's differentiate between the Timeseries and Query Value alerts and add the new Query Value alerts --- src/scripts/createDashboard.ts | 101 +++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 5 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index 115ecef..f51424d 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -20,7 +20,7 @@ const INSPECT_LIST = URLS; const AUDITS = lhConfig.settings.onlyAudits || []; // Auditname -> {warning: "warning value", alert: "alert value"} -const ALERT_MARKERS = { +const ALERT_MARKERS_TIMESERIES = { 'largest-contentful-paint': { "warning": "2500 < y < 4000", "alert": "y > 4000", @@ -42,22 +42,113 @@ const ALERT_MARKERS = { "alert": "y > 5.8", }, }; +const ALERT_MARKERS_QUERY_VALUE = { + 'largest-contentful-paint': [ + { + "comparator": ">", + "value": 4000, + "palette": "white_on_red" + }, + { + "comparator": ">=", + "value": 2500, + "palette": "white_on_yellow" + }, + { + "comparator": "<", + "value": 2500, + "palette": "white_on_green" + } + ], + 'first-contentful-paint': [ + { + "comparator": ">", + "value": 3, + "palette": "white_on_red" + }, + { + "comparator": ">=", + "value": 1.8, + "palette": "white_on_yellow" + }, + { + "comparator": "<", + "value": 1.8, + "palette": "white_on_green" + } + ], + 'cumulative-layout-shift': [ + { + "comparator": ">", + "value": 0.25, + "palette": "white_on_red" + }, + { + "comparator": ">=", + "value": 0.1, + "palette": "white_on_yellow" + }, + { + "comparator": "<", + "value": 0.1, + "palette": "white_on_green" + } + ], + 'total-blocking-time': [ + { + "comparator": ">", + "value": 600, + "palette": "white_on_red" + }, + { + "comparator": ">=", + "value": 200, + "palette": "white_on_yellow" + }, + { + "comparator": "<", + "value": 200, + "palette": "white_on_green" + } + ], + 'speed-index': [ + { + "comparator": ">", + "value": 5.8, + "palette": "white_on_red" + }, + { + "comparator": ">=", + "value": 3.4, + "palette": "white_on_yellow" + }, + { + "comparator": "<", + "value": 3.4, + "palette": "white_on_green" + } + ], +}; -function fetchAlertMarkersForAudit(auditName: string) { - return ALERT_MARKERS.hasOwnProperty(auditName) ? [ +function fetchTimeseriesAlertMarkersForAudit(auditName: string) { + return ALERT_MARKERS_TIMESERIES.hasOwnProperty(auditName) ? [ { "label": "Alert", - "value": ALERT_MARKERS[auditName].alert, + "value": ALERT_MARKERS_TIMESERIES[auditName].alert, "display_type": "error dashed" }, { "label": "Warning", - "value": ALERT_MARKERS[auditName].warning, + "value": ALERT_MARKERS_TIMESERIES[auditName].warning, "display_type": "warning dashed" } ] : [] } +function fetchQueryValueAlertMarkersForAudit(auditName: string) { + return ALERT_MARKERS_QUERY_VALUE.hasOwnProperty(auditName) ? ALERT_MARKERS_QUERY_VALUE[auditName] : [] +} + function createWidgetRequestsForMetric(audit: string, pageType: string): v1.TimeseriesWidgetRequest[] { return [ { From 64151276889fddba7a88ea628fdefdc047b6a38e Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:07:39 -0800 Subject: [PATCH 03/10] feat(createDashboard): create request for the query value widget Similar to the timeseries request function, we will create a new function to handle the creation of requests for the query value widget. One thing to note, unlike the timeseries widget, the query value widget implements the alerts through the requests definition. --- src/scripts/createDashboard.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index f51424d..fdf42c4 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -149,7 +149,7 @@ function fetchQueryValueAlertMarkersForAudit(auditName: string) { return ALERT_MARKERS_QUERY_VALUE.hasOwnProperty(auditName) ? ALERT_MARKERS_QUERY_VALUE[auditName] : [] } -function createWidgetRequestsForMetric(audit: string, pageType: string): v1.TimeseriesWidgetRequest[] { +function createWidgetRequestsForTimeseriesMetric(audit: string, pageType: string): v1.TimeseriesWidgetRequest[] { return [ { responseFormat: "timeseries", @@ -171,8 +171,23 @@ function createWidgetRequestsForMetric(audit: string, pageType: string): v1.Time ] } -function createTimeseriesWidget(widget: TimeseriesWidgetDefinition): v1.Widget { - +function createWidgetRequestsForQueryValueMetric(audit: string, pageType: string, alerts: v1.WidgetConditionalFormat[]): [v1.QueryValueWidgetRequest] { + return [ + { + responseFormat: "scalar", + queries: [ + { + name: "query1", + dataSource: "metrics", + query: `avg:lighthouse.${formatMetricNameForDatadog(audit)}{host:${HOST},page_type:${formatMetricNameForDatadog(pageType)},$FormFactor}`, + aggregator: "avg" + } + ], + formulas: [ {formula: "query1"} ], + conditionalFormats: alerts + } + ] +} return { definition: { title: "", From 263a93de0eb63dd364c15f6c6adf4f3bd8ec3027 Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:08:17 -0800 Subject: [PATCH 04/10] fix(createDashboard): fix the queries for the widgets We've updated the lighthouse metrics to send both a 'score' and 'value' datapoint for each audit. As such we need to update the queries to specify the new subpath. For most of the audits, the 'value' datapoint is more useful than the 'score' datapoint. The only audits that we would use the 'score' datapoint are those with null 'value' datapoints like the 'is_crawlable' audit. --- src/scripts/createDashboard.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index fdf42c4..9786c77 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -157,7 +157,7 @@ function createWidgetRequestsForTimeseriesMetric(audit: string, pageType: string { name: "query1", dataSource: "metrics", - query: `avg:lighthouse.${formatMetricNameForDatadog(audit)}{host:${HOST},page_type:${formatMetricNameForDatadog(pageType)},$FormFactor} by {page_type,url,form_factor}` + query: `avg:lighthouse.${formatMetricNameForDatadog(audit)}.value{host:${HOST},page_type:${formatMetricNameForDatadog(pageType)},$FormFactor} by {url, form_factor}` }, ], formulas: [ {formula: "query1"} ], @@ -179,7 +179,7 @@ function createWidgetRequestsForQueryValueMetric(audit: string, pageType: string { name: "query1", dataSource: "metrics", - query: `avg:lighthouse.${formatMetricNameForDatadog(audit)}{host:${HOST},page_type:${formatMetricNameForDatadog(pageType)},$FormFactor}`, + query: `avg:lighthouse.${formatMetricNameForDatadog(audit)}.value{host:${HOST},page_type:${formatMetricNameForDatadog(pageType)},$FormFactor}`, aggregator: "avg" } ], From 097c6b6c75fbf698a220a286326c5585c73f95ef Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:18:03 -0800 Subject: [PATCH 05/10] feat(createDashboard): add query value widget function Similar to the Timeseries widget function, this adds a new function to create a Query Value widget. --- src/scripts/createDashboard.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index 9786c77..a296b2d 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -15,6 +15,10 @@ type TimeseriesWidgetDefinition = Partial & { requests: v1.TimeseriesWidgetRequest[] } +type QueryValueWidgetDefinition = Partial & { + requests: [v1.QueryValueWidgetRequest] +} + const HOST = process.env.HOST || 'docker-container'; const INSPECT_LIST = URLS; const AUDITS = lhConfig.settings.onlyAudits || []; @@ -188,6 +192,22 @@ function createWidgetRequestsForQueryValueMetric(audit: string, pageType: string } ] } + +function createQueryValueWidget(widget: QueryValueWidgetDefinition): v1.Widget { + return { + definition: { + title: "", + titleSize: "16", + titleAlign: "left", + type: "query_value", + autoscale: true, + precision: 2, + ...widget + } + } +} + +function createTimeseriesWidget(widget: TimeseriesWidgetDefinition): v1.Widget { return { definition: { title: "", From ccc9caaf535a64fb829a054d8b45a7c967f65476 Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:32:07 -0800 Subject: [PATCH 06/10] feat(createDashboard): add layout parameter for widgets We will need to pair up the query value widget and timeseries widget together in the same row. On top of that, we don't want the widgets to expand the full width. Therefore, we will pass a parameter to modify the size and position of the widgets. --- src/scripts/createDashboard.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index a296b2d..4ac89c5 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -193,7 +193,7 @@ function createWidgetRequestsForQueryValueMetric(audit: string, pageType: string ] } -function createQueryValueWidget(widget: QueryValueWidgetDefinition): v1.Widget { +function createQueryValueWidget(widget: QueryValueWidgetDefinition, width: number, height: number, x: number, y: number): v1.Widget { return { definition: { title: "", @@ -203,11 +203,17 @@ function createQueryValueWidget(widget: QueryValueWidgetDefinition): v1.Widget { autoscale: true, precision: 2, ...widget + }, + layout: { + width: width, + height: height, + x: x, + y: y } } } -function createTimeseriesWidget(widget: TimeseriesWidgetDefinition): v1.Widget { +function createTimeseriesWidget(widget: TimeseriesWidgetDefinition, width: number, height: number, x: number, y: number): v1.Widget { return { definition: { title: "", @@ -216,6 +222,12 @@ function createTimeseriesWidget(widget: TimeseriesWidgetDefinition): v1.Widget { type: "timeseries", showLegend: true, ...widget + }, + layout: { + width: width, + height: height, + x: x, + y: y } } } From e9575534d48f01a1126f633d6aebef8cd78f4987 Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:34:37 -0800 Subject: [PATCH 07/10] feat(createDashboard): add query value widgets to iteration This modifies the createWidgetForAllPageTypes function to create a query value widget at the same time as the timeseries widget. On top of that, we will keep track of the position of the widgets to layout them out properly per group. --- src/scripts/createDashboard.ts | 40 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index 4ac89c5..e23f021 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -244,13 +244,39 @@ function createGroupWidget(widget: GroupWidgetDefinition): v1.Widget { } } -function createTimeseriesWidgetsForAllPageTypes(audit: string): v1.Widget[] { +function createWidgetPairsForAllPageTypes(audit: string): v1.Widget[] { const pageTypes = Object.keys(INSPECT_LIST); - const alertMarkers = fetchAlertMarkersForAudit(audit); - const widgetDefinitions = pageTypes.map(pageType => { - const requests = createWidgetRequestsForMetric(audit, pageType); - return createTimeseriesWidget({title: pageType, requests: requests, markers: alertMarkers}) - }) + + const queryValueSize = {width: 2, height: 2}; + const timeseriesSize = {width: 4, height: 2}; + + const queryValueAlertMarkers = fetchQueryValueAlertMarkersForAudit(audit); + const timeseriesAlertMarkers = fetchTimeseriesAlertMarkersForAudit(audit); + + const widgetDefinitions:v1.Widget[] = []; + + let x = 0; + let y = 0; + + for (const pageType of pageTypes) { + if (x === 12) { + x = 0; + y += 2; + } + + const queryValueRequests = createWidgetRequestsForQueryValueMetric(audit, pageType, queryValueAlertMarkers); + const timeseriesRequests = createWidgetRequestsForTimeseriesMetric(audit, pageType); + + const queryValueWidget = createQueryValueWidget({title: pageType, requests: queryValueRequests }, queryValueSize.width, queryValueSize.height, x, y); + + x += queryValueSize.width; + + const timeseriesWidget = createTimeseriesWidget({title: pageType, requests: timeseriesRequests, markers: timeseriesAlertMarkers}, timeseriesSize.width, timeseriesSize.height, x, y); + + x += timeseriesSize.width; + + widgetDefinitions.push(queryValueWidget, timeseriesWidget); + } return widgetDefinitions } @@ -258,7 +284,7 @@ function createTimeseriesWidgetsForAllPageTypes(audit: string): v1.Widget[] { function getWidgetForAllAudits(): v1.Widget[] { const widgetDefinitions: v1.Widget[] = AUDITS.map(audit => { const title = formatAuditName(audit); - const childWidgets = createTimeseriesWidgetsForAllPageTypes(audit); + const childWidgets = createWidgetPairsForAllPageTypes(audit); return createGroupWidget({title: title, widgets: childWidgets}) }) From 0883ef5936c27485391e5192bea3081dbd8f0bad Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 11:44:13 -0800 Subject: [PATCH 08/10] feat(createDashboard): add custom link for timeseries This adds a custom link to the timeseries widgets such that we can link the actual webpage for the datapoint being displayed. This will make it convenient to debug and investigate the data without having to manually type out the URL. --- src/scripts/createDashboard.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index e23f021..1ae8063 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -271,7 +271,23 @@ function createWidgetPairsForAllPageTypes(audit: string): v1.Widget[] { x += queryValueSize.width; - const timeseriesWidget = createTimeseriesWidget({title: pageType, requests: timeseriesRequests, markers: timeseriesAlertMarkers}, timeseriesSize.width, timeseriesSize.height, x, y); + const timeseriesWidget = createTimeseriesWidget( + { + title: pageType, + requests: timeseriesRequests, + markers: timeseriesAlertMarkers, + customLinks: [ + { + "label": "Visit Webpage", + "link": "{{url.value}}" + } + ] + }, + timeseriesSize.width, + timeseriesSize.height, + x, + y + ); x += timeseriesSize.width; From ba54cde5150b8abd3a97334c29abbf201d03468a Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 14:18:18 -0800 Subject: [PATCH 09/10] fix(createDashboard): fix comparator value units For some of the audits, we weren't scaling the comparator value to the correct unit types of the metrics being sent. Most of the metrics are in milliseconds, so we need to scale the comparator value to milliseconds as well as these values assumed the metric was in seconds. --- src/scripts/createDashboard.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index 1ae8063..e2f237f 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -30,8 +30,8 @@ const ALERT_MARKERS_TIMESERIES = { "alert": "y > 4000", }, 'first-contentful-paint': { - "warning": "1.8 < y < 3", - "alert": "y > 3", + "warning": "100 < y < 300", + "alert": "y > 300", }, 'cumulative-layout-shift': { "warning": "0.1 < y < 0.25", @@ -42,8 +42,8 @@ const ALERT_MARKERS_TIMESERIES = { "alert": "y > 600", }, 'speed-index': { - "warning": "3.4 < y < 5.8", - "alert": "y > 5.8", + "warning": "3400 < y < 5800", + "alert": "y > 5800", }, }; const ALERT_MARKERS_QUERY_VALUE = { @@ -67,17 +67,17 @@ const ALERT_MARKERS_QUERY_VALUE = { 'first-contentful-paint': [ { "comparator": ">", - "value": 3, + "value": 300, "palette": "white_on_red" }, { "comparator": ">=", - "value": 1.8, + "value": 100, "palette": "white_on_yellow" }, { "comparator": "<", - "value": 1.8, + "value": 100, "palette": "white_on_green" } ], @@ -118,17 +118,17 @@ const ALERT_MARKERS_QUERY_VALUE = { 'speed-index': [ { "comparator": ">", - "value": 5.8, + "value": 5800, "palette": "white_on_red" }, { "comparator": ">=", - "value": 3.4, + "value": 3400, "palette": "white_on_yellow" }, { "comparator": "<", - "value": 3.4, + "value": 3400, "palette": "white_on_green" } ], From b03cde20ebff5609d46b07ac5eb80be0107c616f Mon Sep 17 00:00:00 2001 From: Angel de la Torre Date: Wed, 14 Feb 2024 14:19:53 -0800 Subject: [PATCH 10/10] fix(createDashboard): fix alert display type naming convention The alert display type is CamelCase, not snake_case. This caused the warning marker to not have the correct display type. --- src/scripts/createDashboard.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripts/createDashboard.ts b/src/scripts/createDashboard.ts index e2f237f..bc09cac 100644 --- a/src/scripts/createDashboard.ts +++ b/src/scripts/createDashboard.ts @@ -139,12 +139,12 @@ function fetchTimeseriesAlertMarkersForAudit(auditName: string) { { "label": "Alert", "value": ALERT_MARKERS_TIMESERIES[auditName].alert, - "display_type": "error dashed" + "displayType": "error dashed" }, { "label": "Warning", "value": ALERT_MARKERS_TIMESERIES[auditName].warning, - "display_type": "warning dashed" + "displayType": "warning dashed" } ] : [] }