From ce8db48a91d6d8a3a1396a9924610cea514c5b6e Mon Sep 17 00:00:00 2001 From: Maciej Bodek Date: Tue, 3 Dec 2024 18:23:09 +0100 Subject: [PATCH] Abstract out widgets, add Commit Rate --- .../web-console/assets/metric-commit-rate.svg | 64 ++++++++++++++++ .../Editor/Metrics/add-metric-dialog.tsx | 32 ++------ .../src/scenes/Editor/Metrics/metric.tsx | 75 +++++-------------- .../src/scenes/Editor/Metrics/utils.ts | 74 +++++++++--------- .../Editor/Metrics/widgets/commitRate.ts | 60 +++++++++++++++ .../scenes/Editor/Metrics/widgets/index.ts | 12 +++ .../scenes/Editor/Metrics/widgets/latency.ts | 44 +++++++++++ .../scenes/Editor/Metrics/widgets/utils.ts | 31 ++++++++ .../Metrics/widgets/writeAmplification.ts | 54 +++++++++++++ .../Editor/Metrics/widgets/writeThroughput.ts | 41 ++++++++++ 10 files changed, 371 insertions(+), 116 deletions(-) create mode 100644 packages/web-console/assets/metric-commit-rate.svg create mode 100644 packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts create mode 100644 packages/web-console/src/scenes/Editor/Metrics/widgets/index.ts create mode 100644 packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts create mode 100644 packages/web-console/src/scenes/Editor/Metrics/widgets/utils.ts create mode 100644 packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts create mode 100644 packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts diff --git a/packages/web-console/assets/metric-commit-rate.svg b/packages/web-console/assets/metric-commit-rate.svg new file mode 100644 index 000000000..e9198eba0 --- /dev/null +++ b/packages/web-console/assets/metric-commit-rate.svg @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/web-console/src/scenes/Editor/Metrics/add-metric-dialog.tsx b/packages/web-console/src/scenes/Editor/Metrics/add-metric-dialog.tsx index f59927b14..b7c1c2163 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/add-metric-dialog.tsx +++ b/packages/web-console/src/scenes/Editor/Metrics/add-metric-dialog.tsx @@ -6,14 +6,14 @@ import { Button, Overlay, } from "@questdb/react-components" -import { Undo } from "@styled-icons/boxicons-regular" import { Text } from "../../../components/Text" import styled from "styled-components" import { AddChart } from "@styled-icons/material" -import { MetricType, metricTypeLabel } from "./utils" +import { MetricType } from "./utils" import { useEditor } from "../../../providers" import merge from "lodash.merge" import { defaultColor, getColorForNewMetric } from "./color-palette" +import { widgets } from "./widgets" const StyledDescription = styled(Dialog.Description)` display: grid; @@ -116,37 +116,21 @@ export const AddMetricDialog = ({ open, onOpenChange }: Props) => { - {[ - { - label: metricTypeLabel[MetricType.LATENCY], - value: MetricType.LATENCY, - iconUrl: "/assets/metric-read-latency.svg", - }, - { - label: metricTypeLabel[MetricType.WRITE_AMPLIFICATION], - value: MetricType.WRITE_AMPLIFICATION, - iconUrl: "/assets/metric-write-amplification.svg", - }, - { - label: metricTypeLabel[MetricType.ROWS_APPLIED], - value: MetricType.ROWS_APPLIED, - iconUrl: "/assets/metric-rows-applied.svg", - }, - ].map((metric) => ( + {Object.entries(widgets).map(([metricType, widget]) => ( handleSelectMetric(metric.value)} + key={metricType} + onClick={() => handleSelectMetric(metricType as MetricType)} > {`${metric.label} - {metric.label} + {widget.label} ))} diff --git a/packages/web-console/src/scenes/Editor/Metrics/metric.tsx b/packages/web-console/src/scenes/Editor/Metrics/metric.tsx index ad07daa65..6dc490678 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/metric.tsx +++ b/packages/web-console/src/scenes/Editor/Metrics/metric.tsx @@ -8,21 +8,13 @@ import React, { import { Metric as MetricItem } from "../../../store/buffers" import { durationInMinutes, - graphDataConfigs, MetricDuration, MetricType, - Latency, - RowsApplied, - metricTypeLabel, LastNotNull, + ResultType, } from "./utils" +import { widgets } from "./widgets" import { QuestContext } from "../../../providers" -import { - latency as latencySQL, - rowsApplied as rowsAppliedSQL, - latencyLastNotNull as latencyLasNotNullSQL, - rowsAppliedLastNotNull as rowsAppliedLastNotNullSQL, -} from "./queries" import * as QuestDB from "../../../utils/questdb" import { Graph } from "./graph" import uPlot from "uplot" @@ -79,64 +71,37 @@ export const Metric = ({ const { autoRefreshTables } = useLocalStorage() + const widgetConfig = widgets[metric.metricType] + const minuteDurations: [MetricDuration, number][] = Object.entries( durationInMinutes, ) as [MetricDuration, number][] - const fetchLatency = async () => { - if (!metric.tableId) return Promise.reject() - return quest.query( - latencySQL(metric.tableId, metricDurationRef.current), - ) - } - - const fetchLatencyLastNotNull = async () => { - if (!metric.tableId) return Promise.reject() - return quest.query(latencyLasNotNullSQL(metric.tableId)) - } - - const fetchRowsApplied = async () => { - if (!metric.tableId) return Promise.reject() - return quest.query( - rowsAppliedSQL(metric.tableId, metricDurationRef.current), - ) - } - - const fetchRowsAppliedLastNotNull = async () => { - if (!metric.tableId) return Promise.reject() - return quest.query(rowsAppliedLastNotNullSQL(metric.tableId)) - } - - const fetchers = { - [MetricType.LATENCY]: fetchLatency, - [MetricType.ROWS_APPLIED]: fetchRowsApplied, - [MetricType.WRITE_AMPLIFICATION]: fetchRowsApplied, - } - - const fetchersLastNotNull = { - [MetricType.LATENCY]: fetchLatencyLastNotNull, - [MetricType.ROWS_APPLIED]: fetchRowsAppliedLastNotNull, - [MetricType.WRITE_AMPLIFICATION]: fetchRowsAppliedLastNotNull, - } - const fetchMetric = async () => { setLoading(true) try { const responses = await Promise.all< - | QuestDB.QueryResult - | QuestDB.QueryResult + | QuestDB.QueryResult | QuestDB.QueryResult >([ - fetchers[metric.metricType](), - fetchersLastNotNull[metric.metricType](), + quest.query( + widgetConfig.getQuery({ + tableId: metric.tableId, + metricDuration: metricDurationRef.current, + }), + ), + quest.query( + widgetConfig.getQueryLastNotNull(metric.tableId), + ), ]) if (responses[0] && responses[0].type === QuestDB.Type.DQL) { - const alignedData = graphDataConfigs[metric.metricType].alignData( - responses[0].data as any, + const alignedData = widgetConfig.alignData( + responses[0].data as unknown as ResultType[MetricType], ) setData(alignedData) } + if (responses[1] && responses[1].type === QuestDB.Type.DQL) { const lastNotNull = responses[1].data[0] as LastNotNull if (lastNotNull) { @@ -202,7 +167,7 @@ export const Metric = ({ return ( - Cannot load metric: {metricTypeLabel[metric.metricType]} + Cannot load metric: {widgetConfig.label} ) @@ -224,8 +189,8 @@ export const Metric = ({ colors={[metric.color]} loading={loading} duration={metricDuration} - label={metricTypeLabel[metric.metricType]} - yValue={graphDataConfigs[metric.metricType].mapYValue} + label={widgetConfig.label} + yValue={widgetConfig.mapYValue} beforeLabel={ = { - [MetricType.ROWS_APPLIED]: "Write throughput", - [MetricType.LATENCY]: "WAL apply latency in ms", - [MetricType.WRITE_AMPLIFICATION]: "Write amplification", +export type Widget = { + label: string + iconUrl: string + getQuery: ({ + tableId, + metricDuration, + sampleBy, + }: { + tableId?: number + metricDuration: MetricDuration + sampleBy?: SampleBy + }) => string + getQueryLastNotNull: (id?: number) => string + alignData: (data: any) => uPlot.AlignedData + mapYValue: (rawValue: number) => string } export enum MetricDuration { @@ -50,6 +62,17 @@ export const mappedSampleBy: Record = { [MetricDuration.SEVEN_DAYS]: SampleBy.ONE_MINUTE, } +export type CommitRate = { + created: string + commit_rate: string + commit_rate_smooth: string +} + +export type WriteAmplification = { + created: string + writeAmplification: string +} + export type RowsApplied = { time: string numOfWalApplies: string @@ -68,6 +91,13 @@ export type LastNotNull = { created: string } +export type ResultType = { + [MetricType.COMMIT_RATE]: CommitRate + [MetricType.LATENCY]: Latency + [MetricType.WRITE_THROUGHPUT]: RowsApplied + [MetricType.WRITE_AMPLIFICATION]: RowsApplied +} + export const minutesToDays = (durationInMinutes: number) => durationInMinutes / 60 / 24 @@ -94,12 +124,12 @@ export const xAxisFormat = { utcToLocal(rawValue, "dd/MM"), } -const sqlValueToFixed = (value: string, decimals: number = 2) => { +export const sqlValueToFixed = (value: string, decimals: number = 2) => { const parsed = parseFloat(value) return Number(parsed.toFixed(decimals)) as unknown as number } -const formatNumbers = (value: number) => { +export const formatNumbers = (value: number) => { if (value >= 1e6) { return (value / 1e6).toFixed(1).replace(/\.0$/, "") + " M" } else if (value >= 1e3) { @@ -107,33 +137,3 @@ const formatNumbers = (value: number) => { } return value.toString() } - -export const graphDataConfigs = { - [MetricType.LATENCY]: { - alignData: (latency: Latency[]): uPlot.AlignedData => [ - latency.map((l) => new Date(l.time).getTime()), - latency.map((l) => sqlValueToFixed(l.avg_latency)), - ], - mapYValue: (rawValue: number) => { - if (rawValue >= 1000) { - const seconds = rawValue / 1000 - return `${seconds.toFixed(2)} s` - } - return `${rawValue} ms` - }, - }, - [MetricType.ROWS_APPLIED]: { - alignData: (rowsApplied: RowsApplied[]): uPlot.AlignedData => [ - rowsApplied.map((l) => new Date(l.time).getTime()), - rowsApplied.map((l) => sqlValueToFixed(l.numOfRowsApplied)), - ], - mapYValue: (rawValue: number) => formatNumbers(rawValue), - }, - [MetricType.WRITE_AMPLIFICATION]: { - alignData: (rowsApplied: RowsApplied[]): uPlot.AlignedData => [ - rowsApplied.map((l) => new Date(l.time).getTime()), - rowsApplied.map((l) => sqlValueToFixed(l.avgWalAmplification)), - ], - mapYValue: (rawValue: number) => formatNumbers(rawValue), - }, -} diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts new file mode 100644 index 000000000..b6b838593 --- /dev/null +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts @@ -0,0 +1,60 @@ +import uPlot from "uplot" +import { Widget, durationInMinutes } from "../utils" +import type { CommitRate } from "../utils" +import { TelemetryTable } from "../../../../consts" +import { getTimeFilter, sqlValueToFixed, formatNumbers } from "./utils" + +export const commitRate: Widget = { + label: "Commit rate per second", + iconUrl: "/assets/metric-commit-rate.svg", + getQuery: ({ tableId, metricDuration, sampleBy }) => { + const minutes = durationInMinutes[metricDuration] + return ` + select + created, + -- the chart can display both values for user convenience + -- they may want to toggle these values on and off + commit_rate, + commit_rate_smooth + from ( + select + created + , commit_rate + -- here the value 59 refers to 1 min moving sum(), it is linked to sampling interval + -- should the sampling interval change, the number of rows should be adjusted to cover 1 min window + , avg(commit_rate) over(order by created rows BETWEEN 59 PRECEDING AND CURRENT ROW) commit_rate_smooth + from ( + select -- calculates coarse commit_rate (commits per second) + created + , count() commit_rate + from ${TelemetryTable.WAL} + where ${tableId ? `tableId = ${tableId} and ` : ""} + event = 103 + ${getTimeFilter(minutes)} + -- it is important this is 1s, should this value change + -- the "commit_rate" value will have to be adjusted to rate/s + sample by 1s + -- fill(0) + ) + -- there is a bug in QuestDB, which does not sort the window dataset + -- once the bug is fixed the order can be removed + order by 1 +); + ` + }, + // TODO: Sample, change! + getQueryLastNotNull: (tableId) => ` +select + created +from ${TelemetryTable.WAL} +where ${tableId ? `tableId = ${tableId} and ` : ""} +event = 103 +and physicalRowCount != null +limit -1 +`, + alignData: (rowsApplied: CommitRate[]): uPlot.AlignedData => [ + rowsApplied.map((l) => new Date(l.created).getTime()), + rowsApplied.map((l) => sqlValueToFixed(l.commit_rate_smooth)), + ], + mapYValue: (rawValue: number) => formatNumbers(rawValue), +} diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/index.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/index.ts new file mode 100644 index 000000000..adf967298 --- /dev/null +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/index.ts @@ -0,0 +1,12 @@ +import { MetricType } from "../utils" +import { latency } from "./latency" +import { writeAmplification } from "./writeAmplification" +import { writeThroughput } from "./writeThroughput" +import { commitRate } from "./commitRate" + +export const widgets = { + [MetricType.COMMIT_RATE]: commitRate, + [MetricType.LATENCY]: latency, + [MetricType.WRITE_THROUGHPUT]: writeThroughput, + [MetricType.WRITE_AMPLIFICATION]: writeAmplification, +} diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts new file mode 100644 index 000000000..29aa9dda7 --- /dev/null +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts @@ -0,0 +1,44 @@ +import uPlot from "uplot" +import { Latency, sqlValueToFixed } from "../utils" +import { Widget, mappedSampleBy, durationInMinutes } from "../utils" +import { getTimeFilter } from "./utils" +import { TelemetryTable } from "../../../../consts" + +export const latency: Widget = { + label: "WAL apply latency in ms", + iconUrl: "/assets/metric-read-latency.svg", + getQuery: ({ tableId, metricDuration, sampleBy }) => { + const minutes = durationInMinutes[metricDuration] + return ` +select + created time, + count(latency) / 2 numOfWalApplies, + avg(latency) * 2 avg_latency +from ${TelemetryTable.WAL} +where ${tableId ? `tableId = ${tableId} and ` : ""} +(event = 105 or event = 103) +${getTimeFilter(minutes)} +sample by ${sampleBy ?? mappedSampleBy[metricDuration]} +` + }, + getQueryLastNotNull: (tableId) => ` +select + created +from ${TelemetryTable.WAL} +where ${tableId ? `tableId = ${tableId} and ` : ""} +(event = 105 or event = 103) +and latency != null +limit -1 +`, + alignData: (latency: Latency[]): uPlot.AlignedData => [ + latency.map((l) => new Date(l.time).getTime()), + latency.map((l) => sqlValueToFixed(l.avg_latency)), + ], + mapYValue: (rawValue: number) => { + if (rawValue >= 1000) { + const seconds = rawValue / 1000 + return `${seconds.toFixed(2)} s` + } + return `${rawValue} ms` + }, +} diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/utils.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/utils.ts new file mode 100644 index 000000000..dd98319b4 --- /dev/null +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/utils.ts @@ -0,0 +1,31 @@ +export const sqlValueToFixed = (value: string, decimals: number = 2) => { + const parsed = parseFloat(value) + return Number(parsed.toFixed(decimals)) as unknown as number +} + +export const formatNumbers = (value: number) => { + if (value >= 1e6) { + return (value / 1e6).toFixed(1).replace(/\.0$/, "") + " M" + } else if (value >= 1e3) { + return (value / 1e3).toFixed(1).replace(/\.0$/, "") + " k" + } + return value.toString() +} + +export const minutesToDays = (durationInMinutes: number) => + durationInMinutes / 60 / 24 + +export const minutesToHours = (durationInMinutes: number) => + durationInMinutes / 60 + +export const minutesToSeconds = (durationInMinutes: number) => + durationInMinutes * 60 + +export const getTimeFilter = ( + minutes: number, +) => `and created > date_trunc('minute', dateadd('${ + minutes >= 1440 ? "d" : "h" +}', -${ + minutes >= 1440 ? minutesToDays(minutes) : minutesToHours(minutes) +}, now())) +and created < date_trunc('minute', now())` diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts new file mode 100644 index 000000000..c4f27001b --- /dev/null +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts @@ -0,0 +1,54 @@ +import uPlot from "uplot" +import { sqlValueToFixed, formatNumbers, WriteAmplification } from "../utils" +import { Widget, mappedSampleBy, durationInMinutes } from "../utils" +import { TelemetryTable } from "../../../../consts" +import { getTimeFilter } from "./utils" + +export const writeAmplification: Widget = { + label: "Write amplification", + iconUrl: "/assets/metric-write-amplification.svg", + getQuery: ({ tableId, metricDuration, sampleBy }) => { + const minutes = durationInMinutes[metricDuration] + return ` +select + created, + -- coars, actual write amplification bucketed in 1s buckets + phy_row_count/row_count writeAmplification +from ( + select + created, + sum(phy_row_count) over (order by created rows between 59 PRECEDING and CURRENT row) phy_row_count, + sum(row_count) over (order by created rows between 59 PRECEDING and CURRENT row) row_count + from ( + select + created, + sum(rowcount) row_count, + sum(physicalRowCount) phy_row_count, + from ${TelemetryTable.WAL} + where ${tableId ? `tableId = ${tableId} and ` : ""} + event = 105 + and rowCount > 0 -- this is fixed clause, we have rows with - rowCount logged + ${getTimeFilter(minutes)} + sample by ${sampleBy ?? mappedSampleBy[metricDuration]} + -- fill with null to avoid spurious values and division by 0 + fill(null,null) + ) +); + ` + }, + getQueryLastNotNull: (tableId) => ` +select + created +from ${TelemetryTable.WAL} +where ${tableId ? `tableId = ${tableId} and ` : ""} +event = 105 +and rowCount != null +and physicalRowCount != null +limit -1 +`, + alignData: (data: WriteAmplification[]): uPlot.AlignedData => [ + data.map((l) => new Date(l.created).getTime()), + data.map((l) => sqlValueToFixed(l.writeAmplification)), + ], + mapYValue: (rawValue: number) => formatNumbers(rawValue), +} diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts new file mode 100644 index 000000000..77a81bac4 --- /dev/null +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts @@ -0,0 +1,41 @@ +import uPlot from "uplot" +import { RowsApplied, sqlValueToFixed, formatNumbers } from "../utils" +import { Widget, mappedSampleBy, durationInMinutes } from "../utils" +import { TelemetryTable } from "../../../../consts" +import { getTimeFilter } from "./utils" + +export const writeThroughput: Widget = { + label: "Write throughput", + iconUrl: "/assets/metric-rows-applied.svg", + getQuery: ({ tableId, metricDuration, sampleBy }) => { + const minutes = durationInMinutes[metricDuration] + + return ` +select + created time, + count(rowCount) numOfWalApplies, + sum(rowCount) numOfRowsApplied, + sum(physicalRowCount) numOfRowsWritten, + coalesce(avg(physicalRowCount/rowCount), 1) avgWalAmplification +from ${TelemetryTable.WAL} +where ${tableId ? `tableId = ${tableId} and ` : ""} +event = 105 +${getTimeFilter(minutes)} +sample by ${sampleBy ?? mappedSampleBy[metricDuration]}` + }, + getQueryLastNotNull: (tableId) => ` +select + created +from ${TelemetryTable.WAL} +where ${tableId ? `tableId = ${tableId} and ` : ""} +event = 105 +and rowCount != null +and physicalRowCount != null +limit -1 +`, + alignData: (rowsApplied: RowsApplied[]): uPlot.AlignedData => [ + rowsApplied.map((l) => new Date(l.time).getTime()), + rowsApplied.map((l) => sqlValueToFixed(l.numOfRowsApplied)), + ], + mapYValue: (rawValue: number) => formatNumbers(rawValue), +}