From e5b0cb936b3696d4e83f1ca77ae43ed1e1e3ca44 Mon Sep 17 00:00:00 2001 From: Maciej Bodek Date: Thu, 12 Dec 2024 13:27:40 +0100 Subject: [PATCH] Refactor time filter to use FROM TO, refactor rolling append --- packages/web-console/src/consts/index.ts | 2 +- .../src/scenes/Editor/Metrics/graph.tsx | 10 ++--- .../src/scenes/Editor/Metrics/index.tsx | 21 ++++++++- .../src/scenes/Editor/Metrics/metric.tsx | 33 +++++++++----- .../scenes/Editor/Metrics/useGraphOptions.ts | 10 +++-- .../src/scenes/Editor/Metrics/utils.ts | 45 ++++++++++--------- .../Editor/Metrics/widgets/commitRate.ts | 15 +++---- .../scenes/Editor/Metrics/widgets/latency.ts | 16 +++---- .../Metrics/widgets/writeAmplification.ts | 14 +++--- .../Editor/Metrics/widgets/writeThroughput.ts | 16 +++---- packages/web-console/src/utils/dateTime.ts | 6 ++- 11 files changed, 107 insertions(+), 81 deletions(-) diff --git a/packages/web-console/src/consts/index.ts b/packages/web-console/src/consts/index.ts index 25c4baa22..481421172 100644 --- a/packages/web-console/src/consts/index.ts +++ b/packages/web-console/src/consts/index.ts @@ -36,6 +36,6 @@ export const API = `https://${BASE}.questdb.io` // to be included in all requests // so server-side can construct a reply // the console will understand -export const API_VERSION = "2"; +export const API_VERSION = "2" export const BUTTON_ICON_SIZE = "26px" diff --git a/packages/web-console/src/scenes/Editor/Metrics/graph.tsx b/packages/web-console/src/scenes/Editor/Metrics/graph.tsx index 54f6ecc3c..5ae5579ea 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/graph.tsx +++ b/packages/web-console/src/scenes/Editor/Metrics/graph.tsx @@ -91,6 +91,8 @@ const LabelValue = styled.span` ` type Props = { + dateFrom: Date + dateNow: Date lastRefresh?: number tableId?: number tableName?: string @@ -106,6 +108,8 @@ type Props = { } export const Graph = ({ + dateFrom, + dateNow, lastRefresh, tableId, tableName, @@ -122,7 +126,6 @@ export const Graph = ({ const timeRef = useRef(null) const valueRef = useRef(null) const uPlotRef = useRef() - const [dateNow, setDateNow] = useState(new Date()) const { isTableMetric, mapYValue, label } = widgetConfig @@ -135,6 +138,7 @@ export const Graph = ({ const graphOptions = useGraphOptions({ data, + dateFrom, dateNow, colors, duration, @@ -156,10 +160,6 @@ export const Graph = ({ } }, [graphRootRef.current]) - useEffect(() => { - setDateNow(new Date()) - }, [lastRefresh, data]) - return (
diff --git a/packages/web-console/src/scenes/Editor/Metrics/index.tsx b/packages/web-console/src/scenes/Editor/Metrics/index.tsx index bd3ed6da1..8916e4516 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/index.tsx +++ b/packages/web-console/src/scenes/Editor/Metrics/index.tsx @@ -13,6 +13,7 @@ import { MetricViewMode, FetchMode, SampleBy, + durationInMinutes, } from "./utils" import { GridAlt, @@ -35,6 +36,7 @@ import { IconWithTooltip } from "../../../components/IconWithTooltip" import { useLocalStorage } from "../../../providers/LocalStorageProvider" import { eventBus } from "../../../modules/EventBus" import { EventType } from "../../../modules/EventBus/types" +import { subMinutes } from "date-fns" const Root = styled.div` display: flex; @@ -115,11 +117,17 @@ const formatSampleByLabel = (sampleBy: SampleBy, duration: MetricDuration) => { export const Metrics = () => { const { activeBuffer, updateBuffer, buffers } = useEditor() - const [metricDuration, setMetricDuration] = useState() const [metricViewMode, setMetricViewMode] = useState( MetricViewMode.GRID, ) + const [dateFrom, setDateFrom] = useState( + subMinutes( + new Date(), + durationInMinutes[metricDuration || MetricDuration.ONE_HOUR], + ), + ) + const [dateNow, setDateNow] = useState(new Date()) const [refreshRate, setRefreshRate] = useState() const [sampleBy, setSampleBy] = useState() const [dialogOpen, setDialogOpen] = useState(false) @@ -205,6 +213,9 @@ export const Metrics = () => { }, []) const setupListeners = () => { + if (intervalRef.current) { + clearInterval(intervalRef.current) + } if (autoRefreshTables && refreshRate && refreshRate !== RefreshRate.OFF) { intervalRef.current = setInterval( () => { @@ -272,6 +283,12 @@ export const Metrics = () => { } }, [metricDuration, refreshRate, metricViewMode, sampleBy]) + useEffect(() => { + const now = new Date() + setDateFrom(subMinutes(now, durationInMinutes[duration])) + setDateNow(now) + }, [lastRefresh]) + useEffect(() => { if (refreshRate) { refreshRateRef.current = refreshRate @@ -444,6 +461,8 @@ export const Metrics = () => { .sort((a, b) => a.position - b.position) .map((metric, index) => ( void @@ -79,6 +84,8 @@ export const Metric = ({ const fetchMetric = async () => { setLoading(true) try { + const subtracted = subMinutes(dateNow, durationInMinutes[metricDuration]) + const timeFilter = getTimeFilter(subtracted, dateNow) const responses = await Promise.all< | QuestDB.QueryResult | QuestDB.QueryResult @@ -88,9 +95,11 @@ export const Metric = ({ tableId: metric.tableId, metricDuration, sampleBy, - // ...(fetchMode === FetchMode.ROLLING_APPEND && { - // limit: -rollingAppendLimit, - // }), + timeFilter, + ...(widgetConfig.querySupportsRollingAppend && + fetchMode === FetchMode.ROLLING_APPEND && { + limit: -rollingAppendLimit, + }), }), ), quest.query( @@ -102,13 +111,13 @@ export const Metric = ({ const alignedData = widgetConfig.alignData( responses[0].data as unknown as ResultType[MetricType], ) - if (fetchMode === FetchMode.ROLLING_APPEND) { - setData(alignedData) - // console.log( - // metric.metricType, - // mergeRollingData(data, alignedData, rollingAppendLimit), - // ) - // setData(mergeRollingData(data, alignedData, rollingAppendLimit)) + if ( + data && + widgetConfig.querySupportsRollingAppend && + fetchMode === FetchMode.ROLLING_APPEND + ) { + console.log(mergeRollingData(data, alignedData, dateFrom)) + setData(mergeRollingData(data, alignedData, dateFrom)) } else { setData(alignedData) } @@ -158,13 +167,15 @@ export const Metric = ({ tableName && lastNotNull ? lastNotNull >= subMinutes( - new Date(), + dateNow, minuteDurations[minuteDurations.length - 1][1], ).getTime() : false return ( string - mapXValue: (rawValue: number, index: number, ticks: number[]) => string | null + mapXValue: (rawValue: number, index: number, ticks: number[]) => string mapYValue: (rawValue: number) => string timeRef: React.RefObject valueRef: React.RefObject @@ -31,11 +32,11 @@ const valuePlugin = ( const { idx } = u.cursor const x = idx !== null && idx !== undefined ? u.data[0][idx] : null const y = idx !== null && idx !== undefined ? u.data[1][idx] : null - if ([y, x].every(Boolean)) { + if ([y, x].every((v) => v !== null)) { timeRef.current!.textContent = utcToLocal( x as number, "dd/MM/yyyy HH:mm:ss", - ) + ) as string valueRef.current!.textContent = mapYValue(y as number) } else { timeRef.current!.textContent = null @@ -47,6 +48,7 @@ const valuePlugin = ( export const useGraphOptions = ({ data, + dateFrom, dateNow, colors, duration, @@ -58,7 +60,7 @@ export const useGraphOptions = ({ }: Params): Omit => { const theme = useContext(ThemeContext) - const start = subMinutes(dateNow, durationInMinutes[duration]).getTime() + const start = dateFrom.getTime() const end = dateNow.getTime() diff --git a/packages/web-console/src/scenes/Editor/Metrics/utils.ts b/packages/web-console/src/scenes/Editor/Metrics/utils.ts index 0ae8d4d66..cdd47c0c8 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/utils.ts +++ b/packages/web-console/src/scenes/Editor/Metrics/utils.ts @@ -1,3 +1,4 @@ +import { formatISO, subMinutes } from "date-fns" import { utcToLocal } from "../../../utils/dateTime" import uPlot from "uplot" @@ -22,8 +23,10 @@ export type Widget = { metricDuration: MetricDuration sampleBy?: SampleBy limit?: number + timeFilter?: string }) => string getQueryLastNotNull: (id?: number) => string + querySupportsRollingAppend: boolean alignData: (data: any) => uPlot.AlignedData mapYValue: (rawValue: number) => string } @@ -211,18 +214,14 @@ export const formatNumbers = (value: number) => { return value.toString() } -export const getTimeFilter = ( - minutes: number, -) => `created > date_trunc('minute', dateadd('${ - minutes >= 1440 ? "d" : minutes >= 60 ? "h" : "s" -}', -${ - minutes >= 1440 - ? minutesToDays(minutes) - : minutes >= 60 - ? minutesToHours(minutes) - : minutesToSeconds(minutes) -}, now())) -and created < date_trunc('${minutes >= 60 ? "minute" : "second"}', now())` +const formatToISOIfNeeded = (date: Date | string) => { + if (date instanceof Date) return formatISO(date) + return date +} + +export const getTimeFilter = (from: Date | string, to: Date | string) => { + return `FROM '${formatToISOIfNeeded(from)}' TO '${formatToISOIfNeeded(to)}'` +} export const getRollingAppendRowLimit = ( refreshRateInSeconds: number, @@ -237,16 +236,22 @@ export const hasData = (data?: uPlot.AlignedData) => { } export const mergeRollingData = ( - oldData: uPlot.AlignedData | undefined, - alignedData: uPlot.AlignedData, - rollingAppendLimit: number, + oldData: uPlot.AlignedData, + newData: uPlot.AlignedData, + dateFrom: Date, ) => { - const slicedOldData: uPlot.AlignedData = oldData - ? oldData.map((d) => d.slice(rollingAppendLimit)) - : Array(alignedData.length).fill([]) + const from = dateFrom.getTime() - return alignedData.map((d, i) => [ - ...slicedOldData[i], + const mergedData = newData.map((d, i) => [ + ...oldData[i], ...d, ]) as uPlot.AlignedData + + return mergedData.map((arr, arrIndex) => + arrIndex === 0 + ? Array.from(arr).filter((time) => time && time >= from) + : Array.from(arr).filter( + (_, index) => mergedData[0] && mergedData[0][index] >= from, + ), + ) as uPlot.AlignedData } diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts index 9cfb60347..bf7606875 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts @@ -1,11 +1,6 @@ import uPlot from "uplot" import type { Widget } from "../utils" -import { - durationInMinutes, - getTimeFilter, - sqlValueToFixed, - formatNumbers, -} from "../utils" +import { sqlValueToFixed, formatNumbers } from "../utils" import { CommitRate, defaultSampleByForDuration } from "../utils" import { TelemetryTable } from "../../../../consts" @@ -13,8 +8,8 @@ export const commitRate: Widget = { label: "Commit rate", iconUrl: "/assets/metric-commit-rate.svg", isTableMetric: true, - getQuery: ({ tableId, metricDuration, sampleBy, limit }) => { - const minutes = durationInMinutes[metricDuration] + querySupportsRollingAppend: true, + getQuery: ({ tableId, metricDuration, sampleBy, limit, timeFilter }) => { return ` select created, @@ -36,10 +31,10 @@ export const commitRate: Widget = { from ${TelemetryTable.WAL} where ${tableId ? `tableId = ${tableId} and ` : ""} event = 103 - and ${getTimeFilter(minutes)} -- it is important this is 1s, should this value change -- the "commit_rate" value will have to be adjusted to rate/s sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]} + ${timeFilter ? timeFilter : ""} fill(0) ) -- there is a bug in QuestDB, which does not sort the window dataset @@ -61,7 +56,7 @@ limit -1 `, alignData: (data: CommitRate[]): uPlot.AlignedData => [ data.map((l) => new Date(l.created).getTime()), - data.map((l) => sqlValueToFixed(l.commit_rate_smooth)), + data.map((l) => sqlValueToFixed(l.commit_rate)), ], mapYValue: (rawValue: number) => formatNumbers(rawValue), } diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts index 7dcc826c6..c76aba7e9 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts @@ -1,29 +1,23 @@ import uPlot from "uplot" import type { Widget } from "../utils" -import { - Latency, - defaultSampleByForDuration, - durationInMinutes, - sqlValueToFixed, - getTimeFilter, -} from "../utils" +import { Latency, defaultSampleByForDuration, sqlValueToFixed } from "../utils" import { TelemetryTable } from "../../../../consts" export const latency: Widget = { label: "WAL apply latency in ms", iconUrl: "/assets/metric-read-latency.svg", isTableMetric: true, - getQuery: ({ tableId, metricDuration, sampleBy, limit }) => { - const minutes = durationInMinutes[metricDuration] + querySupportsRollingAppend: true, + getQuery: ({ tableId, metricDuration, sampleBy, limit, timeFilter }) => { return ` select created, approx_percentile(latency, 0.9, 3) latency - from - (select * from ${TelemetryTable.WAL} where ${getTimeFilter(minutes)}) + from ${TelemetryTable.WAL} where event = 105 -- event is fixed and rowCount > 0 -- this is fixed clause, we have rows with - rowCount logged ${tableId ? `and tableId = ${tableId}` : ""} sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]} + ${timeFilter ? timeFilter : ""} fill(0) ${limit ? `limit ${limit}` : ""} ` diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts index bcbc9fd48..d50ec5d28 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts @@ -3,10 +3,8 @@ import type { Widget } from "../utils" import { WriteAmplification } from "../utils" import { defaultSampleByForDuration, - durationInMinutes, sqlValueToFixed, formatNumbers, - getTimeFilter, } from "../utils" import { TelemetryTable } from "../../../../consts" @@ -14,8 +12,8 @@ export const writeAmplification: Widget = { label: "Write amplification", iconUrl: "/assets/metric-write-amplification.svg", isTableMetric: true, - getQuery: ({ tableId, metricDuration, sampleBy, limit }) => { - const minutes = durationInMinutes[metricDuration] + querySupportsRollingAppend: true, + getQuery: ({ tableId, metricDuration, sampleBy, limit, timeFilter }) => { return ` select created, @@ -35,10 +33,10 @@ from ( where ${tableId ? `tableId = ${tableId} and ` : ""} event = 105 and rowCount > 0 -- this is fixed clause, we have rows with - rowCount logged - and ${getTimeFilter(minutes)} sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]} + ${timeFilter ? timeFilter : ""} -- fill with null to avoid spurious values and division by 0 - fill(null,null) + fill(null) ${limit ? `limit ${limit}` : ""} ) ); @@ -56,7 +54,9 @@ limit -1 `, alignData: (data: WriteAmplification[]): uPlot.AlignedData => [ data.map((l) => new Date(l.created).getTime()), - data.map((l) => sqlValueToFixed(l.writeAmplification)), + data.map((l) => + l.writeAmplification ? sqlValueToFixed(l.writeAmplification) : 0, + ), ], mapYValue: (rawValue: number) => formatNumbers(rawValue), } diff --git a/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts index cafdafaff..48b685802 100644 --- a/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts +++ b/packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts @@ -3,10 +3,8 @@ import type { Widget } from "../utils" import { RowsApplied, defaultSampleByForDuration, - durationInMinutes, sqlValueToFixed, formatNumbers, - getTimeFilter, } from "../utils" import { TelemetryTable } from "../../../../consts" @@ -14,21 +12,19 @@ export const writeThroughput: Widget = { label: "Write throughput", iconUrl: "/assets/metric-rows-applied.svg", isTableMetric: true, - getQuery: ({ tableId, metricDuration, sampleBy, limit }) => { - const minutes = durationInMinutes[metricDuration] - + querySupportsRollingAppend: true, + getQuery: ({ tableId, metricDuration, sampleBy, limit, timeFilter }) => { return ` select created time, count(rowCount) numOfWalApplies, sum(rowCount) numOfRowsApplied, - sum(physicalRowCount) numOfRowsWritten, - coalesce(avg(physicalRowCount/rowCount), 1) avgWalAmplification + sum(physicalRowCount) numOfRowsWritten from ${TelemetryTable.WAL} where ${tableId ? `tableId = ${tableId} and ` : ""} event = 105 -and ${getTimeFilter(minutes)} sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]} +${timeFilter ? timeFilter : ""} fill(null) ${limit ? `limit ${limit}` : ""}` }, @@ -44,7 +40,9 @@ limit -1 `, alignData: (data: RowsApplied[]): uPlot.AlignedData => [ data.map((l) => new Date(l.time).getTime()), - data.map((l) => sqlValueToFixed(l.numOfRowsApplied)), + data.map((l) => + l.numOfRowsApplied ? sqlValueToFixed(l.numOfRowsApplied) : 0, + ), ], mapYValue: (rawValue: number) => formatNumbers(rawValue), } diff --git a/packages/web-console/src/utils/dateTime.ts b/packages/web-console/src/utils/dateTime.ts index 5481c9ba6..1cfff3e72 100644 --- a/packages/web-console/src/utils/dateTime.ts +++ b/packages/web-console/src/utils/dateTime.ts @@ -5,5 +5,7 @@ export const getLocalTimeZone = () => { return Intl.DateTimeFormat().resolvedOptions().timeZone } -export const utcToLocal = (utcDate: number, dateFormat: string) => - format(new TZDate(utcDate, getLocalTimeZone()), dateFormat) +export const utcToLocal = (utcDate: number, dateFormat?: string): string => + dateFormat + ? format(new TZDate(utcDate, getLocalTimeZone()), dateFormat) + : new TZDate(utcDate, getLocalTimeZone()).toISOString()