Cleanup, add rollingAppendRowLimit logic for data refresh

packages/web-console/src/scenes/Editor/Metrics/graph.tsx

@@ -1,4 +1,4 @@
-import React, { useContext, useEffect, useRef, useState } from "react"
+import React, { useEffect, useRef, useState } from "react"
 import styled from "styled-components"
 import { MetricDuration, Widget, xAxisFormat } from "./utils"
 import { useGraphOptions } from "./useGraphOptions"

packages/web-console/src/scenes/Editor/Metrics/index.tsx

@@ -7,7 +7,9 @@ import {
   MetricDuration,
   RefreshRate,
   autoRefreshRates,
-  refreshRates,
+  refreshRatesInSeconds,
+  defaultSampleByForDuration,
+  getRollingAppendRowLimit,
 } from "./utils"
 import { Time, Refresh } from "@styled-icons/boxicons-regular"
 import { AddMetricDialog } from "./add-metric-dialog"
@@ -113,6 +115,17 @@ export const Metrics = () => {
 
   const buffer = buffers.find((b) => b.id === activeBuffer?.id)
 
+  const refreshRateInSec = refreshRate
+    ? refreshRate === RefreshRate.AUTO
+      ? refreshRatesInSeconds[autoRefreshRates[metricDuration]] * 1000
+      : refreshRatesInSeconds[refreshRate]
+    : 0
+
+  const rollingAppendLimit = getRollingAppendRowLimit(
+    refreshRateInSec,
+    defaultSampleByForDuration[metricDuration],
+  )
+
   const updateMetrics = (metrics: Metric[]) => {
     if (buffer?.id) {
       updateBuffer(buffer?.id, {
@@ -172,9 +185,7 @@ export const Metrics = () => {
           if (!tabInFocusRef.current) return
           setLastRefresh(new Date().getTime())
         },
-        refreshRate === RefreshRate.AUTO
-          ? refreshRates[autoRefreshRates[metricDuration]]
-          : refreshRates[refreshRate],
+        refreshRateInSec > 0 ? refreshRateInSec * 1000 : 0,
       )
     } else {
       clearInterval(intervalRef.current)

packages/web-console/src/scenes/Editor/Metrics/utils.ts

@@ -16,10 +16,12 @@ export type Widget = {
     tableId,
     metricDuration,
     sampleBy,
+    limit,
   }: {
     tableId?: number
     metricDuration: MetricDuration
     sampleBy?: SampleBy
+    limit?: number
   }) => string
   getQueryLastNotNull: (id?: number) => string
   alignData: (data: any) => uPlot.AlignedData
@@ -56,14 +58,14 @@ export enum RefreshRate {
   OFF = "Off",
 }
 
-export const refreshRates: Record<RefreshRate, number> = {
+export const refreshRatesInSeconds: Record<RefreshRate, number> = {
   [RefreshRate.AUTO]: 0,
   [RefreshRate.OFF]: 0,
-  [RefreshRate.ONE_SECOND]: 1 * 1000,
-  [RefreshRate.FIVE_SECONDS]: 5 * 1000,
-  [RefreshRate.TEN_SECONDS]: 10 * 1000,
-  [RefreshRate.THIRTY_SECONDS]: 30 * 1000,
-  [RefreshRate.ONE_MINUTE]: 60 * 1000,
+  [RefreshRate.ONE_SECOND]: 1,
+  [RefreshRate.FIVE_SECONDS]: 5,
+  [RefreshRate.TEN_SECONDS]: 10,
+  [RefreshRate.THIRTY_SECONDS]: 30,
+  [RefreshRate.ONE_MINUTE]: 60,
 }
 
 export const autoRefreshRates: Record<
@@ -105,6 +107,14 @@ export const defaultSampleByForDuration: Record<MetricDuration, SampleBy> = {
   [MetricDuration.SEVEN_DAYS]: SampleBy.FIFTEEN_MINUTES,
 }
 
+export const sampleByInSeconds: Record<SampleBy, number> = {
+  [SampleBy.ONE_SECOND]: 1,
+  [SampleBy.ONE_MINUTE]: 60,
+  [SampleBy.FIVE_MINUTES]: 60 * 5,
+  [SampleBy.FIFTEEN_MINUTES]: 60 * 15,
+  [SampleBy.ONE_HOUR]: 60 * 60,
+}
+
 export type CommitRate = {
   created: string
   commit_rate: string
@@ -183,3 +193,21 @@ export const formatNumbers = (value: number) => {
   }
   return value.toString()
 }
+
+export const getTimeFilter = (
+  minutes: number,
+) => `created > date_trunc('minute', dateadd('${
+  minutes >= 1440 ? "d" : minutes >= 60 ? "h" : "s"
+}', -${
+  minutes >= 1440
+    ? minutesToDays(minutes)
+    : minutes >= 60
+    ? minutesToHours(minutes)
+    : minutesToSeconds(minutes)
+}, now()))
+and created < date_trunc('${minutes >= 60 ? "minute" : "second"}', now())`
+
+export const getRollingAppendRowLimit = (
+  refreshRateInSeconds: number,
+  sampleBy: SampleBy,
+) => Math.ceil(refreshRateInSeconds / sampleByInSeconds[sampleBy])

packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts

@@ -1,14 +1,19 @@
 import uPlot from "uplot"
-import { Widget, durationInMinutes } from "../utils"
+import type { Widget } from "../utils"
+import {
+  durationInMinutes,
+  getTimeFilter,
+  sqlValueToFixed,
+  formatNumbers,
+} from "../utils"
 import type { CommitRate } from "../utils"
 import { TelemetryTable } from "../../../../consts"
-import { getTimeFilter, sqlValueToFixed, formatNumbers } from "./utils"
 
 export const commitRate: Widget = {
   label: "Commit rate per second",
   iconUrl: "/assets/metric-commit-rate.svg",
   isTableMetric: true,
-  getQuery: ({ tableId, metricDuration, sampleBy }) => {
+  getQuery: ({ tableId, metricDuration, sampleBy, limit }) => {
     const minutes = durationInMinutes[metricDuration]
     return `
     select 
@@ -40,6 +45,7 @@ export const commitRate: Widget = {
     -- there is a bug in QuestDB, which does not sort the window dataset
     -- once the bug is fixed the order can be removed
     order by 1
+    ${limit ? `limit ${limit}` : ""}
 );
     `
   },
@@ -53,9 +59,9 @@ event = 103
 and physicalRowCount != null
 limit -1
 `,
-  alignData: (rowsApplied: CommitRate[]): uPlot.AlignedData => [
-    rowsApplied.map((l) => new Date(l.created).getTime()),
-    rowsApplied.map((l) => sqlValueToFixed(l.commit_rate_smooth)),
+  alignData: (data: CommitRate[]): uPlot.AlignedData => [
+    data.map((l) => new Date(l.created).getTime()),
+    data.map((l) => sqlValueToFixed(l.commit_rate_smooth)),
   ],
   mapYValue: (rawValue: number) => formatNumbers(rawValue),
 }

packages/web-console/src/scenes/Editor/Metrics/widgets/latency.ts

@@ -1,14 +1,19 @@
 import uPlot from "uplot"
-import { Latency, sqlValueToFixed } from "../utils"
-import { Widget, defaultSampleByForDuration, durationInMinutes } from "../utils"
-import { getTimeFilter } from "./utils"
+import type { Widget } from "../utils"
+import {
+  Latency,
+  defaultSampleByForDuration,
+  durationInMinutes,
+  sqlValueToFixed,
+  getTimeFilter,
+} from "../utils"
 import { TelemetryTable } from "../../../../consts"
 
 export const latency: Widget = {
   label: "WAL apply latency in ms",
   iconUrl: "/assets/metric-read-latency.svg",
   isTableMetric: true,
-  getQuery: ({ tableId, metricDuration, sampleBy }) => {
+  getQuery: ({ tableId, metricDuration, sampleBy, limit }) => {
     const minutes = durationInMinutes[metricDuration]
     return `
 select created, approx_percentile(latency, 0.9, 3) latency
@@ -20,6 +25,7 @@ select created, approx_percentile(latency, 0.9, 3) latency
       ${tableId ? `and tableId = ${tableId}` : ""}
   sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]}
   fill(0)
+  ${limit ? `limit ${limit}` : ""}
   `
   },
   getQueryLastNotNull: (tableId) => `
@@ -31,9 +37,9 @@ event = 105
 and latency != null and rowCount > 0
 limit -1
 `,
-  alignData: (latency: Latency[]): uPlot.AlignedData => [
-    latency.map((l) => new Date(l.created).getTime()),
-    latency.map((l) => sqlValueToFixed(l.latency)),
+  alignData: (data: Latency[]): uPlot.AlignedData => [
+    data.map((l) => new Date(l.created).getTime()),
+    data.map((l) => sqlValueToFixed(l.latency)),
   ],
   mapYValue: (rawValue: number) => {
     if (rawValue >= 1000) {

packages/web-console/src/scenes/Editor/Metrics/widgets/writeAmplification.ts

@@ -1,14 +1,20 @@
 import uPlot from "uplot"
-import { sqlValueToFixed, formatNumbers, WriteAmplification } from "../utils"
-import { Widget, defaultSampleByForDuration, durationInMinutes } from "../utils"
+import type { Widget } from "../utils"
+import { WriteAmplification } from "../utils"
+import {
+  defaultSampleByForDuration,
+  durationInMinutes,
+  sqlValueToFixed,
+  formatNumbers,
+  getTimeFilter,
+} from "../utils"
 import { TelemetryTable } from "../../../../consts"
-import { getTimeFilter } from "./utils"
 
 export const writeAmplification: Widget = {
   label: "Write amplification",
   iconUrl: "/assets/metric-write-amplification.svg",
   isTableMetric: true,
-  getQuery: ({ tableId, metricDuration, sampleBy }) => {
+  getQuery: ({ tableId, metricDuration, sampleBy, limit }) => {
     const minutes = durationInMinutes[metricDuration]
     return `
 select 
@@ -33,6 +39,7 @@ from (
       sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]}      
       -- fill with null to avoid spurious values and division by 0
       fill(null,null)
+      ${limit ? `limit ${limit}` : ""}
   )
 );
     `

packages/web-console/src/scenes/Editor/Metrics/widgets/writeThroughput.ts

@@ -1,14 +1,20 @@
 import uPlot from "uplot"
-import { RowsApplied, sqlValueToFixed, formatNumbers } from "../utils"
-import { Widget, defaultSampleByForDuration, durationInMinutes } from "../utils"
+import type { Widget } from "../utils"
+import {
+  RowsApplied,
+  defaultSampleByForDuration,
+  durationInMinutes,
+  sqlValueToFixed,
+  formatNumbers,
+  getTimeFilter,
+} from "../utils"
 import { TelemetryTable } from "../../../../consts"
-import { getTimeFilter } from "./utils"
 
 export const writeThroughput: Widget = {
   label: "Write throughput",
   iconUrl: "/assets/metric-rows-applied.svg",
   isTableMetric: true,
-  getQuery: ({ tableId, metricDuration, sampleBy }) => {
+  getQuery: ({ tableId, metricDuration, sampleBy, limit }) => {
     const minutes = durationInMinutes[metricDuration]
 
     return `
@@ -22,7 +28,8 @@ from ${TelemetryTable.WAL}
 where ${tableId ? `tableId = ${tableId} and ` : ""}
 event = 105
 and ${getTimeFilter(minutes)}
-sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]}`
+sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]}
+${limit ? `limit ${limit}` : ""}`
   },
   getQueryLastNotNull: (tableId) => `
 select
@@ -34,9 +41,9 @@ and rowCount != null
 and physicalRowCount != null
 limit -1
 `,
-  alignData: (rowsApplied: RowsApplied[]): uPlot.AlignedData => [
-    rowsApplied.map((l) => new Date(l.time).getTime()),
-    rowsApplied.map((l) => sqlValueToFixed(l.numOfRowsApplied)),
+  alignData: (data: RowsApplied[]): uPlot.AlignedData => [
+    data.map((l) => new Date(l.time).getTime()),
+    data.map((l) => sqlValueToFixed(l.numOfRowsApplied)),
   ],
   mapYValue: (rawValue: number) => formatNumbers(rawValue),
 }