Refactor time filter to use FROM TO, refactor rolling append

packages/web-console/src/consts/index.ts

@@ -36,6 +36,6 @@ export const API = `https://${BASE}.questdb.io`
 // to be included in all requests
 // so server-side can construct a reply
 // the console will understand
-export const API_VERSION = "2";
+export const API_VERSION = "2"
 
 export const BUTTON_ICON_SIZE = "26px"

packages/web-console/src/scenes/Editor/Metrics/graph.tsx

@@ -91,6 +91,8 @@ const LabelValue = styled.span`
 `
 
 type Props = {
+  dateFrom: Date
+  dateNow: Date
   lastRefresh?: number
   tableId?: number
   tableName?: string
@@ -106,6 +108,8 @@ type Props = {
 }
 
 export const Graph = ({
+  dateFrom,
+  dateNow,
   lastRefresh,
   tableId,
   tableName,
@@ -122,7 +126,6 @@ export const Graph = ({
   const timeRef = useRef(null)
   const valueRef = useRef(null)
   const uPlotRef = useRef<uPlot>()
-  const [dateNow, setDateNow] = useState(new Date())
 
   const { isTableMetric, mapYValue, label } = widgetConfig
 
@@ -135,6 +138,7 @@ export const Graph = ({
 
   const graphOptions = useGraphOptions({
     data,
+    dateFrom,
     dateNow,
     colors,
     duration,
@@ -156,10 +160,6 @@ export const Graph = ({
     }
   }, [graphRootRef.current])
 
-  useEffect(() => {
-    setDateNow(new Date())
-  }, [lastRefresh, data])
-
   return (
     <Root ref={graphRootRef}>
       <Header>

packages/web-console/src/scenes/Editor/Metrics/index.tsx

@@ -13,6 +13,7 @@ import {
   MetricViewMode,
   FetchMode,
   SampleBy,
+  durationInMinutes,
 } from "./utils"
 import {
   GridAlt,
@@ -35,6 +36,7 @@ import { IconWithTooltip } from "../../../components/IconWithTooltip"
 import { useLocalStorage } from "../../../providers/LocalStorageProvider"
 import { eventBus } from "../../../modules/EventBus"
 import { EventType } from "../../../modules/EventBus/types"
+import { subMinutes } from "date-fns"
 
 const Root = styled.div`
   display: flex;
@@ -115,11 +117,17 @@ const formatSampleByLabel = (sampleBy: SampleBy, duration: MetricDuration) => {
 
 export const Metrics = () => {
   const { activeBuffer, updateBuffer, buffers } = useEditor()
-
   const [metricDuration, setMetricDuration] = useState<MetricDuration>()
   const [metricViewMode, setMetricViewMode] = useState<MetricViewMode>(
     MetricViewMode.GRID,
   )
+  const [dateFrom, setDateFrom] = useState(
+    subMinutes(
+      new Date(),
+      durationInMinutes[metricDuration || MetricDuration.ONE_HOUR],
+    ),
+  )
+  const [dateNow, setDateNow] = useState(new Date())
   const [refreshRate, setRefreshRate] = useState<RefreshRate>()
   const [sampleBy, setSampleBy] = useState<SampleBy>()
   const [dialogOpen, setDialogOpen] = useState(false)
@@ -205,6 +213,9 @@ export const Metrics = () => {
   }, [])
 
   const setupListeners = () => {
+    if (intervalRef.current) {
+      clearInterval(intervalRef.current)
+    }
     if (autoRefreshTables && refreshRate && refreshRate !== RefreshRate.OFF) {
       intervalRef.current = setInterval(
         () => {
@@ -272,6 +283,12 @@ export const Metrics = () => {
     }
   }, [metricDuration, refreshRate, metricViewMode, sampleBy])
 
+  useEffect(() => {
+    const now = new Date()
+    setDateFrom(subMinutes(now, durationInMinutes[duration]))
+    setDateNow(now)
+  }, [lastRefresh])
+
   useEffect(() => {
     if (refreshRate) {
       refreshRateRef.current = refreshRate
@@ -444,6 +461,8 @@ export const Metrics = () => {
             .sort((a, b) => a.position - b.position)
             .map((metric, index) => (
               <MetricComponent
+                dateFrom={dateFrom}
+                dateNow={dateNow}
                 key={index}
                 metric={metric}
                 metricDuration={duration}

packages/web-console/src/scenes/Editor/Metrics/metric.tsx

@@ -10,6 +10,7 @@ import {
   FetchMode,
   mergeRollingData,
   SampleBy,
+  getTimeFilter,
 } from "./utils"
 import { widgets } from "./widgets"
 import { QuestContext } from "../../../providers"
@@ -40,6 +41,8 @@ const ActionButton = styled(Button)`
 `
 
 export const Metric = ({
+  dateFrom,
+  dateNow,
   metric,
   metricDuration,
   onRemove,
@@ -51,6 +54,8 @@ export const Metric = ({
   rollingAppendLimit,
   sampleBy,
 }: {
+  dateFrom: Date
+  dateNow: Date
   metric: MetricItem
   metricDuration: MetricDuration
   onRemove: (metric: MetricItem) => void
@@ -79,6 +84,8 @@ export const Metric = ({
   const fetchMetric = async () => {
     setLoading(true)
     try {
+      const subtracted = subMinutes(dateNow, durationInMinutes[metricDuration])
+      const timeFilter = getTimeFilter(subtracted, dateNow)
       const responses = await Promise.all<
         | QuestDB.QueryResult<ResultType[MetricType]>
         | QuestDB.QueryResult<LastNotNull>
@@ -88,9 +95,11 @@ export const Metric = ({
             tableId: metric.tableId,
             metricDuration,
             sampleBy,
-            // ...(fetchMode === FetchMode.ROLLING_APPEND && {
-            //   limit: -rollingAppendLimit,
-            // }),
+            timeFilter,
+            ...(widgetConfig.querySupportsRollingAppend &&
+              fetchMode === FetchMode.ROLLING_APPEND && {
+                limit: -rollingAppendLimit,
+              }),
           }),
         ),
         quest.query<LastNotNull>(
@@ -102,13 +111,13 @@ export const Metric = ({
         const alignedData = widgetConfig.alignData(
           responses[0].data as unknown as ResultType[MetricType],
         )
-        if (fetchMode === FetchMode.ROLLING_APPEND) {
-          setData(alignedData)
-          // console.log(
-          //   metric.metricType,
-          //   mergeRollingData(data, alignedData, rollingAppendLimit),
-          // )
-          // setData(mergeRollingData(data, alignedData, rollingAppendLimit))
+        if (
+          data &&
+          widgetConfig.querySupportsRollingAppend &&
+          fetchMode === FetchMode.ROLLING_APPEND
+        ) {
+          console.log(mergeRollingData(data, alignedData, dateFrom))
+          setData(mergeRollingData(data, alignedData, dateFrom))
         } else {
           setData(alignedData)
         }
@@ -158,13 +167,15 @@ export const Metric = ({
     tableName && lastNotNull
       ? lastNotNull >=
         subMinutes(
-          new Date(),
+          dateNow,
           minuteDurations[minuteDurations.length - 1][1],
         ).getTime()
       : false
 
   return (
     <Graph
+      dateFrom={dateFrom}
+      dateNow={dateNow}
       lastRefresh={lastRefresh}
       data={metric.tableId && hasData(data) ? data : [[], []]}
       canZoomToData={canZoomToData}

packages/web-console/src/scenes/Editor/Metrics/useGraphOptions.ts

@@ -8,11 +8,12 @@ import { MetricDuration, durationInMinutes } from "./utils"
 
 type Params = {
   data: uPlot.AlignedData
+  dateFrom: Date
   dateNow: Date
   colors: string[]
   duration: MetricDuration
   tickValue?: (rawValue: number) => string
-  mapXValue: (rawValue: number, index: number, ticks: number[]) => string | null
+  mapXValue: (rawValue: number, index: number, ticks: number[]) => string
   mapYValue: (rawValue: number) => string
   timeRef: React.RefObject<HTMLSpanElement>
   valueRef: React.RefObject<HTMLSpanElement>
@@ -31,11 +32,11 @@ const valuePlugin = (
       const { idx } = u.cursor
       const x = idx !== null && idx !== undefined ? u.data[0][idx] : null
       const y = idx !== null && idx !== undefined ? u.data[1][idx] : null
-      if ([y, x].every(Boolean)) {
+      if ([y, x].every((v) => v !== null)) {
         timeRef.current!.textContent = utcToLocal(
           x as number,
           "dd/MM/yyyy HH:mm:ss",
-        )
+        ) as string
         valueRef.current!.textContent = mapYValue(y as number)
       } else {
         timeRef.current!.textContent = null
@@ -47,6 +48,7 @@ const valuePlugin = (
 
 export const useGraphOptions = ({
   data,
+  dateFrom,
   dateNow,
   colors,
   duration,
@@ -58,7 +60,7 @@ export const useGraphOptions = ({
 }: Params): Omit<uPlot.Options, "width" | "height"> => {
   const theme = useContext(ThemeContext)
 
-  const start = subMinutes(dateNow, durationInMinutes[duration]).getTime()
+  const start = dateFrom.getTime()
 
   const end = dateNow.getTime()
 

packages/web-console/src/scenes/Editor/Metrics/utils.ts

@@ -1,3 +1,4 @@
+import { formatISO, subMinutes } from "date-fns"
 import { utcToLocal } from "../../../utils/dateTime"
 import uPlot from "uplot"
 
@@ -22,8 +23,10 @@ export type Widget = {
     metricDuration: MetricDuration
     sampleBy?: SampleBy
     limit?: number
+    timeFilter?: string
   }) => string
   getQueryLastNotNull: (id?: number) => string
+  querySupportsRollingAppend: boolean
   alignData: (data: any) => uPlot.AlignedData
   mapYValue: (rawValue: number) => string
 }
@@ -211,18 +214,14 @@ export const formatNumbers = (value: number) => {
   return value.toString()
 }
 
-export const getTimeFilter = (
-  minutes: number,
-) => `created > date_trunc('minute', dateadd('${
-  minutes >= 1440 ? "d" : minutes >= 60 ? "h" : "s"
-}', -${
-  minutes >= 1440
-    ? minutesToDays(minutes)
-    : minutes >= 60
-    ? minutesToHours(minutes)
-    : minutesToSeconds(minutes)
-}, now()))
-and created < date_trunc('${minutes >= 60 ? "minute" : "second"}', now())`
+const formatToISOIfNeeded = (date: Date | string) => {
+  if (date instanceof Date) return formatISO(date)
+  return date
+}
+
+export const getTimeFilter = (from: Date | string, to: Date | string) => {
+  return `FROM '${formatToISOIfNeeded(from)}' TO '${formatToISOIfNeeded(to)}'`
+}
 
 export const getRollingAppendRowLimit = (
   refreshRateInSeconds: number,
@@ -237,16 +236,22 @@ export const hasData = (data?: uPlot.AlignedData) => {
 }
 
 export const mergeRollingData = (
-  oldData: uPlot.AlignedData | undefined,
-  alignedData: uPlot.AlignedData,
-  rollingAppendLimit: number,
+  oldData: uPlot.AlignedData,
+  newData: uPlot.AlignedData,
+  dateFrom: Date,
 ) => {
-  const slicedOldData: uPlot.AlignedData = oldData
-    ? oldData.map((d) => d.slice(rollingAppendLimit))
-    : Array(alignedData.length).fill([])
+  const from = dateFrom.getTime()
 
-  return alignedData.map((d, i) => [
-    ...slicedOldData[i],
+  const mergedData = newData.map((d, i) => [
+    ...oldData[i],
     ...d,
   ]) as uPlot.AlignedData
+
+  return mergedData.map((arr, arrIndex) =>
+    arrIndex === 0
+      ? Array.from(arr).filter((time) => time && time >= from)
+      : Array.from(arr).filter(
+          (_, index) => mergedData[0] && mergedData[0][index] >= from,
+        ),
+  ) as uPlot.AlignedData
 }

packages/web-console/src/scenes/Editor/Metrics/widgets/commitRate.ts

@@ -1,20 +1,15 @@
 import uPlot from "uplot"
 import type { Widget } from "../utils"
-import {
-  durationInMinutes,
-  getTimeFilter,
-  sqlValueToFixed,
-  formatNumbers,
-} from "../utils"
+import { sqlValueToFixed, formatNumbers } from "../utils"
 import { CommitRate, defaultSampleByForDuration } from "../utils"
 import { TelemetryTable } from "../../../../consts"
 
 export const commitRate: Widget = {
   label: "Commit rate",
   iconUrl: "/assets/metric-commit-rate.svg",
   isTableMetric: true,
-  getQuery: ({ tableId, metricDuration, sampleBy, limit }) => {
-    const minutes = durationInMinutes[metricDuration]
+  querySupportsRollingAppend: true,
+  getQuery: ({ tableId, metricDuration, sampleBy, limit, timeFilter }) => {
     return `
     select 
    created, 
@@ -36,10 +31,10 @@ export const commitRate: Widget = {
       from ${TelemetryTable.WAL}
       where ${tableId ? `tableId = ${tableId} and ` : ""}
       event = 103
-      and ${getTimeFilter(minutes)}
       -- it is important this is 1s, should this value change
       -- the "commit_rate" value will have to be adjusted to rate/s
       sample by ${sampleBy ?? defaultSampleByForDuration[metricDuration]}
+      ${timeFilter ? timeFilter : ""}
       fill(0)
     )
     -- there is a bug in QuestDB, which does not sort the window dataset
@@ -61,7 +56,7 @@ limit -1
 `,
   alignData: (data: CommitRate[]): uPlot.AlignedData => [
     data.map((l) => new Date(l.created).getTime()),
-    data.map((l) => sqlValueToFixed(l.commit_rate_smooth)),
+    data.map((l) => sqlValueToFixed(l.commit_rate)),
   ],
   mapYValue: (rawValue: number) => formatNumbers(rawValue),
 }