Skip to content

Commit

Permalink
Implement zoom to data
Browse files Browse the repository at this point in the history
  • Loading branch information
insmac committed Nov 29, 2024
1 parent b0aa844 commit ee6537f
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 55 deletions.
2 changes: 1 addition & 1 deletion packages/browser-tests/questdb
Submodule questdb updated 51 files
+3 −2 ci/templates/start-self-hosted-job.yml
+3 −3 ci/test-pipeline.yml
+1 −1 core/pom.xml
+10 −1 core/src/main/java/io/questdb/cairo/AbstractTableNameRegistry.java
+4 −4 core/src/main/java/io/questdb/cairo/CairoEngine.java
+6 −1 core/src/main/java/io/questdb/cairo/CairoException.java
+0 −3 core/src/main/java/io/questdb/cairo/DdlListener.java
+10 −2 core/src/main/java/io/questdb/cairo/TableNameRegistry.java
+42 −10 core/src/main/java/io/questdb/cairo/wal/ApplyWal2TableJob.java
+0 −4 core/src/main/java/io/questdb/cairo/wal/WalWriter.java
+21 −23 core/src/main/java/io/questdb/cutlass/pgwire/modern/PGConnectionContextModern.java
+198 −102 core/src/main/java/io/questdb/cutlass/pgwire/modern/PGPipelineEntry.java
+3 −3 core/src/main/java/io/questdb/griffin/QueryBuilder.java
+44 −27 core/src/main/java/io/questdb/griffin/SqlCompilerImpl.java
+13 −2 core/src/main/java/io/questdb/griffin/SqlException.java
+21 −13 core/src/main/java/io/questdb/griffin/SqlParser.java
+8 −8 core/src/main/java/io/questdb/griffin/SqlParserCallback.java
+94 −82 core/src/main/java/io/questdb/griffin/engine/functions/date/TimestampAddFunctionFactory.java
+57 −1 core/src/main/java/io/questdb/griffin/engine/functions/window/AbstractWindowFunctionFactory.java
+7 −38 core/src/main/java/io/questdb/griffin/engine/functions/window/AvgDoubleWindowFunctionFactory.java
+1 −1 core/src/main/java/io/questdb/griffin/engine/functions/window/CountConstWindowFunctionFactory.java
+1 −1 core/src/main/java/io/questdb/griffin/engine/functions/window/CountDoubleWindowFunctionFactory.java
+8 −37 core/src/main/java/io/questdb/griffin/engine/functions/window/CountFunctionFactoryHelper.java
+1 −1 core/src/main/java/io/questdb/griffin/engine/functions/window/CountSymbolWindowFunctionFactory.java
+1 −2 core/src/main/java/io/questdb/griffin/engine/functions/window/CountVarcharWindowFunctionFactory.java
+7 −35 core/src/main/java/io/questdb/griffin/engine/functions/window/FirstValueDoubleWindowFunctionFactory.java
+1,703 −0 core/src/main/java/io/questdb/griffin/engine/functions/window/MaxDoubleWindowFunctionFactory.java
+327 −0 core/src/main/java/io/questdb/griffin/engine/functions/window/MinDoubleWindowFunctionFactory.java
+1 −1 core/src/main/java/io/questdb/griffin/engine/functions/window/SumDoubleWindowFunctionFactory.java
+16 −581 core/src/main/java/io/questdb/griffin/engine/ops/CreateTableOperation.java
+7 −408 core/src/main/java/io/questdb/griffin/engine/ops/CreateTableOperationBuilder.java
+413 −0 core/src/main/java/io/questdb/griffin/engine/ops/CreateTableOperationBuilderImpl.java
+11 −0 core/src/main/java/io/questdb/griffin/engine/ops/CreateTableOperationFuture.java
+618 −0 core/src/main/java/io/questdb/griffin/engine/ops/CreateTableOperationImpl.java
+1 −1 core/src/main/java/io/questdb/griffin/engine/ops/GenericDropOperation.java
+3 −0 core/src/main/java/io/questdb/std/str/Utf8s.java
+2 −0 core/src/main/java/module-info.java
+2 −0 core/src/main/resources/META-INF/services/io.questdb.griffin.FunctionFactory
+2 −2 core/src/test/java/io/questdb/test/cairo/CreateTableTest.java
+3 −0 core/src/test/java/io/questdb/test/cairo/TableNameRegistryTest.java
+239 −0 core/src/test/java/io/questdb/test/cutlass/http/QueryRegistryTest.java
+61 −16 core/src/test/java/io/questdb/test/cutlass/pgwire/PreparedStatementInvalidationTest.java
+13 −0 core/src/test/java/io/questdb/test/griffin/CreateTableAsSelectTest.java
+86 −0 core/src/test/java/io/questdb/test/griffin/DropTableFuzzTest.java
+2 −2 core/src/test/java/io/questdb/test/griffin/GroupByTest.java
+7 −2 core/src/test/java/io/questdb/test/griffin/SqlCompilerImplTest.java
+1 −1 core/src/test/java/io/questdb/test/griffin/SqlParserTest.java
+46 −357 core/src/test/java/io/questdb/test/griffin/engine/functions/date/TimestampAddFunctionFactoryTest.java
+873 −659 core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionTest.java
+420 −11 core/src/test/java/io/questdb/test/griffin/engine/window/WindowFunctionUnitTest.java
+60 −0 core/src/test/resources/sqllogictest/test/sql/test_update_rename_wal_table.test
65 changes: 56 additions & 9 deletions packages/web-console/src/scenes/Editor/Metrics/graph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,40 @@ const GraphWrapper = styled(Box).attrs({
align: "center",
})`
padding: 1rem 0;
.graph-no-data {
position: absolute;
display: flex;
flex-direction: column;
gap: 1rem;
width: 100%;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
width: 100%;
text-align: center;
}
.graph-no-data-text {
color: ${({ theme }) => theme.color.gray2};
font-size: 1.4rem;
text-align: center;
}
.graph-no-data-button {
align-self: center;
background-color: ${({ theme }) => theme.color.selection};
border: none;
color: ${({ theme }) => theme.color.foreground};
border-radius: 0.4rem;
height: 3rem;
padding: 0 1rem;
cursor: pointer;
&:hover {
background-color: ${({ theme }) => theme.color.comment};
}
}
`

const Label = styled.div`
Expand All @@ -63,21 +97,25 @@ type Props = {
beforeLabel?: React.ReactNode
loading?: boolean
data: uPlot.AlignedData
canZoomToData?: boolean
colors: string[]
duration: MetricDuration
yValue: (rawValue: number) => string
actions?: React.ReactNode
onZoomToData?: () => void
}

export const Graph = ({
label,
beforeLabel,
data,
canZoomToData,
colors,
duration,
yValue,
loading,
actions,
onZoomToData,
}: Props) => {
const timeRef = useRef(null)
const valueRef = useRef(null)
Expand Down Expand Up @@ -134,16 +172,25 @@ export const Graph = ({
uPlotRef.current = uplot
if (data[0].length === 0) {
const noData = document.createElement("div")
noData.innerText = "No data available for this period"
noData.style.position = "absolute"
noData.style.left = "50%"
noData.style.top = "50%"
noData.style.transform = "translate(-50%, -50%)"
noData.style.color = theme.color.gray2
noData.style.fontSize = "1.2rem"
noData.style.width = "100%"
noData.style.textAlign = "center"
noData.className = "graph-no-data"
uplot.over.appendChild(noData)

const noDataText = document.createElement("span")
noDataText.innerText = "No data available for this period"
noDataText.className = "graph-no-data-text"
noData.appendChild(noDataText)

if (canZoomToData) {
const zoomToDataButton = document.createElement("button")
zoomToDataButton.className = "graph-no-data-button"
zoomToDataButton.innerText = "Zoom to data"
zoomToDataButton.onclick = () => {
if (onZoomToData) {
onZoomToData()
}
}
noData.appendChild(zoomToDataButton)
}
}
}}
/>
Expand Down
35 changes: 5 additions & 30 deletions packages/web-console/src/scenes/Editor/Metrics/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -197,36 +197,10 @@ export const Metrics = () => {
<Select
name="duration"
value={metricDuration}
options={[
{
label: formatDurationLabel(MetricDuration.ONE_HOUR),
value: MetricDuration.ONE_HOUR,
},
{
label: formatDurationLabel(MetricDuration.THREE_HOURS),
value: MetricDuration.THREE_HOURS,
},
{
label: formatDurationLabel(MetricDuration.SIX_HOURS),
value: MetricDuration.SIX_HOURS,
},
{
label: formatDurationLabel(MetricDuration.TWELVE_HOURS),
value: MetricDuration.TWELVE_HOURS,
},
{
label: formatDurationLabel(MetricDuration.TWENTY_FOUR_HOURS),
value: MetricDuration.TWENTY_FOUR_HOURS,
},
{
label: formatDurationLabel(MetricDuration.THREE_DAYS),
value: MetricDuration.THREE_DAYS,
},
{
label: formatDurationLabel(MetricDuration.SEVEN_DAYS),
value: MetricDuration.SEVEN_DAYS,
},
]}
options={Object.values(MetricDuration).map((duration) => ({
label: formatDurationLabel(duration),
value: duration,
}))}
onChange={(e) =>
setMetricDuration(e.target.value as MetricDuration)
}
Expand Down Expand Up @@ -260,6 +234,7 @@ export const Metrics = () => {
onRemove={handleRemoveMetric}
onTableChange={handleTableChange}
onColorChange={handleColorChange}
onMetricDurationChange={setMetricDuration}
/>
))}
</Charts>
Expand Down
73 changes: 68 additions & 5 deletions packages/web-console/src/scenes/Editor/Metrics/metric.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,21 @@ import React, {
} from "react"
import { Metric as MetricItem } from "../../../store/buffers"
import {
durationInMinutes,
MetricDuration,
MetricType,
Latency,
RowsApplied,
metricTypeLabel,
LastNotNull,
} from "./utils"
import { QuestContext } from "../../../providers"
import { latency as latencySQL, rowsApplied as rowsAppliedSQL } from "./queries"
import {
latency as latencySQL,
rowsApplied as rowsAppliedSQL,
latencyLastNotNull as latencyLasNotNullSQL,
rowsAppliedLastNotNull as rowsAppliedLastNotNullSQL,
} from "./queries"
import * as QuestDB from "../../../utils/questdb"
import { Graph } from "./graph"
import uPlot from "uplot"
Expand All @@ -23,11 +30,11 @@ import { Box, Button, ForwardRef, Popover } from "@questdb/react-components"
import { Error, Palette, Trash } from "@styled-icons/boxicons-regular"
import { useSelector } from "react-redux"
import { selectors } from "../../../store"
import isEqual from "lodash.isequal"
import { useLocalStorage } from "../../../providers/LocalStorageProvider"
import { TableSelector } from "./table-selector"
import { IconWithTooltip } from "../../../components/IconWithTooltip"
import { ColorPalette } from "./color-palette"
import { subMinutes } from "date-fns"

const MetricInfoRoot = styled(Box).attrs({
align: "center",
Expand Down Expand Up @@ -79,16 +86,19 @@ export const Metric = ({
onRemove,
onTableChange,
onColorChange,
onMetricDurationChange,
}: {
metric: MetricItem
metricDuration: MetricDuration
onRemove: (metric: MetricItem) => void
onTableChange: (metric: MetricItem, tableId: number) => void
onColorChange: (metric: MetricItem, color: string) => void
onMetricDurationChange: (duration: MetricDuration) => void
}) => {
const { quest } = useContext(QuestContext)
const [loading, setLoading] = useState(false)
const [data, setData] = useState<uPlot.AlignedData>()
const [lastNotNull, setLastNotNull] = useState<number>()
const [colorPickerOpen, setColorPickerOpen] = useState(false)
const metricDurationRef = useRef(metricDuration)

Expand All @@ -99,43 +109,86 @@ export const Metric = ({

const { autoRefreshTables } = useLocalStorage()

const minuteDurations: [MetricDuration, number][] = Object.entries(
durationInMinutes,
) as [MetricDuration, number][]

const fetchLatency = async () => {
if (!metric.tableId) return Promise.reject()
return quest.query<Latency>(
latencySQL(metric.tableId, metricDurationRef.current),
)
}

const fetchLatencyLastNotNull = async () => {
if (!metric.tableId) return Promise.reject()
return quest.query<LastNotNull>(latencyLasNotNullSQL(metric.tableId))
}

const fetchRowsApplied = async () => {
if (!metric.tableId) return Promise.reject()
return quest.query<RowsApplied>(
rowsAppliedSQL(metric.tableId, metricDurationRef.current),
)
}

const fetchRowsAppliedLastNotNull = async () => {
if (!metric.tableId) return Promise.reject()
return quest.query<LastNotNull>(rowsAppliedLastNotNullSQL(metric.tableId))
}

const fetchers = {
[MetricType.LATENCY]: fetchLatency,
[MetricType.ROWS_APPLIED]: fetchRowsApplied,
[MetricType.WRITE_AMPLIFICATION]: fetchRowsApplied,
}

const fetchersLastNotNull = {
[MetricType.LATENCY]: fetchLatencyLastNotNull,
[MetricType.ROWS_APPLIED]: fetchRowsAppliedLastNotNull,
[MetricType.WRITE_AMPLIFICATION]: fetchRowsAppliedLastNotNull,
}

const fetchMetric = async () => {
setLoading(true)
try {
const response = await fetchers[metric.metricType]()
if (response && response.type === QuestDB.Type.DQL) {
const responses = await Promise.all<
| QuestDB.QueryResult<RowsApplied>
| QuestDB.QueryResult<Latency>
| QuestDB.QueryResult<LastNotNull>
>([
fetchers[metric.metricType](),
fetchersLastNotNull[metric.metricType](),
])

if (responses[0] && responses[0].type === QuestDB.Type.DQL) {
const alignedData = graphDataConfigs[metric.metricType].getData(
response.data as any,
responses[0].data as any,
)
setData(alignedData)
}
if (responses[1] && responses[1].type === QuestDB.Type.DQL) {
const lastNotNull = responses[1].data[0] as LastNotNull
setLastNotNull(new Date(lastNotNull.created).getTime())
}
} catch (err) {
console.error(err)
} finally {
setLoading(false)
}
}

const handleZoomToData = () => {
if (lastNotNull) {
const durationsWithData = minuteDurations.filter(
(d) => lastNotNull >= subMinutes(new Date(), d[1]).getTime(),
)
if (durationsWithData.length) {
onMetricDurationChange(durationsWithData[0][0])
}
}
}

useEffect(() => {
metricDurationRef.current = metricDuration
if (metric.tableId) {
Expand Down Expand Up @@ -181,9 +234,19 @@ export const Metric = ({

const tableName = tables.find((t) => t.id === metric.tableId)?.table_name

const canZoomToData = lastNotNull
? lastNotNull >=
subMinutes(
new Date(),
minuteDurations[minuteDurations.length - 1][1],
).getTime()
: false

return (
<Graph
data={metric.tableId && data ? data : [[], []]}
canZoomToData={canZoomToData}
onZoomToData={handleZoomToData}
colors={[metric.color]}
loading={loading}
duration={metricDuration}
Expand Down
21 changes: 21 additions & 0 deletions packages/web-console/src/scenes/Editor/Metrics/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,17 @@ and created < date_trunc('minute', now())
sample by ${sampleBy ?? mappedSampleBy[metricDuration]}`
}

export const rowsAppliedLastNotNull = (id: number) => `
select
created
from ${TelemetryTable.WAL}
where tableId = ${id}
and event = 105
and rowCount != null
and physicalRowCount != null
limit -1
`

export const latency = (
id: number,
metricDuration: MetricDuration,
Expand All @@ -53,3 +64,13 @@ and created < date_trunc('minute', now())
sample by ${sampleBy ?? mappedSampleBy[metricDuration]}
`
}

export const latencyLastNotNull = (id: number) => `
select
created
from ${TelemetryTable.WAL}
where tableId = ${id}
and (event = 105 or event = 103)
and latency != null
limit -1
`
Loading

0 comments on commit ee6537f

Please sign in to comment.