Skip to content

Commit

Permalink
TTL WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
mtopolnik committed Dec 18, 2024
1 parent 0469576 commit b4c42ae
Show file tree
Hide file tree
Showing 11 changed files with 64 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,14 @@ export const CreateTableDialog = () => {
const { appendQuery } = useEditor()

const handleAddTableSchema = (values: SchemaFormValues) => {
const { name, partitionBy, timestamp, schemaColumns, walEnabled } = values
const { name, partitionBy, timestamp, ttlValue, ttlUnit, schemaColumns, walEnabled } = values
const tableSchemaQuery = formatTableSchemaQuery({
name,
partitionBy,
timestamp,
walEnabled: walEnabled === "true",
ttlValue,
ttlUnit,
schemaColumns: schemaColumns.map((column) => ({
column: column.name,
type: column.type,
Expand Down Expand Up @@ -75,6 +77,8 @@ export const CreateTableDialog = () => {
walEnabled={false}
name=""
partitionBy="NONE"
ttlValue={0}
ttlUnit="HOURS"
schema={[]}
tables={tables}
timestamp=""
Expand Down
10 changes: 10 additions & 0 deletions packages/web-console/src/components/TableSchemaDialog/dialog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ type Props = {
name: string
schema: SchemaColumn[]
partitionBy: string
ttlValue: number
ttlUnit: string
timestamp: string
trigger?: React.ReactNode
tables?: QuestDB.Table[]
Expand All @@ -67,6 +69,8 @@ export const Dialog = ({
schema,
partitionBy,
timestamp,
ttlValue,
ttlUnit,
open,
isEditLocked,
hasWalSetting,
Expand All @@ -82,6 +86,8 @@ export const Dialog = ({
schemaColumns: schema,
partitionBy,
timestamp,
ttlValue,
ttlUnit,
walEnabled: hasWalSetting ? "false" : undefined,
}

Expand All @@ -97,6 +103,8 @@ export const Dialog = ({
schemaColumns: schema,
partitionBy: partitionBy,
timestamp: timestamp,
ttlValue: ttlValue,
ttlUnit: ttlUnit,
walEnabled:
hasWalSetting && walEnabled !== undefined
? walEnabled.toString()
Expand Down Expand Up @@ -136,6 +144,8 @@ export const Dialog = ({
"string.timestampRequired":
"Designated timestamp is required when partitioning is set to anything other than NONE",
}),
ttlValue: Joi.number(),
ttlUnit: Joi.string(),
walEnabled: Joi.any()
.allow(...["true", "false"])
.empty(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,7 @@ export type SchemaFormValues = {
schemaColumns: SchemaColumn[]
partitionBy: string
timestamp: string
ttlValue: number
ttlUnit: string
walEnabled?: string
}
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,8 @@ export const FilesToUpload = ({
name={name}
schema={data.schema}
partitionBy={data.partitionBy}
ttlValue={data.ttlValue}
ttlUnit={data.ttlUnit}
timestamp={data.timestamp}
isEditLocked={
data.exists && data.table_name === data.fileObject.name
Expand Down
18 changes: 18 additions & 0 deletions packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,22 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => {
})()
: "NONE"

const ttlValue =
result.status === FileCheckStatus.EXISTS && tables
? await (async () => {
const table = tables.find((t) => t.table_name === file.name)
return table?.ttlValue ?? 0
})()
: 0

const ttlUnit =
result.status === FileCheckStatus.EXISTS && tables
? await (async () => {
const table = tables.find((t) => t.table_name === file.name)
return table?.ttlUnit ?? "HOURS"
})()
: "HOURS"

const timestamp =
result.status === FileCheckStatus.EXISTS && tables
? await (async () => {
Expand All @@ -110,6 +126,8 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => {
exists: result.status === FileCheckStatus.EXISTS,
schema,
partitionBy,
ttlValue,
ttlUnit,
timestamp,
settings: {
forceHeader: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ export type ProcessedFile = {
schema: SchemaColumn[]
partitionBy: string
timestamp: string
ttlValue: number
ttlUnit: string
isUploading: boolean
uploaded: boolean
uploadResult?: UploadResult
Expand Down
4 changes: 3 additions & 1 deletion packages/web-console/src/scenes/Schema/Table/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,11 @@ const columnRender =
const Table = ({
description,
isScrolling,
designatedTimestamp,
table_name,
designatedTimestamp,
partitionBy,
ttlValue,
ttlUnit,
expanded = false,
walEnabled,
walTableData,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,19 @@ import * as QuestDB from "../../utils/questdb"
export const formatTableSchemaQueryResult = (
name: string,
partitionBy: string,
ttlValue: number,
ttlUnit: string,
columns: QuestDB.Column[],
walEnabled: boolean,
dedup: boolean,
dedup: boolean
): string => {
const findTimestampColumn = columns.find((c) => c.designated)
return formatTableSchemaQuery({
name,
partitionBy,
timestamp: findTimestampColumn ? findTimestampColumn.column : "",
partitionBy,
ttlValue,
ttlUnit,
walEnabled,
dedup,
schemaColumns: columns.map((c) => {
Expand Down
6 changes: 5 additions & 1 deletion packages/web-console/src/scenes/Schema/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,11 @@ const Schema = ({
return formatTableSchemaQueryResult(
tableData.table_name,
tableData.partitionBy,
tableData.ttlValue,
tableData.ttlUnit,
columnResponse.data,
tableData.walEnabled,
tableData.dedup,
tableData.dedup
)
}
} catch (error) {
Expand Down Expand Up @@ -359,6 +361,8 @@ const Schema = ({
table_name={table.table_name}
onChange={handleChange}
partitionBy={table.partitionBy}
ttlValue={table.ttlValue}
ttlUnit={table.ttlUnit}
walEnabled={table.walEnabled}
walTableData={walTables?.find(
(wt) => wt.name === table.table_name,
Expand Down
10 changes: 9 additions & 1 deletion packages/web-console/src/utils/formatTableSchemaQuery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ type Props = {
timestamp: string
dedup: boolean
walEnabled: boolean
ttlValue: number
ttlUnit: string
schemaColumns: Column[]
}

Expand All @@ -27,6 +29,8 @@ export const formatTableSchemaQuery = ({
timestamp,
walEnabled,
dedup,
ttlValue,
ttlUnit,
schemaColumns,
}: Props) => {
const hasValidTimestamp =
Expand Down Expand Up @@ -75,7 +79,11 @@ export const formatTableSchemaQuery = ({
}

if (partitionBy !== "NONE") {
query += ` PARTITION BY ${partitionBy} ${walEnabled ? "WAL" : "BYPASS WAL"}`
query += ` PARTITION BY ${partitionBy}`
if ((ttlValue ?? 0) !== 0) {
query += ` TTL ${ttlValue} ${ttlUnit}`
}
query += ` ${walEnabled ? "WAL" : "BYPASS WAL"}`
}

// For deduplication keys to work, WAL has to be enabled and a designated timestamp has to be set.
Expand Down
2 changes: 2 additions & 0 deletions packages/web-console/src/utils/questdb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ export type Table = {
designatedTimestamp: string
walEnabled: boolean
dedup: boolean
ttlValue: number
ttlUnit: string
}

export type Partition = {
Expand Down

0 comments on commit b4c42ae

Please sign in to comment.