From 0a4e79e00ba6815fcab2f4f000d4b8963de60180 Mon Sep 17 00:00:00 2001 From: Maciej Bodek Date: Mon, 30 Oct 2023 17:26:27 +0100 Subject: [PATCH] Split legacy Import into states --- .../scenes/Import/ImportCSVFiles/dropbox.tsx | 81 +-- .../Import/ImportCSVFiles/files-to-upload.tsx | 467 ++++++++++-------- .../scenes/Import/ImportCSVFiles/index.tsx | 208 ++++---- .../scenes/Import/ImportCSVFiles/upload.tsx | 102 ++++ 4 files changed, 485 insertions(+), 373 deletions(-) create mode 100644 packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx index e92172716..61de03602 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx @@ -1,9 +1,6 @@ import React, { useRef, useState, useEffect } from "react" import styled from "styled-components" -import { Search2 } from "@styled-icons/remix-line" import { Box } from "../../../components/Box" -import { Text } from "@questdb/react-components" -import { Button, Heading } from "@questdb/react-components" import { ProcessedFile } from "./types" const getFileDuplicates = ( @@ -19,6 +16,7 @@ const getFileDuplicates = ( const Root = styled(Box).attrs({ flexDirection: "column" })<{ isDragging: boolean }>` + flex: 1; width: 100%; padding: 4rem 0 0; gap: 2rem; @@ -28,29 +26,22 @@ const Root = styled(Box).attrs({ flexDirection: "column" })<{ transition: all 0.15s ease-in-out; ` -const Caution = styled.div` - margin-top: 2rem; - padding: 2rem; - width: 100%; - background: ${({ theme }) => theme.color.backgroundDarker}; - text-align: center; -` - -const CautionText = styled(Text)` - color: #8b8fa7; - - a { - color: ${({ theme }) => theme.color.foreground}; - } -` - type Props = { files: ProcessedFile[] onFilesDropped: (files: File[]) => void dialogOpen: boolean + render: (props: { + duplicates: File[] + addToQueue: (inputFiles: FileList) => void + }) => React.ReactNode } -export const DropBox = ({ files, onFilesDropped, dialogOpen }: Props) => { +export const DropBox = ({ + files, + onFilesDropped, + dialogOpen, + render, +}: Props) => { const [isDragging, setIsDragging] = useState(false) const [duplicates, setDuplicates] = useState([]) const uploadInputRef = useRef(null) @@ -112,55 +103,7 @@ export const DropBox = ({ files, onFilesDropped, dialogOpen }: Props) => { onDrop={handleDrop} isDragging={isDragging} > - File upload icon - Drag CSV files here or paste from clipboard - { - if (e.target.files === null) return - addToQueue(e.target.files) - }} - multiple={true} - ref={uploadInputRef} - style={{ display: "none" }} - value="" - /> - - {duplicates.length > 0 && ( - - File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} - {duplicates.map((f) => f.name).join(", ")}. Change target table name - and try again. - - )} - - - Suitable for small batches of CSV file upload. For database - migrations, we recommend the{" "} - - COPY SQL - {" "} - command. - - + {render({ duplicates, addToQueue })} ) } diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx index 5d2f4d770..29c6e1a37 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from "react" +import React, { useEffect, useRef } from "react" import styled from "styled-components" import { Heading, Table, Select } from "@questdb/react-components" import type { Props as TableProps } from "@questdb/react-components/dist/components/Table" @@ -14,6 +14,11 @@ import { RenameTableDialog } from "./rename-table-dialog" import { Dialog as TableSchemaDialog } from "../../../components/TableSchemaDialog/dialog" import { UploadResultDialog } from "./upload-result-dialog" import { shortenText } from "../../../utils" +import { DropBox } from "./dropbox" + +const Root = styled(Box).attrs({ flexDirection: "column", gap: "2rem" })` + padding: 2rem; +` const StyledTable = styled(Table)` width: 100%; @@ -55,12 +60,23 @@ const FileTextBox = styled(Box)` padding: 0 1.1rem; ` +const BrowseTextLink = styled.span` + text-decoration: underline; + cursor: pointer; + + &:hover { + text-decoration: none; + } +` + type Props = { files: ProcessedFile[] onDialogToggle: (open: boolean) => void onFileRemove: (id: string) => void onFileUpload: (id: string) => void onFilePropertyChange: (id: string, file: Partial) => void + onFilesDropped: (files: File[]) => void + dialogOpen: boolean } export const FilesToUpload = ({ @@ -69,7 +85,10 @@ export const FilesToUpload = ({ onFileRemove, onFilePropertyChange, onFileUpload, + onFilesDropped, + dialogOpen, }: Props) => { + const uploadInputRef = useRef(null) const [renameDialogOpen, setRenameDialogOpen] = React.useState< string | undefined >() @@ -85,214 +104,250 @@ export const FilesToUpload = ({ }, [renameDialogOpen, schemaDialogOpen]) return ( - - Upload queue - >> - columns={[ - { - header: "File", - align: "flex-start", - ...(files.length > 0 && { width: "350px" }), - render: ({ data }) => { - const file = ( - - - {shortenText(data.fileObject.name, 20)} - + ( + + Upload queue + { + if (e.target.files === null) return + addToQueue(e.target.files) + }} + multiple={true} + ref={uploadInputRef} + style={{ display: "none" }} + value="" + /> + + You can drag and drop more files or{" "} + { + uploadInputRef.current?.click() + }} + > + browse from disk + + + {duplicates.length > 0 && ( + + File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} + {duplicates.map((f) => f.name).join(", ")}. Change target table + name and try again. + + )} + >> + columns={[ + { + header: "File", + align: "flex-start", + ...(files.length > 0 && { width: "350px" }), + render: ({ data }) => { + const file = ( + + + {shortenText(data.fileObject.name, 20)} + - - {bytesWithSuffix(data.fileObject.size)} - - - ) - return ( - - - - {data.fileObject.name.length > 20 && ( - - {data.fileObject.name} - - )} - {data.fileObject.name.length <= 20 && file} - - - {!data.isUploading && data.uploadResult && ( - - )} - - {(data.uploadResult && - data.uploadResult.rowsRejected > 0) || - (data.error && ( - - {data.uploadResult && - data.uploadResult.rowsRejected > 0 && ( - - {data.uploadResult.rowsRejected.toLocaleString()}{" "} - row - {data.uploadResult.rowsRejected > 1 - ? "s" - : ""}{" "} - rejected - - )} - {data.error && ( - - {data.error} - + + {bytesWithSuffix(data.fileObject.size)} + + + ) + return ( + + + + {data.fileObject.name.length > 20 && ( + + {data.fileObject.name} + + )} + {data.fileObject.name.length <= 20 && file} + + + {!data.isUploading && data.uploadResult && ( + )} - - ))} - - - ) - }, - }, - { - header: "Table name", - align: "flex-end", - width: "200px", - render: ({ data }) => { - return ( - setRenameDialogOpen(f?.id)} - onNameChange={(name) => { - onFilePropertyChange(data.id, { - table_name: name, - }) - }} - file={data} - /> - ) - }, - }, - { - header: ( - - Schema - - - } - > - - Optional. By default, QuestDB will infer schema from the CSV - file structure - - - ), + + {(data.uploadResult && + data.uploadResult.rowsRejected > 0) || + (data.error && ( + + {data.uploadResult && + data.uploadResult.rowsRejected > 0 && ( + + {data.uploadResult.rowsRejected.toLocaleString()}{" "} + row + {data.uploadResult.rowsRejected > 1 + ? "s" + : ""}{" "} + rejected + + )} + {data.error && ( + + {data.error} + + )} + + ))} + + + ) + }, + }, + { + header: "Table name", + align: "flex-end", + width: "200px", + render: ({ data }) => { + return ( + setRenameDialogOpen(f?.id)} + onNameChange={(name) => { + onFilePropertyChange(data.id, { + table_name: name, + }) + }} + file={data} + /> + ) + }, + }, + { + header: ( + + Schema + + + } + > + + Optional. By default, QuestDB will infer schema from the + CSV file structure + + + ), - align: "center", - width: "150px", - render: ({ data }) => { - const name = data.table_name ?? data.fileObject.name - return ( - - setSchemaDialogOpen(name ? data.id : undefined) - } - onSchemaChange={(schema) => { - onFilePropertyChange(data.id, { - schema: schema.schemaColumns, - partitionBy: schema.partitionBy, - timestamp: schema.timestamp, - }) - }} - name={name} - schema={data.schema} - partitionBy={data.partitionBy} - timestamp={data.timestamp} - isEditLocked={ - data.exists && data.table_name === data.fileObject.name - } - hasWalSetting={false} - ctaText="Save" - /> - ) - }, - }, - { - header: ( - - Write mode - - - } - > - - Append: data will be appended to the set. -
- Overwrite: any existing data or structure - will be overwritten. Required for partitioning and timestamp - related changes. - - - ), - align: "center", - width: "150px", - render: ({ data }) => ( - ) => + onFilePropertyChange(data.id, { + settings: { + ...data.settings, + overwrite: e.target.value === "true", + }, + }) + } + options={[ + { + label: "Append", + value: "false", + }, + { + label: "Overwrite", + value: "true", + }, + ]} + /> + ), + }, + { + align: "flex-end", + width: "300px", + render: ({ data }) => ( + { + onFilePropertyChange(data.fileObject.name, { + settings, + }) + }} + /> + ), + }, + ]} + rows={files} + /> + {files.length === 0 && ( + + + No files in queue + + + )} + )} - + /> ) } diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx index 6077e6968..dfafe254e 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx @@ -20,13 +20,16 @@ import { mapColumnTypeToUI, uuid, } from "./utils" +import { Upload } from "./upload" + +type State = "upload" | "list" type Props = { onImported: (result: UploadResult) => void } const Root = styled(Box).attrs({ gap: "4rem", flexDirection: "column" })` - padding: 2rem; + flex: 1; ` export const ImportCSVFiles = ({ onImported }: Props) => { @@ -36,6 +39,7 @@ export const ImportCSVFiles = ({ onImported }: Props) => { const tables = useSelector(selectors.query.getTables) const rootRef = useRef(null) const isVisible = useIsVisible(rootRef) + const [state, setState] = useState("upload") const setFileProperties = (id: string, file: Partial) => { setFilesDropped((files) => @@ -127,6 +131,7 @@ export const ImportCSVFiles = ({ onImported }: Props) => { const handleDrop = async (files: File[]) => { const fileConfigs = await getFileConfigs(files) setFilesDropped((filesDropped) => [...filesDropped, ...fileConfigs]) + setState("list") } const handleVisible = async () => { @@ -150,107 +155,114 @@ export const ImportCSVFiles = ({ onImported }: Props) => { return ( - - { - const file = filesDropped.find((f) => f.id === id) as ProcessedFile + {state === "upload" && ( + + )} - if (file.isUploading) { - return - } - setIsUploading(file, true) - try { - const response = await quest.uploadCSVFile({ - file: file.fileObject, - name: file.table_name, - settings: file.settings, - schema: file.schema.map(mapColumnTypeToQuestDB), - partitionBy: file.partitionBy, - timestamp: file.timestamp, - onProgress: (progress) => { - setFileProperties(file.id, { - uploadProgress: progress, - }) - }, - }) - setFileProperties(file.id, { - uploaded: response.status === "OK", - uploadResult: response.status === "OK" ? response : undefined, - schema: - response.status === "OK" - ? response.columns.map((c) => { - // Schema response only contains name and type, - // so we look for the pattern provided before upload and augment the schema - const match = file.schema.find((s) => s.name === c.name) - return { - ...pick(c, ["name"]), - ...{ - type: mapColumnTypeToUI(c.type), - pattern: - c.type === "TIMESTAMP" - ? match?.pattern + {state === "list" && ( + { + const file = filesDropped.find((f) => f.id === id) as ProcessedFile + + if (file.isUploading) { + return + } + setIsUploading(file, true) + try { + const response = await quest.uploadCSVFile({ + file: file.fileObject, + name: file.table_name, + settings: file.settings, + schema: file.schema.map(mapColumnTypeToQuestDB), + partitionBy: file.partitionBy, + timestamp: file.timestamp, + onProgress: (progress) => { + setFileProperties(file.id, { + uploadProgress: progress, + }) + }, + }) + setFileProperties(file.id, { + uploaded: response.status === "OK", + uploadResult: response.status === "OK" ? response : undefined, + schema: + response.status === "OK" + ? response.columns.map((c) => { + // Schema response only contains name and type, + // so we look for the pattern provided before upload and augment the schema + const match = file.schema.find((s) => s.name === c.name) + return { + ...pick(c, ["name"]), + ...{ + type: mapColumnTypeToUI(c.type), + pattern: + c.type === "TIMESTAMP" ? match?.pattern - : DEFAULT_TIMESTAMP_FORMAT - : "", - precision: isGeoHash(c.type) - ? extractPrecionFromGeohash(c.type) - : undefined, - }, - } as SchemaColumn - }) - : file.schema, - error: response.status === "OK" ? undefined : response.status, - }) - if (response.status === "OK") { - onImported(response) + ? match?.pattern + : DEFAULT_TIMESTAMP_FORMAT + : "", + precision: isGeoHash(c.type) + ? extractPrecionFromGeohash(c.type) + : undefined, + }, + } as SchemaColumn + }) + : file.schema, + error: response.status === "OK" ? undefined : response.status, + }) + if (response.status === "OK") { + onImported(response) + } + setIsUploading(file, false) + } catch (err) { + setIsUploading(file, false) + setFileProperties(file.id, { + uploaded: false, + uploadResult: undefined, + uploadProgress: 0, + error: "Upload error", + }) } - setIsUploading(file, false) - } catch (err) { - setIsUploading(file, false) - setFileProperties(file.id, { - uploaded: false, - uploadResult: undefined, - uploadProgress: 0, - error: "Upload error", - }) - } - }} - onFileRemove={(id) => { - const file = filesDropped.find((f) => f.id === id) as ProcessedFile - setFilesDropped( - filesDropped.filter( - (f) => f.fileObject.name !== file.fileObject.name, - ), - ) - }} - onFilePropertyChange={async (id, partialFile) => { - const processedFiles = await Promise.all( - filesDropped.map(async (file) => { - if (file.id === id) { - // Only check for file existence if table name is changed - const result = partialFile.table_name - ? await quest.checkCSVFile(partialFile.table_name) - : await Promise.resolve({ status: file.status }) - return { - ...file, - ...partialFile, - status: result.status, - error: partialFile.table_name ? undefined : file.error, // reset prior error if table name is changed + }} + onFileRemove={(id) => { + const file = filesDropped.find((f) => f.id === id) as ProcessedFile + setFilesDropped( + filesDropped.filter( + (f) => f.fileObject.name !== file.fileObject.name, + ), + ) + }} + onFilePropertyChange={async (id, partialFile) => { + const processedFiles = await Promise.all( + filesDropped.map(async (file) => { + if (file.id === id) { + // Only check for file existence if table name is changed + const result = partialFile.table_name + ? await quest.checkCSVFile(partialFile.table_name) + : await Promise.resolve({ status: file.status }) + return { + ...file, + ...partialFile, + status: result.status, + error: partialFile.table_name ? undefined : file.error, // reset prior error if table name is changed + } + } else { + return file } - } else { - return file - } - }), - ) - setFilesDropped(processedFiles) - }} - /> + }), + ) + setFilesDropped(processedFiles) + }} + /> + )} ) } diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx new file mode 100644 index 000000000..8fd14372e --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx @@ -0,0 +1,102 @@ +import React, { useRef } from "react" +import styled from "styled-components" +import { ProcessedFile } from "./types" +import { DropBox } from "./dropbox" +import { Search2 } from "@styled-icons/remix-line" +import { Box } from "../../../components/Box" +import { Text } from "@questdb/react-components" +import { Button, Heading } from "@questdb/react-components" + +const Actions = styled(Box).attrs({ flexDirection: "column" })` + margin: auto; +` + +const Caution = styled.div` + margin-top: auto; + padding: 2rem; + width: 100%; + background: ${({ theme }) => theme.color.backgroundDarker}; + text-align: center; +` + +const CautionText = styled(Text)` + color: #8b8fa7; + + a { + color: ${({ theme }) => theme.color.foreground}; + } +` + +type Props = { + files: ProcessedFile[] + onFilesDropped: (files: File[]) => void + dialogOpen: boolean +} +export const Upload = ({ files, onFilesDropped, dialogOpen }: Props) => { + const uploadInputRef = useRef(null) + + return ( + ( + + + File upload icon + + Drag CSV files here or paste from clipboard + + { + if (e.target.files === null) return + addToQueue(e.target.files) + }} + multiple={true} + ref={uploadInputRef} + style={{ display: "none" }} + value="" + /> + + {duplicates.length > 0 && ( + + File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} + {duplicates.map((f) => f.name).join(", ")}. Change target table + name and try again. + + )} + + + + Suitable for small batches of CSV file upload. For database + migrations, we recommend the{" "} + + COPY SQL + {" "} + command. + + + + )} + /> + ) +}