diff --git a/package-lock.json b/package-lock.json index f54ce57cf..f37338690 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,7 +37,8 @@ "redux": "4.0.x", "redux-logic": "3.x", "reselect": "4.0.x", - "string-natural-compare": "3.0.x" + "string-natural-compare": "3.0.x", + "zarrita": "^0.3.2" }, "devDependencies": { "@babel/cli": "7.x", @@ -4089,6 +4090,40 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "node_modules/@zarrita/core": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.0.3.tgz", + "integrity": "sha512-fWv51b+xbYnws1pkNDPwJQoDa76aojxplHyMup82u11UAiet3gURMsrrkhM6YbeTgSY1A8oGxDOrvar3SiZpLA==", + "dependencies": { + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1", + "numcodecs": "^0.2.2" + } + }, + "node_modules/@zarrita/indexing": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.0.3.tgz", + "integrity": "sha512-Q61d9MYX6dsK1DLltEpwx4mJWCZHj0TXiaEN4QpxNDtToa/EoyytP/pYHPypO4GXBscZooJ6eZkKT5FMx9PVfg==", + "dependencies": { + "@zarrita/core": "^0.0.3", + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1" + } + }, + "node_modules/@zarrita/storage": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.0.2.tgz", + "integrity": "sha512-uFt4abAoiOYLroalNDAnVaQxA17zGKrQ0waYKmTVm+bNonz8ggKZP+0FqMhgUZITGChqoANHuYTazbuU5AFXWA==", + "dependencies": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.3.6" + } + }, + "node_modules/@zarrita/typedarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.0.1.tgz", + "integrity": "sha512-ZdvNjYP1bEuQXoSTVkemV99w42jHYrJ3nh9golCLd4MVBlrVbfZo4wWgBslU4JZUaDikhFSH+GWMDgAq/rI32g==" + }, "node_modules/7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -13880,6 +13915,14 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/numcodecs": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.2.2.tgz", + "integrity": "sha512-Y5K8mv80yb4MgVpcElBkUeMZqeE4TrovxRit/dTZvoRl6YkB6WEjY+fiUjGCblITnt3T3fmrDg8yRWu0gOLjhQ==", + "engines": { + "node": ">=12" + } + }, "node_modules/nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -15941,6 +15984,11 @@ "redux": ">=3.5.2" } }, + "node_modules/reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "node_modules/refractor": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", @@ -18141,6 +18189,17 @@ "yaku": "^0.16.6" } }, + "node_modules/unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "dependencies": { + "uzip-module": "^1.0.2" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -18218,6 +18277,11 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "node_modules/v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -19318,6 +19382,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zarrita": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.3.2.tgz", + "integrity": "sha512-Zx9nS28C2tXZhF1BmQkgQGi0M/Z5JiM/KCMa+fEYtr/MnIzyizR4sKRA/sXjDP1iuylILWTJAWWBJD//0ONXCA==", + "dependencies": { + "@zarrita/core": "^0.0.3", + "@zarrita/indexing": "^0.0.3", + "@zarrita/storage": "^0.0.2" + } + }, "packages/desktop": { "name": "fms-file-explorer-desktop", "version": "7.0.0", @@ -22477,6 +22551,40 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "@zarrita/core": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.0.3.tgz", + "integrity": "sha512-fWv51b+xbYnws1pkNDPwJQoDa76aojxplHyMup82u11UAiet3gURMsrrkhM6YbeTgSY1A8oGxDOrvar3SiZpLA==", + "requires": { + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1", + "numcodecs": "^0.2.2" + } + }, + "@zarrita/indexing": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.0.3.tgz", + "integrity": "sha512-Q61d9MYX6dsK1DLltEpwx4mJWCZHj0TXiaEN4QpxNDtToa/EoyytP/pYHPypO4GXBscZooJ6eZkKT5FMx9PVfg==", + "requires": { + "@zarrita/core": "^0.0.3", + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1" + } + }, + "@zarrita/storage": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.0.2.tgz", + "integrity": "sha512-uFt4abAoiOYLroalNDAnVaQxA17zGKrQ0waYKmTVm+bNonz8ggKZP+0FqMhgUZITGChqoANHuYTazbuU5AFXWA==", + "requires": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.3.6" + } + }, + "@zarrita/typedarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.0.1.tgz", + "integrity": "sha512-ZdvNjYP1bEuQXoSTVkemV99w42jHYrJ3nh9golCLd4MVBlrVbfZo4wWgBslU4JZUaDikhFSH+GWMDgAq/rI32g==" + }, "7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -29949,6 +30057,11 @@ } } }, + "numcodecs": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.2.2.tgz", + "integrity": "sha512-Y5K8mv80yb4MgVpcElBkUeMZqeE4TrovxRit/dTZvoRl6YkB6WEjY+fiUjGCblITnt3T3fmrDg8yRWu0gOLjhQ==" + }, "nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -31427,6 +31540,11 @@ "rxjs": "^6.6.6" } }, + "reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "refractor": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", @@ -33139,6 +33257,14 @@ "yaku": "^0.16.6" } }, + "unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "requires": { + "uzip-module": "^1.0.2" + } + }, "uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -33202,6 +33328,11 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, + "uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -34006,6 +34137,16 @@ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true + }, + "zarrita": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.3.2.tgz", + "integrity": "sha512-Zx9nS28C2tXZhF1BmQkgQGi0M/Z5JiM/KCMa+fEYtr/MnIzyizR4sKRA/sXjDP1iuylILWTJAWWBJD//0ONXCA==", + "requires": { + "@zarrita/core": "^0.0.3", + "@zarrita/indexing": "^0.0.3", + "@zarrita/storage": "^0.0.2" + } } } } diff --git a/package.json b/package.json index 4c97188ee..4526050e1 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "redux": "4.0.x", "redux-logic": "3.x", "reselect": "4.0.x", - "string-natural-compare": "3.0.x" + "string-natural-compare": "3.0.x", + "zarrita": "^0.3.2" } } diff --git a/packages/core/App.tsx b/packages/core/App.tsx index eec917b17..033b23a11 100644 --- a/packages/core/App.tsx +++ b/packages/core/App.tsx @@ -3,9 +3,10 @@ import { initializeIcons, loadTheme } from "@fluentui/react"; import classNames from "classnames"; import { uniqueId } from "lodash"; import * as React from "react"; -import { batch, useDispatch, useSelector } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import ContextMenu from "./components/ContextMenu"; +import DataSourcePrompt from "./components/DataSourcePrompt"; import Modal from "./components/Modal"; import DirectoryTree from "./components/DirectoryTree"; import FileDetails from "./components/FileDetails"; @@ -14,7 +15,7 @@ import StatusMessage from "./components/StatusMessage"; import TutorialTooltip from "./components/TutorialTooltip"; import QuerySidebar from "./components/QuerySidebar"; import { FileExplorerServiceBaseUrl } from "./constants"; -import { interaction, metadata, selection } from "./state"; +import { interaction, selection } from "./state"; import "./styles/global.css"; import styles from "./App.module.css"; @@ -42,6 +43,7 @@ export default function App(props: AppProps) { const { fileExplorerServiceBaseUrl = FileExplorerServiceBaseUrl.PRODUCTION } = props; const dispatch = useDispatch(); + const hasQuerySelected = useSelector(selection.selectors.hasQuerySelected); const isDarkTheme = useSelector(selection.selectors.getIsDarkTheme); const shouldDisplaySmallFont = useSelector(selection.selectors.getShouldDisplaySmallFont); const platformDependentServices = useSelector( @@ -70,14 +72,8 @@ export default function App(props: AppProps) { }, [platformDependentServices, dispatch]); // Set data source base urls - // And kick off the process of requesting metadata needed by the application. React.useEffect(() => { - batch(() => { - dispatch(interaction.actions.setFileExplorerServiceBaseUrl(fileExplorerServiceBaseUrl)); - dispatch(metadata.actions.requestAnnotations()); - dispatch(metadata.actions.requestDataSources()); - dispatch(selection.actions.setAnnotationHierarchy([])); - }); + dispatch(interaction.actions.initializeApp(fileExplorerServiceBaseUrl)); }, [dispatch, fileExplorerServiceBaseUrl]); return ( @@ -92,8 +88,14 @@ export default function App(props: AppProps) {
- - + {hasQuerySelected ? ( + <> + + + + ) : ( + + )}
diff --git a/packages/core/components/AnnotationPicker/index.tsx b/packages/core/components/AnnotationPicker/index.tsx index b1486d17e..01ac0be69 100644 --- a/packages/core/components/AnnotationPicker/index.tsx +++ b/packages/core/components/AnnotationPicker/index.tsx @@ -10,17 +10,16 @@ import { metadata, selection } from "../../state"; interface Props { id?: string; - enableAllAnnotations?: boolean; disabledTopLevelAnnotations?: boolean; hasSelectAllCapability?: boolean; disableUnavailableAnnotations?: boolean; className?: string; title?: string; - selections: Annotation[]; + selections: string[]; annotationSubMenuRenderer?: ( item: ListItem ) => React.ReactElement>; - setSelections: (annotations: Annotation[]) => void; + setSelections: (annotations: string[]) => void; } /** @@ -28,25 +27,22 @@ interface Props { * downloading a manifest. */ export default function AnnotationPicker(props: Props) { - const annotations = useSelector(metadata.selectors.getSortedAnnotations).filter( - (annotation) => - !props.disabledTopLevelAnnotations || - !TOP_LEVEL_FILE_ANNOTATION_NAMES.includes(annotation.name) - ); + const annotations = useSelector(metadata.selectors.getSortedAnnotations); const unavailableAnnotations = useSelector( selection.selectors.getUnavailableAnnotationsForHierarchy ); const areAvailableAnnotationLoading = useSelector( selection.selectors.getAvailableAnnotationsForHierarchyLoading ); - const recentAnnotationNames = useSelector(selection.selectors.getRecentAnnotations); + const recentAnnotations = recentAnnotationNames.flatMap((name) => annotations.filter((annotation) => annotation.name === name) ); // Define buffer item const bufferBar = { + name: "buffer", selected: false, disabled: false, isBuffer: true, @@ -55,47 +51,47 @@ export default function AnnotationPicker(props: Props) { }; // combine all annotation lists and buffer item objects - const rawItems = [...recentAnnotations, bufferBar, ...annotations]; - - const items = uniqBy( - rawItems.flatMap((annotation) => { - if (annotation instanceof Annotation) { - return { - selected: props.selections.some( - (selected) => selected.name === annotation.name - ), - disabled: - !props.enableAllAnnotations && - unavailableAnnotations.some( - (unavailable) => unavailable.name === annotation.name - ), - recent: - recentAnnotationNames.includes(annotation.name) && - !props.selections.some((selected) => selected.name === annotation.name), - loading: !props.enableAllAnnotations && areAvailableAnnotationLoading, - description: annotation.description, - data: annotation, - value: annotation.name, - displayValue: annotation.displayName, - }; - } else { - // This is reached if the 'annotation' is a spacer. + const nonUniqueItems = [...recentAnnotations, bufferBar, ...annotations] + .filter( + (annotation) => + !props.disabledTopLevelAnnotations || + !TOP_LEVEL_FILE_ANNOTATION_NAMES.includes(annotation.name) + ) + .map((annotation) => { + // This is reached if the 'annotation' is a spacer. + if (!(annotation instanceof Annotation)) { return annotation; } - }), - "value" - ); + + const isSelected = props.selections.some((selected) => selected === annotation.name); + return { + selected: isSelected, + recent: recentAnnotationNames.includes(annotation.name) && !isSelected, + disabled: + props.disableUnavailableAnnotations && + unavailableAnnotations.some( + (unavailable) => unavailable.name === annotation.name + ), + loading: props.disableUnavailableAnnotations && areAvailableAnnotationLoading, + description: annotation.description, + data: annotation, + value: annotation.name, + displayValue: annotation.displayName, + }; + }); + + const items = uniqBy(nonUniqueItems, "value"); const removeSelection = (item: ListItem) => { props.setSelections( - props.selections.filter((annotation) => annotation.name !== item.data?.name) + props.selections.filter((annotation) => annotation !== item.data?.name) ); }; const addSelection = (item: ListItem) => { // Should never be undefined, included as guard statement to satisfy compiler if (item.data) { - props.setSelections([...props.selections, item.data]); + props.setSelections([...props.selections, item.data.name]); } }; @@ -108,7 +104,9 @@ export default function AnnotationPicker(props: Props) { onDeselect={removeSelection} onSelect={addSelection} onSelectAll={ - props.hasSelectAllCapability ? () => props.setSelections?.(annotations) : undefined + props.hasSelectAllCapability + ? () => props.setSelections?.(annotations.map((a) => a.name)) + : undefined } onDeselectAll={() => props.setSelections([])} subMenuRenderer={props.annotationSubMenuRenderer} diff --git a/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css b/packages/core/components/DataSourcePrompt/DataSourcePrompt.module.css similarity index 98% rename from packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css rename to packages/core/components/DataSourcePrompt/DataSourcePrompt.module.css index 6fa93df73..6734de630 100644 --- a/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css +++ b/packages/core/components/DataSourcePrompt/DataSourcePrompt.module.css @@ -76,7 +76,7 @@ } .text, .warning { - font-size: smaller; + text-align: center; } .warning { diff --git a/packages/core/components/Modal/DataSourcePrompt/index.tsx b/packages/core/components/DataSourcePrompt/index.tsx similarity index 86% rename from packages/core/components/Modal/DataSourcePrompt/index.tsx rename to packages/core/components/DataSourcePrompt/index.tsx index 192e8ad9e..0107a927e 100644 --- a/packages/core/components/Modal/DataSourcePrompt/index.tsx +++ b/packages/core/components/DataSourcePrompt/index.tsx @@ -3,15 +3,13 @@ import { throttle } from "lodash"; import * as React from "react"; import { useDispatch, useSelector } from "react-redux"; -import { ModalProps } from ".."; -import BaseModal from "../BaseModal"; -import { Source } from "../../../entity/FileExplorerURL"; -import { interaction, selection } from "../../../state"; +import { interaction, selection } from "../../state"; +import { Source } from "../../entity/FileExplorerURL"; import styles from "./DataSourcePrompt.module.css"; -interface Props extends ModalProps { - isEditing?: boolean; +interface Props { + hideTitle?: boolean; } const DATA_SOURCE_DETAILS = [ @@ -29,22 +27,26 @@ const DATA_SOURCE_DETAILS = [ /** * Dialog meant to prompt user to select a data source option */ -export default function DataSourcePrompt({ onDismiss }: Props) { +export default function DataSourcePrompt(props: Props) { const dispatch = useDispatch(); - const dataSourceToReplace = useSelector(interaction.selectors.getDataSourceForVisibleModal); + const selectedDataSources = useSelector(selection.selectors.getSelectedDataSources); + const dataSourceInfo = useSelector(interaction.selectors.getDataSourceInfoForVisibleModal); + const { source: sourceToReplace, query } = dataSourceInfo || {}; const [dataSourceURL, setDataSourceURL] = React.useState(""); const [isDataSourceDetailExpanded, setIsDataSourceDetailExpanded] = React.useState(false); const addOrReplaceQuery = (source: Source) => { - if (dataSourceToReplace) { + if (sourceToReplace) { dispatch(selection.actions.replaceDataSource(source)); + } else if (query) { + dispatch(selection.actions.changeDataSources([...selectedDataSources, source])); } else { dispatch( selection.actions.addQuery({ name: `New ${source.name} Query`, - parts: { source }, + parts: { sources: [source] }, }) ); } @@ -62,7 +64,7 @@ export default function DataSourcePrompt({ onDismiss }: Props) { return; } addOrReplaceQuery({ name, type: extension, uri: selectedFile }); - onDismiss(); + dispatch(interaction.actions.hideVisibleModal()); } }; const onEnterURL = throttle( @@ -88,20 +90,20 @@ export default function DataSourcePrompt({ onDismiss }: Props) { type: extensionGuess as "csv" | "json" | "parquet", uri: dataSourceURL, }); - onDismiss(); + dispatch(interaction.actions.hideVisibleModal()); }, 10000, { leading: true, trailing: false } ); - const body = ( + return ( <> - {dataSourceToReplace && ( + {sourceToReplace && (

Notice

There was an error loading the data source file " - {dataSourceToReplace.name}". Please re-select the data source file or a + {sourceToReplace.name}". Please re-select the data source file or a replacement.

@@ -111,10 +113,10 @@ export default function DataSourcePrompt({ onDismiss }: Props) {

)} + {!props.hideTitle &&

Choose a data source

}

- Please provide a ".csv", ".parquet", or ".json" file - containing metadata about some files. See more details for information about what a - data source file should look like... + To get started, load a CSV, Parquet, or JSON file containing metadata (annotations) + about your files to view them.

{isDataSourceDetailExpanded ? (
@@ -180,6 +182,4 @@ export default function DataSourcePrompt({ onDismiss }: Props) {
); - - return ; } diff --git a/packages/core/components/FileDetails/index.tsx b/packages/core/components/FileDetails/index.tsx index a93f7942f..2cc9f7801 100644 --- a/packages/core/components/FileDetails/index.tsx +++ b/packages/core/components/FileDetails/index.tsx @@ -78,6 +78,13 @@ function resizeHandleDoubleClick() { export default function FileDetails(props: Props) { const [windowState, windowDispatch] = React.useReducer(windowStateReducer, INITIAL_STATE); const [fileDetails, isLoading] = useFileDetails(); + const [thumbnailPath, setThumbnailPath] = React.useState(undefined); + + React.useEffect(() => { + if (fileDetails) { + fileDetails.getPathToThumbnail().then(setThumbnailPath); + } + }, [fileDetails]); // If FileDetails pane is minimized, set its width to the width of the WindowActionButtons. Else, let it be // defined by whatever the CSS determines (setting an inline style to undefined will prompt ReactDOM to not apply @@ -85,7 +92,6 @@ export default function FileDetails(props: Props) { const minimizedWidth = windowState.state === WindowState.MINIMIZED ? WINDOW_ACTION_BUTTON_WIDTH : undefined; - const thumbnailPath = fileDetails?.getPathToThumbnail(); let thumbnailHeight = undefined; let thumbnailWidth = undefined; if (windowState.state === WindowState.DEFAULT) { diff --git a/packages/core/components/FileList/ColumnPicker.tsx b/packages/core/components/FileList/ColumnPicker.tsx index 132afe65b..2aec5dc8a 100644 --- a/packages/core/components/FileList/ColumnPicker.tsx +++ b/packages/core/components/FileList/ColumnPicker.tsx @@ -2,25 +2,30 @@ import * as React from "react"; import { useSelector, useDispatch } from "react-redux"; import AnnotationPicker from "../AnnotationPicker"; -import { selection } from "../../state"; +import { metadata, selection } from "../../state"; /** * Picker for selecting which columns to display in the file list. */ export default function ColumnPicker() { const dispatch = useDispatch(); + const annotations = useSelector(metadata.selectors.getAnnotations); const columnAnnotations = useSelector(selection.selectors.getAnnotationsToDisplay); return ( { + title="Select metadata to display as columns" + selections={columnAnnotations.map((a) => a.name)} + setSelections={(selectedAnnotations) => { // Prevent de-selecting all columns - if (!annotations.length) { + if (!selectedAnnotations.length) { dispatch(selection.actions.setDisplayAnnotations([columnAnnotations[0]])); } else { - dispatch(selection.actions.setDisplayAnnotations(annotations)); + dispatch( + selection.actions.setDisplayAnnotations( + annotations.filter((a) => selectedAnnotations.includes(a.name)) + ) + ); } }} /> diff --git a/packages/core/components/FileList/LazilyRenderedThumbnail.tsx b/packages/core/components/FileList/LazilyRenderedThumbnail.tsx index e6329782a..05a23778c 100644 --- a/packages/core/components/FileList/LazilyRenderedThumbnail.tsx +++ b/packages/core/components/FileList/LazilyRenderedThumbnail.tsx @@ -51,6 +51,14 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr const file = fileSet.getFileByIndex(overallIndex); const thumbnailSize = measuredWidth / fileGridColCount - 2 * MARGIN; + const [thumbnailPath, setThumbnailPath] = React.useState(undefined); + + React.useEffect(() => { + if (file) { + file.getPathToThumbnail().then(setThumbnailPath); + } + }, [file]); + const isSelected = React.useMemo(() => { return fileSelection.isSelected(fileSet, overallIndex); }, [fileSelection, fileSet, overallIndex]); @@ -88,7 +96,6 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr let content; if (file) { - const thumbnailPath = file.getPathToThumbnail(); const filenameForRender = clipFileName(file?.name); content = (
; + } + + export class HTTPStore implements Store { + constructor(baseUrl: string); + getItem(key: string): Promise; + } + + export function open(params: { + store: Store; + path: string; + }): Promise<{ getRaw(): Promise }>; +} diff --git a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx index 859520af3..5d1c9959a 100644 --- a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx +++ b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx @@ -13,9 +13,11 @@ describe("", () => { visibleModal: ModalType.CodeSnippet, }, selection: { - dataSource: { - uri: "fake-uri.test", - }, + dataSources: [ + { + uri: "fake-uri.test", + }, + ], }, }); diff --git a/packages/core/components/Modal/DataSource/index.tsx b/packages/core/components/Modal/DataSource/index.tsx new file mode 100644 index 000000000..5ca6a915b --- /dev/null +++ b/packages/core/components/Modal/DataSource/index.tsx @@ -0,0 +1,18 @@ +import * as React from "react"; + +import { ModalProps } from ".."; +import BaseModal from "../BaseModal"; +import DataSourcePrompt from "../../DataSourcePrompt"; + +/** + * Dialog meant to prompt user to select a data source option + */ +export default function DataSource(props: ModalProps) { + return ( + } + title="Choose a data source" + onDismiss={props.onDismiss} + /> + ); +} diff --git a/packages/core/components/Modal/MetadataManifest/index.tsx b/packages/core/components/Modal/MetadataManifest/index.tsx index 1c9b40c91..ee8a9724e 100644 --- a/packages/core/components/Modal/MetadataManifest/index.tsx +++ b/packages/core/components/Modal/MetadataManifest/index.tsx @@ -6,8 +6,7 @@ import { useDispatch, useSelector } from "react-redux"; import { ModalProps } from ".."; import BaseModal from "../BaseModal"; import AnnotationPicker from "../../AnnotationPicker"; -import * as modalSelectors from "../selectors"; -import { interaction } from "../../../state"; +import { interaction, metadata } from "../../../state"; import styles from "./MetadataManifest.module.css"; @@ -17,18 +16,26 @@ import styles from "./MetadataManifest.module.css"; */ export default function MetadataManifest({ onDismiss }: ModalProps) { const dispatch = useDispatch(); - const annotationsPreviouslySelected = useSelector( - modalSelectors.getAnnotationsPreviouslySelected - ); - const [selectedAnnotations, setSelectedAnnotations] = React.useState( - annotationsPreviouslySelected - ); + const annotations = useSelector(metadata.selectors.getAnnotations); + const annotationsPreviouslySelected = useSelector(interaction.selectors.getCsvColumns); const fileTypeForVisibleModal = useSelector(interaction.selectors.getFileTypeForVisibleModal); + const [selectedAnnotations, setSelectedAnnotations] = React.useState([]); + + // Update the selected annotations when the previously selected annotations + // or list of all annotations change like on data source change + React.useEffect(() => { + const annotationsPreviouslySelectedAvailable = ( + annotationsPreviouslySelected || [] + ).filter((annotationName) => + annotations.some((annotation) => annotationName === annotation.name) + ); + setSelectedAnnotations(annotationsPreviouslySelectedAvailable); + }, [annotations, annotationsPreviouslySelected]); + const onDownload = () => { - const selectedAnnotationNames = selectedAnnotations.map((annotation) => annotation.name); dispatch( - interaction.actions.downloadManifest(selectedAnnotationNames, fileTypeForVisibleModal) + interaction.actions.downloadManifest(selectedAnnotations, fileTypeForVisibleModal) ); onDismiss(); }; diff --git a/packages/core/components/Modal/index.tsx b/packages/core/components/Modal/index.tsx index c982f2472..faeb5a141 100644 --- a/packages/core/components/Modal/index.tsx +++ b/packages/core/components/Modal/index.tsx @@ -3,7 +3,7 @@ import { useDispatch, useSelector } from "react-redux"; import { interaction } from "../../state"; import CodeSnippet from "./CodeSnippet"; -import DataSourcePrompt from "./DataSourcePrompt"; +import DataSource from "./DataSource"; import MetadataManifest from "./MetadataManifest"; export interface ModalProps { @@ -12,7 +12,7 @@ export interface ModalProps { export enum ModalType { CodeSnippet = 1, - DataSourcePrompt = 2, + DataSource = 2, MetadataManifest = 3, } @@ -30,8 +30,8 @@ export default function Modal() { switch (visibleModal) { case ModalType.CodeSnippet: return ; - case ModalType.DataSourcePrompt: - return ; + case ModalType.DataSource: + return ; case ModalType.MetadataManifest: return ; default: diff --git a/packages/core/components/Modal/selectors.ts b/packages/core/components/Modal/selectors.ts deleted file mode 100644 index 2ec2bead2..000000000 --- a/packages/core/components/Modal/selectors.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { createSelector } from "reselect"; - -import * as interactionSelectors from "../../state/interaction/selectors"; -import * as metadataSelectors from "../../state/metadata/selectors"; - -/** - * Returns Annotation instances for those annotations that were previously used to generate - * either a CSV manifest or dataset (via Python snippet generation). - */ -export const getAnnotationsPreviouslySelected = createSelector( - [interactionSelectors.getCsvColumns, metadataSelectors.getAnnotations], - (annotationDisplayNames, annotations) => - annotations.filter((annotation) => annotationDisplayNames?.includes(annotation.displayName)) -); diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index dc819fe2e..11339d632 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -1,39 +1,110 @@ +import { List } from "@fluentui/react"; import * as React from "react"; +import { useDispatch, useSelector } from "react-redux"; import QueryPart from "."; +import ListRow, { ListItem } from "../ListPicker/ListRow"; import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; import { Source } from "../../entity/FileExplorerURL"; +import { interaction, metadata, selection } from "../../state"; interface Props { - dataSources: (Source | undefined)[]; + dataSources: Source[]; } /** * Component responsible for rendering the "Data Source" part of the query */ export default function QueryDataSource(props: Props) { + const dispatch = useDispatch(); + const selectedQuery = useSelector(selection.selectors.getSelectedQuery); + const dataSources = useSelector(metadata.selectors.getDataSources); + const selectedDataSources = useSelector(selection.selectors.getSelectedDataSources); + + const addDataSourceOptions: ListItem[] = [ + ...dataSources.map((source) => ({ + displayValue: source.name, + value: source.name, + disabled: + selectedDataSources.length <= 1 && + selectedDataSources.some((selected) => source.name === selected.name), + data: source, + selected: selectedDataSources.some((selected) => source.name === selected.name), + })), + { + displayValue: "New Data Source...", + value: "New Data Source...", + selected: false, + }, + ]; + return (
TODO: To be implemented in another ticket
} - rows={props.dataSources.map((dataSource) => { - // TODO: This should change when we move towards - // having a blank data source only possible - // on an empty load - // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/105 - if (!dataSource) { - return { - id: AICS_FMS_DATA_SOURCE_NAME, - title: AICS_FMS_DATA_SOURCE_NAME, - }; - } - - return { - id: dataSource.name, - title: dataSource.name, - }; - })} + disabled={selectedDataSources[0]?.name === AICS_FMS_DATA_SOURCE_NAME} + onDelete={ + selectedDataSources.length > 1 + ? (dataSource) => + dispatch( + selection.actions.changeDataSources( + selectedDataSources.filter((s) => s.name !== dataSource) + ) + ) + : undefined + } + onRenderAddMenuList={ + selectedDataSources[0]?.name === AICS_FMS_DATA_SOURCE_NAME + ? undefined + : () => ( +
+ String(item.value)} + items={addDataSourceOptions} + onRenderCell={(item) => + item && ( + + item.data + ? dispatch( + selection.actions.changeDataSources([ + ...selectedDataSources, + item.data as Source, + ]) + ) + : dispatch( + interaction.actions.promptForDataSource( + { + query: selectedQuery, + } + ) + ) + } + onDeselect={() => + item.data && + selectedDataSources.length > 1 && + dispatch( + selection.actions.changeDataSources( + selectedDataSources.filter( + (source) => + source.name !== item.data?.name + ) + ) + ) + } + /> + ) + } + /> +
+ ) + } + rows={props.dataSources.map((dataSource) => ({ + id: dataSource.name, + title: dataSource.name, + }))} /> ); } diff --git a/packages/core/components/QueryPart/QueryFilter.tsx b/packages/core/components/QueryPart/QueryFilter.tsx index 8f2497154..02360cc3c 100644 --- a/packages/core/components/QueryPart/QueryFilter.tsx +++ b/packages/core/components/QueryPart/QueryFilter.tsx @@ -11,6 +11,7 @@ import { metadata, selection } from "../../state"; import Annotation from "../../entity/Annotation"; interface Props { + disabled?: boolean; filters: FileFilter[]; } @@ -26,6 +27,7 @@ export default function QueryFilter(props: Props) { return ( @@ -38,14 +40,11 @@ export default function QueryFilter(props: Props) { onRenderAddMenuList={() => ( ( )} - selections={annotations.filter((annotation) => - props.filters.some((f) => f.name === annotation.name) - )} + selections={props.filters.map((filter) => filter.name)} setSelections={() => dispatch(selection.actions.setFileFilters([]))} /> )} diff --git a/packages/core/components/QueryPart/QueryGroup.tsx b/packages/core/components/QueryPart/QueryGroup.tsx index ee393fdf1..a14b5039b 100644 --- a/packages/core/components/QueryPart/QueryGroup.tsx +++ b/packages/core/components/QueryPart/QueryGroup.tsx @@ -1,13 +1,13 @@ import * as React from "react"; -import { useDispatch, useSelector } from "react-redux"; +import { useDispatch } from "react-redux"; import QueryPart from "."; import AnnotationPicker from "../AnnotationPicker"; import Tutorial from "../../entity/Tutorial"; -import { metadata, selection } from "../../state"; -import Annotation from "../../entity/Annotation"; +import { selection } from "../../state"; interface Props { + disabled?: boolean; groups: string[]; } @@ -17,14 +17,6 @@ interface Props { export default function QueryGroup(props: Props) { const dispatch = useDispatch(); - const annotations = useSelector(metadata.selectors.getSortedAnnotations); - - const selectedAnnotations = props.groups - .map((annotationName) => - annotations.find((annotation) => annotation.name === annotationName) - ) - .filter((a) => !!a) as Annotation[]; - const onDelete = (annotationName: string) => { dispatch(selection.actions.removeFromAnnotationHierarchy(annotationName)); }; @@ -36,6 +28,7 @@ export default function QueryGroup(props: Props) { return ( { - dispatch( - selection.actions.setAnnotationHierarchy(annotations.map((a) => a.name)) - ); + dispatch(selection.actions.setAnnotationHierarchy(annotations)); }} /> )} - rows={selectedAnnotations.map((annotation) => ({ - id: annotation.name, - title: annotation.displayName, + // TODO: Should we care about display name?? seems time to make the name of + // annotations just the display name for top level annotations bro + rows={props.groups.map((annotation) => ({ + id: annotation, + title: annotation, }))} /> ); diff --git a/packages/core/components/QueryPart/QueryPart.module.css b/packages/core/components/QueryPart/QueryPart.module.css index e9cfd872c..869ae825c 100644 --- a/packages/core/components/QueryPart/QueryPart.module.css +++ b/packages/core/components/QueryPart/QueryPart.module.css @@ -35,3 +35,8 @@ margin: 0; padding-top: 8px } + +.disabled { + opacity: 0.5; + pointer-events: none; +} diff --git a/packages/core/components/QueryPart/QuerySort.tsx b/packages/core/components/QueryPart/QuerySort.tsx index 0001c6ca0..ede34fc23 100644 --- a/packages/core/components/QueryPart/QuerySort.tsx +++ b/packages/core/components/QueryPart/QuerySort.tsx @@ -1,13 +1,14 @@ import * as React from "react"; -import { useDispatch, useSelector } from "react-redux"; +import { useDispatch } from "react-redux"; import QueryPart from "."; import AnnotationPicker from "../AnnotationPicker"; -import { metadata, selection } from "../../state"; +import { selection } from "../../state"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; interface Props { + disabled?: boolean; sort?: FileSort; } @@ -17,11 +18,10 @@ interface Props { export default function QuerySort(props: Props) { const dispatch = useDispatch(); - const annotations = useSelector(metadata.selectors.getSortedAnnotations); - return ( dispatch(selection.actions.setSortColumn())} @@ -39,13 +39,11 @@ export default function QuerySort(props: Props) { annotation.name === props.sort?.annotationName - )} + selections={props.sort?.annotationName ? [props.sort.annotationName] : []} setSelections={(annotations) => { const newAnnotation = annotations.filter( - (annotation) => annotation.name !== props.sort?.annotationName - )?.[0].name; + (annotation) => annotation !== props.sort?.annotationName + )[0]; dispatch( selection.actions.setSortColumn( newAnnotation diff --git a/packages/core/components/QueryPart/index.tsx b/packages/core/components/QueryPart/index.tsx index bbf80bf29..51323d3bb 100644 --- a/packages/core/components/QueryPart/index.tsx +++ b/packages/core/components/QueryPart/index.tsx @@ -4,6 +4,7 @@ import { IRenderFunction, PrimaryButton, } from "@fluentui/react"; +import classNames from "classnames"; import * as React from "react"; import { DragDropContext, OnDragEndResponder } from "react-beautiful-dnd"; @@ -14,6 +15,7 @@ import styles from "./QueryPart.module.css"; interface Props { title: string; + disabled?: boolean; tutorialId?: string; addButtonIconName: string; rows: QueryPartRowItem[]; @@ -41,10 +43,11 @@ export default function QueryPart(props: Props) { }; return ( -
+
{ setIsExpanded(props.isSelected); }, [props.isSelected]); - const decodedURL = React.useMemo( - () => (props.isSelected ? currentQueryParts : props.query.parts), - [props.query.parts, currentQueryParts, props.isSelected] + const queryComponents = React.useMemo( + () => (props.isSelected ? currentQueryParts : props.query?.parts), + [props.query?.parts, currentQueryParts, props.isSelected] ); const onQueryUpdate = (updatedQuery: QueryType) => { @@ -76,12 +78,13 @@ export default function Query(props: QueryProps) {
{props.isSelected &&
}

- Data Source: {decodedURL.source?.name} + Data Source:{" "} + {queryComponents.sources.map((source) => source.name).join(", ")}

- {!!decodedURL.hierarchy.length && ( + {!!queryComponents.hierarchy.length && (

Groupings:{" "} - {decodedURL.hierarchy + {queryComponents.hierarchy .map( (a) => annotations.find((annotation) => annotation.name === a) @@ -90,18 +93,18 @@ export default function Query(props: QueryProps) { .join(", ")}

)} - {!!decodedURL.filters.length && ( + {!!queryComponents.filters.length && (

Filters:{" "} - {decodedURL.filters + {queryComponents.filters .map((filter) => `${filter.name}: ${filter.value}`) .join(", ")}

)} - {!!decodedURL.sortColumn && ( + {!!queryComponents.sortColumn && (

- Sort: {decodedURL.sortColumn.annotationName} ( - {decodedURL.sortColumn.order}) + Sort: {queryComponents.sortColumn.annotationName} ( + {queryComponents.sortColumn.order})

)}
@@ -134,10 +137,10 @@ export default function Query(props: QueryProps) { />
- - - - + + + +
1} diff --git a/packages/core/components/QuerySidebar/index.tsx b/packages/core/components/QuerySidebar/index.tsx index 95c745362..f5410ed3e 100644 --- a/packages/core/components/QuerySidebar/index.tsx +++ b/packages/core/components/QuerySidebar/index.tsx @@ -7,7 +7,6 @@ import Query from "./Query"; import { HELP_OPTIONS } from "./tutorials"; import { ModalType } from "../Modal"; import SvgIcon from "../SvgIcon"; -import FileExplorerURL, { DEFAULT_AICS_FMS_QUERY } from "../../entity/FileExplorerURL"; import Tutorial from "../../entity/Tutorial"; import { interaction, selection } from "../../state"; import { AICS_LOGO } from "../../icons"; @@ -23,48 +22,12 @@ interface QuerySidebarProps { */ export default function QuerySidebar(props: QuerySidebarProps) { const dispatch = useDispatch(); + const isOnWeb = useSelector(interaction.selectors.isOnWeb); const queries = useSelector(selection.selectors.getQueries); const selectedQuery = useSelector(selection.selectors.getSelectedQuery); - const isAicsEmployee = useSelector(interaction.selectors.isAicsEmployee); - const isOnWeb = useSelector(interaction.selectors.isOnWeb); const dataSources = useSelector(interaction.selectors.getAllDataSources); const currentGlobalURL = useSelector(selection.selectors.getEncodedFileExplorerUrl); - // Select query by default if none is selected - React.useEffect(() => { - if (!selectedQuery && queries.length) { - dispatch(selection.actions.changeQuery(queries[0])); - } - }, [selectedQuery, queries, dispatch]); - - // Determine a default query to render or prompt the user for a data source - // if no default is accessible - React.useEffect(() => { - if (!queries.length) { - if (!window.location.search) { - if (isAicsEmployee === true) { - // If the user is an AICS employee and there is no query in the URL, add a default query - dispatch( - selection.actions.addQuery({ - name: "New AICS Query", - parts: DEFAULT_AICS_FMS_QUERY, - }) - ); - } else if (isAicsEmployee === false) { - // If no query is selected and there is no query in the URL, prompt the user to select a data source - dispatch(interaction.actions.setVisibleModal(ModalType.DataSourcePrompt)); - } - } else if (isAicsEmployee === undefined) { - dispatch( - selection.actions.addQuery({ - name: "New Query", - parts: FileExplorerURL.decode(window.location.search), - }) - ); - } - } - }, [isAicsEmployee, queries, dispatch]); - React.useEffect(() => { if (selectedQuery) { const newurl = @@ -91,7 +54,7 @@ export default function QuerySidebar(props: QuerySidebarProps) { dispatch( selection.actions.addQuery({ name: `New ${source.name} query`, - parts: { source }, + parts: { sources: [source] }, }) ); }, @@ -102,7 +65,7 @@ export default function QuerySidebar(props: QuerySidebarProps) { text: "New Data Source...", iconProps: { iconName: "NewFolder" }, onClick: () => { - dispatch(interaction.actions.setVisibleModal(ModalType.DataSourcePrompt)); + dispatch(interaction.actions.setVisibleModal(ModalType.DataSource)); }, }, ], @@ -161,13 +124,23 @@ export default function QuerySidebar(props: QuerySidebarProps) { data-is-scrollable="true" data-is-focusable="true" > - {queries.map((query) => ( + {queries.length ? ( + queries.map((query) => ( + + )) + ) : ( - ))} + )}
({ + name: item.name, + value: item.type !== "space" ? 0 : null, + })); +} + +export async function renderZarrThumbnailURL(zarrUrl: string): Promise { + try { + const store = new FetchStore(zarrUrl); + const root = zarr.root(store); + const group = await zarr.open(root, { kind: "group" }); + + if ( + !group.attrs || + !Array.isArray(group.attrs.multiscales) || + group.attrs.multiscales.length === 0 + ) { + throw new Error("Invalid multiscales attribute structure"); + } + + const { multiscales } = group.attrs; + const datasets = multiscales[0].datasets; + const lowestResolutionDataset = datasets[datasets.length - 1]; + const lowestResolutionLocation = root.resolve(lowestResolutionDataset.path); + const lowestResolution = await zarr.open(lowestResolutionLocation, { kind: "array" }); + + // Determine Slice + const axes = transformAxes(multiscales[0].axes); + const zIndex = axes.findIndex((item) => item.name === "z"); + if (zIndex !== -1) { + const zSliceIndex = Math.ceil(lowestResolution.shape[zIndex] / 2); + axes[zIndex].value = zSliceIndex; + } + + const lowestResolutionView = await zarrGet( + lowestResolution, + axes.map((item) => item.value) + ); + const u16data = lowestResolutionView.data as Uint16Array; + + // Normalize Data + const min = Math.min(...u16data); + const max = Math.max(...u16data); + const normalizedData = new Uint8Array(u16data.length); + for (let i = 0; i < u16data.length; i++) { + normalizedData[i] = Math.round((255 * (u16data[i] - min)) / (max - min)); + } + + // Build Canvas + const width = lowestResolution.shape[axes.findIndex((item) => item.name === "x")]; + const height = lowestResolution.shape[axes.findIndex((item) => item.name === "y")]; + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + + if (!context) { + throw new Error("Failed to get canvas context"); + } + + // Draw data + const imageData = context.createImageData(width, height); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const idx = (y * width + x) * 4; + const value = normalizedData[y * width + x]; + imageData.data[idx] = value; // Red + imageData.data[idx + 1] = value; // Green + imageData.data[idx + 2] = value; // Blue + imageData.data[idx + 3] = 255; // Alpha + } + } + + context.putImageData(imageData, 0, 0); + + // Convert data to data URL + return canvas.toDataURL("image/png"); + } catch (error) { + console.error("Error reading Zarr image:", error); + throw error; + } +} diff --git a/packages/core/entity/FileDetail/index.ts b/packages/core/entity/FileDetail/index.ts index c950572ee..7484042ba 100644 --- a/packages/core/entity/FileDetail/index.ts +++ b/packages/core/entity/FileDetail/index.ts @@ -1,11 +1,11 @@ import { FmsFileAnnotation } from "../../services/FileService"; +import { renderZarrThumbnailURL } from "./RenderZarrThumbnailURL"; const RENDERABLE_IMAGE_FORMATS = [".jpg", ".jpeg", ".png", ".gif"]; // Should probably record this somewhere we can dynamically adjust to, or perhaps just in the file // document itself, alas for now this will do. const HARD_CODED_AICS_S3_BUCKET_PATH = "http://production.files.allencell.org.s3.amazonaws.com"; - const AICS_FMS_FILES_NGINX_SERVER = "http://aics.corp.alleninstitute.org/labkey/fmsfiles/image"; /** @@ -160,8 +160,7 @@ export default class FileDetail { return this.fileDetail.annotations.find((annotation) => annotation.name === annotationName); } - public getPathToThumbnail(): string | undefined { - // If no thumbnail present try to render the file itself as the thumbnail + public async getPathToThumbnail(): Promise { if (!this.thumbnail) { const isFileRenderableImage = RENDERABLE_IMAGE_FORMATS.some((format) => this.name.toLowerCase().endsWith(format) @@ -178,6 +177,17 @@ export default class FileDetail { if (this.thumbnail?.startsWith("/allen")) { return `${AICS_FMS_FILES_NGINX_SERVER}${this.thumbnail}`; } + + if (this.cloudPath.endsWith(".zarr")) { + try { + // const thumbnailURL = await renderZarrThumbnailURL(this.cloudPath); + const thumbnailURL = await renderZarrThumbnailURL(this.cloudPath); + return thumbnailURL; + } catch (error) { + console.error("Error generating Zarr thumbnail:", error); + throw new Error("Unable to generate Zarr thumbnail"); + } + } return this.thumbnail; } } diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index 679ed3016..f61e58d96 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -6,14 +6,14 @@ import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; export interface Source { name: string; - type: "csv" | "json" | "parquet"; + type?: "csv" | "json" | "parquet"; uri?: string | File; } // Components of the application state this captures export interface FileExplorerURLComponents { hierarchy: string[]; - source?: Source; + sources: Source[]; filters: FileFilter[]; openFolders: FileFolder[]; sortColumn?: FileSort; @@ -23,6 +23,7 @@ export const EMPTY_QUERY_COMPONENTS: FileExplorerURLComponents = { hierarchy: [], filters: [], openFolders: [], + sources: [], }; const BEGINNING_OF_TODAY = new Date(); @@ -44,6 +45,7 @@ export const PAST_YEAR_FILTER = new FileFilter( export const DEFAULT_AICS_FMS_QUERY: FileExplorerURLComponents = { hierarchy: [], openFolders: [], + sources: [{ name: AICS_FMS_DATA_SOURCE_NAME }], filters: [PAST_YEAR_FILTER], sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), }; @@ -72,12 +74,18 @@ export default class FileExplorerURL { urlComponents.openFolders?.map((folder) => { params.append("openFolder", JSON.stringify(folder.fileFolder)); }); + urlComponents.sources?.map((source) => { + params.append( + "source", + JSON.stringify({ + ...source, + uri: source.uri instanceof String ? source.uri : undefined, + }) + ); + }); if (urlComponents.sortColumn) { params.append("sort", JSON.stringify(urlComponents.sortColumn.toJSON())); } - if (urlComponents.source) { - params.append("source", JSON.stringify(urlComponents.source)); - } return params.toString(); } @@ -91,7 +99,7 @@ export default class FileExplorerURL { const unparsedOpenFolders = params.getAll("openFolder"); const unparsedFilters = params.getAll("filter"); - const unparsedSource = params.get("source"); + const unparsedSources = params.getAll("source"); const hierarchy = params.getAll("group"); const unparsedSort = params.get("sort"); const hierarchyDepth = hierarchy.length; @@ -112,7 +120,7 @@ export default class FileExplorerURL { filters: unparsedFilters .map((unparsedFilter) => JSON.parse(unparsedFilter)) .map((parsedFilter) => new FileFilter(parsedFilter.name, parsedFilter.value)), - source: unparsedSource ? JSON.parse(unparsedSource) : undefined, + sources: unparsedSources.map((unparsedSource) => JSON.parse(unparsedSource)), openFolders: unparsedOpenFolders .map((unparsedFolder) => JSON.parse(unparsedFolder)) .filter((parsedFolder) => parsedFolder.length <= hierarchyDepth) @@ -125,12 +133,13 @@ export default class FileExplorerURL { userOS: string ) { if ( - urlComponents?.source?.name === AICS_FMS_DATA_SOURCE_NAME || - !urlComponents?.source?.uri + (urlComponents?.sources?.length && urlComponents.sources.length > 1) || + urlComponents?.sources?.[0]?.name === AICS_FMS_DATA_SOURCE_NAME || + !urlComponents?.sources?.[0]?.uri ) { return "# Coming soon"; } - const sourceString = this.convertDataSourceToPython(urlComponents?.source, userOS); + const sourceString = this.convertDataSourceToPython(urlComponents?.sources?.[0], userOS); const groupByQueryString = urlComponents.hierarchy ?.map((annotation) => this.convertGroupByToPython(annotation)) diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index 86b2269e2..725927b51 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -38,7 +38,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.FILE_SIZE, SortOrder.DESC), - source: mockSource, + sources: [mockSource], }; // Act @@ -46,7 +46,7 @@ describe("FileExplorerURL", () => { // Assert expect(result).to.be.equal( - "group=Cell+Line&group=Donor+Plasmid&group=Lifting%3F&filter=%7B%22name%22%3A%22Cas9%22%2C%22value%22%3A%22spCas9%22%7D&filter=%7B%22name%22%3A%22Donor+Plasmid%22%2C%22value%22%3A%22ACTB-mEGFP%22%7D&openFolder=%5B%22AICS-0%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Cfalse%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Ctrue%5D&sort=%7B%22annotationName%22%3A%22file_size%22%2C%22order%22%3A%22DESC%22%7D&source=%7B%22name%22%3A%22Fake+Collection%22%2C%22type%22%3A%22csv%22%7D" + "group=Cell+Line&group=Donor+Plasmid&group=Lifting%3F&filter=%7B%22name%22%3A%22Cas9%22%2C%22value%22%3A%22spCas9%22%7D&filter=%7B%22name%22%3A%22Donor+Plasmid%22%2C%22value%22%3A%22ACTB-mEGFP%22%7D&openFolder=%5B%22AICS-0%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Cfalse%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Ctrue%5D&source=%7B%22name%22%3A%22Fake+Collection%22%2C%22type%22%3A%22csv%22%7D&sort=%7B%22annotationName%22%3A%22file_size%22%2C%22order%22%3A%22DESC%22%7D" ); }); @@ -56,6 +56,7 @@ describe("FileExplorerURL", () => { hierarchy: [], filters: [], openFolders: [], + sources: [], }; // Act @@ -85,7 +86,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - source: mockSource, + sources: [mockSource], }; const encodedUrl = FileExplorerURL.encode(components); const encodedUrlWithWhitespace = " " + encodedUrl + " "; @@ -104,7 +105,7 @@ describe("FileExplorerURL", () => { filters: [], openFolders: [], sortColumn: undefined, - source: undefined, + sources: [], }; const encodedUrl = FileExplorerURL.encode(components); @@ -121,6 +122,7 @@ describe("FileExplorerURL", () => { hierarchy: ["Cell Line"], filters: [], openFolders: [new FileFolder(["AICS-0"]), new FileFolder(["AICS-0", false])], + sources: [], }; const encodedUrl = FileExplorerURL.encode(components); @@ -147,6 +149,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.FILE_PATH, "Garbage" as any), + sources: [], }; const encodedUrl = FileExplorerURL.encode(components); @@ -161,7 +164,7 @@ describe("FileExplorerURL", () => { const expectedAnnotationNames = ["Cell Line", "Donor Plasmid", "Lifting?"]; const components: Partial = { hierarchy: expectedAnnotationNames, - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedPandasGroups = expectedAnnotationNames.map( (annotation) => `.groupby('${annotation}', group_keys=True).apply(lambda x: x)` @@ -183,7 +186,7 @@ describe("FileExplorerURL", () => { ]; const components: Partial = { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedPandasQueries = expectedFilters.map( (filter) => `\`${filter.name}\`=="${filter.value}"` @@ -205,7 +208,7 @@ describe("FileExplorerURL", () => { ]; const components: Partial = { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedPandasQueries = expectedFilters.map( (filter) => `\`${filter.name}\`=="${filter.value}"` @@ -223,7 +226,7 @@ describe("FileExplorerURL", () => { // Arrange const components: Partial = { sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedPandasSort = `.sort_values(by='${AnnotationName.UPLOADED}', ascending=False`; const expectedResult = `df${expectedPandasSort}`; @@ -238,7 +241,7 @@ describe("FileExplorerURL", () => { it("provides info on converting external data source to pandas dataframe", () => { // Arrange const components: Partial = { - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedResult = `df = pd.read_csv('${mockSourceWithUri.uri}').astype('str')`; @@ -252,7 +255,7 @@ describe("FileExplorerURL", () => { it("adds raw flag in pandas conversion code for Windows OS", () => { // Arrange const components: Partial = { - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedResult = `df = pd.read_csv(r'${mockSourceWithUri.uri}').astype('str')`; @@ -274,7 +277,7 @@ describe("FileExplorerURL", () => { hierarchy: expectedAnnotationNames, filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - source: mockSourceWithUri, + sources: [mockSourceWithUri], }; const expectedResult = /df\.groupby\(.*\)\.query\(.*\)\.query\(.*\)\.sort_values\(.*\)/i; diff --git a/packages/core/entity/SQLBuilder/index.ts b/packages/core/entity/SQLBuilder/index.ts index 5caaa0324..3a6bb13af 100644 --- a/packages/core/entity/SQLBuilder/index.ts +++ b/packages/core/entity/SQLBuilder/index.ts @@ -1,3 +1,5 @@ +import { castArray } from "lodash"; + /** * A simple SQL query builder. */ @@ -20,8 +22,12 @@ export default class SQLBuilder { return this; } - public from(statement: string): SQLBuilder { - this.fromStatement = statement; + public from(statement: string | string[]): SQLBuilder { + const statementAsArray = castArray(statement); + if (!statementAsArray.length) { + throw new Error('"FROM" statement requires at least one argument'); + } + this.fromStatement = statementAsArray.sort().join(", "); return this; } diff --git a/packages/core/errors/DataSourcePreparationError.ts b/packages/core/errors/DataSourcePreparationError.ts new file mode 100644 index 000000000..86dced748 --- /dev/null +++ b/packages/core/errors/DataSourcePreparationError.ts @@ -0,0 +1,14 @@ +export default class DataSourcePreparationError extends Error { + public sourceName: string; + + constructor(message: string, sourceName: string) { + super(message); + + if (Error.captureStackTrace) { + Error.captureStackTrace(this, DataSourcePreparationError); + } + + this.name = "DataSourcePreparationError"; + this.sourceName = sourceName; + } +} diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index e5443eb50..39fa0a5a1 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -3,18 +3,11 @@ import DatabaseService from "../../DatabaseService"; import DatabaseServiceNoop from "../../DatabaseService/DatabaseServiceNoop"; import Annotation from "../../../entity/Annotation"; import FileFilter from "../../../entity/FileFilter"; -import { AnnotationType } from "../../../entity/AnnotationFormatter"; import SQLBuilder from "../../../entity/SQLBuilder"; interface Config { databaseService: DatabaseService; - dataSourceName: string; -} - -interface DescribeQueryResult { - [key: string]: string; - column_name: string; - column_type: string; + dataSourceNames: string[]; } interface SummarizeQueryResult { @@ -28,44 +21,20 @@ interface SummarizeQueryResult { */ export default class DatabaseAnnotationService implements AnnotationService { private readonly databaseService: DatabaseService; - private readonly dataSourceName: string; + private readonly dataSourceNames: string[]; constructor( - config: Config = { dataSourceName: "Unknown", databaseService: new DatabaseServiceNoop() } + config: Config = { dataSourceNames: [], databaseService: new DatabaseServiceNoop() } ) { - this.dataSourceName = config.dataSourceName; + this.dataSourceNames = config.dataSourceNames; this.databaseService = config.databaseService; } - private static columnTypeToAnnotationType(columnType: string): string { - switch (columnType) { - case "INTEGER": - case "BIGINT": - // TODO: Add support for column types - // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 - // return AnnotationType.NUMBER; - case "VARCHAR": - case "TEXT": - default: - return AnnotationType.STRING; - } - } - /** * Fetch all annotations. */ - public async fetchAnnotations(): Promise { - const sql = `DESCRIBE "${this.dataSourceName}"`; - const rows = (await this.databaseService.query(sql)) as DescribeQueryResult[]; - return rows.map( - (row) => - new Annotation({ - annotationDisplayName: row["column_name"], - annotationName: row["column_name"], - description: "", - type: DatabaseAnnotationService.columnTypeToAnnotationType(row["column_type"]), - }) - ); + public fetchAnnotations(): Promise { + return this.databaseService.fetchAnnotations(this.dataSourceNames); } /** @@ -75,7 +44,7 @@ export default class DatabaseAnnotationService implements AnnotationService { const select_key = "select_key"; const sql = new SQLBuilder() .select(`DISTINCT "${annotation}" AS ${select_key}`) - .from(this.dataSourceName) + .from(this.dataSourceNames) .toSQL(); const rows = await this.databaseService.query(sql); return [ @@ -114,7 +83,8 @@ export default class DatabaseAnnotationService implements AnnotationService { const sqlBuilder = new SQLBuilder() .select(`DISTINCT "${hierarchy[path.length]}"`) - .from(this.dataSourceName); + .from(this.dataSourceNames); + Object.keys(filtersByAnnotation).forEach((annotation) => { const annotationValues = filtersByAnnotation[annotation]; if (annotationValues[0] === null) { @@ -125,6 +95,7 @@ export default class DatabaseAnnotationService implements AnnotationService { ); } }); + const rows = await this.databaseService.query(sqlBuilder.toSQL()); return rows.map((row) => row[hierarchy[path.length]]); } @@ -136,7 +107,7 @@ export default class DatabaseAnnotationService implements AnnotationService { public async fetchAvailableAnnotationsForHierarchy(annotations: string[]): Promise { const sql = new SQLBuilder() .summarize() - .from(this.dataSourceName) + .from(this.dataSourceNames) .where(annotations.map((annotation) => `"${annotation}" IS NOT NULL`)) .toSQL(); const rows = (await this.databaseService.query(sql)) as SummarizeQueryResult[]; diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts index be9b5b2ea..2a2c2e0e4 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts @@ -1,34 +1,11 @@ import { expect } from "chai"; -import Annotation from "../../../../entity/Annotation"; import FileFilter from "../../../../entity/FileFilter"; import DatabaseServiceNoop from "../../../DatabaseService/DatabaseServiceNoop"; import DatabaseAnnotationService from ".."; describe("DatabaseAnnotationService", () => { - describe("fetchAnnotations", () => { - const annotations = ["A", "B", "Cc", "dD"].map((name) => ({ - name, - })); - class MockDatabaseService extends DatabaseServiceNoop { - public query(): Promise<{ [key: string]: string }[]> { - return Promise.resolve(annotations); - } - } - const databaseService = new MockDatabaseService(); - - it("issues request for all available Annotations", async () => { - const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", - databaseService, - }); - const actualAnnotations = await annotationService.fetchAnnotations(); - expect(actualAnnotations.length).to.equal(annotations.length); - expect(actualAnnotations[0]).to.be.instanceOf(Annotation); - }); - }); - describe("fetchAnnotationValues", () => { const annotations = ["A", "B", "Cc", "dD"].map((name, index) => ({ select_key: name.toLowerCase() + index, @@ -44,7 +21,7 @@ describe("DatabaseAnnotationService", () => { const annotation = "foo"; const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", + dataSourceNames: ["a", "b or c"], databaseService, }); const actualValues = await annotationService.fetchValues(annotation); @@ -68,7 +45,7 @@ describe("DatabaseAnnotationService", () => { it("issues a request for annotation values for the first level of the annotation hierarchy", async () => { const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", + dataSourceNames: ["d"], databaseService, }); const values = await annotationService.fetchRootHierarchyValues(["foo"], []); @@ -77,7 +54,7 @@ describe("DatabaseAnnotationService", () => { it("issues a request for annotation values for the first level of the annotation hierarchy with filters", async () => { const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", + dataSourceNames: ["e"], databaseService, }); const filter = new FileFilter("bar", "barValue"); @@ -102,7 +79,7 @@ describe("DatabaseAnnotationService", () => { const expectedValues = ["A0", "B1", "Cc2", "dD3"]; const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", + dataSourceNames: ["ghjiasd", "second source"], databaseService, }); const values = await annotationService.fetchHierarchyValuesUnderPath( @@ -117,7 +94,7 @@ describe("DatabaseAnnotationService", () => { const expectedValues = ["A0", "B1", "Cc2", "dD3"]; const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", + dataSourceNames: ["mock1"], databaseService, }); const filter = new FileFilter("bar", "barValue"); @@ -145,7 +122,7 @@ describe("DatabaseAnnotationService", () => { it("issues request for annotations that can be combined with current hierarchy", async () => { const annotationService = new DatabaseAnnotationService({ - dataSourceName: "Unknown", + dataSourceNames: ["mock1"], databaseService, }); const values = await annotationService.fetchAvailableAnnotationsForHierarchy([ diff --git a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts index 84243cf85..de48feaaa 100644 --- a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts +++ b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts @@ -1,8 +1,12 @@ import DatabaseService from "."; -export default class DatabaseServiceNoop implements DatabaseService { - public addDataSource() { - return Promise.reject("DatabaseServiceNoop:addDataSource"); +export default class DatabaseServiceNoop extends DatabaseService { + public execute(): Promise { + return Promise.reject("DatabaseServiceNoop:execute"); + } + + public prepareDataSources() { + return Promise.reject("DatabaseServiceNoop::prepareDataSources"); } public saveQuery(): Promise { @@ -12,4 +16,8 @@ export default class DatabaseServiceNoop implements DatabaseService { public query(): Promise<{ [key: string]: string }[]> { return Promise.reject("DatabaseServiceNoop:query"); } + + protected addDataSource(): Promise { + return Promise.reject("DatabaseServiceNoop:addDataSource"); + } } diff --git a/packages/core/services/DatabaseService/index.ts b/packages/core/services/DatabaseService/index.ts index 98bec99a5..bb1a9fce9 100644 --- a/packages/core/services/DatabaseService/index.ts +++ b/packages/core/services/DatabaseService/index.ts @@ -1,18 +1,159 @@ +import { isEmpty } from "lodash"; + +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; +import Annotation from "../../entity/Annotation"; +import { AnnotationType } from "../../entity/AnnotationFormatter"; +import { Source } from "../../entity/FileExplorerURL"; +import SQLBuilder from "../../entity/SQLBuilder"; + /** * Service reponsible for querying against a database */ -export default interface DatabaseService { - addDataSource( - name: string, - type: "csv" | "json" | "parquet", - uri: File | string - ): Promise; - - saveQuery( - destination: string, - sql: string, - format: "csv" | "parquet" | "json" +export default abstract class DatabaseService { + private currentAggregateSource?: string; + // Initialize with AICS FMS data source name to pretend it always exists + protected readonly existingDataSources = new Set([AICS_FMS_DATA_SOURCE_NAME]); + private readonly dataSourceToAnnotationsMap: Map = new Map(); + + public abstract saveQuery( + _destination: string, + _sql: string, + _format: "csv" | "parquet" | "json" ): Promise; - query(sql: string): Promise<{ [key: string]: string }[]>; + public abstract query(_sql: string): Promise<{ [key: string]: string }[]>; + + protected abstract addDataSource(_dataSource: Source): Promise; + + protected abstract execute(_sql: string): Promise; + + private static columnTypeToAnnotationType(columnType: string): string { + switch (columnType) { + case "INTEGER": + case "BIGINT": + // TODO: Add support for column types + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 + // return AnnotationType.NUMBER; + case "VARCHAR": + case "TEXT": + default: + return AnnotationType.STRING; + } + } + + constructor() { + // 'this' scope gets lost when a higher order class (ex. DatabaseService) + // calls a lower level class (ex. DatabaseServiceWeb) + this.addDataSource = this.addDataSource.bind(this); + this.execute = this.execute.bind(this); + this.query = this.query.bind(this); + } + + public async prepareDataSources(dataSources: Source[]): Promise { + await Promise.all(dataSources.map(this.addDataSource)); + + // Because when querying multiple data sources column differences can complicate the queries + // preparing a table ahead of time that is the aggregate of the data sources is most optimal + // should look toward some way of reducing the memory footprint if that becomes an issue + if (dataSources.length > 1) { + await this.aggregateDataSources(dataSources); + } + } + + protected async deleteDataSource(dataSource: string): Promise { + this.existingDataSources.delete(dataSource); + this.dataSourceToAnnotationsMap.delete(dataSource); + await this.execute(`DROP TABLE IF EXISTS "${dataSource}"`); + } + + private async aggregateDataSources(dataSources: Source[]): Promise { + const viewName = dataSources + .map((source) => source.name) + .sort() + .join(", "); + + if (this.currentAggregateSource) { + // Prevent adding the same data source multiple times by shortcutting out here + if (this.currentAggregateSource === viewName) { + return; + } + + // Otherwise, if an old aggregate exists, delete it + await this.deleteDataSource(this.currentAggregateSource); + } + + const columnsSoFar = new Set(); + for (const dataSource of dataSources) { + // Fetch information about this data source + const annotationsInDataSource = await this.fetchAnnotations([dataSource.name]); + const columnsInDataSource = annotationsInDataSource.map( + (annotation) => annotation.name + ); + const newColumns = columnsInDataSource.filter((column) => !columnsSoFar.has(column)); + + // If there are no columns / data added yet we need to create the table from + // scratch so we can provide an easy shortcut around the default way of adding + // data to a table + if (columnsSoFar.size === 0) { + await this.execute( + `CREATE TABLE "${viewName}" AS SELECT *, '${dataSource.name}' AS "Data source" FROM "${dataSource.name}"` + ); + this.currentAggregateSource = viewName; + } else { + // If adding data to an existing table we will need to add any new columns + // unsure why but seemingly unable to add multiple columns in one alter table + // statement so we will need to loop through and add them one by one + if (newColumns.length) { + const alterTableSQL = newColumns + .map((column) => `ALTER TABLE "${viewName}" ADD COLUMN "${column}" VARCHAR`) + .join("; "); + await this.execute(alterTableSQL); + } + + // After we have added any new columns to the table schema we just need + // to insert the data from the new table to this table replacing any non-existent + // columns with an empty value (null) + const columnsSoFarArr = [...columnsSoFar, ...newColumns]; + await this.execute(` + INSERT INTO "${viewName}" ("${columnsSoFarArr.join('", "')}", "Data source") + SELECT ${columnsSoFarArr + .map((column) => + columnsInDataSource.includes(column) ? `"${column}"` : "NULL" + ) + .join(", ")}, '${dataSource.name}' AS "Data source" + FROM "${dataSource.name}" + `); + } + + // Add the new columns from this data source to the existing columns + // to avoid adding duplicate columns + newColumns.forEach((column) => columnsSoFar.add(column)); + } + } + + public async fetchAnnotations(dataSourceNames: string[]): Promise { + const aggregateDataSourceName = dataSourceNames.sort().join(", "); + if (!this.dataSourceToAnnotationsMap.has(aggregateDataSourceName)) { + const sql = new SQLBuilder() + .from('information_schema"."columns') + .where(`table_name = '${aggregateDataSourceName}'`) + .toSQL(); + const rows = await this.query(sql); + if (isEmpty(rows)) { + throw new Error(`Unable to fetch annotations for ${aggregateDataSourceName}`); + } + const annotations = rows.map( + (row) => + new Annotation({ + annotationDisplayName: row["column_name"], + annotationName: row["column_name"], + description: "", + type: DatabaseService.columnTypeToAnnotationType(row["data_type"]), + }) + ); + this.dataSourceToAnnotationsMap.set(aggregateDataSourceName, annotations); + } + + return this.dataSourceToAnnotationsMap.get(aggregateDataSourceName) || []; + } } diff --git a/packages/core/services/DatabaseService/test/DatabaseService.test.ts b/packages/core/services/DatabaseService/test/DatabaseService.test.ts new file mode 100644 index 000000000..145bda4ba --- /dev/null +++ b/packages/core/services/DatabaseService/test/DatabaseService.test.ts @@ -0,0 +1,34 @@ +import { expect } from "chai"; + +import DatabaseService from ".."; + +describe("DatabaseService", () => { + describe("fetchAnnotations", () => { + const annotations = ["A", "B", "Cc", "dD"].map((name) => ({ + name, + })); + + // DatabaseService is abstract so we need a dummy impl to test + // implemented methods + class DatabaseServiceDummyImpl extends DatabaseService { + saveQuery(): Promise { + throw new Error("Not implemented in dummy impl"); + } + execute(): Promise { + throw new Error("Not implemented in dummy impl"); + } + addDataSource(): Promise { + throw new Error("Not implemented in dummy impl"); + } + query(): Promise { + return Promise.resolve(annotations); + } + } + + it("issues request for all available Annotations", async () => { + const service = new DatabaseServiceDummyImpl(); + const actualAnnotations = await service.fetchAnnotations(["foo"]); + expect(actualAnnotations.length).to.equal(4); + }); + }); +}); diff --git a/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts b/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts index bfcbf9b92..1b721b8f1 100644 --- a/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts +++ b/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts @@ -16,9 +16,9 @@ export default class FileDownloadServiceNoop implements FileDownloadService { }); } - prepareHttpResourceForDownload(): Promise { - return Promise.resolve( - "Triggered prepareHttpResourceForDownload on FileDownloadServiceNoop; returning without triggering a download." + prepareHttpResourceForDownload(): Promise { + return Promise.reject( + "Triggered prepareHttpResourceForDownload on FileDownloadServiceNoop" ); } diff --git a/packages/core/services/FileDownloadService/index.ts b/packages/core/services/FileDownloadService/index.ts index 0f1256943..66f468702 100644 --- a/packages/core/services/FileDownloadService/index.ts +++ b/packages/core/services/FileDownloadService/index.ts @@ -41,7 +41,7 @@ export default interface FileDownloadService { /** * Retrieve a Blob from a server over HTTP. */ - prepareHttpResourceForDownload(url: string, postBody: string): Promise; + prepareHttpResourceForDownload(url: string, postBody: string): Promise; /** * Attempt to cancel an active download request, deleting the downloaded artifact if present. diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index 598f419c0..07abde0d4 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -12,7 +12,7 @@ import SQLBuilder from "../../../entity/SQLBuilder"; interface Config { databaseService: DatabaseService; - dataSourceName: string; + dataSourceNames: string[]; downloadService: FileDownloadService; } @@ -22,7 +22,7 @@ interface Config { export default class DatabaseFileService implements FileService { private readonly databaseService: DatabaseService; private readonly downloadService: FileDownloadService; - private readonly dataSourceName: string; + private readonly dataSourceNames: string[]; private static convertDatabaseRowToFileDetail( row: { [key: string]: string }, @@ -51,24 +51,32 @@ export default class DatabaseFileService implements FileService { return new FileDetail({ annotations: [ ...annotations, - ...Object.entries(omit(row, ...annotations.keys())).map(([name, values]: any) => ({ - name, - values: `${values}`.split(",").map((value: string) => value.trim()), - })), + ...Object.entries(omit(row, ...annotations.keys())).flatMap(([name, values]: any) => + values !== null + ? [ + { + name, + values: `${values}` + .split(",") + .map((value: string) => value.trim()), + }, + ] + : [] + ), ], }); } constructor( config: Config = { - dataSourceName: "Unknown", + dataSourceNames: [], databaseService: new DatabaseServiceNoop(), downloadService: new FileDownloadServiceNoop(), } ) { this.databaseService = config.databaseService; this.downloadService = config.downloadService; - this.dataSourceName = config.dataSourceName; + this.dataSourceNames = config.dataSourceNames; } public async getCountOfMatchingFiles(fileSet: FileSet): Promise { @@ -76,10 +84,11 @@ export default class DatabaseFileService implements FileService { const sql = fileSet .toQuerySQLBuilder() .select(`COUNT(*) AS ${select_key}`) - .from(this.dataSourceName) + .from(this.dataSourceNames) // Remove sort if present .orderBy() .toSQL(); + const rows = await this.databaseService.query(sql); return parseInt(rows[0][select_key], 10); } @@ -103,10 +112,11 @@ export default class DatabaseFileService implements FileService { public async getFiles(request: GetFilesRequest): Promise { const sql = request.fileSet .toQuerySQLBuilder() - .from(this.dataSourceName) + .from(this.dataSourceNames) .offset(request.from * request.limit) .limit(request.limit) .toSQL(); + const rows = await this.databaseService.query(sql); return rows.map((row, index) => DatabaseFileService.convertDatabaseRowToFileDetail( @@ -126,17 +136,17 @@ export default class DatabaseFileService implements FileService { ): Promise { const sqlBuilder = new SQLBuilder() .select(annotations.map((annotation) => `"${annotation}"`).join(", ")) - .from(this.dataSourceName); + .from(this.dataSourceNames); selections.forEach((selection) => { selection.indexRanges.forEach((indexRange) => { const subQuery = new SQLBuilder() .select('"File Path"') - .from(this.dataSourceName as string) + .from(this.dataSourceNames) .whereOr( Object.entries(selection.filters).map(([column, values]) => { const commaSeperatedValues = values.map((v) => `'${v}'`).join(", "); - return `"${column}" IN (${commaSeperatedValues}}`; + return `"${column}" IN (${commaSeperatedValues})`; }) ) .offset(indexRange.start) @@ -150,7 +160,7 @@ export default class DatabaseFileService implements FileService { ); } - sqlBuilder.whereOr(`"File Path" IN (${subQuery})`); + sqlBuilder.whereOr(`"File Path" IN (${subQuery.toSQL()})`); }); }); diff --git a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts index 2fdb2cb7d..ddce55286 100644 --- a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts +++ b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts @@ -28,7 +28,7 @@ describe("DatabaseFileService", () => { describe("getFiles", () => { it("issues request for files that match given parameters", async () => { const databaseFileService = new DatabaseFileService({ - dataSourceName: "Unknown", + dataSourceNames: ["whatever", "and another"], databaseService, downloadService: new FileDownloadServiceNoop(), }); @@ -83,7 +83,7 @@ describe("DatabaseFileService", () => { it("issues request for aggregated information about given files", async () => { // Arrange const fileService = new DatabaseFileService({ - dataSourceName: "Unknown", + dataSourceNames: ["whatever"], databaseService, downloadService: new FileDownloadServiceNoop(), }); @@ -105,7 +105,7 @@ describe("DatabaseFileService", () => { describe("getCountOfMatchingFiles", () => { it("issues request for count of files matching given parameters", async () => { const fileService = new DatabaseFileService({ - dataSourceName: "Unknown", + dataSourceNames: ["whatever"], databaseService, downloadService: new FileDownloadServiceNoop(), }); diff --git a/packages/core/services/FileService/HttpFileService/index.ts b/packages/core/services/FileService/HttpFileService/index.ts index b8c79761b..b46308dfc 100644 --- a/packages/core/services/FileService/HttpFileService/index.ts +++ b/packages/core/services/FileService/HttpFileService/index.ts @@ -115,7 +115,7 @@ export default class HttpFileService extends HttpServiceBase implements FileServ const postData = JSON.stringify({ annotations, selections }); const url = `${this.baseUrl}/${HttpFileService.BASE_CSV_DOWNLOAD_URL}${this.pathSuffix}`; - const manifestAsString = await this.downloadService.prepareHttpResourceForDownload( + const manifestAsJSON = await this.downloadService.prepareHttpResourceForDownload( url, postData ); @@ -125,7 +125,7 @@ export default class HttpFileService extends HttpServiceBase implements FileServ name, id: name, path: url, - data: manifestAsString, + data: new Blob([manifestAsJSON as BlobPart], { type: "application/json" }), }, uniqueId() ); diff --git a/packages/core/state/interaction/actions.ts b/packages/core/state/interaction/actions.ts index f568de9d8..57d9f7ca8 100644 --- a/packages/core/state/interaction/actions.ts +++ b/packages/core/state/interaction/actions.ts @@ -21,15 +21,20 @@ export const PROMPT_FOR_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "prompt-fo type PartialSource = Omit; +export interface DataSourcePromptInfo { + source?: PartialSource; + query?: string; +} + export interface PromptForDataSource { type: string; - payload: PartialSource; + payload: DataSourcePromptInfo; } -export function promptForDataSource(dataSource: PartialSource): PromptForDataSource { +export function promptForDataSource(info: DataSourcePromptInfo): PromptForDataSource { return { type: PROMPT_FOR_DATA_SOURCE, - payload: dataSource, + payload: info, }; } @@ -205,21 +210,18 @@ export function setIsAicsEmployee(isAicsEmployee: boolean): SetIsAicsEmployee { } /** - * SET CONNECTION CONFIGURATION FOR THE FILE EXPLORER SERVICE + * Set connection configuration and kick off any tasks to initialize the app */ -export const SET_FILE_EXPLORER_SERVICE_BASE_URL = makeConstant( - STATE_BRANCH_NAME, - "set-file-explorer-service-connection-config" -); +export const INITIALIZE_APP = makeConstant(STATE_BRANCH_NAME, "initialize-app"); -export interface SetFileExplorerServiceBaseUrl { +export interface InitializeApp { type: string; payload: string; } -export function setFileExplorerServiceBaseUrl(baseUrl: string): SetFileExplorerServiceBaseUrl { +export function initializeApp(baseUrl: string): InitializeApp { return { - type: SET_FILE_EXPLORER_SERVICE_BASE_URL, + type: INITIALIZE_APP, payload: baseUrl, }; } diff --git a/packages/core/state/interaction/logics.ts b/packages/core/state/interaction/logics.ts index 74d86c3f8..869472f10 100644 --- a/packages/core/state/interaction/logics.ts +++ b/packages/core/state/interaction/logics.ts @@ -25,7 +25,7 @@ import { OpenWithDefaultAction, PROMPT_FOR_NEW_EXECUTABLE, setUserSelectedApplication, - SET_FILE_EXPLORER_SERVICE_BASE_URL, + INITIALIZE_APP, setIsAicsEmployee, } from "./actions"; import * as interactionSelectors from "./selectors"; @@ -41,15 +41,45 @@ import FileDetail from "../../entity/FileDetail"; import { AnnotationName } from "../../entity/Annotation"; import FileSelection from "../../entity/FileSelection"; import NumericRange from "../../entity/NumericRange"; +import FileExplorerURL, { DEFAULT_AICS_FMS_QUERY } from "../../entity/FileExplorerURL"; /** * Interceptor responsible for checking if the user is able to access the AICS network */ const checkAicsEmployee = createLogic({ - type: SET_FILE_EXPLORER_SERVICE_BASE_URL, + type: INITIALIZE_APP, async process(deps: ReduxLogicDeps, dispatch, done) { + const queries = selection.selectors.getQueries(deps.getState()); + const selectedQuery = selection.selectors.getSelectedQuery(deps.getState()); const fileService = interactionSelectors.getHttpFileService(deps.getState()); + + // Redimentary check to see if the user is an AICS Employee by + // checking if the AICS network is accessible const isAicsEmployee = await fileService.isNetworkAccessible(); + + // If no query is currently selected attempt to choose one for the user + if (!selectedQuery) { + // If there are query args representing a query we can extract that + // into the query to render (ex. when refreshing a page) + if (window.location.search) { + dispatch( + selection.actions.addQuery({ + name: "New Query", + parts: FileExplorerURL.decode(window.location.search), + }) + ); + } else if (queries.length) { + dispatch(selection.actions.changeQuery(queries[0])); + } else if (isAicsEmployee) { + dispatch( + selection.actions.addQuery({ + name: "New AICS FMS Query", + parts: DEFAULT_AICS_FMS_QUERY, + }) + ); + } + } + dispatch(setIsAicsEmployee(isAicsEmployee) as AnyAction); done(); }, @@ -448,18 +478,18 @@ const refresh = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { try { const { getState } = deps; + const hierarchy = selection.selectors.getAnnotationHierarchy(getState()); const annotationService = interactionSelectors.getAnnotationService(getState()); // Refresh list of annotations & which annotations are available - const hierarchy = selection.selectors.getAnnotationHierarchy(getState()); const [annotations, availableAnnotations] = await Promise.all([ annotationService.fetchAnnotations(), annotationService.fetchAvailableAnnotationsForHierarchy(hierarchy), ]); dispatch(metadata.actions.receiveAnnotations(annotations)); dispatch(selection.actions.setAvailableAnnotations(availableAnnotations)); - } catch (e) { - console.error("Error encountered while refreshing"); + } catch (err) { + console.error(`Error encountered while refreshing: ${err}`); const annotations = metadata.selectors.getAnnotations(deps.getState()); dispatch(selection.actions.setAvailableAnnotations(annotations.map((a) => a.name))); } finally { diff --git a/packages/core/state/interaction/reducer.ts b/packages/core/state/interaction/reducer.ts index c126ff36e..5f24a3aed 100644 --- a/packages/core/state/interaction/reducer.ts +++ b/packages/core/state/interaction/reducer.ts @@ -8,7 +8,7 @@ import { REFRESH, REMOVE_STATUS, SET_USER_SELECTED_APPLICATIONS, - SET_FILE_EXPLORER_SERVICE_BASE_URL, + INITIALIZE_APP, SET_STATUS, SET_VISIBLE_MODAL, SHOW_CONTEXT_MENU, @@ -20,10 +20,11 @@ import { PROMPT_FOR_DATA_SOURCE, DownloadManifestAction, DOWNLOAD_MANIFEST, + DataSourcePromptInfo, + PromptForDataSource, } from "./actions"; import { ContextMenuItem, PositionReference } from "../../components/ContextMenu"; import { ModalType } from "../../components/Modal"; -import { Source } from "../../entity/FileExplorerURL"; import FileFilter from "../../entity/FileFilter"; import { PlatformDependentServices } from "../../services"; import ApplicationInfoServiceNoop from "../../services/ApplicationInfoService/ApplicationInfoServiceNoop"; @@ -37,12 +38,12 @@ import DatabaseServiceNoop from "../../services/DatabaseService/DatabaseServiceN export interface InteractionStateBranch { applicationVersion?: string; - dataSourceForVisibleModal?: Source; contextMenuIsVisible: boolean; contextMenuItems: ContextMenuItem[]; contextMenuPositionReference: PositionReference; contextMenuOnDismiss?: () => void; csvColumns?: string[]; + dataSourceInfoForVisibleModal?: DataSourcePromptInfo; fileExplorerServiceBaseUrl: string; fileTypeForVisibleModal: "csv" | "json" | "parquet"; fileFiltersForVisibleModal: FileFilter[]; @@ -147,7 +148,7 @@ export default makeReducer( ...state, csvColumns: action.payload.annotations, }), - [SET_FILE_EXPLORER_SERVICE_BASE_URL]: (state, action) => ({ + [INITIALIZE_APP]: (state, action) => ({ ...state, fileExplorerServiceBaseUrl: action.payload, }), @@ -162,10 +163,10 @@ export default makeReducer( fileTypeForVisibleModal: action.payload.fileType, fileFiltersForVisibleModal: action.payload.fileFilters, }), - [PROMPT_FOR_DATA_SOURCE]: (state, action) => ({ + [PROMPT_FOR_DATA_SOURCE]: (state, action: PromptForDataSource) => ({ ...state, - visibleModal: ModalType.DataSourcePrompt, - dataSourceForVisibleModal: action.payload, + visibleModal: ModalType.DataSource, + dataSourceInfoForVisibleModal: action.payload, }), }, initialState diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index 8bfb67ba5..d4de838d3 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -2,7 +2,7 @@ import { uniqBy } from "lodash"; import { createSelector } from "reselect"; import { State } from "../"; -import { getDataSource, getPythonConversion } from "../selection/selectors"; +import { getSelectedDataSources, getPythonConversion } from "../selection/selectors"; import { AnnotationService, FileService } from "../../services"; import DatasetService, { DataSource, @@ -22,8 +22,8 @@ export const getContextMenuPositionReference = (state: State) => state.interaction.contextMenuPositionReference; export const getContextMenuOnDismiss = (state: State) => state.interaction.contextMenuOnDismiss; export const getCsvColumns = (state: State) => state.interaction.csvColumns; -export const getDataSourceForVisibleModal = (state: State) => - state.interaction.dataSourceForVisibleModal; +export const getDataSourceInfoForVisibleModal = (state: State) => + state.interaction.dataSourceInfoForVisibleModal; export const getFileExplorerServiceBaseUrl = (state: State) => state.interaction.fileExplorerServiceBaseUrl; export const getFileFiltersForVisibleModal = (state: State) => @@ -105,12 +105,12 @@ export const getHttpFileService = createSelector( ); export const getFileService = createSelector( - [getHttpFileService, getDataSource, getPlatformDependentServices, getRefreshKey], - (httpFileService, dataSource, platformDependentServices): FileService => { - if (dataSource && dataSource?.name !== AICS_FMS_DATA_SOURCE_NAME) { + [getHttpFileService, getSelectedDataSources, getPlatformDependentServices, getRefreshKey], + (httpFileService, dataSourceNames, platformDependentServices): FileService => { + if (dataSourceNames[0]?.name !== AICS_FMS_DATA_SOURCE_NAME) { return new DatabaseFileService({ databaseService: platformDependentServices.databaseService, - dataSourceName: dataSource.name, + dataSourceNames: dataSourceNames.map((source) => source.name), downloadService: platformDependentServices.fileDownloadService, }); } @@ -124,7 +124,7 @@ export const getAnnotationService = createSelector( getApplicationVersion, getUserName, getFileExplorerServiceBaseUrl, - getDataSource, + getSelectedDataSources, getPlatformDependentServices, getRefreshKey, ], @@ -132,13 +132,13 @@ export const getAnnotationService = createSelector( applicationVersion, userName, fileExplorerBaseUrl, - dataSource, + dataSources, platformDependentServices ): AnnotationService => { - if (dataSource && dataSource?.name !== AICS_FMS_DATA_SOURCE_NAME) { + if (dataSources.length && dataSources[0]?.name !== AICS_FMS_DATA_SOURCE_NAME) { return new DatabaseAnnotationService({ databaseService: platformDependentServices.databaseService, - dataSourceName: dataSource.name, + dataSourceNames: dataSources.map((source) => source.name), }); } return new HttpAnnotationService({ diff --git a/packages/core/state/interaction/test/logics.test.ts b/packages/core/state/interaction/test/logics.test.ts index e3e4ef4eb..17dd32195 100644 --- a/packages/core/state/interaction/test/logics.test.ts +++ b/packages/core/state/interaction/test/logics.test.ts @@ -43,6 +43,7 @@ import NotificationServiceNoop from "../../../services/NotificationService/Notif import HttpFileService from "../../../services/FileService/HttpFileService"; import HttpAnnotationService from "../../../services/AnnotationService/HttpAnnotationService"; import FileDetail, { FmsFile } from "../../../entity/FileDetail"; +import DatabaseServiceNoop from "../../../services/DatabaseService/DatabaseServiceNoop"; describe("Interaction logics", () => { const fileSelection = new FileSelection().select({ @@ -57,6 +58,12 @@ describe("Interaction logics", () => { } } + class MockDatabaseService extends DatabaseServiceNoop { + saveQuery() { + return Promise.resolve(new Uint8Array()); + } + } + describe("downloadManifest", () => { const sandbox = createSandbox(); @@ -103,10 +110,12 @@ describe("Interaction logics", () => { const state = mergeState(initialState, { interaction: { platformDependentServices: { + databaseService: new MockDatabaseService(), fileDownloadService: new FileDownloadServiceNoop(), }, }, selection: { + dataSources: [{ name: "mock" }], fileSelection, }, }); diff --git a/packages/core/state/selection/actions.ts b/packages/core/state/selection/actions.ts index 642f19270..8aa036180 100644 --- a/packages/core/state/selection/actions.ts +++ b/packages/core/state/selection/actions.ts @@ -8,7 +8,6 @@ import FileSet from "../../entity/FileSet"; import FileSort from "../../entity/FileSort"; import NumericRange from "../../entity/NumericRange"; import Tutorial from "../../entity/Tutorial"; -import { DataSource } from "../../services/DataSourceService"; import { EMPTY_QUERY_COMPONENTS, FileExplorerURLComponents, @@ -551,21 +550,21 @@ export function decodeFileExplorerURL(decodedFileExplorerURL: string): DecodeFil } /** - * CHANGE_DATA_SOURCE + * CHANGE_DATA_SOURCES * - * Intention to update the data source queries are run against. + * Intention to update the data sources queries are run against. */ -export const CHANGE_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "change-data-source"); +export const CHANGE_DATA_SOURCES = makeConstant(STATE_BRANCH_NAME, "change-data-sources"); -export interface ChangeDataSourceAction { - payload?: DataSource; +export interface ChangeDataSourcesAction { + payload: Source[]; type: string; } -export function changeDataSource(dataSource?: DataSource): ChangeDataSourceAction { +export function changeDataSources(dataSources: Source[]): ChangeDataSourcesAction { return { - payload: dataSource, - type: CHANGE_DATA_SOURCE, + payload: dataSources, + type: CHANGE_DATA_SOURCES, }; } diff --git a/packages/core/state/selection/logics.ts b/packages/core/state/selection/logics.ts index 9c13195b9..c69076634 100644 --- a/packages/core/state/selection/logics.ts +++ b/packages/core/state/selection/logics.ts @@ -20,10 +20,7 @@ import { SET_ANNOTATION_HIERARCHY, SELECT_NEARBY_FILE, setSortColumn, - changeDataSource, - CHANGE_DATA_SOURCE, CHANGE_QUERY, - ChangeDataSourceAction, SetAnnotationHierarchyAction, RemoveFromAnnotationHierarchyAction, ReorderAnnotationHierarchyAction, @@ -36,6 +33,9 @@ import { REPLACE_DATA_SOURCE, ReplaceDataSource, REMOVE_QUERY, + changeDataSources, + ChangeDataSourcesAction, + CHANGE_DATA_SOURCES, } from "./actions"; import { interaction, metadata, ReduxLogicDeps, selection } from "../"; import * as selectionSelectors from "./selectors"; @@ -46,6 +46,8 @@ import FileFolder from "../../entity/FileFolder"; import FileSelection from "../../entity/FileSelection"; import FileSet from "../../entity/FileSet"; import HttpAnnotationService from "../../services/AnnotationService/HttpAnnotationService"; +import { DataSource } from "../../services/DataSourceService"; +import DataSourcePreparationError from "../../errors/DataSourcePreparationError"; /** * Interceptor responsible for transforming payload of SELECT_FILE actions to account for whether the intention is to @@ -294,25 +296,12 @@ const toggleFileFolderCollapse = createLogic({ const decodeFileExplorerURLLogics = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const encodedURL = deps.action.payload; - const dataSources = interaction.selectors.getAllDataSources(deps.getState()); - const { hierarchy, filters, openFolders, sortColumn, source } = FileExplorerURL.decode( + const { hierarchy, filters, openFolders, sortColumn, sources } = FileExplorerURL.decode( encodedURL ); - let selectedDataSource = dataSources.find((c) => c.name === source?.name); - // It is possible the user was sent a novel data source in the URL - if (source && !selectedDataSource) { - const newDataSource = { - ...source, - id: source.name, - version: 1, - }; - dispatch(metadata.actions.receiveDataSources([...dataSources, newDataSource])); - selectedDataSource = newDataSource; - } - batch(() => { - dispatch(changeDataSource(selectedDataSource)); + dispatch(changeDataSources(sources)); dispatch(setAnnotationHierarchy(hierarchy)); dispatch(setFileFilters(filters)); dispatch(setOpenFileFolders(openFolders)); @@ -445,18 +434,52 @@ const selectNearbyFile = createLogic({ * a refresh action so that the resources pertain to the current data source */ const changeDataSourceLogic = createLogic({ + type: CHANGE_DATA_SOURCES, async process(deps: ReduxLogicDeps, dispatch, done) { - const action: ChangeDataSourceAction = deps.action; - const dataSource = action.payload; + const { payload: selectedDataSources } = deps.action as ChangeDataSourcesAction; const dataSources = interaction.selectors.getAllDataSources(deps.getState()); - if (dataSource && !dataSources.some((dataSource) => dataSource.id === dataSource.id)) { - dispatch(metadata.actions.receiveDataSources([...dataSources, dataSource])); + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); + + const newSelectedDataSources: DataSource[] = []; + const existingSelectedDataSources: DataSource[] = []; + selectedDataSources.forEach((source) => { + const existingSource = dataSources.find((s) => s.name === source.name); + if (existingSource) { + existingSelectedDataSources.push(existingSource); + } else { + newSelectedDataSources.push({ ...source, id: source.name }); + } + }); + + // It is possible the user was sent a novel data source in the URL + if (selectedDataSources.length > existingSelectedDataSources.length) { + dispatch( + metadata.actions.receiveDataSources([...dataSources, ...newSelectedDataSources]) + ); + } + + // Prepare the data sources ahead of querying against them below + try { + await databaseService.prepareDataSources(selectedDataSources); + } catch (err) { + const errMsg = `Error encountered while preparing data sources (Full error: ${ + (err as Error).message + })`; + console.error(errMsg); + if (err instanceof DataSourcePreparationError) { + dispatch( + interaction.actions.promptForDataSource({ source: { name: err.sourceName } }) + ); + } else { + alert(errMsg); + } } dispatch(interaction.actions.refresh() as AnyAction); done(); }, - type: CHANGE_DATA_SOURCE, }); /** @@ -464,6 +487,28 @@ const changeDataSourceLogic = createLogic({ */ const addQueryLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { + const { payload: newQuery } = deps.action as AddQuery; + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); + + // Prepare the data sources ahead of querying against them below + try { + await databaseService.prepareDataSources(newQuery.parts.sources); + } catch (err) { + const errMsg = `Error encountered while preparing data sources (Full error: ${ + (err as Error).message + })`; + console.error(errMsg); + if (err instanceof DataSourcePreparationError) { + dispatch( + interaction.actions.promptForDataSource({ source: { name: err.sourceName } }) + ); + } else { + alert(errMsg); + } + } + dispatch(changeQuery(deps.action.payload)); done(); }, @@ -498,9 +543,6 @@ const addQueryLogic = createLogic({ const changeQueryLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const { payload: newlySelectedQuery } = deps.action as ChangeQuery; - const { databaseService } = interaction.selectors.getPlatformDependentServices( - deps.getState() - ); const currentQueries = selectionSelectors.getQueries(deps.getState()); const currentQueryParts = selectionSelectors.getCurrentQueryParts(deps.getState()); const updatedQueries = currentQueries.map((query) => ({ @@ -511,19 +553,6 @@ const changeQueryLogic = createLogic({ : query.parts, })); - if (newlySelectedQuery.parts.source?.uri) { - try { - await databaseService.addDataSource( - newlySelectedQuery.parts.source.name, - newlySelectedQuery.parts.source.type, - newlySelectedQuery.parts.source.uri - ); - } catch (error) { - console.error("Failed to add data source, prompting for replacement", error); - dispatch(interaction.actions.promptForDataSource(newlySelectedQuery.parts.source)); - } - } - dispatch( decodeFileExplorerURL(FileExplorerURL.encode(newlySelectedQuery.parts)) as AnyAction ); @@ -549,25 +578,27 @@ const removeQueryLogic = createLogic({ const replaceDataSourceLogic = createLogic({ type: REPLACE_DATA_SOURCE, async process(deps: ReduxLogicDeps, dispatch, done) { - const { - payload: { name, type, uri }, - } = deps.ctx.replaceDataSourceAction as ReplaceDataSource; + const { payload: replacementSource } = deps.ctx + .replaceDataSourceAction as ReplaceDataSource; const { databaseService } = interaction.selectors.getPlatformDependentServices( deps.getState() ); + // Prepare the data sources ahead of querying against them below try { - if (uri) { - await databaseService.addDataSource(name, type, uri); + await databaseService.prepareDataSources([replacementSource]); + } catch (err) { + const errMsg = `Error encountered while replacing data sources (Full error: ${ + (err as Error).message + })`; + console.error(errMsg); + if (err instanceof DataSourcePreparationError) { + dispatch( + interaction.actions.promptForDataSource({ source: { name: err.sourceName } }) + ); + } else { + alert(errMsg); } - } catch (error) { - console.error("Failed to add data source, prompting for replacement", error); - dispatch( - interaction.actions.promptForDataSource({ - name, - uri, - }) - ); } dispatch(interaction.actions.refresh() as AnyAction); @@ -578,7 +609,7 @@ const replaceDataSourceLogic = createLogic({ deps.ctx.replaceDataSourceAction = deps.action; const queries = selectionSelectors.getQueries(deps.getState()); const updatedQueries = queries.map((query) => { - if (query.parts.source?.name !== replacementDataSource.name) { + if (query.parts.sources[0]?.name !== replacementDataSource.name) { return query; } diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index 1693faa4a..33277c6fa 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -1,5 +1,5 @@ import { makeReducer } from "@aics/redux-utils"; -import { castArray, omit, uniq } from "lodash"; +import { castArray, omit, uniq, uniqBy } from "lodash"; import interaction from "../interaction"; import { THUMBNAIL_SIZE_TO_NUM_COLUMNS } from "../../constants"; @@ -19,7 +19,7 @@ import { RESET_COLUMN_WIDTH, SORT_COLUMN, SET_SORT_COLUMN, - CHANGE_DATA_SOURCE, + CHANGE_DATA_SOURCES, SELECT_TUTORIAL, ADJUST_GLOBAL_FONT_SIZE, Query, @@ -32,12 +32,13 @@ import { SET_FILE_GRID_COLUMN_COUNT, REMOVE_QUERY, RemoveQuery, + ChangeDataSourcesAction, SetSortColumnAction, SetFileFiltersAction, } from "./actions"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; -import { DataSource } from "../../services/DataSourceService"; +import { Source } from "../../entity/FileExplorerURL"; export interface SelectionStateBranch { annotationHierarchy: string[]; @@ -46,7 +47,7 @@ export interface SelectionStateBranch { columnWidths: { [index: string]: number; // columnName to widthPercent mapping }; - dataSource?: DataSource; + dataSources: Source[]; displayAnnotations: Annotation[]; fileGridColumnCount: number; fileSelection: FileSelection; @@ -65,13 +66,14 @@ export interface SelectionStateBranch { export const initialState = { annotationHierarchy: [], availableAnnotationsForHierarchy: [], - availableAnnotationsForHierarchyLoading: false, + availableAnnotationsForHierarchyLoading: true, columnWidths: { [AnnotationName.FILE_NAME]: 0.4, [AnnotationName.KIND]: 0.2, [AnnotationName.TYPE]: 0.25, [AnnotationName.FILE_SIZE]: 0.15, }, + dataSources: [], displayAnnotations: [], isDarkTheme: true, fileGridColumnCount: THUMBNAIL_SIZE_TO_NUM_COLUMNS.LARGE, @@ -137,11 +139,9 @@ export default makeReducer( sortColumn: new FileSort(action.payload, SortOrder.DESC), }; }, - [CHANGE_DATA_SOURCE]: (state, action) => ({ + [CHANGE_DATA_SOURCES]: (state, action: ChangeDataSourcesAction) => ({ ...state, - annotationHierarchy: [], - dataSource: action.payload, - filters: [], + dataSources: uniqBy(action.payload, "name"), fileSelection: new FileSelection(), openFileFolders: [], }), @@ -211,7 +211,7 @@ export default makeReducer( ...state, openFileFolders: action.payload, }), - [interaction.actions.SET_FILE_EXPLORER_SERVICE_BASE_URL]: (state) => ({ + [interaction.actions.INITIALIZE_APP]: (state) => ({ ...state, // Reset file selections when pointed at a new backend diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index fb500cecd..c8b49f71d 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -5,9 +5,6 @@ import { State } from "../"; import Annotation from "../../entity/Annotation"; import FileExplorerURL, { FileExplorerURLComponents } from "../../entity/FileExplorerURL"; import FileFilter from "../../entity/FileFilter"; -import FileFolder from "../../entity/FileFolder"; -import FileSort from "../../entity/FileSort"; -import { DataSource } from "../../services/DataSourceService"; import { getAnnotations } from "../metadata/selectors"; import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; @@ -19,13 +16,13 @@ export const getAvailableAnnotationsForHierarchy = (state: State) => export const getAvailableAnnotationsForHierarchyLoading = (state: State) => state.selection.availableAnnotationsForHierarchyLoading; export const getColumnWidths = (state: State) => state.selection.columnWidths; -export const getDataSource = (state: State) => state.selection.dataSource; export const getFileGridColumnCount = (state: State) => state.selection.fileGridColumnCount; export const getFileFilters = (state: State) => state.selection.filters; export const getFileSelection = (state: State) => state.selection.fileSelection; export const getIsDarkTheme = (state: State) => state.selection.isDarkTheme; export const getOpenFileFolders = (state: State) => state.selection.openFileFolders; export const getRecentAnnotations = (state: State) => state.selection.recentAnnotations; +export const getSelectedDataSources = (state: State) => state.selection.dataSources; export const getSelectedQuery = (state: State) => state.selection.selectedQuery; export const getShouldDisplaySmallFont = (state: State) => state.selection.shouldDisplaySmallFont; export const getShouldDisplayThumbnailView = (state: State) => @@ -36,25 +33,27 @@ export const getQueries = (state: State) => state.selection.queries; const getPlatformDependentServices = (state: State) => state.interaction.platformDependentServices; // Importing normally creates a circular dependency // COMPOSED SELECTORS +export const hasQuerySelected = createSelector([getSelectedQuery], (query): boolean => !!query); + export const isQueryingAicsFms = createSelector( - [getDataSource], - (dataSource): boolean => !dataSource || dataSource.name === AICS_FMS_DATA_SOURCE_NAME + [getSelectedDataSources], + (dataSources): boolean => dataSources[0]?.name === AICS_FMS_DATA_SOURCE_NAME ); export const getCurrentQueryParts = createSelector( - [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getDataSource], - ( - hierarchy: string[], - filters: FileFilter[], - openFolders: FileFolder[], - sortColumn?: FileSort, - source?: DataSource - ): FileExplorerURLComponents => ({ + [ + getAnnotationHierarchy, + getFileFilters, + getOpenFileFolders, + getSortColumn, + getSelectedDataSources, + ], + (hierarchy, filters, openFolders, sortColumn, sources): FileExplorerURLComponents => ({ hierarchy, filters, openFolders, sortColumn, - source, + sources, }) ); @@ -70,23 +69,16 @@ export const getPythonConversion = createSelector( getFileFilters, getOpenFileFolders, getSortColumn, - getDataSource, + getSelectedDataSources, ], - ( - platformDependentServices, - hierarchy: string[], - filters: FileFilter[], - openFolders: FileFolder[], - sortColumn?: FileSort, - source?: DataSource - ) => { + (platformDependentServices, hierarchy, filters, openFolders, sortColumn, sources) => { return FileExplorerURL.convertToPython( { hierarchy, filters, openFolders, sortColumn, - source, + sources, }, platformDependentServices.executionEnvService.getOS() ); diff --git a/packages/core/state/selection/test/logics.test.ts b/packages/core/state/selection/test/logics.test.ts index 83f673245..31e120fae 100644 --- a/packages/core/state/selection/test/logics.test.ts +++ b/packages/core/state/selection/test/logics.test.ts @@ -23,7 +23,7 @@ import { setAnnotationHierarchy, selectNearbyFile, SET_SORT_COLUMN, - changeDataSource, + changeDataSources, } from "../actions"; import { initialState, interaction } from "../../"; import Annotation, { AnnotationName } from "../../../entity/Annotation"; @@ -31,7 +31,7 @@ import FileFilter from "../../../entity/FileFilter"; import selectionLogics from "../logics"; import { annotationsJson } from "../../../entity/Annotation/mocks"; import NumericRange from "../../../entity/NumericRange"; -import FileExplorerURL, { Source } from "../../../entity/FileExplorerURL"; +import FileExplorerURL from "../../../entity/FileExplorerURL"; import FileFolder from "../../../entity/FileFolder"; import FileSet from "../../../entity/FileSet"; import FileSelection from "../../../entity/FileSelection"; @@ -686,7 +686,7 @@ describe("Selection logics", () => { }); // Act - store.dispatch(changeDataSource({} as any)); + store.dispatch(changeDataSources([{}] as any[])); await logicMiddleware.whenComplete(); // Assert @@ -925,13 +925,14 @@ describe("Selection logics", () => { }); describe("decodeFileExplorerURL", () => { - const mockDataSource: DataSource = { - id: "1234148", - name: "Test Data Source", - version: 1, - type: "csv", - uri: "", - }; + const mockDataSources: DataSource[] = [ + { + id: "1234148", + name: "Test Data Source", + version: 1, + type: "csv", + }, + ]; beforeEach(() => { const datasetService = new DatasetService(); @@ -948,7 +949,7 @@ describe("Selection logics", () => { const state = mergeState(initialState, { metadata: { annotations, - dataSources: [mockDataSource], + dataSources: mockDataSources, }, }); const { store, logicMiddleware, actions } = configureMockStore({ @@ -959,17 +960,12 @@ describe("Selection logics", () => { const filters = [new FileFilter(annotations[3].name, "20x")]; const openFolders = [["a"], ["a", false]].map((folder) => new FileFolder(folder)); const sortColumn = new FileSort(AnnotationName.UPLOADED, SortOrder.DESC); - const source: Source = { - name: mockDataSource.name, - uri: "", - type: "csv", - }; const encodedURL = FileExplorerURL.encode({ hierarchy, filters, openFolders, sortColumn, - source, + sources: mockDataSources, }); // Act @@ -1001,7 +997,7 @@ describe("Selection logics", () => { payload: sortColumn, }) ).to.be.true; - expect(actions.includesMatch(changeDataSource(mockDataSource))).to.be.true; + expect(actions.includesMatch(changeDataSources(mockDataSources))).to.be.true; }); }); }); diff --git a/packages/core/state/selection/test/reducer.test.ts b/packages/core/state/selection/test/reducer.test.ts index 2f3ce0ce6..0ae120167 100644 --- a/packages/core/state/selection/test/reducer.test.ts +++ b/packages/core/state/selection/test/reducer.test.ts @@ -15,16 +15,10 @@ import { DataSource } from "../../../services/DataSourceService"; describe("Selection reducer", () => { [ - { - actionConstant: selection.actions.SET_ANNOTATION_HIERARCHY, - expectedAction: selection.actions.setAnnotationHierarchy([]), - }, - { - actionConstant: interaction.actions.SET_FILE_EXPLORER_SERVICE_BASE_URL, - expectedAction: interaction.actions.setFileExplorerServiceBaseUrl("base"), - }, - ].forEach(({ actionConstant, expectedAction }) => - it(`clears selected file state when ${actionConstant} is fired`, () => { + selection.actions.setAnnotationHierarchy([]), + interaction.actions.initializeApp("base"), + ].forEach((expectedAction) => + it(`clears selected file state when ${expectedAction.type} is fired`, () => { // arrange const prevSelection = new FileSelection().select({ fileSet: new FileSet(), @@ -47,8 +41,8 @@ describe("Selection reducer", () => { }) ); - describe(selection.actions.CHANGE_DATA_SOURCE, () => { - it("clears hierarchy, filters, file selection, and open folders", () => { + describe(selection.actions.CHANGE_DATA_SOURCES, () => { + it("clears file selection and open folders", () => { // Arrange const state = { ...selection.initialState, @@ -61,22 +55,25 @@ describe("Selection reducer", () => { filters: [new FileFilter("file_id", "1238401234")], openFileFolders: [new FileFolder(["AICS-11"])], }; - const dataSource: DataSource = { - name: "My Tiffs", - version: 2, - type: "csv", - id: "13123019", - uri: "", - }; + const dataSources: DataSource[] = [ + { + name: "My Tiffs", + version: 2, + type: "csv", + id: "13123019", + uri: "", + }, + ]; // Act - const actual = selection.reducer(state, selection.actions.changeDataSource(dataSource)); + const actual = selection.reducer( + state, + selection.actions.changeDataSources(dataSources) + ); // Assert - expect(actual.annotationHierarchy).to.be.empty; - expect(actual.dataSource).to.deep.equal(dataSource); + expect(actual.dataSources).to.deep.equal(dataSources); expect(actual.fileSelection.count()).to.equal(0); - expect(actual.filters).to.be.empty; expect(actual.openFileFolders).to.be.empty; }); }); @@ -252,14 +249,6 @@ describe("Selection reducer", () => { // Arrange const initialSelectionState = { ...selection.initialState }; - // (sanity-check) available annotations are not loading before refresh - expect( - selection.selectors.getAvailableAnnotationsForHierarchyLoading({ - ...initialState, - selection: initialSelectionState, - }) - ).to.be.false; - // Act const nextSelectionState = selection.reducer( initialSelectionState, diff --git a/packages/desktop/src/services/DatabaseServiceElectron.ts b/packages/desktop/src/services/DatabaseServiceElectron.ts index 1373da2b2..7cdea694e 100644 --- a/packages/desktop/src/services/DatabaseServiceElectron.ts +++ b/packages/desktop/src/services/DatabaseServiceElectron.ts @@ -5,26 +5,93 @@ import * as path from "path"; import duckdb from "duckdb"; import { DatabaseService } from "../../../core/services"; +import { Source } from "../../../core/entity/FileExplorerURL"; +import DataSourcePreparationError from "../../../core/errors/DataSourcePreparationError"; -export default class DatabaseServiceElectron implements DatabaseService { +export default class DatabaseServiceElectron extends DatabaseService { private database: duckdb.Database; - private readonly existingDataSources = new Set(); constructor() { + super(); this.database = new duckdb.Database(":memory:"); } - public async addDataSource( - name: string, - type: "csv" | "json" | "parquet", - uri: File | string - ): Promise { + /** + * Saves the result of the query to the designated location. + * May return a value if the location is not a physical location but rather + * a temporary database location (buffer) + */ + public saveQuery( + destination: string, + sql: string, + format: "csv" | "json" | "parquet" + ): Promise { + const saveOptions = [`FORMAT '${format}'`]; + if (format === "csv") { + saveOptions.push("HEADER"); + } + return new Promise((resolve, reject) => { + this.database.run( + `COPY (${sql}) TO '${destination}.${format}' (${saveOptions.join(", ")});`, + (err: any, result: any) => { + if (err) { + reject(err.message); + } else { + resolve(result); + } + } + ); + }); + } + + public query(sql: string): Promise { + return new Promise((resolve, reject) => { + try { + this.database.all(sql, (err: any, tableData: any) => { + if (err) { + reject(err.message); + } else { + resolve(tableData); + } + }); + } catch (error) { + return Promise.reject(`${error}`); + } + }); + } + + public async reset(): Promise { + await this.close(); + this.database = new duckdb.Database(":memory:"); + } + + public close(): Promise { + return new Promise((resolve, reject) => { + this.database.close((err) => { + if (err) { + reject(err.message); + } else { + resolve(); + } + }); + }); + } + + protected async addDataSource(dataSource: Source): Promise { + const { name, type, uri } = dataSource; if (this.existingDataSources.has(name)) { return; // no-op } + if (!type || !uri) { + throw new DataSourcePreparationError( + "Data source type and URI are missing", + dataSource.name + ); + } let source: string; let tempLocation; + this.existingDataSources.add(name); try { if (typeof uri === "string") { source = uri; @@ -71,8 +138,9 @@ export default class DatabaseServiceElectron implements DatabaseService { ); } }); - - this.existingDataSources.add(name); + } catch (err) { + await this.deleteDataSource(name); + throw new DataSourcePreparationError((err as Error).message, name); } finally { if (tempLocation) { await fs.promises.unlink(tempLocation); @@ -80,34 +148,14 @@ export default class DatabaseServiceElectron implements DatabaseService { } } - /** - * Saves the result of the query to the designated location. - * May return a value if the location is not a physical location but rather - * a temporary database location (buffer) - */ - public saveQuery(destination: string, sql: string, format: string): Promise { - return new Promise((resolve, reject) => { - this.database.run( - `COPY (${sql}) TO '${destination}.${format}' (FORMAT '${format}');`, - (err: any, result: any) => { - if (err) { - reject(err.message); - } else { - resolve(result); - } - } - ); - }); - } - - public query(sql: string): Promise { + protected async execute(sql: string): Promise { return new Promise((resolve, reject) => { try { - this.database.all(sql, (err: any, tableData: any) => { + this.database.exec(sql, (err: any) => { if (err) { reject(err.message); } else { - resolve(tableData); + resolve(); } }); } catch (error) { @@ -115,21 +163,4 @@ export default class DatabaseServiceElectron implements DatabaseService { } }); } - - public async reset(): Promise { - await this.close(); - this.database = new duckdb.Database(":memory:"); - } - - public close(): Promise { - return new Promise((resolve, reject) => { - this.database.close((err) => { - if (err) { - reject(err.message); - } else { - resolve(); - } - }); - }); - } } diff --git a/packages/desktop/src/services/FileDownloadServiceElectron.ts b/packages/desktop/src/services/FileDownloadServiceElectron.ts index cf6c23e03..46c25d015 100644 --- a/packages/desktop/src/services/FileDownloadServiceElectron.ts +++ b/packages/desktop/src/services/FileDownloadServiceElectron.ts @@ -317,9 +317,8 @@ export default class FileDownloadServiceElectron }); } - public async prepareHttpResourceForDownload(url: string, postBody: string): Promise { - const responseAsJSON = await this.rawPost(url, postBody); - return JSON.stringify(responseAsJSON); + public prepareHttpResourceForDownload(url: string, postBody: string): Promise { + return this.rawPost(url, postBody); } public cancelActiveRequest(downloadRequestId: string) { diff --git a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts index b29aaba91..b18595a37 100644 --- a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts +++ b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts @@ -23,7 +23,7 @@ describe("DatabaseServiceElectron", () => { await service.close(); }); - describe("addDataSource", () => { + describe("prepareDataSources", () => { it("creates table from file of type csv", async () => { // Arrange const tempFileName = "test.csv"; @@ -31,7 +31,7 @@ describe("DatabaseServiceElectron", () => { await fs.promises.writeFile(tempFile, "color\nblue\ngreen\norange"); // Act - await service.addDataSource(tempFileName, "csv", tempFile); + await service.prepareDataSources([{ name: tempFileName, type: "csv", uri: tempFile }]); // Assert const result = await service.query(`SELECT * FROM "${tempFileName}"`); @@ -48,7 +48,7 @@ describe("DatabaseServiceElectron", () => { ); // Act - await service.addDataSource(tempFileName, "json", tempFile); + await service.prepareDataSources([{ name: tempFileName, type: "json", uri: tempFile }]); // Assert const result = await service.query(`SELECT * FROM "${tempFileName}"`); @@ -79,7 +79,7 @@ describe("DatabaseServiceElectron", () => { // Assert const fileStat = await fs.promises.stat(`${destination}.${format}`); - expect(fileStat.size).to.equal(0); + expect(fileStat.size).to.equal(215); }); }); }); diff --git a/packages/web/src/services/DatabaseServiceWeb.ts b/packages/web/src/services/DatabaseServiceWeb.ts index d65b03782..b4b9d5e74 100644 --- a/packages/web/src/services/DatabaseServiceWeb.ts +++ b/packages/web/src/services/DatabaseServiceWeb.ts @@ -1,10 +1,11 @@ import * as duckdb from "@duckdb/duckdb-wasm"; import { DatabaseService } from "../../../core/services"; +import DataSourcePreparationError from "../../../core/errors/DataSourcePreparationError"; +import { Source } from "../../../core/entity/FileExplorerURL"; -export default class DatabaseServiceWeb implements DatabaseService { +export default class DatabaseServiceWeb extends DatabaseService { private database: duckdb.AsyncDuckDB | undefined; - private readonly existingDataSources = new Set(); public async initialize(logLevel: duckdb.LogLevel = duckdb.LogLevel.INFO) { const allBundles = duckdb.getJsDelivrBundles(); @@ -25,53 +26,6 @@ export default class DatabaseServiceWeb implements DatabaseService { URL.revokeObjectURL(worker_url); } - public async addDataSource( - name: string, - type: "csv" | "json" | "parquet", - uri: File | string - ): Promise { - if (!this.database) { - throw new Error("Database failed to initialize"); - } - if (!this.existingDataSources.has(name)) { - if (uri instanceof File) { - await this.database.registerFileHandle( - name, - uri, - duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, - true - ); - } else { - const protocol = uri.startsWith("s3") - ? duckdb.DuckDBDataProtocol.S3 - : duckdb.DuckDBDataProtocol.HTTP; - - await this.database.registerFileURL(name, uri, protocol, false); - } - - const connection = await this.database.connect(); - try { - if (type === "parquet") { - await connection.query( - `CREATE TABLE "${name}" AS FROM parquet_scan('${name}');` - ); - } else if (type === "json") { - await connection.query( - `CREATE TABLE "${name}" AS FROM read_json_auto('${name}');` - ); - } else { - // Default to CSV - await connection.query( - `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` - ); - } - this.existingDataSources.add(name); - } finally { - await connection.close(); - } - } - } - /** * Saves the result of the query to the designated location. * Returns an array representating the data from the query in the format designated @@ -118,4 +72,69 @@ export default class DatabaseServiceWeb implements DatabaseService { public async close(): Promise { this.database?.detach(); } + + public async addDataSource(dataSource: Source): Promise { + const { name, type, uri } = dataSource; + if (!this.database) { + throw new Error("Database failed to initialize"); + } + if (this.existingDataSources.has(name)) { + return; + } + if (!type || !uri) { + throw new DataSourcePreparationError( + "Data source type and URI are missing", + dataSource.name + ); + } + + this.existingDataSources.add(name); + try { + if (uri instanceof File) { + await this.database.registerFileHandle( + name, + uri, + duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, + true + ); + } else if ((uri as any) instanceof String) { + const protocol = uri.startsWith("s3") + ? duckdb.DuckDBDataProtocol.S3 + : duckdb.DuckDBDataProtocol.HTTP; + + await this.database.registerFileURL(name, uri, protocol, false); + } else { + throw new Error( + `URI is of unexpected type, should be File instance or String: ${uri}` + ); + } + + if (type === "parquet") { + await this.execute(`CREATE TABLE "${name}" AS FROM parquet_scan('${name}');`); + } else if (type === "json") { + await this.execute(`CREATE TABLE "${name}" AS FROM read_json_auto('${name}');`); + } else { + // Default to CSV + await this.execute( + `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` + ); + } + } catch (err) { + await this.deleteDataSource(name); + throw new DataSourcePreparationError((err as Error).message, name); + } + } + + protected async execute(sql: string): Promise { + if (!this.database) { + throw new Error("Database failed to initialize"); + } + + const connection = await this.database.connect(); + try { + await connection.query(sql); + } finally { + await connection.close(); + } + } } diff --git a/packages/web/src/services/FileDownloadServiceWeb.ts b/packages/web/src/services/FileDownloadServiceWeb.ts index 33ab5df10..1921503b9 100644 --- a/packages/web/src/services/FileDownloadServiceWeb.ts +++ b/packages/web/src/services/FileDownloadServiceWeb.ts @@ -39,9 +39,8 @@ export default class FileDownloadServiceWeb extends HttpServiceBase implements F } } - public async prepareHttpResourceForDownload(url: string, postBody: string): Promise { - const responseAsJSON = await this.rawPost(url, postBody); - return JSON.stringify(responseAsJSON); + public prepareHttpResourceForDownload(url: string, postBody: string): Promise { + return this.rawPost(url, postBody); } public getDefaultDownloadDirectory(): Promise {