From 85f628ffbb2c532932d55e25bfb2b5dd7b622bf0 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Mon, 20 May 2024 20:33:22 -0700 Subject: [PATCH 01/20] Add multi data source support --- packages/core/App.tsx | 24 ++--- .../DataSourcePrompt.module.css | 2 +- .../{Modal => }/DataSourcePrompt/index.tsx | 28 +++--- .../components/Modal/DataSource/index.tsx | 18 ++++ packages/core/components/Modal/index.tsx | 8 +- .../components/QueryPart/QueryDataSource.tsx | 96 +++++++++++++++---- .../core/components/QueryPart/QueryFilter.tsx | 2 + .../core/components/QueryPart/QueryGroup.tsx | 2 + .../components/QueryPart/QueryPart.module.css | 5 + .../core/components/QueryPart/QuerySort.tsx | 2 + packages/core/components/QueryPart/index.tsx | 5 +- .../core/components/QuerySidebar/Query.tsx | 17 ++-- .../core/components/QuerySidebar/index.tsx | 61 ++++-------- packages/core/entity/FileExplorerURL/index.ts | 17 ++-- .../test/fileexplorerurl.test.ts | 9 +- packages/core/entity/SQLBuilder/index.ts | 18 +++- .../DatabaseAnnotationService/index.ts | 19 ++-- .../test/DatabaseAnnotationService.test.ts | 35 +++++-- .../FileService/DatabaseFileService/index.ts | 16 ++-- .../test/DatabaseFileService.test.ts | 6 +- packages/core/state/interaction/actions.ts | 13 +-- packages/core/state/interaction/logics.ts | 34 ++++++- packages/core/state/interaction/reducer.ts | 6 +- packages/core/state/interaction/selectors.ts | 18 ++-- packages/core/state/selection/actions.ts | 55 +++++++++-- packages/core/state/selection/logics.ts | 79 ++++++++------- packages/core/state/selection/reducer.ts | 16 ++-- packages/core/state/selection/selectors.ts | 29 +++--- .../core/state/selection/test/logics.test.ts | 20 ++-- .../core/state/selection/test/reducer.test.ts | 83 ++++++++-------- 30 files changed, 462 insertions(+), 281 deletions(-) rename packages/core/components/{Modal => }/DataSourcePrompt/DataSourcePrompt.module.css (98%) rename packages/core/components/{Modal => }/DataSourcePrompt/index.tsx (90%) create mode 100644 packages/core/components/Modal/DataSource/index.tsx diff --git a/packages/core/App.tsx b/packages/core/App.tsx index eec917b17..6666b9a48 100644 --- a/packages/core/App.tsx +++ b/packages/core/App.tsx @@ -3,9 +3,10 @@ import { initializeIcons, loadTheme } from "@fluentui/react"; import classNames from "classnames"; import { uniqueId } from "lodash"; import * as React from "react"; -import { batch, useDispatch, useSelector } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import ContextMenu from "./components/ContextMenu"; +import DataSourcePrompt from "./components/DataSourcePrompt"; import Modal from "./components/Modal"; import DirectoryTree from "./components/DirectoryTree"; import FileDetails from "./components/FileDetails"; @@ -14,7 +15,7 @@ import StatusMessage from "./components/StatusMessage"; import TutorialTooltip from "./components/TutorialTooltip"; import QuerySidebar from "./components/QuerySidebar"; import { FileExplorerServiceBaseUrl } from "./constants"; -import { interaction, metadata, selection } from "./state"; +import { interaction, selection } from "./state"; import "./styles/global.css"; import styles from "./App.module.css"; @@ -42,6 +43,7 @@ export default function App(props: AppProps) { const { fileExplorerServiceBaseUrl = FileExplorerServiceBaseUrl.PRODUCTION } = props; const dispatch = useDispatch(); + const selectedQuery = useSelector(selection.selectors.hasQuerySelected); const isDarkTheme = useSelector(selection.selectors.getIsDarkTheme); const shouldDisplaySmallFont = useSelector(selection.selectors.getShouldDisplaySmallFont); const platformDependentServices = useSelector( @@ -70,14 +72,8 @@ export default function App(props: AppProps) { }, [platformDependentServices, dispatch]); // Set data source base urls - // And kick off the process of requesting metadata needed by the application. React.useEffect(() => { - batch(() => { - dispatch(interaction.actions.setFileExplorerServiceBaseUrl(fileExplorerServiceBaseUrl)); - dispatch(metadata.actions.requestAnnotations()); - dispatch(metadata.actions.requestDataSources()); - dispatch(selection.actions.setAnnotationHierarchy([])); - }); + dispatch(interaction.actions.initializeApp(fileExplorerServiceBaseUrl)); }, [dispatch, fileExplorerServiceBaseUrl]); return ( @@ -92,8 +88,14 @@ export default function App(props: AppProps) {
- - + {selectedQuery ? ( + <> + + + + ) : ( + + )}
diff --git a/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css b/packages/core/components/DataSourcePrompt/DataSourcePrompt.module.css similarity index 98% rename from packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css rename to packages/core/components/DataSourcePrompt/DataSourcePrompt.module.css index 6fa93df73..6734de630 100644 --- a/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css +++ b/packages/core/components/DataSourcePrompt/DataSourcePrompt.module.css @@ -76,7 +76,7 @@ } .text, .warning { - font-size: smaller; + text-align: center; } .warning { diff --git a/packages/core/components/Modal/DataSourcePrompt/index.tsx b/packages/core/components/DataSourcePrompt/index.tsx similarity index 90% rename from packages/core/components/Modal/DataSourcePrompt/index.tsx rename to packages/core/components/DataSourcePrompt/index.tsx index 192e8ad9e..32c5df5ee 100644 --- a/packages/core/components/Modal/DataSourcePrompt/index.tsx +++ b/packages/core/components/DataSourcePrompt/index.tsx @@ -3,15 +3,14 @@ import { throttle } from "lodash"; import * as React from "react"; import { useDispatch, useSelector } from "react-redux"; -import { ModalProps } from ".."; -import BaseModal from "../BaseModal"; -import { Source } from "../../../entity/FileExplorerURL"; -import { interaction, selection } from "../../../state"; +import { interaction, selection } from "../../state"; +import { Source } from "../../entity/FileExplorerURL"; import styles from "./DataSourcePrompt.module.css"; -interface Props extends ModalProps { - isEditing?: boolean; +interface Props { + hideTitle?: boolean; + onDismiss?: () => void; } const DATA_SOURCE_DETAILS = [ @@ -29,7 +28,7 @@ const DATA_SOURCE_DETAILS = [ /** * Dialog meant to prompt user to select a data source option */ -export default function DataSourcePrompt({ onDismiss }: Props) { +export default function DataSourcePrompt(props: Props) { const dispatch = useDispatch(); const dataSourceToReplace = useSelector(interaction.selectors.getDataSourceForVisibleModal); @@ -44,7 +43,7 @@ export default function DataSourcePrompt({ onDismiss }: Props) { dispatch( selection.actions.addQuery({ name: `New ${source.name} Query`, - parts: { source }, + parts: { sources: [source] }, }) ); } @@ -62,7 +61,7 @@ export default function DataSourcePrompt({ onDismiss }: Props) { return; } addOrReplaceQuery({ name, type: extension, uri: selectedFile }); - onDismiss(); + props.onDismiss?.(); } }; const onEnterURL = throttle( @@ -88,13 +87,12 @@ export default function DataSourcePrompt({ onDismiss }: Props) { type: extensionGuess as "csv" | "json" | "parquet", uri: dataSourceURL, }); - onDismiss(); }, 10000, { leading: true, trailing: false } ); - const body = ( + return ( <> {dataSourceToReplace && (
@@ -111,10 +109,10 @@ export default function DataSourcePrompt({ onDismiss }: Props) {

)} + {!props.hideTitle &&

Choose a data source

}

- Please provide a ".csv", ".parquet", or ".json" file - containing metadata about some files. See more details for information about what a - data source file should look like... + To get started, load a CSV, Parquet, or JSON file containing metadata (annotations) + about your files to view them.

{isDataSourceDetailExpanded ? (
@@ -180,6 +178,4 @@ export default function DataSourcePrompt({ onDismiss }: Props) {
); - - return ; } diff --git a/packages/core/components/Modal/DataSource/index.tsx b/packages/core/components/Modal/DataSource/index.tsx new file mode 100644 index 000000000..c485bfcaf --- /dev/null +++ b/packages/core/components/Modal/DataSource/index.tsx @@ -0,0 +1,18 @@ +import * as React from "react"; + +import { ModalProps } from ".."; +import BaseModal from "../BaseModal"; +import DataSourcePrompt from "../../DataSourcePrompt"; + +/** + * Dialog meant to prompt user to select a data source option + */ +export default function DataSource(props: ModalProps) { + return ( + } + title="Choose a data source" + onDismiss={props.onDismiss} + /> + ); +} diff --git a/packages/core/components/Modal/index.tsx b/packages/core/components/Modal/index.tsx index c982f2472..faeb5a141 100644 --- a/packages/core/components/Modal/index.tsx +++ b/packages/core/components/Modal/index.tsx @@ -3,7 +3,7 @@ import { useDispatch, useSelector } from "react-redux"; import { interaction } from "../../state"; import CodeSnippet from "./CodeSnippet"; -import DataSourcePrompt from "./DataSourcePrompt"; +import DataSource from "./DataSource"; import MetadataManifest from "./MetadataManifest"; export interface ModalProps { @@ -12,7 +12,7 @@ export interface ModalProps { export enum ModalType { CodeSnippet = 1, - DataSourcePrompt = 2, + DataSource = 2, MetadataManifest = 3, } @@ -30,8 +30,8 @@ export default function Modal() { switch (visibleModal) { case ModalType.CodeSnippet: return ; - case ModalType.DataSourcePrompt: - return ; + case ModalType.DataSource: + return ; case ModalType.MetadataManifest: return ; default: diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index dc819fe2e..c01af8440 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -1,39 +1,95 @@ +import { List } from "@fluentui/react"; import * as React from "react"; +import { useDispatch, useSelector } from "react-redux"; import QueryPart from "."; -import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; +import { ModalType } from "../Modal"; import { Source } from "../../entity/FileExplorerURL"; +import { interaction, metadata, selection } from "../../state"; +import ListRow, { ListItem } from "../ListPicker/ListRow"; interface Props { - dataSources: (Source | undefined)[]; + dataSources: Source[]; } /** * Component responsible for rendering the "Data Source" part of the query */ export default function QueryDataSource(props: Props) { + const dispatch = useDispatch(); + const dataSources = useSelector(metadata.selectors.getDataSources); + const selectedDataSources = useSelector(selection.selectors.getSelectedDataSources); + + const addDataSourceOptions: ListItem[] = React.useMemo( + () => [ + ...dataSources.map((source) => ({ + displayValue: source.name, + value: source.name, + data: source, + selected: selectedDataSources.some((selected) => source.name === selected.name), + iconProps: { iconName: "Folder" }, + onClick: () => { + dispatch(selection.actions.addDataSource(source)); + }, + secondaryText: "Data Source", + })), + { + displayValue: "New Data Source...", + value: "New Data Source...", + selected: false, + iconProps: { iconName: "NewFolder" }, + onClick: () => { + dispatch(interaction.actions.setVisibleModal(ModalType.DataSource)); + }, + }, + ], + [dispatch, dataSources, selectedDataSources] + ); + return (
TODO: To be implemented in another ticket
} - rows={props.dataSources.map((dataSource) => { - // TODO: This should change when we move towards - // having a blank data source only possible - // on an empty load - // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/105 - if (!dataSource) { - return { - id: AICS_FMS_DATA_SOURCE_NAME, - title: AICS_FMS_DATA_SOURCE_NAME, - }; - } - - return { - id: dataSource.name, - title: dataSource.name, - }; - })} + onDelete={(dataSource) => dispatch(selection.actions.removeDataSource(dataSource))} + onRenderAddMenuList={() => ( +
+ String(item.value)} + items={addDataSourceOptions} + // onShouldVirtualize={() => filteredItems.length > 100} + onRenderCell={(item) => + item && ( + + item.data + ? dispatch( + selection.actions.addDataSource( + item.data as Source + ) + ) + : dispatch( + interaction.actions.setVisibleModal( + ModalType.DataSource + ) + ) + } + onDeselect={() => + item.data && + dispatch(selection.actions.removeDataSource(item.data.name)) + } + /> + ) + } + /> +
+ )} + rows={props.dataSources.map((dataSource) => ({ + id: dataSource.name, + title: dataSource.name, + }))} /> ); } diff --git a/packages/core/components/QueryPart/QueryFilter.tsx b/packages/core/components/QueryPart/QueryFilter.tsx index 8f2497154..5a0d3e9e8 100644 --- a/packages/core/components/QueryPart/QueryFilter.tsx +++ b/packages/core/components/QueryPart/QueryFilter.tsx @@ -11,6 +11,7 @@ import { metadata, selection } from "../../state"; import Annotation from "../../entity/Annotation"; interface Props { + disabled?: boolean; filters: FileFilter[]; } @@ -26,6 +27,7 @@ export default function QueryFilter(props: Props) { return ( diff --git a/packages/core/components/QueryPart/QueryGroup.tsx b/packages/core/components/QueryPart/QueryGroup.tsx index ee393fdf1..354353f4d 100644 --- a/packages/core/components/QueryPart/QueryGroup.tsx +++ b/packages/core/components/QueryPart/QueryGroup.tsx @@ -8,6 +8,7 @@ import { metadata, selection } from "../../state"; import Annotation from "../../entity/Annotation"; interface Props { + disabled?: boolean; groups: string[]; } @@ -36,6 +37,7 @@ export default function QueryGroup(props: Props) { return ( dispatch(selection.actions.setSortColumn())} diff --git a/packages/core/components/QueryPart/index.tsx b/packages/core/components/QueryPart/index.tsx index bbf80bf29..51323d3bb 100644 --- a/packages/core/components/QueryPart/index.tsx +++ b/packages/core/components/QueryPart/index.tsx @@ -4,6 +4,7 @@ import { IRenderFunction, PrimaryButton, } from "@fluentui/react"; +import classNames from "classnames"; import * as React from "react"; import { DragDropContext, OnDragEndResponder } from "react-beautiful-dnd"; @@ -14,6 +15,7 @@ import styles from "./QueryPart.module.css"; interface Props { title: string; + disabled?: boolean; tutorialId?: string; addButtonIconName: string; rows: QueryPartRowItem[]; @@ -41,10 +43,11 @@ export default function QueryPart(props: Props) { }; return ( -
+
{ setIsExpanded(props.isSelected); }, [props.isSelected]); const decodedURL = React.useMemo( - () => (props.isSelected ? currentQueryParts : props.query.parts), - [props.query.parts, currentQueryParts, props.isSelected] + () => (props.isSelected ? currentQueryParts : props.query?.parts), + [props.query?.parts, currentQueryParts, props.isSelected] ); const onQueryUpdate = (updatedQuery: QueryType) => { @@ -76,7 +78,8 @@ export default function Query(props: QueryProps) {
{props.isSelected &&
}

- Data Source: {decodedURL.source?.name} + Data Source:{" "} + {decodedURL.sources.map((source) => source.name).join(", ")}

{!!decodedURL.hierarchy.length && (

@@ -134,10 +137,10 @@ export default function Query(props: QueryProps) { />

- - - - + + + +
1} diff --git a/packages/core/components/QuerySidebar/index.tsx b/packages/core/components/QuerySidebar/index.tsx index 97ac9721e..6b4247b5e 100644 --- a/packages/core/components/QuerySidebar/index.tsx +++ b/packages/core/components/QuerySidebar/index.tsx @@ -7,7 +7,6 @@ import Query from "./Query"; import { HELP_OPTIONS } from "./tutorials"; import { ModalType } from "../Modal"; import SvgIcon from "../SvgIcon"; -import FileExplorerURL, { DEFAULT_AICS_FMS_QUERY } from "../../entity/FileExplorerURL"; import Tutorial from "../../entity/Tutorial"; import { interaction, selection } from "../../state"; import { AICS_LOGO } from "../../icons"; @@ -25,45 +24,9 @@ export default function QuerySidebar(props: QuerySidebarProps) { const dispatch = useDispatch(); const queries = useSelector(selection.selectors.getQueries); const selectedQuery = useSelector(selection.selectors.getSelectedQuery); - const isAicsEmployee = useSelector(interaction.selectors.isAicsEmployee); const dataSources = useSelector(interaction.selectors.getAllDataSources); const currentGlobalURL = useSelector(selection.selectors.getEncodedFileExplorerUrl); - // Select query by default if none is selected - React.useEffect(() => { - if (!selectedQuery && queries.length) { - dispatch(selection.actions.changeQuery(queries[0])); - } - }, [selectedQuery, queries, dispatch]); - - // Determine a default query to render or prompt the user for a data source - // if no default is accessible - React.useEffect(() => { - if (!queries.length) { - if (!window.location.search) { - if (isAicsEmployee === true) { - // If the user is an AICS employee and there is no query in the URL, add a default query - dispatch( - selection.actions.addQuery({ - name: "New AICS Query", - parts: DEFAULT_AICS_FMS_QUERY, - }) - ); - } else if (isAicsEmployee === false) { - // If no query is selected and there is no query in the URL, prompt the user to select a data source - dispatch(interaction.actions.setVisibleModal(ModalType.DataSourcePrompt)); - } - } else if (isAicsEmployee === undefined) { - dispatch( - selection.actions.addQuery({ - name: "New Query", - parts: FileExplorerURL.decode(window.location.search), - }) - ); - } - } - }, [isAicsEmployee, queries, dispatch]); - React.useEffect(() => { if (selectedQuery) { const newurl = @@ -90,7 +53,7 @@ export default function QuerySidebar(props: QuerySidebarProps) { dispatch( selection.actions.addQuery({ name: `New ${source.name} query`, - parts: { source }, + parts: { sources: [source] }, }) ); }, @@ -101,7 +64,7 @@ export default function QuerySidebar(props: QuerySidebarProps) { text: "New Data Source...", iconProps: { iconName: "NewFolder" }, onClick: () => { - dispatch(interaction.actions.setVisibleModal(ModalType.DataSourcePrompt)); + dispatch(interaction.actions.setVisibleModal(ModalType.DataSource)); }, }, ], @@ -156,13 +119,23 @@ export default function QuerySidebar(props: QuerySidebarProps) { data-is-scrollable="true" data-is-focusable="true" > - {queries.map((query) => ( + {queries.length ? ( + queries.map((query) => ( + + )) + ) : ( - ))} + )}
{ params.append("openFolder", JSON.stringify(folder.fileFolder)); }); + urlComponents.sources?.map((source) => { + params.append("source", JSON.stringify(source)); + }); if (urlComponents.sortColumn) { params.append("sort", JSON.stringify(urlComponents.sortColumn.toJSON())); } - if (urlComponents.source) { - params.append("source", JSON.stringify(urlComponents.source)); - } return params.toString(); } @@ -90,7 +93,7 @@ export default class FileExplorerURL { const unparsedOpenFolders = params.getAll("openFolder"); const unparsedFilters = params.getAll("filter"); - const unparsedSource = params.get("source"); + const unparsedSources = params.getAll("source"); const hierarchy = params.getAll("group"); const unparsedSort = params.get("sort"); const hierarchyDepth = hierarchy.length; @@ -111,7 +114,7 @@ export default class FileExplorerURL { filters: unparsedFilters .map((unparsedFilter) => JSON.parse(unparsedFilter)) .map((parsedFilter) => new FileFilter(parsedFilter.name, parsedFilter.value)), - source: unparsedSource ? JSON.parse(unparsedSource) : undefined, + sources: unparsedSources.map((unparsedSource) => JSON.parse(unparsedSource)), openFolders: unparsedOpenFolders .map((unparsedFolder) => JSON.parse(unparsedFolder)) .filter((parsedFolder) => parsedFolder.length <= hierarchyDepth) diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index cd626fa50..588829de4 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -31,7 +31,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.FILE_SIZE, SortOrder.DESC), - source: mockSource, + sources: [mockSource], }; // Act @@ -49,6 +49,7 @@ describe("FileExplorerURL", () => { hierarchy: [], filters: [], openFolders: [], + sources: [], }; // Act @@ -78,7 +79,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - source: mockSource, + sources: [mockSource], }; const encodedUrl = FileExplorerURL.encode(components); const encodedUrlWithWhitespace = " " + encodedUrl + " "; @@ -97,7 +98,7 @@ describe("FileExplorerURL", () => { filters: [], openFolders: [], sortColumn: undefined, - source: undefined, + sources: [], }; const encodedUrl = FileExplorerURL.encode(components); @@ -114,6 +115,7 @@ describe("FileExplorerURL", () => { hierarchy: ["Cell Line"], filters: [], openFolders: [new FileFolder(["AICS-0"]), new FileFolder(["AICS-0", false])], + sources: [], }; const encodedUrl = FileExplorerURL.encode(components); @@ -140,6 +142,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.FILE_PATH, "Garbage" as any), + sources: [], }; const encodedUrl = FileExplorerURL.encode(components); diff --git a/packages/core/entity/SQLBuilder/index.ts b/packages/core/entity/SQLBuilder/index.ts index 5caaa0324..60e0a040a 100644 --- a/packages/core/entity/SQLBuilder/index.ts +++ b/packages/core/entity/SQLBuilder/index.ts @@ -1,3 +1,5 @@ +import { castArray } from "lodash"; + /** * A simple SQL query builder. */ @@ -5,6 +7,7 @@ export default class SQLBuilder { private isSummarizing = false; private selectStatement = "*"; private fromStatement?: string; + private joinStatements?: string[]; private readonly whereClauses: string[] = []; private orderByClause?: string; private offsetNum?: number; @@ -20,8 +23,18 @@ export default class SQLBuilder { return this; } - public from(statement: string): SQLBuilder { - this.fromStatement = statement; + public from(statement: string | string[]): SQLBuilder { + const statementAsArray = castArray(statement); + if (!statementAsArray.length) { + throw new Error('"FROM" statement requires at least one argument'); + } + this.fromStatement = statementAsArray[0]; + this.joinStatements = statementAsArray + .slice(1) + .map( + (table, idx) => + `JOIN ${table} ON ${table}."File Path" == ${statementAsArray[idx]}."File Path"` + ); return this; } @@ -75,6 +88,7 @@ export default class SQLBuilder { ${this.isSummarizing ? "SUMMARIZE" : ""} SELECT ${this.selectStatement} FROM "${this.fromStatement}" + ${this.joinStatements?.length ? this.joinStatements : ""} ${this.whereClauses.length ? `WHERE (${this.whereClauses.join(") AND (")})` : ""} ${this.orderByClause ? `ORDER BY ${this.orderByClause}` : ""} ${this.offsetNum !== undefined ? `OFFSET ${this.offsetNum}` : ""} diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index e5443eb50..f57b7c89e 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -8,7 +8,7 @@ import SQLBuilder from "../../../entity/SQLBuilder"; interface Config { databaseService: DatabaseService; - dataSourceName: string; + dataSourceNames: string[]; } interface DescribeQueryResult { @@ -28,12 +28,12 @@ interface SummarizeQueryResult { */ export default class DatabaseAnnotationService implements AnnotationService { private readonly databaseService: DatabaseService; - private readonly dataSourceName: string; + private readonly dataSourceNames: string[]; constructor( - config: Config = { dataSourceName: "Unknown", databaseService: new DatabaseServiceNoop() } + config: Config = { dataSourceNames: [], databaseService: new DatabaseServiceNoop() } ) { - this.dataSourceName = config.dataSourceName; + this.dataSourceNames = config.dataSourceNames; this.databaseService = config.databaseService; } @@ -55,7 +55,10 @@ export default class DatabaseAnnotationService implements AnnotationService { * Fetch all annotations. */ public async fetchAnnotations(): Promise { - const sql = `DESCRIBE "${this.dataSourceName}"`; + let sql = `DESCRIBE "${this.dataSourceNames[0]}" `; + this.dataSourceNames.slice(1).forEach((source, i) => { + sql += ` JOIN "${source}" ON ${this.dataSourceNames[i]}."File Path" == ${source}."File Path"`; + }); const rows = (await this.databaseService.query(sql)) as DescribeQueryResult[]; return rows.map( (row) => @@ -75,7 +78,7 @@ export default class DatabaseAnnotationService implements AnnotationService { const select_key = "select_key"; const sql = new SQLBuilder() .select(`DISTINCT "${annotation}" AS ${select_key}`) - .from(this.dataSourceName) + .from(this.dataSourceNames) .toSQL(); const rows = await this.databaseService.query(sql); return [ @@ -114,7 +117,7 @@ export default class DatabaseAnnotationService implements AnnotationService { const sqlBuilder = new SQLBuilder() .select(`DISTINCT "${hierarchy[path.length]}"`) - .from(this.dataSourceName); + .from(this.dataSourceNames); Object.keys(filtersByAnnotation).forEach((annotation) => { const annotationValues = filtersByAnnotation[annotation]; if (annotationValues[0] === null) { @@ -136,7 +139,7 @@ export default class DatabaseAnnotationService implements AnnotationService { public async fetchAvailableAnnotationsForHierarchy(annotations: string[]): Promise { const sql = new SQLBuilder() .summarize() - .from(this.dataSourceName) + .from(this.dataSourceNames) .where(annotations.map((annotation) => `"${annotation}" IS NOT NULL`)) .toSQL(); const rows = (await this.databaseService.query(sql)) as SummarizeQueryResult[]; diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts index 039c7c4a4..205f01ece 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts @@ -19,7 +19,10 @@ describe("DatabaseAnnotationService", () => { const databaseService = new MockDatabaseService(); it("issues request for all available Annotations", async () => { - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const actualAnnotations = await annotationService.fetchAnnotations(); expect(actualAnnotations.length).to.equal(annotations.length); expect(actualAnnotations[0]).to.be.instanceOf(Annotation); @@ -40,7 +43,10 @@ describe("DatabaseAnnotationService", () => { it("issues request for 'foo' values", async () => { const annotation = "foo"; - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const actualValues = await annotationService.fetchValues(annotation); expect(actualValues).to.be.deep.equal(["a0", "b1", "cc2", "dd3"]); }); @@ -61,13 +67,19 @@ describe("DatabaseAnnotationService", () => { const databaseService = new MockDatabaseService(); it("issues a request for annotation values for the first level of the annotation hierarchy", async () => { - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const values = await annotationService.fetchRootHierarchyValues(["foo"], []); expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Whatever2"]); }); it("issues a request for annotation values for the first level of the annotation hierarchy with filters", async () => { - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const filter = new FileFilter("bar", "barValue"); const values = await annotationService.fetchRootHierarchyValues(["foo"], [filter]); expect(values).to.deep.equal(["Cell Line0", "Is Split Scene1", "Whatever2"]); @@ -89,7 +101,10 @@ describe("DatabaseAnnotationService", () => { it("issues request for hierarchy values under a specific path within the hierarchy", async () => { const expectedValues = ["A0", "B1", "Cc2", "dD3"]; - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const values = await annotationService.fetchHierarchyValuesUnderPath( ["foo", "bar"], ["baz"], @@ -101,7 +116,10 @@ describe("DatabaseAnnotationService", () => { it("issues request for hierarchy values under a specific path within the hierarchy with filters", async () => { const expectedValues = ["A0", "B1", "Cc2", "dD3"]; - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const filter = new FileFilter("bar", "barValue"); const values = await annotationService.fetchHierarchyValuesUnderPath( ["foo", "bar"], @@ -126,7 +144,10 @@ describe("DatabaseAnnotationService", () => { const databaseService = new MockDatabaseService(); it("issues request for annotations that can be combined with current hierarchy", async () => { - const annotationService = new DatabaseAnnotationService({ dataSourceName: "Unknown", databaseService }); + const annotationService = new DatabaseAnnotationService({ + dataSourceNames: [], + databaseService, + }); const values = await annotationService.fetchAvailableAnnotationsForHierarchy([ "cell_line", "cas9", diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index 598f419c0..09b7fe040 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -12,7 +12,7 @@ import SQLBuilder from "../../../entity/SQLBuilder"; interface Config { databaseService: DatabaseService; - dataSourceName: string; + dataSourceNames: string[]; downloadService: FileDownloadService; } @@ -22,7 +22,7 @@ interface Config { export default class DatabaseFileService implements FileService { private readonly databaseService: DatabaseService; private readonly downloadService: FileDownloadService; - private readonly dataSourceName: string; + private readonly dataSourceNames: string[]; private static convertDatabaseRowToFileDetail( row: { [key: string]: string }, @@ -61,14 +61,14 @@ export default class DatabaseFileService implements FileService { constructor( config: Config = { - dataSourceName: "Unknown", + dataSourceNames: [], databaseService: new DatabaseServiceNoop(), downloadService: new FileDownloadServiceNoop(), } ) { this.databaseService = config.databaseService; this.downloadService = config.downloadService; - this.dataSourceName = config.dataSourceName; + this.dataSourceNames = config.dataSourceNames; } public async getCountOfMatchingFiles(fileSet: FileSet): Promise { @@ -76,7 +76,7 @@ export default class DatabaseFileService implements FileService { const sql = fileSet .toQuerySQLBuilder() .select(`COUNT(*) AS ${select_key}`) - .from(this.dataSourceName) + .from(this.dataSourceNames) // Remove sort if present .orderBy() .toSQL(); @@ -103,7 +103,7 @@ export default class DatabaseFileService implements FileService { public async getFiles(request: GetFilesRequest): Promise { const sql = request.fileSet .toQuerySQLBuilder() - .from(this.dataSourceName) + .from(this.dataSourceNames) .offset(request.from * request.limit) .limit(request.limit) .toSQL(); @@ -126,13 +126,13 @@ export default class DatabaseFileService implements FileService { ): Promise { const sqlBuilder = new SQLBuilder() .select(annotations.map((annotation) => `"${annotation}"`).join(", ")) - .from(this.dataSourceName); + .from(this.dataSourceNames); selections.forEach((selection) => { selection.indexRanges.forEach((indexRange) => { const subQuery = new SQLBuilder() .select('"File Path"') - .from(this.dataSourceName as string) + .from(this.dataSourceNames) .whereOr( Object.entries(selection.filters).map(([column, values]) => { const commaSeperatedValues = values.map((v) => `'${v}'`).join(", "); diff --git a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts index 2fdb2cb7d..cc1bde50f 100644 --- a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts +++ b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts @@ -28,7 +28,7 @@ describe("DatabaseFileService", () => { describe("getFiles", () => { it("issues request for files that match given parameters", async () => { const databaseFileService = new DatabaseFileService({ - dataSourceName: "Unknown", + dataSourceNames: [], databaseService, downloadService: new FileDownloadServiceNoop(), }); @@ -83,7 +83,7 @@ describe("DatabaseFileService", () => { it("issues request for aggregated information about given files", async () => { // Arrange const fileService = new DatabaseFileService({ - dataSourceName: "Unknown", + dataSourceNames: [], databaseService, downloadService: new FileDownloadServiceNoop(), }); @@ -105,7 +105,7 @@ describe("DatabaseFileService", () => { describe("getCountOfMatchingFiles", () => { it("issues request for count of files matching given parameters", async () => { const fileService = new DatabaseFileService({ - dataSourceName: "Unknown", + dataSourceNames: [], databaseService, downloadService: new FileDownloadServiceNoop(), }); diff --git a/packages/core/state/interaction/actions.ts b/packages/core/state/interaction/actions.ts index f568de9d8..8e456fc39 100644 --- a/packages/core/state/interaction/actions.ts +++ b/packages/core/state/interaction/actions.ts @@ -205,21 +205,18 @@ export function setIsAicsEmployee(isAicsEmployee: boolean): SetIsAicsEmployee { } /** - * SET CONNECTION CONFIGURATION FOR THE FILE EXPLORER SERVICE + * Set connection configuration and kick off any tasks to initialize the app */ -export const SET_FILE_EXPLORER_SERVICE_BASE_URL = makeConstant( - STATE_BRANCH_NAME, - "set-file-explorer-service-connection-config" -); +export const INITIALIZE_APP = makeConstant(STATE_BRANCH_NAME, "initialize-app"); -export interface SetFileExplorerServiceBaseUrl { +export interface InitializeApp { type: string; payload: string; } -export function setFileExplorerServiceBaseUrl(baseUrl: string): SetFileExplorerServiceBaseUrl { +export function initializeApp(baseUrl: string): InitializeApp { return { - type: SET_FILE_EXPLORER_SERVICE_BASE_URL, + type: INITIALIZE_APP, payload: baseUrl, }; } diff --git a/packages/core/state/interaction/logics.ts b/packages/core/state/interaction/logics.ts index 74d86c3f8..27607f8d8 100644 --- a/packages/core/state/interaction/logics.ts +++ b/packages/core/state/interaction/logics.ts @@ -25,7 +25,7 @@ import { OpenWithDefaultAction, PROMPT_FOR_NEW_EXECUTABLE, setUserSelectedApplication, - SET_FILE_EXPLORER_SERVICE_BASE_URL, + INITIALIZE_APP, setIsAicsEmployee, } from "./actions"; import * as interactionSelectors from "./selectors"; @@ -41,15 +41,45 @@ import FileDetail from "../../entity/FileDetail"; import { AnnotationName } from "../../entity/Annotation"; import FileSelection from "../../entity/FileSelection"; import NumericRange from "../../entity/NumericRange"; +import FileExplorerURL, { DEFAULT_AICS_FMS_QUERY } from "../../entity/FileExplorerURL"; /** * Interceptor responsible for checking if the user is able to access the AICS network */ const checkAicsEmployee = createLogic({ - type: SET_FILE_EXPLORER_SERVICE_BASE_URL, + type: INITIALIZE_APP, async process(deps: ReduxLogicDeps, dispatch, done) { + const queries = selection.selectors.getQueries(deps.getState()); + const selectedQuery = selection.selectors.getSelectedQuery(deps.getState()); const fileService = interactionSelectors.getHttpFileService(deps.getState()); + + // Redimentary check to see if the user is an AICS Employee by + // checking if the AICS network is accessible const isAicsEmployee = await fileService.isNetworkAccessible(); + + // If no query is currently selected attempt to choose one for the user + if (!selectedQuery) { + // If there are query args representing a query we can extract that + // into the query to render (ex. when refreshing a page) + if (window.location.search) { + dispatch( + selection.actions.addQuery({ + name: "New Query", + parts: FileExplorerURL.decode(window.location.search), + }) + ); + } else if (queries.length) { + dispatch(selection.actions.changeQuery(queries[0])); + } else if (isAicsEmployee) { + dispatch( + selection.actions.addQuery({ + name: "New AICS FMS Query", + parts: DEFAULT_AICS_FMS_QUERY, + }) + ); + } + } + dispatch(setIsAicsEmployee(isAicsEmployee) as AnyAction); done(); }, diff --git a/packages/core/state/interaction/reducer.ts b/packages/core/state/interaction/reducer.ts index c126ff36e..8b425cc54 100644 --- a/packages/core/state/interaction/reducer.ts +++ b/packages/core/state/interaction/reducer.ts @@ -8,7 +8,7 @@ import { REFRESH, REMOVE_STATUS, SET_USER_SELECTED_APPLICATIONS, - SET_FILE_EXPLORER_SERVICE_BASE_URL, + INITIALIZE_APP, SET_STATUS, SET_VISIBLE_MODAL, SHOW_CONTEXT_MENU, @@ -147,7 +147,7 @@ export default makeReducer( ...state, csvColumns: action.payload.annotations, }), - [SET_FILE_EXPLORER_SERVICE_BASE_URL]: (state, action) => ({ + [INITIALIZE_APP]: (state, action) => ({ ...state, fileExplorerServiceBaseUrl: action.payload, }), @@ -164,7 +164,7 @@ export default makeReducer( }), [PROMPT_FOR_DATA_SOURCE]: (state, action) => ({ ...state, - visibleModal: ModalType.DataSourcePrompt, + visibleModal: ModalType.DataSource, dataSourceForVisibleModal: action.payload, }), }, diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index bd7412994..4612bf64c 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -2,7 +2,7 @@ import { uniqBy } from "lodash"; import { createSelector } from "reselect"; import { State } from "../"; -import { getDataSource } from "../selection/selectors"; +import { getSelectedDataSources } from "../selection/selectors"; import { AnnotationService, FileService } from "../../services"; import DatasetService, { DataSource, @@ -106,12 +106,12 @@ export const getHttpFileService = createSelector( ); export const getFileService = createSelector( - [getHttpFileService, getDataSource, getPlatformDependentServices, getRefreshKey], - (httpFileService, dataSource, platformDependentServices): FileService => { - if (dataSource && dataSource?.name !== AICS_FMS_DATA_SOURCE_NAME) { + [getHttpFileService, getSelectedDataSources, getPlatformDependentServices, getRefreshKey], + (httpFileService, dataSourceNames, platformDependentServices): FileService => { + if (dataSourceNames[0]?.name !== AICS_FMS_DATA_SOURCE_NAME) { return new DatabaseFileService({ databaseService: platformDependentServices.databaseService, - dataSourceName: dataSource.name, + dataSourceNames: dataSourceNames.map((source) => source.name), downloadService: platformDependentServices.fileDownloadService, }); } @@ -125,7 +125,7 @@ export const getAnnotationService = createSelector( getApplicationVersion, getUserName, getFileExplorerServiceBaseUrl, - getDataSource, + getSelectedDataSources, getPlatformDependentServices, getRefreshKey, ], @@ -133,13 +133,13 @@ export const getAnnotationService = createSelector( applicationVersion, userName, fileExplorerBaseUrl, - dataSource, + dataSources, platformDependentServices ): AnnotationService => { - if (dataSource && dataSource?.name !== AICS_FMS_DATA_SOURCE_NAME) { + if (dataSources.length && dataSources[0]?.name !== AICS_FMS_DATA_SOURCE_NAME) { return new DatabaseAnnotationService({ databaseService: platformDependentServices.databaseService, - dataSourceName: dataSource.name, + dataSourceNames: dataSources.map((source) => source.name), }); } return new HttpAnnotationService({ diff --git a/packages/core/state/selection/actions.ts b/packages/core/state/selection/actions.ts index 642f19270..02e8b5d16 100644 --- a/packages/core/state/selection/actions.ts +++ b/packages/core/state/selection/actions.ts @@ -8,7 +8,6 @@ import FileSet from "../../entity/FileSet"; import FileSort from "../../entity/FileSort"; import NumericRange from "../../entity/NumericRange"; import Tutorial from "../../entity/Tutorial"; -import { DataSource } from "../../services/DataSourceService"; import { EMPTY_QUERY_COMPONENTS, FileExplorerURLComponents, @@ -17,6 +16,44 @@ import { const STATE_BRANCH_NAME = "selection"; +/** + * ADD_DATA_SOURCE + * + * Intention is to add a data source to the current query + */ +export const ADD_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "add-data-source"); + +export interface AddDataSource { + payload: Source; + type: string; +} + +export function addDataSource(source: Source): AddDataSource { + return { + payload: source, + type: ADD_DATA_SOURCE, + }; +} + +/** + * REMOVE_DATA_SOURCE + * + * Intention is to remove a data source from the current query + */ +export const REMOVE_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "remove-data-source"); + +export interface RemoveDataSource { + payload: string; + type: string; +} + +export function removeDataSource(sourceName: string): RemoveDataSource { + return { + payload: sourceName, + type: ADD_DATA_SOURCE, + }; +} + /** * SET_FILE_FILTERS * @@ -551,21 +588,21 @@ export function decodeFileExplorerURL(decodedFileExplorerURL: string): DecodeFil } /** - * CHANGE_DATA_SOURCE + * CHANGE_DATA_SOURCES * - * Intention to update the data source queries are run against. + * Intention to update the data sources queries are run against. */ -export const CHANGE_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "change-data-source"); +export const CHANGE_DATA_SOURCES = makeConstant(STATE_BRANCH_NAME, "change-data-sources"); -export interface ChangeDataSourceAction { - payload?: DataSource; +export interface ChangeDataSourcesAction { + payload: Source[]; type: string; } -export function changeDataSource(dataSource?: DataSource): ChangeDataSourceAction { +export function changeDataSources(dataSources: Source[]): ChangeDataSourcesAction { return { - payload: dataSource, - type: CHANGE_DATA_SOURCE, + payload: dataSources, + type: CHANGE_DATA_SOURCES, }; } diff --git a/packages/core/state/selection/logics.ts b/packages/core/state/selection/logics.ts index 9c13195b9..17d41d841 100644 --- a/packages/core/state/selection/logics.ts +++ b/packages/core/state/selection/logics.ts @@ -20,10 +20,7 @@ import { SET_ANNOTATION_HIERARCHY, SELECT_NEARBY_FILE, setSortColumn, - changeDataSource, - CHANGE_DATA_SOURCE, CHANGE_QUERY, - ChangeDataSourceAction, SetAnnotationHierarchyAction, RemoveFromAnnotationHierarchyAction, ReorderAnnotationHierarchyAction, @@ -36,6 +33,9 @@ import { REPLACE_DATA_SOURCE, ReplaceDataSource, REMOVE_QUERY, + changeDataSources, + ChangeDataSourcesAction, + CHANGE_DATA_SOURCES, } from "./actions"; import { interaction, metadata, ReduxLogicDeps, selection } from "../"; import * as selectionSelectors from "./selectors"; @@ -46,6 +46,7 @@ import FileFolder from "../../entity/FileFolder"; import FileSelection from "../../entity/FileSelection"; import FileSet from "../../entity/FileSet"; import HttpAnnotationService from "../../services/AnnotationService/HttpAnnotationService"; +import { DataSource } from "../../services/DataSourceService"; /** * Interceptor responsible for transforming payload of SELECT_FILE actions to account for whether the intention is to @@ -294,25 +295,12 @@ const toggleFileFolderCollapse = createLogic({ const decodeFileExplorerURLLogics = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const encodedURL = deps.action.payload; - const dataSources = interaction.selectors.getAllDataSources(deps.getState()); - const { hierarchy, filters, openFolders, sortColumn, source } = FileExplorerURL.decode( + const { hierarchy, filters, openFolders, sortColumn, sources } = FileExplorerURL.decode( encodedURL ); - let selectedDataSource = dataSources.find((c) => c.name === source?.name); - // It is possible the user was sent a novel data source in the URL - if (source && !selectedDataSource) { - const newDataSource = { - ...source, - id: source.name, - version: 1, - }; - dispatch(metadata.actions.receiveDataSources([...dataSources, newDataSource])); - selectedDataSource = newDataSource; - } - batch(() => { - dispatch(changeDataSource(selectedDataSource)); + dispatch(changeDataSources(sources)); dispatch(setAnnotationHierarchy(hierarchy)); dispatch(setFileFilters(filters)); dispatch(setOpenFileFolders(openFolders)); @@ -446,17 +434,31 @@ const selectNearbyFile = createLogic({ */ const changeDataSourceLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { - const action: ChangeDataSourceAction = deps.action; - const dataSource = action.payload; + const { payload: selectedDataSources } = deps.action as ChangeDataSourcesAction; const dataSources = interaction.selectors.getAllDataSources(deps.getState()); - if (dataSource && !dataSources.some((dataSource) => dataSource.id === dataSource.id)) { - dispatch(metadata.actions.receiveDataSources([...dataSources, dataSource])); + + const newSelectedDataSources: DataSource[] = []; + const existingSelectedDataSources: DataSource[] = []; + selectedDataSources.forEach((source) => { + const existingSource = dataSources.find((s) => s.name === source.name); + if (existingSource) { + existingSelectedDataSources.push(existingSource); + } else { + newSelectedDataSources.push({ ...source, id: source.name }); + } + }); + + // It is possible the user was sent a novel data source in the URL + if (selectedDataSources.length > existingSelectedDataSources.length) { + dispatch( + metadata.actions.receiveDataSources([...dataSources, ...newSelectedDataSources]) + ); } dispatch(interaction.actions.refresh() as AnyAction); done(); }, - type: CHANGE_DATA_SOURCE, + type: CHANGE_DATA_SOURCES, }); /** @@ -511,18 +513,21 @@ const changeQueryLogic = createLogic({ : query.parts, })); - if (newlySelectedQuery.parts.source?.uri) { - try { - await databaseService.addDataSource( - newlySelectedQuery.parts.source.name, - newlySelectedQuery.parts.source.type, - newlySelectedQuery.parts.source.uri - ); - } catch (error) { - console.error("Failed to add data source, prompting for replacement", error); - dispatch(interaction.actions.promptForDataSource(newlySelectedQuery.parts.source)); - } - } + await Promise.all( + newlySelectedQuery.parts.sources.map(async (source) => { + if (source.uri && source.type) { + try { + await databaseService.addDataSource(source.name, source.type, source.uri); + } catch (error) { + console.error( + "Failed to add data source, prompting for replacement", + error + ); + dispatch(interaction.actions.promptForDataSource(source)); + } + } + }) + ); dispatch( decodeFileExplorerURL(FileExplorerURL.encode(newlySelectedQuery.parts)) as AnyAction @@ -557,7 +562,7 @@ const replaceDataSourceLogic = createLogic({ ); try { - if (uri) { + if (uri && type) { await databaseService.addDataSource(name, type, uri); } } catch (error) { @@ -578,7 +583,7 @@ const replaceDataSourceLogic = createLogic({ deps.ctx.replaceDataSourceAction = deps.action; const queries = selectionSelectors.getQueries(deps.getState()); const updatedQueries = queries.map((query) => { - if (query.parts.source?.name !== replacementDataSource.name) { + if (query.parts.sources[0]?.name !== replacementDataSource.name) { return query; } diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index e43a187e0..6d1b4873d 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -19,7 +19,7 @@ import { RESET_COLUMN_WIDTH, SORT_COLUMN, SET_SORT_COLUMN, - CHANGE_DATA_SOURCE, + CHANGE_DATA_SOURCES, SELECT_TUTORIAL, ADJUST_GLOBAL_FONT_SIZE, Query, @@ -32,10 +32,11 @@ import { SET_FILE_GRID_COLUMN_COUNT, REMOVE_QUERY, RemoveQuery, + ChangeDataSourcesAction, } from "./actions"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; -import { DataSource } from "../../services/DataSourceService"; +import { Source } from "../../entity/FileExplorerURL"; export interface SelectionStateBranch { annotationHierarchy: string[]; @@ -44,7 +45,7 @@ export interface SelectionStateBranch { columnWidths: { [index: string]: number; // columnName to widthPercent mapping }; - dataSource?: DataSource; + dataSources: Source[]; displayAnnotations: Annotation[]; fileGridColumnCount: number; fileSelection: FileSelection; @@ -62,13 +63,14 @@ export interface SelectionStateBranch { export const initialState = { annotationHierarchy: [], availableAnnotationsForHierarchy: [], - availableAnnotationsForHierarchyLoading: false, + availableAnnotationsForHierarchyLoading: true, columnWidths: { [AnnotationName.FILE_NAME]: 0.4, [AnnotationName.KIND]: 0.2, [AnnotationName.TYPE]: 0.25, [AnnotationName.FILE_SIZE]: 0.15, }, + dataSources: [], displayAnnotations: [], isDarkTheme: true, fileGridColumnCount: THUMBNAIL_SIZE_TO_NUM_COLUMNS.LARGE, @@ -129,10 +131,10 @@ export default makeReducer( sortColumn: new FileSort(action.payload, SortOrder.DESC), }; }, - [CHANGE_DATA_SOURCE]: (state, action) => ({ + [CHANGE_DATA_SOURCES]: (state, action: ChangeDataSourcesAction) => ({ ...state, annotationHierarchy: [], - dataSource: action.payload, + dataSources: action.payload, filters: [], fileSelection: new FileSelection(), openFileFolders: [], @@ -198,7 +200,7 @@ export default makeReducer( ...state, openFileFolders: action.payload, }), - [interaction.actions.SET_FILE_EXPLORER_SERVICE_BASE_URL]: (state) => ({ + [interaction.actions.INITIALIZE_APP]: (state) => ({ ...state, // Reset file selections when pointed at a new backend diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index ec484a4d6..fe9ae4add 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -5,9 +5,6 @@ import { State } from "../"; import Annotation from "../../entity/Annotation"; import FileExplorerURL, { FileExplorerURLComponents } from "../../entity/FileExplorerURL"; import FileFilter from "../../entity/FileFilter"; -import FileFolder from "../../entity/FileFolder"; -import FileSort from "../../entity/FileSort"; -import { DataSource } from "../../services/DataSourceService"; import { getAnnotations } from "../metadata/selectors"; import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; @@ -19,12 +16,12 @@ export const getAvailableAnnotationsForHierarchy = (state: State) => export const getAvailableAnnotationsForHierarchyLoading = (state: State) => state.selection.availableAnnotationsForHierarchyLoading; export const getColumnWidths = (state: State) => state.selection.columnWidths; -export const getDataSource = (state: State) => state.selection.dataSource; export const getFileGridColumnCount = (state: State) => state.selection.fileGridColumnCount; export const getFileFilters = (state: State) => state.selection.filters; export const getFileSelection = (state: State) => state.selection.fileSelection; export const getIsDarkTheme = (state: State) => state.selection.isDarkTheme; export const getOpenFileFolders = (state: State) => state.selection.openFileFolders; +export const getSelectedDataSources = (state: State) => state.selection.dataSources; export const getSelectedQuery = (state: State) => state.selection.selectedQuery; export const getShouldDisplaySmallFont = (state: State) => state.selection.shouldDisplaySmallFont; export const getShouldDisplayThumbnailView = (state: State) => @@ -34,25 +31,27 @@ export const getTutorial = (state: State) => state.selection.tutorial; export const getQueries = (state: State) => state.selection.queries; // COMPOSED SELECTORS +export const hasQuerySelected = createSelector([getSelectedQuery], (query): boolean => !!query); + export const isQueryingAicsFms = createSelector( - [getDataSource], - (dataSource): boolean => !dataSource || dataSource.name === AICS_FMS_DATA_SOURCE_NAME + [getSelectedDataSources], + (dataSources): boolean => dataSources[0]?.name === AICS_FMS_DATA_SOURCE_NAME ); export const getCurrentQueryParts = createSelector( - [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getDataSource], - ( - hierarchy: string[], - filters: FileFilter[], - openFolders: FileFolder[], - sortColumn?: FileSort, - source?: DataSource - ): FileExplorerURLComponents => ({ + [ + getAnnotationHierarchy, + getFileFilters, + getOpenFileFolders, + getSortColumn, + getSelectedDataSources, + ], + (hierarchy, filters, openFolders, sortColumn, sources): FileExplorerURLComponents => ({ hierarchy, filters, openFolders, sortColumn, - source, + sources, }) ); diff --git a/packages/core/state/selection/test/logics.test.ts b/packages/core/state/selection/test/logics.test.ts index 83f673245..a87381267 100644 --- a/packages/core/state/selection/test/logics.test.ts +++ b/packages/core/state/selection/test/logics.test.ts @@ -23,7 +23,7 @@ import { setAnnotationHierarchy, selectNearbyFile, SET_SORT_COLUMN, - changeDataSource, + changeDataSources, } from "../actions"; import { initialState, interaction } from "../../"; import Annotation, { AnnotationName } from "../../../entity/Annotation"; @@ -686,7 +686,7 @@ describe("Selection logics", () => { }); // Act - store.dispatch(changeDataSource({} as any)); + store.dispatch(changeDataSources([{}] as any[])); await logicMiddleware.whenComplete(); // Assert @@ -959,17 +959,19 @@ describe("Selection logics", () => { const filters = [new FileFilter(annotations[3].name, "20x")]; const openFolders = [["a"], ["a", false]].map((folder) => new FileFolder(folder)); const sortColumn = new FileSort(AnnotationName.UPLOADED, SortOrder.DESC); - const source: Source = { - name: mockDataSource.name, - uri: "", - type: "csv", - }; + const sources: Source[] = [ + { + name: mockDataSource.name, + uri: "", + type: "csv", + }, + ]; const encodedURL = FileExplorerURL.encode({ hierarchy, filters, openFolders, sortColumn, - source, + sources, }); // Act @@ -1001,7 +1003,7 @@ describe("Selection logics", () => { payload: sortColumn, }) ).to.be.true; - expect(actions.includesMatch(changeDataSource(mockDataSource))).to.be.true; + expect(actions.includesMatch(changeDataSources([mockDataSource]))).to.be.true; }); }); }); diff --git a/packages/core/state/selection/test/reducer.test.ts b/packages/core/state/selection/test/reducer.test.ts index 2e421c376..13d7f2e26 100644 --- a/packages/core/state/selection/test/reducer.test.ts +++ b/packages/core/state/selection/test/reducer.test.ts @@ -14,40 +14,38 @@ import FileFolder from "../../../entity/FileFolder"; import { DataSource } from "../../../services/DataSourceService"; describe("Selection reducer", () => { - [ - selection.actions.SET_ANNOTATION_HIERARCHY, - interaction.actions.SET_FILE_EXPLORER_SERVICE_BASE_URL, - ].forEach((actionConstant) => - it(`clears selected file state when ${actionConstant} is fired`, () => { - // arrange - const prevSelection = new FileSelection().select({ - fileSet: new FileSet(), - index: new NumericRange(1, 3), - sortOrder: 0, - }); - const initialSelectionState = { - ...selection.initialState, - fileSelection: prevSelection, - }; - - const action = { - type: actionConstant, - }; - - // act - const nextSelectionState = selection.reducer(initialSelectionState, action); - const nextSelection = selection.selectors.getFileSelection({ - ...initialState, - selection: nextSelectionState, - }); + [selection.actions.SET_ANNOTATION_HIERARCHY, interaction.actions.INITIALIZE_APP].forEach( + (actionConstant) => + it(`clears selected file state when ${actionConstant} is fired`, () => { + // arrange + const prevSelection = new FileSelection().select({ + fileSet: new FileSet(), + index: new NumericRange(1, 3), + sortOrder: 0, + }); + const initialSelectionState = { + ...selection.initialState, + fileSelection: prevSelection, + }; + + const action = { + type: actionConstant, + }; + + // act + const nextSelectionState = selection.reducer(initialSelectionState, action); + const nextSelection = selection.selectors.getFileSelection({ + ...initialState, + selection: nextSelectionState, + }); - // assert - expect(prevSelection.count()).to.equal(3); // sanity-check - expect(nextSelection.count()).to.equal(0); - }) + // assert + expect(prevSelection.count()).to.equal(3); // sanity-check + expect(nextSelection.count()).to.equal(0); + }) ); - describe(selection.actions.CHANGE_DATA_SOURCE, () => { + describe(selection.actions.CHANGE_DATA_SOURCES, () => { it("clears hierarchy, filters, file selection, and open folders", () => { // Arrange const state = { @@ -61,20 +59,25 @@ describe("Selection reducer", () => { filters: [new FileFilter("file_id", "1238401234")], openFileFolders: [new FileFolder(["AICS-11"])], }; - const dataSource: DataSource = { - name: "My Tiffs", - version: 2, - type: "csv", - id: "13123019", - uri: "", - }; + const dataSources: DataSource[] = [ + { + name: "My Tiffs", + version: 2, + type: "csv", + id: "13123019", + uri: "", + }, + ]; // Act - const actual = selection.reducer(state, selection.actions.changeDataSource(dataSource)); + const actual = selection.reducer( + state, + selection.actions.changeDataSources(dataSources) + ); // Assert expect(actual.annotationHierarchy).to.be.empty; - expect(actual.dataSource).to.deep.equal(dataSource); + expect(actual.dataSources).to.deep.equal(dataSources); expect(actual.fileSelection.count()).to.equal(0); expect(actual.filters).to.be.empty; expect(actual.openFileFolders).to.be.empty; From fe48abb1d7bb4ef7d64f9dcf3cb151cc2cf4de0d Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Tue, 21 May 2024 10:03:04 -0700 Subject: [PATCH 02/20] UI prompts and logics for multi data source working --- .../components/DataSourcePrompt/index.tsx | 11 ++-- .../components/QueryPart/QueryDataSource.tsx | 51 ++++++++----------- packages/core/state/interaction/actions.ts | 11 ++-- packages/core/state/interaction/reducer.ts | 9 ++-- packages/core/state/interaction/selectors.ts | 4 +- packages/core/state/selection/logics.ts | 31 +++++++++-- packages/core/state/selection/reducer.ts | 14 ++++- 7 files changed, 84 insertions(+), 47 deletions(-) diff --git a/packages/core/components/DataSourcePrompt/index.tsx b/packages/core/components/DataSourcePrompt/index.tsx index 32c5df5ee..cecc8a19b 100644 --- a/packages/core/components/DataSourcePrompt/index.tsx +++ b/packages/core/components/DataSourcePrompt/index.tsx @@ -31,14 +31,17 @@ const DATA_SOURCE_DETAILS = [ export default function DataSourcePrompt(props: Props) { const dispatch = useDispatch(); - const dataSourceToReplace = useSelector(interaction.selectors.getDataSourceForVisibleModal); + const dataSourceInfo = useSelector(interaction.selectors.getDataSourceInfoForVisibleModal); + const { source: sourceToReplace, query } = dataSourceInfo || {}; const [dataSourceURL, setDataSourceURL] = React.useState(""); const [isDataSourceDetailExpanded, setIsDataSourceDetailExpanded] = React.useState(false); const addOrReplaceQuery = (source: Source) => { - if (dataSourceToReplace) { + if (sourceToReplace) { dispatch(selection.actions.replaceDataSource(source)); + } else if (query) { + dispatch(selection.actions.addDataSource(source)); } else { dispatch( selection.actions.addQuery({ @@ -94,12 +97,12 @@ export default function DataSourcePrompt(props: Props) { return ( <> - {dataSourceToReplace && ( + {sourceToReplace && (

Notice

There was an error loading the data source file " - {dataSourceToReplace.name}". Please re-select the data source file or a + {sourceToReplace.name}". Please re-select the data source file or a replacement.

diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index c01af8440..7beeeaa05 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -3,7 +3,6 @@ import * as React from "react"; import { useDispatch, useSelector } from "react-redux"; import QueryPart from "."; -import { ModalType } from "../Modal"; import { Source } from "../../entity/FileExplorerURL"; import { interaction, metadata, selection } from "../../state"; import ListRow, { ListItem } from "../ListPicker/ListRow"; @@ -17,34 +16,26 @@ interface Props { */ export default function QueryDataSource(props: Props) { const dispatch = useDispatch(); + const selectedQuery = useSelector(selection.selectors.getSelectedQuery); const dataSources = useSelector(metadata.selectors.getDataSources); const selectedDataSources = useSelector(selection.selectors.getSelectedDataSources); - const addDataSourceOptions: ListItem[] = React.useMemo( - () => [ - ...dataSources.map((source) => ({ - displayValue: source.name, - value: source.name, - data: source, - selected: selectedDataSources.some((selected) => source.name === selected.name), - iconProps: { iconName: "Folder" }, - onClick: () => { - dispatch(selection.actions.addDataSource(source)); - }, - secondaryText: "Data Source", - })), - { - displayValue: "New Data Source...", - value: "New Data Source...", - selected: false, - iconProps: { iconName: "NewFolder" }, - onClick: () => { - dispatch(interaction.actions.setVisibleModal(ModalType.DataSource)); - }, - }, - ], - [dispatch, dataSources, selectedDataSources] - ); + const addDataSourceOptions: ListItem[] = [ + ...dataSources.map((source) => ({ + displayValue: source.name, + value: source.name, + disabled: + selectedDataSources.length <= 1 && + selectedDataSources.some((selected) => source.name === selected.name), + data: source, + selected: selectedDataSources.some((selected) => source.name === selected.name), + })), + { + displayValue: "New Data Source...", + value: "New Data Source...", + selected: false, + }, + ]; return ( String(item.value)} items={addDataSourceOptions} - // onShouldVirtualize={() => filteredItems.length > 100} onRenderCell={(item) => item && ( item.data && + selectedDataSources.length > 1 && dispatch(selection.actions.removeDataSource(item.data.name)) } /> diff --git a/packages/core/state/interaction/actions.ts b/packages/core/state/interaction/actions.ts index 8e456fc39..57d9f7ca8 100644 --- a/packages/core/state/interaction/actions.ts +++ b/packages/core/state/interaction/actions.ts @@ -21,15 +21,20 @@ export const PROMPT_FOR_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "prompt-fo type PartialSource = Omit; +export interface DataSourcePromptInfo { + source?: PartialSource; + query?: string; +} + export interface PromptForDataSource { type: string; - payload: PartialSource; + payload: DataSourcePromptInfo; } -export function promptForDataSource(dataSource: PartialSource): PromptForDataSource { +export function promptForDataSource(info: DataSourcePromptInfo): PromptForDataSource { return { type: PROMPT_FOR_DATA_SOURCE, - payload: dataSource, + payload: info, }; } diff --git a/packages/core/state/interaction/reducer.ts b/packages/core/state/interaction/reducer.ts index 8b425cc54..5f24a3aed 100644 --- a/packages/core/state/interaction/reducer.ts +++ b/packages/core/state/interaction/reducer.ts @@ -20,10 +20,11 @@ import { PROMPT_FOR_DATA_SOURCE, DownloadManifestAction, DOWNLOAD_MANIFEST, + DataSourcePromptInfo, + PromptForDataSource, } from "./actions"; import { ContextMenuItem, PositionReference } from "../../components/ContextMenu"; import { ModalType } from "../../components/Modal"; -import { Source } from "../../entity/FileExplorerURL"; import FileFilter from "../../entity/FileFilter"; import { PlatformDependentServices } from "../../services"; import ApplicationInfoServiceNoop from "../../services/ApplicationInfoService/ApplicationInfoServiceNoop"; @@ -37,12 +38,12 @@ import DatabaseServiceNoop from "../../services/DatabaseService/DatabaseServiceN export interface InteractionStateBranch { applicationVersion?: string; - dataSourceForVisibleModal?: Source; contextMenuIsVisible: boolean; contextMenuItems: ContextMenuItem[]; contextMenuPositionReference: PositionReference; contextMenuOnDismiss?: () => void; csvColumns?: string[]; + dataSourceInfoForVisibleModal?: DataSourcePromptInfo; fileExplorerServiceBaseUrl: string; fileTypeForVisibleModal: "csv" | "json" | "parquet"; fileFiltersForVisibleModal: FileFilter[]; @@ -162,10 +163,10 @@ export default makeReducer( fileTypeForVisibleModal: action.payload.fileType, fileFiltersForVisibleModal: action.payload.fileFilters, }), - [PROMPT_FOR_DATA_SOURCE]: (state, action) => ({ + [PROMPT_FOR_DATA_SOURCE]: (state, action: PromptForDataSource) => ({ ...state, visibleModal: ModalType.DataSource, - dataSourceForVisibleModal: action.payload, + dataSourceInfoForVisibleModal: action.payload, }), }, initialState diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index 4612bf64c..5bdf70ac5 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -22,8 +22,8 @@ export const getContextMenuPositionReference = (state: State) => state.interaction.contextMenuPositionReference; export const getContextMenuOnDismiss = (state: State) => state.interaction.contextMenuOnDismiss; export const getCsvColumns = (state: State) => state.interaction.csvColumns; -export const getDataSourceForVisibleModal = (state: State) => - state.interaction.dataSourceForVisibleModal; +export const getDataSourceInfoForVisibleModal = (state: State) => + state.interaction.dataSourceInfoForVisibleModal; export const getFileExplorerServiceBaseUrl = (state: State) => state.interaction.fileExplorerServiceBaseUrl; export const getFileFiltersForVisibleModal = (state: State) => diff --git a/packages/core/state/selection/logics.ts b/packages/core/state/selection/logics.ts index 17d41d841..17e110077 100644 --- a/packages/core/state/selection/logics.ts +++ b/packages/core/state/selection/logics.ts @@ -36,6 +36,8 @@ import { changeDataSources, ChangeDataSourcesAction, CHANGE_DATA_SOURCES, + ADD_DATA_SOURCE, + AddDataSource, } from "./actions"; import { interaction, metadata, ReduxLogicDeps, selection } from "../"; import * as selectionSelectors from "./selectors"; @@ -523,7 +525,7 @@ const changeQueryLogic = createLogic({ "Failed to add data source, prompting for replacement", error ); - dispatch(interaction.actions.promptForDataSource(source)); + dispatch(interaction.actions.promptForDataSource({ source })); } } }) @@ -551,6 +553,26 @@ const removeQueryLogic = createLogic({ }, }); +const addDataSourceLogic = createLogic({ + type: ADD_DATA_SOURCE, + async process(deps: ReduxLogicDeps, dispatch, done) { + const { payload: source } = deps.action as AddDataSource; + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); + + if (source.uri && source.type) { + try { + await databaseService.addDataSource(source.name, source.type, source.uri); + } catch (error) { + console.error("Failed to add data source, prompting for replacement", error); + dispatch(interaction.actions.promptForDataSource({ source })); + } + } + done(); + }, +}); + const replaceDataSourceLogic = createLogic({ type: REPLACE_DATA_SOURCE, async process(deps: ReduxLogicDeps, dispatch, done) { @@ -569,8 +591,10 @@ const replaceDataSourceLogic = createLogic({ console.error("Failed to add data source, prompting for replacement", error); dispatch( interaction.actions.promptForDataSource({ - name, - uri, + source: { + name, + uri, + }, }) ); } @@ -601,6 +625,7 @@ const replaceDataSourceLogic = createLogic({ export default [ selectFile, + addDataSourceLogic, modifyAnnotationHierarchy, modifyFileFilters, toggleFileFolderCollapse, diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index 6d1b4873d..f3e917450 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -1,5 +1,5 @@ import { makeReducer } from "@aics/redux-utils"; -import { omit } from "lodash"; +import { omit, uniqBy } from "lodash"; import interaction from "../interaction"; import { THUMBNAIL_SIZE_TO_NUM_COLUMNS } from "../../constants"; @@ -33,6 +33,10 @@ import { REMOVE_QUERY, RemoveQuery, ChangeDataSourcesAction, + ADD_DATA_SOURCE, + AddDataSource, + RemoveDataSource, + REMOVE_DATA_SOURCE, } from "./actions"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; @@ -143,6 +147,14 @@ export default makeReducer( ...state, queries: [action.payload, ...state.queries], }), + [ADD_DATA_SOURCE]: (state, action: AddDataSource) => ({ + ...state, + dataSources: uniqBy([...state.dataSources, action.payload], "name"), + }), + [REMOVE_DATA_SOURCE]: (state, action: RemoveDataSource) => ({ + ...state, + dataSources: state.dataSources.filter((source) => source.name === action.payload), + }), [CHANGE_QUERY]: (state, action: ChangeQuery) => ({ ...state, selectedQuery: action.payload.name, From 2888478bc22361ff1b9431a40bac659a7783f901 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Tue, 21 May 2024 11:05:11 -0700 Subject: [PATCH 03/20] Ensure lack of dupe data --- packages/core/App.tsx | 4 +- packages/core/entity/SQLBuilder/index.ts | 2 +- .../DatabaseAnnotationService/index.ts | 22 +++---- .../FileService/DatabaseFileService/index.ts | 11 +++- packages/core/state/interaction/logics.ts | 4 +- packages/core/state/selection/logics.ts | 1 + .../web/src/services/DatabaseServiceWeb.ts | 64 +++++++++---------- 7 files changed, 56 insertions(+), 52 deletions(-) diff --git a/packages/core/App.tsx b/packages/core/App.tsx index 6666b9a48..033b23a11 100644 --- a/packages/core/App.tsx +++ b/packages/core/App.tsx @@ -43,7 +43,7 @@ export default function App(props: AppProps) { const { fileExplorerServiceBaseUrl = FileExplorerServiceBaseUrl.PRODUCTION } = props; const dispatch = useDispatch(); - const selectedQuery = useSelector(selection.selectors.hasQuerySelected); + const hasQuerySelected = useSelector(selection.selectors.hasQuerySelected); const isDarkTheme = useSelector(selection.selectors.getIsDarkTheme); const shouldDisplaySmallFont = useSelector(selection.selectors.getShouldDisplaySmallFont); const platformDependentServices = useSelector( @@ -88,7 +88,7 @@ export default function App(props: AppProps) {

- {selectedQuery ? ( + {hasQuerySelected ? ( <> diff --git a/packages/core/entity/SQLBuilder/index.ts b/packages/core/entity/SQLBuilder/index.ts index 60e0a040a..4791c0415 100644 --- a/packages/core/entity/SQLBuilder/index.ts +++ b/packages/core/entity/SQLBuilder/index.ts @@ -33,7 +33,7 @@ export default class SQLBuilder { .slice(1) .map( (table, idx) => - `JOIN ${table} ON ${table}."File Path" == ${statementAsArray[idx]}."File Path"` + `FULL JOIN "${table}" ON "${table}"."File Path" = "${statementAsArray[idx]}"."File Path"` ); return this; } diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index f57b7c89e..a750e83f4 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -1,3 +1,5 @@ +import { uniqBy } from "lodash"; + import AnnotationService, { AnnotationValue } from ".."; import DatabaseService from "../../DatabaseService"; import DatabaseServiceNoop from "../../DatabaseService/DatabaseServiceNoop"; @@ -11,12 +13,6 @@ interface Config { dataSourceNames: string[]; } -interface DescribeQueryResult { - [key: string]: string; - column_name: string; - column_type: string; -} - interface SummarizeQueryResult { [key: string]: string; column_name: string; @@ -55,18 +51,18 @@ export default class DatabaseAnnotationService implements AnnotationService { * Fetch all annotations. */ public async fetchAnnotations(): Promise { - let sql = `DESCRIBE "${this.dataSourceNames[0]}" `; - this.dataSourceNames.slice(1).forEach((source, i) => { - sql += ` JOIN "${source}" ON ${this.dataSourceNames[i]}."File Path" == ${source}."File Path"`; - }); - const rows = (await this.databaseService.query(sql)) as DescribeQueryResult[]; - return rows.map( + const sql = new SQLBuilder() + .from('information_schema"."columns') + .where(`table_name IN ('${this.dataSourceNames.join("', '")}')`) + .toSQL(); + const rows = await this.databaseService.query(sql); + return uniqBy(rows, "column_name").map( (row) => new Annotation({ annotationDisplayName: row["column_name"], annotationName: row["column_name"], description: "", - type: DatabaseAnnotationService.columnTypeToAnnotationType(row["column_type"]), + type: DatabaseAnnotationService.columnTypeToAnnotationType(row["data_type"]), }) ); } diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index 09b7fe040..e2bd8d6c0 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -103,6 +103,11 @@ export default class DatabaseFileService implements FileService { public async getFiles(request: GetFilesRequest): Promise { const sql = request.fileSet .toQuerySQLBuilder() + .select( + `*, COALESCE(${this.dataSourceNames + .map((source) => `"${source}"."File Path"`) + .join(", ")}) AS "File Path"` + ) .from(this.dataSourceNames) .offset(request.from * request.limit) .limit(request.limit) @@ -131,7 +136,11 @@ export default class DatabaseFileService implements FileService { selections.forEach((selection) => { selection.indexRanges.forEach((indexRange) => { const subQuery = new SQLBuilder() - .select('"File Path"') + .select( + `COALESCE(${this.dataSourceNames + .map((source) => `"${source}"."File Path"`) + .join(", ")}) AS "File Path"` + ) .from(this.dataSourceNames) .whereOr( Object.entries(selection.filters).map(([column, values]) => { diff --git a/packages/core/state/interaction/logics.ts b/packages/core/state/interaction/logics.ts index 27607f8d8..f406efb47 100644 --- a/packages/core/state/interaction/logics.ts +++ b/packages/core/state/interaction/logics.ts @@ -488,8 +488,8 @@ const refresh = createLogic({ ]); dispatch(metadata.actions.receiveAnnotations(annotations)); dispatch(selection.actions.setAvailableAnnotations(availableAnnotations)); - } catch (e) { - console.error("Error encountered while refreshing"); + } catch (err) { + console.error(`Error encountered while refreshing: ${err}`); const annotations = metadata.selectors.getAnnotations(deps.getState()); dispatch(selection.actions.setAvailableAnnotations(annotations.map((a) => a.name))); } finally { diff --git a/packages/core/state/selection/logics.ts b/packages/core/state/selection/logics.ts index 17e110077..dabeabfdb 100644 --- a/packages/core/state/selection/logics.ts +++ b/packages/core/state/selection/logics.ts @@ -564,6 +564,7 @@ const addDataSourceLogic = createLogic({ if (source.uri && source.type) { try { await databaseService.addDataSource(source.name, source.type, source.uri); + dispatch(interaction.actions.refresh() as AnyAction); } catch (error) { console.error("Failed to add data source, prompting for replacement", error); dispatch(interaction.actions.promptForDataSource({ source })); diff --git a/packages/web/src/services/DatabaseServiceWeb.ts b/packages/web/src/services/DatabaseServiceWeb.ts index d65b03782..f84411462 100644 --- a/packages/web/src/services/DatabaseServiceWeb.ts +++ b/packages/web/src/services/DatabaseServiceWeb.ts @@ -33,42 +33,40 @@ export default class DatabaseServiceWeb implements DatabaseService { if (!this.database) { throw new Error("Database failed to initialize"); } - if (!this.existingDataSources.has(name)) { - if (uri instanceof File) { - await this.database.registerFileHandle( - name, - uri, - duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, - true - ); - } else { - const protocol = uri.startsWith("s3") - ? duckdb.DuckDBDataProtocol.S3 - : duckdb.DuckDBDataProtocol.HTTP; + if (this.existingDataSources.has(name)) { + return; + } - await this.database.registerFileURL(name, uri, protocol, false); - } + if (uri instanceof File) { + await this.database.registerFileHandle( + name, + uri, + duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, + true + ); + } else { + const protocol = uri.startsWith("s3") + ? duckdb.DuckDBDataProtocol.S3 + : duckdb.DuckDBDataProtocol.HTTP; + + await this.database.registerFileURL(name, uri, protocol, false); + } - const connection = await this.database.connect(); - try { - if (type === "parquet") { - await connection.query( - `CREATE TABLE "${name}" AS FROM parquet_scan('${name}');` - ); - } else if (type === "json") { - await connection.query( - `CREATE TABLE "${name}" AS FROM read_json_auto('${name}');` - ); - } else { - // Default to CSV - await connection.query( - `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` - ); - } - this.existingDataSources.add(name); - } finally { - await connection.close(); + const connection = await this.database.connect(); + try { + if (type === "parquet") { + await connection.query(`CREATE TABLE "${name}" AS FROM parquet_scan('${name}');`); + } else if (type === "json") { + await connection.query(`CREATE TABLE "${name}" AS FROM read_json_auto('${name}');`); + } else { + // Default to CSV + await connection.query( + `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` + ); } + this.existingDataSources.add(name); + } finally { + await connection.close(); } } From d8c1ad1b3ab4cb24e19ade508ee754afef5b5cec Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Tue, 21 May 2024 12:17:45 -0700 Subject: [PATCH 04/20] Improve UX of adding/removing sources --- .../components/DataSourcePrompt/index.tsx | 4 +- .../components/Modal/DataSource/index.tsx | 2 +- .../components/QueryPart/QueryDataSource.tsx | 87 +++++++++++-------- .../core/components/QuerySidebar/Query.tsx | 26 +++--- packages/core/state/selection/actions.ts | 2 +- packages/core/state/selection/reducer.ts | 2 +- 6 files changed, 69 insertions(+), 54 deletions(-) diff --git a/packages/core/components/DataSourcePrompt/index.tsx b/packages/core/components/DataSourcePrompt/index.tsx index cecc8a19b..1a771e651 100644 --- a/packages/core/components/DataSourcePrompt/index.tsx +++ b/packages/core/components/DataSourcePrompt/index.tsx @@ -10,7 +10,6 @@ import styles from "./DataSourcePrompt.module.css"; interface Props { hideTitle?: boolean; - onDismiss?: () => void; } const DATA_SOURCE_DETAILS = [ @@ -64,7 +63,7 @@ export default function DataSourcePrompt(props: Props) { return; } addOrReplaceQuery({ name, type: extension, uri: selectedFile }); - props.onDismiss?.(); + dispatch(interaction.actions.hideVisibleModal()); } }; const onEnterURL = throttle( @@ -90,6 +89,7 @@ export default function DataSourcePrompt(props: Props) { type: extensionGuess as "csv" | "json" | "parquet", uri: dataSourceURL, }); + dispatch(interaction.actions.hideVisibleModal()); }, 10000, { leading: true, trailing: false } diff --git a/packages/core/components/Modal/DataSource/index.tsx b/packages/core/components/Modal/DataSource/index.tsx index c485bfcaf..5ca6a915b 100644 --- a/packages/core/components/Modal/DataSource/index.tsx +++ b/packages/core/components/Modal/DataSource/index.tsx @@ -10,7 +10,7 @@ import DataSourcePrompt from "../../DataSourcePrompt"; export default function DataSource(props: ModalProps) { return ( } + body={} title="Choose a data source" onDismiss={props.onDismiss} /> diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index 7beeeaa05..859d4533b 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -3,9 +3,10 @@ import * as React from "react"; import { useDispatch, useSelector } from "react-redux"; import QueryPart from "."; +import ListRow, { ListItem } from "../ListPicker/ListRow"; +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; import { Source } from "../../entity/FileExplorerURL"; import { interaction, metadata, selection } from "../../state"; -import ListRow, { ListItem } from "../ListPicker/ListRow"; interface Props { dataSources: Source[]; @@ -41,42 +42,56 @@ export default function QueryDataSource(props: Props) { dispatch(selection.actions.removeDataSource(dataSource))} - onRenderAddMenuList={() => ( -
- String(item.value)} - items={addDataSourceOptions} - onRenderCell={(item) => - item && ( - - item.data - ? dispatch( - selection.actions.addDataSource( - item.data as Source + onDelete={ + selectedDataSources.length > 1 + ? (dataSource) => dispatch(selection.actions.removeDataSource(dataSource)) + : undefined + } + onRenderAddMenuList={ + selectedDataSources[0]?.name === AICS_FMS_DATA_SOURCE_NAME + ? undefined + : () => ( +
+ String(item.value)} + items={addDataSourceOptions} + onRenderCell={(item) => + item && ( + + item.data + ? dispatch( + selection.actions.addDataSource( + item.data as Source + ) + ) + : dispatch( + interaction.actions.promptForDataSource( + { + query: selectedQuery, + } + ) + ) + } + onDeselect={() => + item.data && + selectedDataSources.length > 1 && + dispatch( + selection.actions.removeDataSource( + item.data.name + ) ) - ) - : dispatch( - interaction.actions.promptForDataSource({ - query: selectedQuery, - }) - ) - } - onDeselect={() => - item.data && - selectedDataSources.length > 1 && - dispatch(selection.actions.removeDataSource(item.data.name)) - } - /> - ) - } - /> -
- )} + } + /> + ) + } + /> +
+ ) + } rows={props.dataSources.map((dataSource) => ({ id: dataSource.name, title: dataSource.name, diff --git a/packages/core/components/QuerySidebar/Query.tsx b/packages/core/components/QuerySidebar/Query.tsx index 60fc04a32..2efbb43c8 100644 --- a/packages/core/components/QuerySidebar/Query.tsx +++ b/packages/core/components/QuerySidebar/Query.tsx @@ -35,7 +35,7 @@ export default function Query(props: QueryProps) { setIsExpanded(props.isSelected); }, [props.isSelected]); - const decodedURL = React.useMemo( + const queryComponents = React.useMemo( () => (props.isSelected ? currentQueryParts : props.query?.parts), [props.query?.parts, currentQueryParts, props.isSelected] ); @@ -79,12 +79,12 @@ export default function Query(props: QueryProps) { {props.isSelected &&
}

Data Source:{" "} - {decodedURL.sources.map((source) => source.name).join(", ")} + {queryComponents.sources.map((source) => source.name).join(", ")}

- {!!decodedURL.hierarchy.length && ( + {!!queryComponents.hierarchy.length && (

Groupings:{" "} - {decodedURL.hierarchy + {queryComponents.hierarchy .map( (a) => annotations.find((annotation) => annotation.name === a) @@ -93,18 +93,18 @@ export default function Query(props: QueryProps) { .join(", ")}

)} - {!!decodedURL.filters.length && ( + {!!queryComponents.filters.length && (

Filters:{" "} - {decodedURL.filters + {queryComponents.filters .map((filter) => `${filter.name}: ${filter.value}`) .join(", ")}

)} - {!!decodedURL.sortColumn && ( + {!!queryComponents.sortColumn && (

- Sort: {decodedURL.sortColumn.annotationName} ( - {decodedURL.sortColumn.order}) + Sort: {queryComponents.sortColumn.annotationName} ( + {queryComponents.sortColumn.order})

)}
@@ -137,10 +137,10 @@ export default function Query(props: QueryProps) { />
- - - - + + + +
1} diff --git a/packages/core/state/selection/actions.ts b/packages/core/state/selection/actions.ts index 02e8b5d16..50c9b9834 100644 --- a/packages/core/state/selection/actions.ts +++ b/packages/core/state/selection/actions.ts @@ -50,7 +50,7 @@ export interface RemoveDataSource { export function removeDataSource(sourceName: string): RemoveDataSource { return { payload: sourceName, - type: ADD_DATA_SOURCE, + type: REMOVE_DATA_SOURCE, }; } diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index f3e917450..e12f4845d 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -153,7 +153,7 @@ export default makeReducer( }), [REMOVE_DATA_SOURCE]: (state, action: RemoveDataSource) => ({ ...state, - dataSources: state.dataSources.filter((source) => source.name === action.payload), + dataSources: state.dataSources.filter((source) => source.name !== action.payload), }), [CHANGE_QUERY]: (state, action: ChangeQuery) => ({ ...state, From 709a17994418e19a4fff656b920e18bdc1a9c00c Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Tue, 21 May 2024 12:35:02 -0700 Subject: [PATCH 05/20] Adjust unit tests --- .../test/fileexplorerurl.test.ts | 2 +- .../test/DatabaseAnnotationService.test.ts | 14 ++++----- .../test/DatabaseFileService.test.ts | 6 ++-- .../state/interaction/test/logics.test.ts | 15 +++++++++ .../core/state/selection/test/logics.test.ts | 31 ++++++++----------- .../core/state/selection/test/reducer.test.ts | 8 ----- 6 files changed, 39 insertions(+), 37 deletions(-) diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index 588829de4..c0973c4bb 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -39,7 +39,7 @@ describe("FileExplorerURL", () => { // Assert expect(result).to.be.equal( - "group=Cell+Line&group=Donor+Plasmid&group=Lifting%3F&filter=%7B%22name%22%3A%22Cas9%22%2C%22value%22%3A%22spCas9%22%7D&filter=%7B%22name%22%3A%22Donor+Plasmid%22%2C%22value%22%3A%22ACTB-mEGFP%22%7D&openFolder=%5B%22AICS-0%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Cfalse%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Ctrue%5D&sort=%7B%22annotationName%22%3A%22file_size%22%2C%22order%22%3A%22DESC%22%7D&source=%7B%22name%22%3A%22Fake+Collection%22%2C%22type%22%3A%22csv%22%7D" + "group=Cell+Line&group=Donor+Plasmid&group=Lifting%3F&filter=%7B%22name%22%3A%22Cas9%22%2C%22value%22%3A%22spCas9%22%7D&filter=%7B%22name%22%3A%22Donor+Plasmid%22%2C%22value%22%3A%22ACTB-mEGFP%22%7D&openFolder=%5B%22AICS-0%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Cfalse%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Ctrue%5D&source=%7B%22name%22%3A%22Fake+Collection%22%2C%22type%22%3A%22csv%22%7D&sort=%7B%22annotationName%22%3A%22file_size%22%2C%22order%22%3A%22DESC%22%7D" ); }); diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts index 205f01ece..95a2fdcc7 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts @@ -24,7 +24,7 @@ describe("DatabaseAnnotationService", () => { databaseService, }); const actualAnnotations = await annotationService.fetchAnnotations(); - expect(actualAnnotations.length).to.equal(annotations.length); + expect(actualAnnotations.length).to.equal(1); expect(actualAnnotations[0]).to.be.instanceOf(Annotation); }); }); @@ -44,7 +44,7 @@ describe("DatabaseAnnotationService", () => { const annotation = "foo"; const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], + dataSourceNames: ["a", "b or c"], databaseService, }); const actualValues = await annotationService.fetchValues(annotation); @@ -68,7 +68,7 @@ describe("DatabaseAnnotationService", () => { it("issues a request for annotation values for the first level of the annotation hierarchy", async () => { const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], + dataSourceNames: ["d"], databaseService, }); const values = await annotationService.fetchRootHierarchyValues(["foo"], []); @@ -77,7 +77,7 @@ describe("DatabaseAnnotationService", () => { it("issues a request for annotation values for the first level of the annotation hierarchy with filters", async () => { const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], + dataSourceNames: ["e"], databaseService, }); const filter = new FileFilter("bar", "barValue"); @@ -102,7 +102,7 @@ describe("DatabaseAnnotationService", () => { const expectedValues = ["A0", "B1", "Cc2", "dD3"]; const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], + dataSourceNames: ["ghjiasd", "second source"], databaseService, }); const values = await annotationService.fetchHierarchyValuesUnderPath( @@ -117,7 +117,7 @@ describe("DatabaseAnnotationService", () => { const expectedValues = ["A0", "B1", "Cc2", "dD3"]; const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], + dataSourceNames: ["mock1"], databaseService, }); const filter = new FileFilter("bar", "barValue"); @@ -145,7 +145,7 @@ describe("DatabaseAnnotationService", () => { it("issues request for annotations that can be combined with current hierarchy", async () => { const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], + dataSourceNames: ["mock1"], databaseService, }); const values = await annotationService.fetchAvailableAnnotationsForHierarchy([ diff --git a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts index cc1bde50f..ddce55286 100644 --- a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts +++ b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts @@ -28,7 +28,7 @@ describe("DatabaseFileService", () => { describe("getFiles", () => { it("issues request for files that match given parameters", async () => { const databaseFileService = new DatabaseFileService({ - dataSourceNames: [], + dataSourceNames: ["whatever", "and another"], databaseService, downloadService: new FileDownloadServiceNoop(), }); @@ -83,7 +83,7 @@ describe("DatabaseFileService", () => { it("issues request for aggregated information about given files", async () => { // Arrange const fileService = new DatabaseFileService({ - dataSourceNames: [], + dataSourceNames: ["whatever"], databaseService, downloadService: new FileDownloadServiceNoop(), }); @@ -105,7 +105,7 @@ describe("DatabaseFileService", () => { describe("getCountOfMatchingFiles", () => { it("issues request for count of files matching given parameters", async () => { const fileService = new DatabaseFileService({ - dataSourceNames: [], + dataSourceNames: ["whatever"], databaseService, downloadService: new FileDownloadServiceNoop(), }); diff --git a/packages/core/state/interaction/test/logics.test.ts b/packages/core/state/interaction/test/logics.test.ts index e3e4ef4eb..d13d7f8ca 100644 --- a/packages/core/state/interaction/test/logics.test.ts +++ b/packages/core/state/interaction/test/logics.test.ts @@ -38,6 +38,7 @@ import FileDownloadService, { } from "../../../services/FileDownloadService"; import FileViewerService from "../../../services/FileViewerService"; import { annotationsJson } from "../../../entity/Annotation/mocks"; +import { DatabaseService } from "../../../services"; import FileDownloadServiceNoop from "../../../services/FileDownloadService/FileDownloadServiceNoop"; import NotificationServiceNoop from "../../../services/NotificationService/NotificationServiceNoop"; import HttpFileService from "../../../services/FileService/HttpFileService"; @@ -57,6 +58,18 @@ describe("Interaction logics", () => { } } + class MockDatabaseService implements DatabaseService { + saveQuery() { + return Promise.resolve(new Uint8Array()); + } + addDataSource() { + return Promise.reject("addDataSource mock"); + } + query() { + return Promise.reject("query mock"); + } + } + describe("downloadManifest", () => { const sandbox = createSandbox(); @@ -103,10 +116,12 @@ describe("Interaction logics", () => { const state = mergeState(initialState, { interaction: { platformDependentServices: { + databaseService: new MockDatabaseService(), fileDownloadService: new FileDownloadServiceNoop(), }, }, selection: { + dataSources: [{ name: "mock" }], fileSelection, }, }); diff --git a/packages/core/state/selection/test/logics.test.ts b/packages/core/state/selection/test/logics.test.ts index a87381267..8f3665559 100644 --- a/packages/core/state/selection/test/logics.test.ts +++ b/packages/core/state/selection/test/logics.test.ts @@ -31,7 +31,7 @@ import FileFilter from "../../../entity/FileFilter"; import selectionLogics from "../logics"; import { annotationsJson } from "../../../entity/Annotation/mocks"; import NumericRange from "../../../entity/NumericRange"; -import FileExplorerURL, { Source } from "../../../entity/FileExplorerURL"; +import FileExplorerURL from "../../../entity/FileExplorerURL"; import FileFolder from "../../../entity/FileFolder"; import FileSet from "../../../entity/FileSet"; import FileSelection from "../../../entity/FileSelection"; @@ -925,13 +925,15 @@ describe("Selection logics", () => { }); describe("decodeFileExplorerURL", () => { - const mockDataSource: DataSource = { - id: "1234148", - name: "Test Data Source", - version: 1, - type: "csv", - uri: "", - }; + const mockDataSources: DataSource[] = [ + { + id: "1234148", + name: "Test Data Source", + version: 1, + type: "csv", + uri: "", + }, + ]; beforeEach(() => { const datasetService = new DatasetService(); @@ -948,7 +950,7 @@ describe("Selection logics", () => { const state = mergeState(initialState, { metadata: { annotations, - dataSources: [mockDataSource], + dataSources: mockDataSources, }, }); const { store, logicMiddleware, actions } = configureMockStore({ @@ -959,19 +961,12 @@ describe("Selection logics", () => { const filters = [new FileFilter(annotations[3].name, "20x")]; const openFolders = [["a"], ["a", false]].map((folder) => new FileFolder(folder)); const sortColumn = new FileSort(AnnotationName.UPLOADED, SortOrder.DESC); - const sources: Source[] = [ - { - name: mockDataSource.name, - uri: "", - type: "csv", - }, - ]; const encodedURL = FileExplorerURL.encode({ hierarchy, filters, openFolders, sortColumn, - sources, + sources: mockDataSources, }); // Act @@ -1003,7 +998,7 @@ describe("Selection logics", () => { payload: sortColumn, }) ).to.be.true; - expect(actions.includesMatch(changeDataSources([mockDataSource]))).to.be.true; + expect(actions.includesMatch(changeDataSources(mockDataSources))).to.be.true; }); }); }); diff --git a/packages/core/state/selection/test/reducer.test.ts b/packages/core/state/selection/test/reducer.test.ts index 13d7f2e26..af6eaf9bb 100644 --- a/packages/core/state/selection/test/reducer.test.ts +++ b/packages/core/state/selection/test/reducer.test.ts @@ -255,14 +255,6 @@ describe("Selection reducer", () => { // Arrange const initialSelectionState = { ...selection.initialState }; - // (sanity-check) available annotations are not loading before refresh - expect( - selection.selectors.getAvailableAnnotationsForHierarchyLoading({ - ...initialState, - selection: initialSelectionState, - }) - ).to.be.false; - // Act const nextSelectionState = selection.reducer( initialSelectionState, From 2a921e422e1263ead97d2d2ebb147a86acda0e9b Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Fri, 31 May 2024 10:11:21 -0700 Subject: [PATCH 06/20] Create view before multi-data source --- .../components/AnnotationPicker/index.tsx | 19 +-- .../core/components/FileList/ColumnPicker.tsx | 17 ++- .../Modal/MetadataManifest/index.tsx | 10 +- packages/core/components/Modal/selectors.ts | 14 -- .../core/components/QueryPart/QueryFilter.tsx | 5 +- .../core/components/QueryPart/QueryGroup.tsx | 27 ++-- .../core/components/QueryPart/QuerySort.tsx | 14 +- packages/core/entity/SQLBuilder/index.ts | 2 +- .../DatabaseAnnotationService/index.ts | 10 +- .../DatabaseService/DatabaseServiceNoop.ts | 4 + .../core/services/DatabaseService/index.ts | 2 + .../FileService/DatabaseFileService/index.ts | 76 ++++++----- .../state/interaction/test/logics.test.ts | 3 + .../src/services/DatabaseServiceElectron.ts | 4 + .../web/src/services/DatabaseServiceWeb.ts | 125 +++++++++++++++--- 15 files changed, 211 insertions(+), 121 deletions(-) delete mode 100644 packages/core/components/Modal/selectors.ts diff --git a/packages/core/components/AnnotationPicker/index.tsx b/packages/core/components/AnnotationPicker/index.tsx index 48a5d6b44..4a7df5ac5 100644 --- a/packages/core/components/AnnotationPicker/index.tsx +++ b/packages/core/components/AnnotationPicker/index.tsx @@ -9,17 +9,16 @@ import { metadata, selection } from "../../state"; interface Props { id?: string; - enableAllAnnotations?: boolean; disabledTopLevelAnnotations?: boolean; hasSelectAllCapability?: boolean; disableUnavailableAnnotations?: boolean; className?: string; title?: string; - selections: Annotation[]; + selections: string[]; annotationSubMenuRenderer?: ( item: ListItem ) => React.ReactElement>; - setSelections: (annotations: Annotation[]) => void; + setSelections: (annotations: string[]) => void; } /** @@ -42,11 +41,11 @@ export default function AnnotationPicker(props: Props) { !TOP_LEVEL_FILE_ANNOTATION_NAMES.includes(annotation.name) ) .map((annotation) => ({ - selected: props.selections.some((selected) => selected.name === annotation.name), + selected: props.selections.some((selected) => selected === annotation.name), disabled: - !props.enableAllAnnotations && + props.disableUnavailableAnnotations && unavailableAnnotations.some((unavailable) => unavailable.name === annotation.name), - loading: !props.enableAllAnnotations && areAvailableAnnotationLoading, + loading: props.disableUnavailableAnnotations && areAvailableAnnotationLoading, description: annotation.description, data: annotation, value: annotation.name, @@ -55,14 +54,14 @@ export default function AnnotationPicker(props: Props) { const removeSelection = (item: ListItem) => { props.setSelections( - props.selections.filter((annotation) => annotation.name !== item.data?.name) + props.selections.filter((annotation) => annotation !== item.data?.name) ); }; const addSelection = (item: ListItem) => { // Should never be undefined, included as guard statement to satisfy compiler if (item.data) { - props.setSelections([...props.selections, item.data]); + props.setSelections([...props.selections, item.data.name]); } }; @@ -75,7 +74,9 @@ export default function AnnotationPicker(props: Props) { onDeselect={removeSelection} onSelect={addSelection} onSelectAll={ - props.hasSelectAllCapability ? () => props.setSelections?.(annotations) : undefined + props.hasSelectAllCapability + ? () => props.setSelections?.(annotations.map((a) => a.name)) + : undefined } onDeselectAll={() => props.setSelections([])} subMenuRenderer={props.annotationSubMenuRenderer} diff --git a/packages/core/components/FileList/ColumnPicker.tsx b/packages/core/components/FileList/ColumnPicker.tsx index 132afe65b..2aec5dc8a 100644 --- a/packages/core/components/FileList/ColumnPicker.tsx +++ b/packages/core/components/FileList/ColumnPicker.tsx @@ -2,25 +2,30 @@ import * as React from "react"; import { useSelector, useDispatch } from "react-redux"; import AnnotationPicker from "../AnnotationPicker"; -import { selection } from "../../state"; +import { metadata, selection } from "../../state"; /** * Picker for selecting which columns to display in the file list. */ export default function ColumnPicker() { const dispatch = useDispatch(); + const annotations = useSelector(metadata.selectors.getAnnotations); const columnAnnotations = useSelector(selection.selectors.getAnnotationsToDisplay); return ( { + title="Select metadata to display as columns" + selections={columnAnnotations.map((a) => a.name)} + setSelections={(selectedAnnotations) => { // Prevent de-selecting all columns - if (!annotations.length) { + if (!selectedAnnotations.length) { dispatch(selection.actions.setDisplayAnnotations([columnAnnotations[0]])); } else { - dispatch(selection.actions.setDisplayAnnotations(annotations)); + dispatch( + selection.actions.setDisplayAnnotations( + annotations.filter((a) => selectedAnnotations.includes(a.name)) + ) + ); } }} /> diff --git a/packages/core/components/Modal/MetadataManifest/index.tsx b/packages/core/components/Modal/MetadataManifest/index.tsx index 1c9b40c91..9e5eeb8ce 100644 --- a/packages/core/components/Modal/MetadataManifest/index.tsx +++ b/packages/core/components/Modal/MetadataManifest/index.tsx @@ -6,7 +6,6 @@ import { useDispatch, useSelector } from "react-redux"; import { ModalProps } from ".."; import BaseModal from "../BaseModal"; import AnnotationPicker from "../../AnnotationPicker"; -import * as modalSelectors from "../selectors"; import { interaction } from "../../../state"; import styles from "./MetadataManifest.module.css"; @@ -17,18 +16,15 @@ import styles from "./MetadataManifest.module.css"; */ export default function MetadataManifest({ onDismiss }: ModalProps) { const dispatch = useDispatch(); - const annotationsPreviouslySelected = useSelector( - modalSelectors.getAnnotationsPreviouslySelected - ); + const annotationsPreviouslySelected = useSelector(interaction.selectors.getCsvColumns); const [selectedAnnotations, setSelectedAnnotations] = React.useState( - annotationsPreviouslySelected + annotationsPreviouslySelected || [] ); const fileTypeForVisibleModal = useSelector(interaction.selectors.getFileTypeForVisibleModal); const onDownload = () => { - const selectedAnnotationNames = selectedAnnotations.map((annotation) => annotation.name); dispatch( - interaction.actions.downloadManifest(selectedAnnotationNames, fileTypeForVisibleModal) + interaction.actions.downloadManifest(selectedAnnotations, fileTypeForVisibleModal) ); onDismiss(); }; diff --git a/packages/core/components/Modal/selectors.ts b/packages/core/components/Modal/selectors.ts deleted file mode 100644 index 2ec2bead2..000000000 --- a/packages/core/components/Modal/selectors.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { createSelector } from "reselect"; - -import * as interactionSelectors from "../../state/interaction/selectors"; -import * as metadataSelectors from "../../state/metadata/selectors"; - -/** - * Returns Annotation instances for those annotations that were previously used to generate - * either a CSV manifest or dataset (via Python snippet generation). - */ -export const getAnnotationsPreviouslySelected = createSelector( - [interactionSelectors.getCsvColumns, metadataSelectors.getAnnotations], - (annotationDisplayNames, annotations) => - annotations.filter((annotation) => annotationDisplayNames?.includes(annotation.displayName)) -); diff --git a/packages/core/components/QueryPart/QueryFilter.tsx b/packages/core/components/QueryPart/QueryFilter.tsx index 5a0d3e9e8..02360cc3c 100644 --- a/packages/core/components/QueryPart/QueryFilter.tsx +++ b/packages/core/components/QueryPart/QueryFilter.tsx @@ -40,14 +40,11 @@ export default function QueryFilter(props: Props) { onRenderAddMenuList={() => ( ( )} - selections={annotations.filter((annotation) => - props.filters.some((f) => f.name === annotation.name) - )} + selections={props.filters.map((filter) => filter.name)} setSelections={() => dispatch(selection.actions.setFileFilters([]))} /> )} diff --git a/packages/core/components/QueryPart/QueryGroup.tsx b/packages/core/components/QueryPart/QueryGroup.tsx index 354353f4d..a14b5039b 100644 --- a/packages/core/components/QueryPart/QueryGroup.tsx +++ b/packages/core/components/QueryPart/QueryGroup.tsx @@ -1,11 +1,10 @@ import * as React from "react"; -import { useDispatch, useSelector } from "react-redux"; +import { useDispatch } from "react-redux"; import QueryPart from "."; import AnnotationPicker from "../AnnotationPicker"; import Tutorial from "../../entity/Tutorial"; -import { metadata, selection } from "../../state"; -import Annotation from "../../entity/Annotation"; +import { selection } from "../../state"; interface Props { disabled?: boolean; @@ -18,14 +17,6 @@ interface Props { export default function QueryGroup(props: Props) { const dispatch = useDispatch(); - const annotations = useSelector(metadata.selectors.getSortedAnnotations); - - const selectedAnnotations = props.groups - .map((annotationName) => - annotations.find((annotation) => annotation.name === annotationName) - ) - .filter((a) => !!a) as Annotation[]; - const onDelete = (annotationName: string) => { dispatch(selection.actions.removeFromAnnotationHierarchy(annotationName)); }; @@ -47,17 +38,17 @@ export default function QueryGroup(props: Props) { disabledTopLevelAnnotations disableUnavailableAnnotations title="Select metadata to group by" - selections={selectedAnnotations} + selections={props.groups} setSelections={(annotations) => { - dispatch( - selection.actions.setAnnotationHierarchy(annotations.map((a) => a.name)) - ); + dispatch(selection.actions.setAnnotationHierarchy(annotations)); }} /> )} - rows={selectedAnnotations.map((annotation) => ({ - id: annotation.name, - title: annotation.displayName, + // TODO: Should we care about display name?? seems time to make the name of + // annotations just the display name for top level annotations bro + rows={props.groups.map((annotation) => ({ + id: annotation, + title: annotation, }))} /> ); diff --git a/packages/core/components/QueryPart/QuerySort.tsx b/packages/core/components/QueryPart/QuerySort.tsx index 805773fee..ede34fc23 100644 --- a/packages/core/components/QueryPart/QuerySort.tsx +++ b/packages/core/components/QueryPart/QuerySort.tsx @@ -1,9 +1,9 @@ import * as React from "react"; -import { useDispatch, useSelector } from "react-redux"; +import { useDispatch } from "react-redux"; import QueryPart from "."; import AnnotationPicker from "../AnnotationPicker"; -import { metadata, selection } from "../../state"; +import { selection } from "../../state"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; @@ -18,8 +18,6 @@ interface Props { export default function QuerySort(props: Props) { const dispatch = useDispatch(); - const annotations = useSelector(metadata.selectors.getSortedAnnotations); - return ( annotation.name === props.sort?.annotationName - )} + selections={props.sort?.annotationName ? [props.sort.annotationName] : []} setSelections={(annotations) => { const newAnnotation = annotations.filter( - (annotation) => annotation.name !== props.sort?.annotationName - )?.[0].name; + (annotation) => annotation !== props.sort?.annotationName + )[0]; dispatch( selection.actions.setSortColumn( newAnnotation diff --git a/packages/core/entity/SQLBuilder/index.ts b/packages/core/entity/SQLBuilder/index.ts index 4791c0415..99b234ae5 100644 --- a/packages/core/entity/SQLBuilder/index.ts +++ b/packages/core/entity/SQLBuilder/index.ts @@ -87,7 +87,7 @@ export default class SQLBuilder { return ` ${this.isSummarizing ? "SUMMARIZE" : ""} SELECT ${this.selectStatement} - FROM "${this.fromStatement}" + FROM ${this.fromStatement} ${this.joinStatements?.length ? this.joinStatements : ""} ${this.whereClauses.length ? `WHERE (${this.whereClauses.join(") AND (")})` : ""} ${this.orderByClause ? `ORDER BY ${this.orderByClause}` : ""} diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index a750e83f4..e1c3e8e00 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -52,7 +52,7 @@ export default class DatabaseAnnotationService implements AnnotationService { */ public async fetchAnnotations(): Promise { const sql = new SQLBuilder() - .from('information_schema"."columns') + .from("information_schema.columns") .where(`table_name IN ('${this.dataSourceNames.join("', '")}')`) .toSQL(); const rows = await this.databaseService.query(sql); @@ -74,7 +74,7 @@ export default class DatabaseAnnotationService implements AnnotationService { const select_key = "select_key"; const sql = new SQLBuilder() .select(`DISTINCT "${annotation}" AS ${select_key}`) - .from(this.dataSourceNames) + .from(`"${this.dataSourceNames.join(", ")}"`) .toSQL(); const rows = await this.databaseService.query(sql); return [ @@ -113,7 +113,8 @@ export default class DatabaseAnnotationService implements AnnotationService { const sqlBuilder = new SQLBuilder() .select(`DISTINCT "${hierarchy[path.length]}"`) - .from(this.dataSourceNames); + .from(`"${this.dataSourceNames.join(", ")}"`); + Object.keys(filtersByAnnotation).forEach((annotation) => { const annotationValues = filtersByAnnotation[annotation]; if (annotationValues[0] === null) { @@ -124,6 +125,7 @@ export default class DatabaseAnnotationService implements AnnotationService { ); } }); + const rows = await this.databaseService.query(sqlBuilder.toSQL()); return rows.map((row) => row[hierarchy[path.length]]); } @@ -135,7 +137,7 @@ export default class DatabaseAnnotationService implements AnnotationService { public async fetchAvailableAnnotationsForHierarchy(annotations: string[]): Promise { const sql = new SQLBuilder() .summarize() - .from(this.dataSourceNames) + .from(`"${this.dataSourceNames.join(", ")}"`) .where(annotations.map((annotation) => `"${annotation}" IS NOT NULL`)) .toSQL(); const rows = (await this.databaseService.query(sql)) as SummarizeQueryResult[]; diff --git a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts index 84243cf85..9b900df51 100644 --- a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts +++ b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts @@ -5,6 +5,10 @@ export default class DatabaseServiceNoop implements DatabaseService { return Promise.reject("DatabaseServiceNoop:addDataSource"); } + public createViewOfDataSources() { + return Promise.reject("DatabaseServiceNoop::createViewOfDataSources"); + } + public saveQuery(): Promise { return Promise.reject("DatabaseServiceNoop:saveQuery"); } diff --git a/packages/core/services/DatabaseService/index.ts b/packages/core/services/DatabaseService/index.ts index 98bec99a5..355fd9efe 100644 --- a/packages/core/services/DatabaseService/index.ts +++ b/packages/core/services/DatabaseService/index.ts @@ -8,6 +8,8 @@ export default interface DatabaseService { uri: File | string ): Promise; + createViewOfDataSources(dataSources: string[]): Promise; + saveQuery( destination: string, sql: string, diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index e2bd8d6c0..ece1706a3 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -76,10 +76,11 @@ export default class DatabaseFileService implements FileService { const sql = fileSet .toQuerySQLBuilder() .select(`COUNT(*) AS ${select_key}`) - .from(this.dataSourceNames) + .from(`"${this.dataSourceNames.join(", ")}"`) // Remove sort if present .orderBy() .toSQL(); + const rows = await this.databaseService.query(sql); return parseInt(rows[0][select_key], 10); } @@ -101,17 +102,16 @@ export default class DatabaseFileService implements FileService { * and potentially starting from a particular file_id and limited to a set number of files. */ public async getFiles(request: GetFilesRequest): Promise { + // TODO: temp + await this.databaseService.createViewOfDataSources(this.dataSourceNames); + const sql = request.fileSet .toQuerySQLBuilder() - .select( - `*, COALESCE(${this.dataSourceNames - .map((source) => `"${source}"."File Path"`) - .join(", ")}) AS "File Path"` - ) - .from(this.dataSourceNames) + .from(`"${this.dataSourceNames.join(", ")}"`) .offset(request.from * request.limit) .limit(request.limit) .toSQL(); + const rows = await this.databaseService.query(sql); return rows.map((row, index) => DatabaseFileService.convertDatabaseRowToFileDetail( @@ -131,35 +131,39 @@ export default class DatabaseFileService implements FileService { ): Promise { const sqlBuilder = new SQLBuilder() .select(annotations.map((annotation) => `"${annotation}"`).join(", ")) - .from(this.dataSourceNames); + .from(`"${this.dataSourceNames[0]}"`); selections.forEach((selection) => { selection.indexRanges.forEach((indexRange) => { - const subQuery = new SQLBuilder() - .select( - `COALESCE(${this.dataSourceNames - .map((source) => `"${source}"."File Path"`) - .join(", ")}) AS "File Path"` - ) - .from(this.dataSourceNames) - .whereOr( - Object.entries(selection.filters).map(([column, values]) => { - const commaSeperatedValues = values.map((v) => `'${v}'`).join(", "); - return `"${column}" IN (${commaSeperatedValues}}`; - }) - ) - .offset(indexRange.start) - .limit(indexRange.end - indexRange.start + 1); - - if (selection.sort) { - subQuery.orderBy( - `"${selection.sort.annotationName}" ${ - selection.sort.ascending ? "ASC" : "DESC" - }` - ); - } - - sqlBuilder.whereOr(`"File Path" IN (${subQuery})`); + const subQuerySql = this.dataSourceNames + .map((dataSource) => { + const subQuery = new SQLBuilder() + .select("File Path") + .from(`"${dataSource}"`) + .whereOr( + Object.entries(selection.filters).map(([column, values]) => { + const commaSeperatedValues = values + .map((v) => `'${v}'`) + .join(", "); + return `"${column}" IN (${commaSeperatedValues}}`; + }) + ) + .offset(indexRange.start) + .limit(indexRange.end - indexRange.start + 1); + + if (selection.sort) { + subQuery.orderBy( + `"${selection.sort.annotationName}" ${ + selection.sort.ascending ? "ASC" : "DESC" + }` + ); + } + + return subQuery.toSQL(); + }) + .join(" UNION "); + + sqlBuilder.whereOr(`"File Path" IN (${subQuerySql})`); }); }); @@ -180,7 +184,11 @@ export default class DatabaseFileService implements FileService { }; } - const buffer = await this.databaseService.saveQuery(uniqueId(), sqlBuilder.toSQL(), format); + const buffer = await this.databaseService.saveQuery( + uniqueId(), + `SELECT * FROM "${this.dataSourceNames[0]}" LIMIT 2`, + format + ); const name = `file-selection-${new Date()}.${format}`; return this.downloadService.download( { diff --git a/packages/core/state/interaction/test/logics.test.ts b/packages/core/state/interaction/test/logics.test.ts index d13d7f8ca..cc3729737 100644 --- a/packages/core/state/interaction/test/logics.test.ts +++ b/packages/core/state/interaction/test/logics.test.ts @@ -65,6 +65,9 @@ describe("Interaction logics", () => { addDataSource() { return Promise.reject("addDataSource mock"); } + createViewOfDataSources(): Promise { + return Promise.reject("createViewOfDataSources mock"); + } query() { return Promise.reject("query mock"); } diff --git a/packages/desktop/src/services/DatabaseServiceElectron.ts b/packages/desktop/src/services/DatabaseServiceElectron.ts index 1373da2b2..21b3718a4 100644 --- a/packages/desktop/src/services/DatabaseServiceElectron.ts +++ b/packages/desktop/src/services/DatabaseServiceElectron.ts @@ -80,6 +80,10 @@ export default class DatabaseServiceElectron implements DatabaseService { } } + public createViewOfDataSources() { + return Promise.reject("DatabaseServiceNoop::createViewOfDataSources"); + } + /** * Saves the result of the query to the designated location. * May return a value if the location is not a physical location but rather diff --git a/packages/web/src/services/DatabaseServiceWeb.ts b/packages/web/src/services/DatabaseServiceWeb.ts index f84411462..0a5f51809 100644 --- a/packages/web/src/services/DatabaseServiceWeb.ts +++ b/packages/web/src/services/DatabaseServiceWeb.ts @@ -1,11 +1,28 @@ import * as duckdb from "@duckdb/duckdb-wasm"; import { DatabaseService } from "../../../core/services"; +import SQLBuilder from "../../../core/entity/SQLBuilder"; +import Annotation from "../../../core/entity/Annotation"; +import { AnnotationType } from "../../../core/entity/AnnotationFormatter"; export default class DatabaseServiceWeb implements DatabaseService { private database: duckdb.AsyncDuckDB | undefined; private readonly existingDataSources = new Set(); + private static columnTypeToAnnotationType(columnType: string): string { + switch (columnType) { + case "INTEGER": + case "BIGINT": + // TODO: Add support for column types + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 + // return AnnotationType.NUMBER; + case "VARCHAR": + case "TEXT": + default: + return AnnotationType.STRING; + } + } + public async initialize(logLevel: duckdb.LogLevel = duckdb.LogLevel.INFO) { const allBundles = duckdb.getJsDelivrBundles(); @@ -25,6 +42,76 @@ export default class DatabaseServiceWeb implements DatabaseService { URL.revokeObjectURL(worker_url); } + public async createViewOfDataSources(dataSources: string[]): Promise { + const viewName = dataSources.join(", "); + + // Prevent adding the same data source multiple times by shortcutting out here + if (this.existingDataSources.has(viewName)) { + return; + } + + const columnsSoFar = new Set(); + for (const dataSource of dataSources) { + // Fetch information about this data source + const annotationsInDataSource = await this.fetchAnnotations(dataSource); + const columnsInDataSource = annotationsInDataSource.map( + (annotation) => annotation.name + ); + const newColumns = columnsInDataSource.filter((column) => !columnsSoFar.has(column)); + + // If there are no columns / data added yet we need to create the table from + // scratch so we can provide an easy shortcut around the default way of adding + // data to a table + if (columnsSoFar.size === 0) { + await this.execute(`CREATE TABLE "${viewName}" AS FROM "${dataSource}"`); + this.existingDataSources.add(viewName); + } else { + // If adding data to an existing table we will need to add any new columns + if (newColumns.length) { + await this.execute(` + ALTER TABLE "${viewName}" + ADD ${newColumns.map((c) => `"${c}" VARCHAR`).join(", ")} + `); + } + + // After we have added any new columns to the table schema we just need + // to insert the data from the new table to this table replacing any non-existent + // columns with an empty value (null) + const columnsSoFarArr = [...columnsSoFar, ...newColumns]; + await this.execute(` + INSERT INTO "${viewName}" ("${columnsSoFarArr.join('", "')}") + SELECT ${columnsSoFarArr + .map((column) => + columnsInDataSource.includes(column) ? `"${column}"` : "NULL" + ) + .join(", ")} + FROM "${dataSource}" + `); + } + + // Add the new columns from this data source to the existing columns + // to avoid adding duplicate columns + newColumns.forEach((column) => columnsSoFar.add(column)); + } + } + + private async fetchAnnotations(dataSource: string): Promise { + const sql = new SQLBuilder() + .from("information_schema.columns") + .where(`table_name = '${dataSource}'`) + .toSQL(); + const rows = (await this.query(sql)) as any[]; // TODO: so many things to do + return rows.map( + (row) => + new Annotation({ + annotationDisplayName: row["column_name"], + annotationName: row["column_name"], + description: "", + type: DatabaseServiceWeb.columnTypeToAnnotationType(row["data_type"]), + }) + ); + } + public async addDataSource( name: string, type: "csv" | "json" | "parquet", @@ -52,22 +139,17 @@ export default class DatabaseServiceWeb implements DatabaseService { await this.database.registerFileURL(name, uri, protocol, false); } - const connection = await this.database.connect(); - try { - if (type === "parquet") { - await connection.query(`CREATE TABLE "${name}" AS FROM parquet_scan('${name}');`); - } else if (type === "json") { - await connection.query(`CREATE TABLE "${name}" AS FROM read_json_auto('${name}');`); - } else { - // Default to CSV - await connection.query( - `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` - ); - } - this.existingDataSources.add(name); - } finally { - await connection.close(); + if (type === "parquet") { + await this.execute(`CREATE TABLE "${name}" AS FROM parquet_scan('${name}');`); + } else if (type === "json") { + await this.execute(`CREATE TABLE "${name}" AS FROM read_json_auto('${name}');`); + } else { + // Default to CSV + await this.execute( + `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` + ); } + this.existingDataSources.add(name); } /** @@ -116,4 +198,17 @@ export default class DatabaseServiceWeb implements DatabaseService { public async close(): Promise { this.database?.detach(); } + + private async execute(sql: string): Promise { + if (!this.database) { + throw new Error("Database failed to initialize"); + } + + const connection = await this.database.connect(); + try { + await connection.query(sql); + } finally { + await connection.close(); + } + } } From c943ae1db5f136885f3902fabfd4f563bfa953a9 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Thu, 6 Jun 2024 14:46:59 -0700 Subject: [PATCH 07/20] Refactor DatabaseService into abstract class --- .../components/DataSourcePrompt/index.tsx | 3 +- .../components/QueryPart/QueryDataSource.tsx | 22 ++- packages/core/entity/FileExplorerURL/index.ts | 8 +- packages/core/entity/SQLBuilder/index.ts | 12 +- .../core/errors/DataSourcePreparationError.ts | 14 ++ .../DatabaseAnnotationService/index.ts | 38 +--- .../DatabaseService/DatabaseServiceNoop.ts | 14 +- .../core/services/DatabaseService/index.ts | 147 ++++++++++++-- .../FileService/DatabaseFileService/index.ts | 65 +++--- packages/core/state/interaction/logics.ts | 8 +- .../state/interaction/test/logics.test.ts | 13 +- packages/core/state/selection/actions.ts | 38 ---- packages/core/state/selection/logics.ts | 118 +++++------ packages/core/state/selection/reducer.ts | 16 +- .../src/services/DatabaseServiceElectron.ts | 125 +++++++----- .../test/DatabaseServiceElectron.test.ts | 6 +- .../web/src/services/DatabaseServiceWeb.ts | 186 ++++++------------ 17 files changed, 408 insertions(+), 425 deletions(-) create mode 100644 packages/core/errors/DataSourcePreparationError.ts diff --git a/packages/core/components/DataSourcePrompt/index.tsx b/packages/core/components/DataSourcePrompt/index.tsx index 1a771e651..0107a927e 100644 --- a/packages/core/components/DataSourcePrompt/index.tsx +++ b/packages/core/components/DataSourcePrompt/index.tsx @@ -30,6 +30,7 @@ const DATA_SOURCE_DETAILS = [ export default function DataSourcePrompt(props: Props) { const dispatch = useDispatch(); + const selectedDataSources = useSelector(selection.selectors.getSelectedDataSources); const dataSourceInfo = useSelector(interaction.selectors.getDataSourceInfoForVisibleModal); const { source: sourceToReplace, query } = dataSourceInfo || {}; @@ -40,7 +41,7 @@ export default function DataSourcePrompt(props: Props) { if (sourceToReplace) { dispatch(selection.actions.replaceDataSource(source)); } else if (query) { - dispatch(selection.actions.addDataSource(source)); + dispatch(selection.actions.changeDataSources([...selectedDataSources, source])); } else { dispatch( selection.actions.addQuery({ diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index 859d4533b..3ed51bab3 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -44,7 +44,12 @@ export default function QueryDataSource(props: Props) { addButtonIconName="Folder" onDelete={ selectedDataSources.length > 1 - ? (dataSource) => dispatch(selection.actions.removeDataSource(dataSource)) + ? (dataSource) => + dispatch( + selection.actions.changeDataSources( + selectedDataSources.filter((s) => s.name !== dataSource) + ) + ) : undefined } onRenderAddMenuList={ @@ -60,13 +65,13 @@ export default function QueryDataSource(props: Props) { item && ( item.data ? dispatch( - selection.actions.addDataSource( - item.data as Source - ) + selection.actions.changeDataSources([ + ...selectedDataSources, + item.data as Source, + ]) ) : dispatch( interaction.actions.promptForDataSource( @@ -80,8 +85,11 @@ export default function QueryDataSource(props: Props) { item.data && selectedDataSources.length > 1 && dispatch( - selection.actions.removeDataSource( - item.data.name + selection.actions.changeDataSources( + selectedDataSources.filter( + (source) => + source.name !== item.data?.name + ) ) ) } diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index 91459870d..028a390a2 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -75,7 +75,13 @@ export default class FileExplorerURL { params.append("openFolder", JSON.stringify(folder.fileFolder)); }); urlComponents.sources?.map((source) => { - params.append("source", JSON.stringify(source)); + params.append( + "source", + JSON.stringify({ + ...source, + uri: source.uri instanceof String ? source.uri : undefined, + }) + ); }); if (urlComponents.sortColumn) { params.append("sort", JSON.stringify(urlComponents.sortColumn.toJSON())); diff --git a/packages/core/entity/SQLBuilder/index.ts b/packages/core/entity/SQLBuilder/index.ts index 99b234ae5..3a6bb13af 100644 --- a/packages/core/entity/SQLBuilder/index.ts +++ b/packages/core/entity/SQLBuilder/index.ts @@ -7,7 +7,6 @@ export default class SQLBuilder { private isSummarizing = false; private selectStatement = "*"; private fromStatement?: string; - private joinStatements?: string[]; private readonly whereClauses: string[] = []; private orderByClause?: string; private offsetNum?: number; @@ -28,13 +27,7 @@ export default class SQLBuilder { if (!statementAsArray.length) { throw new Error('"FROM" statement requires at least one argument'); } - this.fromStatement = statementAsArray[0]; - this.joinStatements = statementAsArray - .slice(1) - .map( - (table, idx) => - `FULL JOIN "${table}" ON "${table}"."File Path" = "${statementAsArray[idx]}"."File Path"` - ); + this.fromStatement = statementAsArray.sort().join(", "); return this; } @@ -87,8 +80,7 @@ export default class SQLBuilder { return ` ${this.isSummarizing ? "SUMMARIZE" : ""} SELECT ${this.selectStatement} - FROM ${this.fromStatement} - ${this.joinStatements?.length ? this.joinStatements : ""} + FROM "${this.fromStatement}" ${this.whereClauses.length ? `WHERE (${this.whereClauses.join(") AND (")})` : ""} ${this.orderByClause ? `ORDER BY ${this.orderByClause}` : ""} ${this.offsetNum !== undefined ? `OFFSET ${this.offsetNum}` : ""} diff --git a/packages/core/errors/DataSourcePreparationError.ts b/packages/core/errors/DataSourcePreparationError.ts new file mode 100644 index 000000000..86dced748 --- /dev/null +++ b/packages/core/errors/DataSourcePreparationError.ts @@ -0,0 +1,14 @@ +export default class DataSourcePreparationError extends Error { + public sourceName: string; + + constructor(message: string, sourceName: string) { + super(message); + + if (Error.captureStackTrace) { + Error.captureStackTrace(this, DataSourcePreparationError); + } + + this.name = "DataSourcePreparationError"; + this.sourceName = sourceName; + } +} diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index e1c3e8e00..66b494081 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -1,11 +1,8 @@ -import { uniqBy } from "lodash"; - import AnnotationService, { AnnotationValue } from ".."; import DatabaseService from "../../DatabaseService"; import DatabaseServiceNoop from "../../DatabaseService/DatabaseServiceNoop"; import Annotation from "../../../entity/Annotation"; import FileFilter from "../../../entity/FileFilter"; -import { AnnotationType } from "../../../entity/AnnotationFormatter"; import SQLBuilder from "../../../entity/SQLBuilder"; interface Config { @@ -33,38 +30,11 @@ export default class DatabaseAnnotationService implements AnnotationService { this.databaseService = config.databaseService; } - private static columnTypeToAnnotationType(columnType: string): string { - switch (columnType) { - case "INTEGER": - case "BIGINT": - // TODO: Add support for column types - // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 - // return AnnotationType.NUMBER; - case "VARCHAR": - case "TEXT": - default: - return AnnotationType.STRING; - } - } - /** * Fetch all annotations. */ public async fetchAnnotations(): Promise { - const sql = new SQLBuilder() - .from("information_schema.columns") - .where(`table_name IN ('${this.dataSourceNames.join("', '")}')`) - .toSQL(); - const rows = await this.databaseService.query(sql); - return uniqBy(rows, "column_name").map( - (row) => - new Annotation({ - annotationDisplayName: row["column_name"], - annotationName: row["column_name"], - description: "", - type: DatabaseAnnotationService.columnTypeToAnnotationType(row["data_type"]), - }) - ); + return this.databaseService.fetchAnnotations(this.dataSourceNames); } /** @@ -74,7 +44,7 @@ export default class DatabaseAnnotationService implements AnnotationService { const select_key = "select_key"; const sql = new SQLBuilder() .select(`DISTINCT "${annotation}" AS ${select_key}`) - .from(`"${this.dataSourceNames.join(", ")}"`) + .from(this.dataSourceNames) .toSQL(); const rows = await this.databaseService.query(sql); return [ @@ -113,7 +83,7 @@ export default class DatabaseAnnotationService implements AnnotationService { const sqlBuilder = new SQLBuilder() .select(`DISTINCT "${hierarchy[path.length]}"`) - .from(`"${this.dataSourceNames.join(", ")}"`); + .from(this.dataSourceNames); Object.keys(filtersByAnnotation).forEach((annotation) => { const annotationValues = filtersByAnnotation[annotation]; @@ -137,7 +107,7 @@ export default class DatabaseAnnotationService implements AnnotationService { public async fetchAvailableAnnotationsForHierarchy(annotations: string[]): Promise { const sql = new SQLBuilder() .summarize() - .from(`"${this.dataSourceNames.join(", ")}"`) + .from(this.dataSourceNames) .where(annotations.map((annotation) => `"${annotation}" IS NOT NULL`)) .toSQL(); const rows = (await this.databaseService.query(sql)) as SummarizeQueryResult[]; diff --git a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts index 9b900df51..de48feaaa 100644 --- a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts +++ b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts @@ -1,12 +1,12 @@ import DatabaseService from "."; -export default class DatabaseServiceNoop implements DatabaseService { - public addDataSource() { - return Promise.reject("DatabaseServiceNoop:addDataSource"); +export default class DatabaseServiceNoop extends DatabaseService { + public execute(): Promise { + return Promise.reject("DatabaseServiceNoop:execute"); } - public createViewOfDataSources() { - return Promise.reject("DatabaseServiceNoop::createViewOfDataSources"); + public prepareDataSources() { + return Promise.reject("DatabaseServiceNoop::prepareDataSources"); } public saveQuery(): Promise { @@ -16,4 +16,8 @@ export default class DatabaseServiceNoop implements DatabaseService { public query(): Promise<{ [key: string]: string }[]> { return Promise.reject("DatabaseServiceNoop:query"); } + + protected addDataSource(): Promise { + return Promise.reject("DatabaseServiceNoop:addDataSource"); + } } diff --git a/packages/core/services/DatabaseService/index.ts b/packages/core/services/DatabaseService/index.ts index 355fd9efe..4e9275f8e 100644 --- a/packages/core/services/DatabaseService/index.ts +++ b/packages/core/services/DatabaseService/index.ts @@ -1,20 +1,139 @@ +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; +import Annotation from "../../entity/Annotation"; +import { AnnotationType } from "../../entity/AnnotationFormatter"; +import { Source } from "../../entity/FileExplorerURL"; +import SQLBuilder from "../../entity/SQLBuilder"; + /** * Service reponsible for querying against a database */ -export default interface DatabaseService { - addDataSource( - name: string, - type: "csv" | "json" | "parquet", - uri: File | string - ): Promise; - - createViewOfDataSources(dataSources: string[]): Promise; - - saveQuery( - destination: string, - sql: string, - format: "csv" | "parquet" | "json" +export default abstract class DatabaseService { + private currentAggregateSource?: string; + // Initialize with AICS FMS data source name to pretend it always exists + protected readonly existingDataSources = new Set([AICS_FMS_DATA_SOURCE_NAME]); + + public abstract saveQuery( + _destination: string, + _sql: string, + _format: "csv" | "parquet" | "json" ): Promise; - query(sql: string): Promise<{ [key: string]: string }[]>; + public abstract query(_sql: string): Promise<{ [key: string]: string }[]>; + + protected abstract addDataSource(_dataSource: Source): Promise; + + protected abstract execute(_sql: string): Promise; + + private static columnTypeToAnnotationType(columnType: string): string { + switch (columnType) { + case "INTEGER": + case "BIGINT": + // TODO: Add support for column types + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 + // return AnnotationType.NUMBER; + case "VARCHAR": + case "TEXT": + default: + return AnnotationType.STRING; + } + } + + constructor() { + // 'this' scope gets lost when a higher order class (ex. DatabaseService) + // calls a lower level class (ex. DatabaseServiceWeb) + this.addDataSource = this.addDataSource.bind(this); + this.execute = this.execute.bind(this); + this.query = this.query.bind(this); + } + + public async prepareDataSources(dataSources: Source[]): Promise { + await Promise.all(dataSources.map(this.addDataSource)); + + // Because when querying multiple data sources column differences can complicate the queries + // preparing a table ahead of time that is the aggregate of the data sources is most optimal + // should look toward some way of reducing the memory footprint if that becomes an issue + if (dataSources.length > 1) { + await this.aggregateDataSources(dataSources); + } + } + + protected async deleteDataSource(dataSource: string): Promise { + this.existingDataSources.delete(dataSource); + await this.execute(`DROP TABLE IF EXISTS "${dataSource}"`); + } + + private async aggregateDataSources(dataSources: Source[]): Promise { + const viewName = dataSources.sort().join(", "); + + if (this.currentAggregateSource) { + // Prevent adding the same data source multiple times by shortcutting out here + if (this.currentAggregateSource === viewName) { + return; + } + + // Otherwise, if an old aggregate exists, delete it + await this.deleteDataSource(this.currentAggregateSource); + } + + const columnsSoFar = new Set(); + for (const dataSource of dataSources) { + // Fetch information about this data source + const annotationsInDataSource = await this.fetchAnnotations([dataSource.name]); + const columnsInDataSource = annotationsInDataSource.map( + (annotation) => annotation.name + ); + const newColumns = columnsInDataSource.filter((column) => !columnsSoFar.has(column)); + + // If there are no columns / data added yet we need to create the table from + // scratch so we can provide an easy shortcut around the default way of adding + // data to a table + if (columnsSoFar.size === 0) { + await this.execute(`CREATE TABLE "${viewName}" AS SELECT * FROM "${dataSource}"`); + this.currentAggregateSource = viewName; + } else { + // If adding data to an existing table we will need to add any new columns + if (newColumns.length) { + await this.execute(` + ALTER TABLE "${viewName}" + ADD ${newColumns.map((c) => `"${c}" VARCHAR`).join(", ")} + `); + } + + // After we have added any new columns to the table schema we just need + // to insert the data from the new table to this table replacing any non-existent + // columns with an empty value (null) + const columnsSoFarArr = [...columnsSoFar, ...newColumns]; + await this.execute(` + INSERT INTO "${viewName}" ("${columnsSoFarArr.join('", "')}") + SELECT ${columnsSoFarArr + .map((column) => + columnsInDataSource.includes(column) ? `"${column}"` : "NULL" + ) + .join(", ")} + FROM "${dataSource}" + `); + } + + // Add the new columns from this data source to the existing columns + // to avoid adding duplicate columns + newColumns.forEach((column) => columnsSoFar.add(column)); + } + } + + public async fetchAnnotations(dataSourceNames: string[]): Promise { + const sql = new SQLBuilder() + .from('information_schema"."columns') + .where(`table_name = '${dataSourceNames.sort().join("', '")}'`) + .toSQL(); + const rows = await this.query(sql); + return rows.map( + (row) => + new Annotation({ + annotationDisplayName: row["column_name"], + annotationName: row["column_name"], + description: "", + type: DatabaseService.columnTypeToAnnotationType(row["data_type"]), + }) + ); + } } diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index ece1706a3..4fe4d6bad 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -76,7 +76,7 @@ export default class DatabaseFileService implements FileService { const sql = fileSet .toQuerySQLBuilder() .select(`COUNT(*) AS ${select_key}`) - .from(`"${this.dataSourceNames.join(", ")}"`) + .from(this.dataSourceNames) // Remove sort if present .orderBy() .toSQL(); @@ -102,12 +102,9 @@ export default class DatabaseFileService implements FileService { * and potentially starting from a particular file_id and limited to a set number of files. */ public async getFiles(request: GetFilesRequest): Promise { - // TODO: temp - await this.databaseService.createViewOfDataSources(this.dataSourceNames); - const sql = request.fileSet .toQuerySQLBuilder() - .from(`"${this.dataSourceNames.join(", ")}"`) + .from(this.dataSourceNames) .offset(request.from * request.limit) .limit(request.limit) .toSQL(); @@ -131,39 +128,31 @@ export default class DatabaseFileService implements FileService { ): Promise { const sqlBuilder = new SQLBuilder() .select(annotations.map((annotation) => `"${annotation}"`).join(", ")) - .from(`"${this.dataSourceNames[0]}"`); + .from(this.dataSourceNames); selections.forEach((selection) => { selection.indexRanges.forEach((indexRange) => { - const subQuerySql = this.dataSourceNames - .map((dataSource) => { - const subQuery = new SQLBuilder() - .select("File Path") - .from(`"${dataSource}"`) - .whereOr( - Object.entries(selection.filters).map(([column, values]) => { - const commaSeperatedValues = values - .map((v) => `'${v}'`) - .join(", "); - return `"${column}" IN (${commaSeperatedValues}}`; - }) - ) - .offset(indexRange.start) - .limit(indexRange.end - indexRange.start + 1); - - if (selection.sort) { - subQuery.orderBy( - `"${selection.sort.annotationName}" ${ - selection.sort.ascending ? "ASC" : "DESC" - }` - ); - } - - return subQuery.toSQL(); - }) - .join(" UNION "); - - sqlBuilder.whereOr(`"File Path" IN (${subQuerySql})`); + const subQuery = new SQLBuilder() + .select("File Path") + .from(this.dataSourceNames) + .whereOr( + Object.entries(selection.filters).map(([column, values]) => { + const commaSeperatedValues = values.map((v) => `'${v}'`).join(", "); + return `"${column}" IN (${commaSeperatedValues})`; + }) + ) + .offset(indexRange.start) + .limit(indexRange.end - indexRange.start + 1); + + if (selection.sort) { + subQuery.orderBy( + `"${selection.sort.annotationName}" ${ + selection.sort.ascending ? "ASC" : "DESC" + }` + ); + } + + sqlBuilder.whereOr(`"File Path" IN (${subQuery.toSQL()})`); }); }); @@ -184,11 +173,7 @@ export default class DatabaseFileService implements FileService { }; } - const buffer = await this.databaseService.saveQuery( - uniqueId(), - `SELECT * FROM "${this.dataSourceNames[0]}" LIMIT 2`, - format - ); + const buffer = await this.databaseService.saveQuery(uniqueId(), sqlBuilder.toSQL(), format); const name = `file-selection-${new Date()}.${format}`; return this.downloadService.download( { diff --git a/packages/core/state/interaction/logics.ts b/packages/core/state/interaction/logics.ts index f406efb47..0954f8e1a 100644 --- a/packages/core/state/interaction/logics.ts +++ b/packages/core/state/interaction/logics.ts @@ -476,12 +476,12 @@ const showContextMenu = createLogic({ */ const refresh = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { - try { - const { getState } = deps; - const annotationService = interactionSelectors.getAnnotationService(getState()); + const { getState } = deps; + const hierarchy = selection.selectors.getAnnotationHierarchy(getState()); + const annotationService = interactionSelectors.getAnnotationService(getState()); + try { // Refresh list of annotations & which annotations are available - const hierarchy = selection.selectors.getAnnotationHierarchy(getState()); const [annotations, availableAnnotations] = await Promise.all([ annotationService.fetchAnnotations(), annotationService.fetchAvailableAnnotationsForHierarchy(hierarchy), diff --git a/packages/core/state/interaction/test/logics.test.ts b/packages/core/state/interaction/test/logics.test.ts index cc3729737..17dd32195 100644 --- a/packages/core/state/interaction/test/logics.test.ts +++ b/packages/core/state/interaction/test/logics.test.ts @@ -38,12 +38,12 @@ import FileDownloadService, { } from "../../../services/FileDownloadService"; import FileViewerService from "../../../services/FileViewerService"; import { annotationsJson } from "../../../entity/Annotation/mocks"; -import { DatabaseService } from "../../../services"; import FileDownloadServiceNoop from "../../../services/FileDownloadService/FileDownloadServiceNoop"; import NotificationServiceNoop from "../../../services/NotificationService/NotificationServiceNoop"; import HttpFileService from "../../../services/FileService/HttpFileService"; import HttpAnnotationService from "../../../services/AnnotationService/HttpAnnotationService"; import FileDetail, { FmsFile } from "../../../entity/FileDetail"; +import DatabaseServiceNoop from "../../../services/DatabaseService/DatabaseServiceNoop"; describe("Interaction logics", () => { const fileSelection = new FileSelection().select({ @@ -58,19 +58,10 @@ describe("Interaction logics", () => { } } - class MockDatabaseService implements DatabaseService { + class MockDatabaseService extends DatabaseServiceNoop { saveQuery() { return Promise.resolve(new Uint8Array()); } - addDataSource() { - return Promise.reject("addDataSource mock"); - } - createViewOfDataSources(): Promise { - return Promise.reject("createViewOfDataSources mock"); - } - query() { - return Promise.reject("query mock"); - } } describe("downloadManifest", () => { diff --git a/packages/core/state/selection/actions.ts b/packages/core/state/selection/actions.ts index 50c9b9834..8aa036180 100644 --- a/packages/core/state/selection/actions.ts +++ b/packages/core/state/selection/actions.ts @@ -16,44 +16,6 @@ import { const STATE_BRANCH_NAME = "selection"; -/** - * ADD_DATA_SOURCE - * - * Intention is to add a data source to the current query - */ -export const ADD_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "add-data-source"); - -export interface AddDataSource { - payload: Source; - type: string; -} - -export function addDataSource(source: Source): AddDataSource { - return { - payload: source, - type: ADD_DATA_SOURCE, - }; -} - -/** - * REMOVE_DATA_SOURCE - * - * Intention is to remove a data source from the current query - */ -export const REMOVE_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "remove-data-source"); - -export interface RemoveDataSource { - payload: string; - type: string; -} - -export function removeDataSource(sourceName: string): RemoveDataSource { - return { - payload: sourceName, - type: REMOVE_DATA_SOURCE, - }; -} - /** * SET_FILE_FILTERS * diff --git a/packages/core/state/selection/logics.ts b/packages/core/state/selection/logics.ts index dabeabfdb..c69076634 100644 --- a/packages/core/state/selection/logics.ts +++ b/packages/core/state/selection/logics.ts @@ -36,8 +36,6 @@ import { changeDataSources, ChangeDataSourcesAction, CHANGE_DATA_SOURCES, - ADD_DATA_SOURCE, - AddDataSource, } from "./actions"; import { interaction, metadata, ReduxLogicDeps, selection } from "../"; import * as selectionSelectors from "./selectors"; @@ -49,6 +47,7 @@ import FileSelection from "../../entity/FileSelection"; import FileSet from "../../entity/FileSet"; import HttpAnnotationService from "../../services/AnnotationService/HttpAnnotationService"; import { DataSource } from "../../services/DataSourceService"; +import DataSourcePreparationError from "../../errors/DataSourcePreparationError"; /** * Interceptor responsible for transforming payload of SELECT_FILE actions to account for whether the intention is to @@ -435,9 +434,13 @@ const selectNearbyFile = createLogic({ * a refresh action so that the resources pertain to the current data source */ const changeDataSourceLogic = createLogic({ + type: CHANGE_DATA_SOURCES, async process(deps: ReduxLogicDeps, dispatch, done) { const { payload: selectedDataSources } = deps.action as ChangeDataSourcesAction; const dataSources = interaction.selectors.getAllDataSources(deps.getState()); + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); const newSelectedDataSources: DataSource[] = []; const existingSelectedDataSources: DataSource[] = []; @@ -457,10 +460,26 @@ const changeDataSourceLogic = createLogic({ ); } + // Prepare the data sources ahead of querying against them below + try { + await databaseService.prepareDataSources(selectedDataSources); + } catch (err) { + const errMsg = `Error encountered while preparing data sources (Full error: ${ + (err as Error).message + })`; + console.error(errMsg); + if (err instanceof DataSourcePreparationError) { + dispatch( + interaction.actions.promptForDataSource({ source: { name: err.sourceName } }) + ); + } else { + alert(errMsg); + } + } + dispatch(interaction.actions.refresh() as AnyAction); done(); }, - type: CHANGE_DATA_SOURCES, }); /** @@ -468,6 +487,28 @@ const changeDataSourceLogic = createLogic({ */ const addQueryLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { + const { payload: newQuery } = deps.action as AddQuery; + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); + + // Prepare the data sources ahead of querying against them below + try { + await databaseService.prepareDataSources(newQuery.parts.sources); + } catch (err) { + const errMsg = `Error encountered while preparing data sources (Full error: ${ + (err as Error).message + })`; + console.error(errMsg); + if (err instanceof DataSourcePreparationError) { + dispatch( + interaction.actions.promptForDataSource({ source: { name: err.sourceName } }) + ); + } else { + alert(errMsg); + } + } + dispatch(changeQuery(deps.action.payload)); done(); }, @@ -502,9 +543,6 @@ const addQueryLogic = createLogic({ const changeQueryLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const { payload: newlySelectedQuery } = deps.action as ChangeQuery; - const { databaseService } = interaction.selectors.getPlatformDependentServices( - deps.getState() - ); const currentQueries = selectionSelectors.getQueries(deps.getState()); const currentQueryParts = selectionSelectors.getCurrentQueryParts(deps.getState()); const updatedQueries = currentQueries.map((query) => ({ @@ -515,22 +553,6 @@ const changeQueryLogic = createLogic({ : query.parts, })); - await Promise.all( - newlySelectedQuery.parts.sources.map(async (source) => { - if (source.uri && source.type) { - try { - await databaseService.addDataSource(source.name, source.type, source.uri); - } catch (error) { - console.error( - "Failed to add data source, prompting for replacement", - error - ); - dispatch(interaction.actions.promptForDataSource({ source })); - } - } - }) - ); - dispatch( decodeFileExplorerURL(FileExplorerURL.encode(newlySelectedQuery.parts)) as AnyAction ); @@ -553,51 +575,30 @@ const removeQueryLogic = createLogic({ }, }); -const addDataSourceLogic = createLogic({ - type: ADD_DATA_SOURCE, - async process(deps: ReduxLogicDeps, dispatch, done) { - const { payload: source } = deps.action as AddDataSource; - const { databaseService } = interaction.selectors.getPlatformDependentServices( - deps.getState() - ); - - if (source.uri && source.type) { - try { - await databaseService.addDataSource(source.name, source.type, source.uri); - dispatch(interaction.actions.refresh() as AnyAction); - } catch (error) { - console.error("Failed to add data source, prompting for replacement", error); - dispatch(interaction.actions.promptForDataSource({ source })); - } - } - done(); - }, -}); - const replaceDataSourceLogic = createLogic({ type: REPLACE_DATA_SOURCE, async process(deps: ReduxLogicDeps, dispatch, done) { - const { - payload: { name, type, uri }, - } = deps.ctx.replaceDataSourceAction as ReplaceDataSource; + const { payload: replacementSource } = deps.ctx + .replaceDataSourceAction as ReplaceDataSource; const { databaseService } = interaction.selectors.getPlatformDependentServices( deps.getState() ); + // Prepare the data sources ahead of querying against them below try { - if (uri && type) { - await databaseService.addDataSource(name, type, uri); + await databaseService.prepareDataSources([replacementSource]); + } catch (err) { + const errMsg = `Error encountered while replacing data sources (Full error: ${ + (err as Error).message + })`; + console.error(errMsg); + if (err instanceof DataSourcePreparationError) { + dispatch( + interaction.actions.promptForDataSource({ source: { name: err.sourceName } }) + ); + } else { + alert(errMsg); } - } catch (error) { - console.error("Failed to add data source, prompting for replacement", error); - dispatch( - interaction.actions.promptForDataSource({ - source: { - name, - uri, - }, - }) - ); } dispatch(interaction.actions.refresh() as AnyAction); @@ -626,7 +627,6 @@ const replaceDataSourceLogic = createLogic({ export default [ selectFile, - addDataSourceLogic, modifyAnnotationHierarchy, modifyFileFilters, toggleFileFolderCollapse, diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index e12f4845d..0c4fd2375 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -33,10 +33,6 @@ import { REMOVE_QUERY, RemoveQuery, ChangeDataSourcesAction, - ADD_DATA_SOURCE, - AddDataSource, - RemoveDataSource, - REMOVE_DATA_SOURCE, } from "./actions"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; @@ -137,9 +133,7 @@ export default makeReducer( }, [CHANGE_DATA_SOURCES]: (state, action: ChangeDataSourcesAction) => ({ ...state, - annotationHierarchy: [], - dataSources: action.payload, - filters: [], + dataSources: uniqBy(action.payload, "name"), fileSelection: new FileSelection(), openFileFolders: [], }), @@ -147,14 +141,6 @@ export default makeReducer( ...state, queries: [action.payload, ...state.queries], }), - [ADD_DATA_SOURCE]: (state, action: AddDataSource) => ({ - ...state, - dataSources: uniqBy([...state.dataSources, action.payload], "name"), - }), - [REMOVE_DATA_SOURCE]: (state, action: RemoveDataSource) => ({ - ...state, - dataSources: state.dataSources.filter((source) => source.name !== action.payload), - }), [CHANGE_QUERY]: (state, action: ChangeQuery) => ({ ...state, selectedQuery: action.payload.name, diff --git a/packages/desktop/src/services/DatabaseServiceElectron.ts b/packages/desktop/src/services/DatabaseServiceElectron.ts index 21b3718a4..604671c7f 100644 --- a/packages/desktop/src/services/DatabaseServiceElectron.ts +++ b/packages/desktop/src/services/DatabaseServiceElectron.ts @@ -5,26 +5,85 @@ import * as path from "path"; import duckdb from "duckdb"; import { DatabaseService } from "../../../core/services"; +import { Source } from "../../../core/entity/FileExplorerURL"; +import DataSourcePreparationError from "../../../core/errors/DataSourcePreparationError"; -export default class DatabaseServiceElectron implements DatabaseService { +export default class DatabaseServiceElectron extends DatabaseService { private database: duckdb.Database; - private readonly existingDataSources = new Set(); constructor() { + super(); this.database = new duckdb.Database(":memory:"); } - public async addDataSource( - name: string, - type: "csv" | "json" | "parquet", - uri: File | string - ): Promise { + /** + * Saves the result of the query to the designated location. + * May return a value if the location is not a physical location but rather + * a temporary database location (buffer) + */ + public saveQuery(destination: string, sql: string, format: string): Promise { + return new Promise((resolve, reject) => { + this.database.run( + `COPY (${sql}) TO '${destination}.${format}' (FORMAT '${format}');`, + (err: any, result: any) => { + if (err) { + reject(err.message); + } else { + resolve(result); + } + } + ); + }); + } + + public query(sql: string): Promise { + return new Promise((resolve, reject) => { + try { + this.database.all(sql, (err: any, tableData: any) => { + if (err) { + reject(err.message); + } else { + resolve(tableData); + } + }); + } catch (error) { + return Promise.reject(`${error}`); + } + }); + } + + public async reset(): Promise { + await this.close(); + this.database = new duckdb.Database(":memory:"); + } + + public close(): Promise { + return new Promise((resolve, reject) => { + this.database.close((err) => { + if (err) { + reject(err.message); + } else { + resolve(); + } + }); + }); + } + + protected async addDataSource(dataSource: Source): Promise { + const { name, type, uri } = dataSource; if (this.existingDataSources.has(name)) { return; // no-op } + if (!type || !uri) { + throw new DataSourcePreparationError( + "Data source type and URI are missing", + dataSource.name + ); + } let source: string; let tempLocation; + this.existingDataSources.add(name); try { if (typeof uri === "string") { source = uri; @@ -71,8 +130,9 @@ export default class DatabaseServiceElectron implements DatabaseService { ); } }); - - this.existingDataSources.add(name); + } catch (err) { + await this.deleteDataSource(name); + throw new DataSourcePreparationError((err as Error).message, name); } finally { if (tempLocation) { await fs.promises.unlink(tempLocation); @@ -80,38 +140,14 @@ export default class DatabaseServiceElectron implements DatabaseService { } } - public createViewOfDataSources() { - return Promise.reject("DatabaseServiceNoop::createViewOfDataSources"); - } - - /** - * Saves the result of the query to the designated location. - * May return a value if the location is not a physical location but rather - * a temporary database location (buffer) - */ - public saveQuery(destination: string, sql: string, format: string): Promise { - return new Promise((resolve, reject) => { - this.database.run( - `COPY (${sql}) TO '${destination}.${format}' (FORMAT '${format}');`, - (err: any, result: any) => { - if (err) { - reject(err.message); - } else { - resolve(result); - } - } - ); - }); - } - - public query(sql: string): Promise { + protected async execute(sql: string): Promise { return new Promise((resolve, reject) => { try { - this.database.all(sql, (err: any, tableData: any) => { + this.database.exec(sql, (err: any) => { if (err) { reject(err.message); } else { - resolve(tableData); + resolve(); } }); } catch (error) { @@ -119,21 +155,4 @@ export default class DatabaseServiceElectron implements DatabaseService { } }); } - - public async reset(): Promise { - await this.close(); - this.database = new duckdb.Database(":memory:"); - } - - public close(): Promise { - return new Promise((resolve, reject) => { - this.database.close((err) => { - if (err) { - reject(err.message); - } else { - resolve(); - } - }); - }); - } } diff --git a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts index b29aaba91..c739aca26 100644 --- a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts +++ b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts @@ -23,7 +23,7 @@ describe("DatabaseServiceElectron", () => { await service.close(); }); - describe("addDataSource", () => { + describe("prepareDataSources", () => { it("creates table from file of type csv", async () => { // Arrange const tempFileName = "test.csv"; @@ -31,7 +31,7 @@ describe("DatabaseServiceElectron", () => { await fs.promises.writeFile(tempFile, "color\nblue\ngreen\norange"); // Act - await service.addDataSource(tempFileName, "csv", tempFile); + await service.prepareDataSources([{ name: tempFileName, type: "csv", uri: tempFile }]); // Assert const result = await service.query(`SELECT * FROM "${tempFileName}"`); @@ -48,7 +48,7 @@ describe("DatabaseServiceElectron", () => { ); // Act - await service.addDataSource(tempFileName, "json", tempFile); + await service.prepareDataSources([{ name: tempFileName, type: "json", uri: tempFile }]); // Assert const result = await service.query(`SELECT * FROM "${tempFileName}"`); diff --git a/packages/web/src/services/DatabaseServiceWeb.ts b/packages/web/src/services/DatabaseServiceWeb.ts index 0a5f51809..b4b9d5e74 100644 --- a/packages/web/src/services/DatabaseServiceWeb.ts +++ b/packages/web/src/services/DatabaseServiceWeb.ts @@ -1,27 +1,11 @@ import * as duckdb from "@duckdb/duckdb-wasm"; import { DatabaseService } from "../../../core/services"; -import SQLBuilder from "../../../core/entity/SQLBuilder"; -import Annotation from "../../../core/entity/Annotation"; -import { AnnotationType } from "../../../core/entity/AnnotationFormatter"; +import DataSourcePreparationError from "../../../core/errors/DataSourcePreparationError"; +import { Source } from "../../../core/entity/FileExplorerURL"; -export default class DatabaseServiceWeb implements DatabaseService { +export default class DatabaseServiceWeb extends DatabaseService { private database: duckdb.AsyncDuckDB | undefined; - private readonly existingDataSources = new Set(); - - private static columnTypeToAnnotationType(columnType: string): string { - switch (columnType) { - case "INTEGER": - case "BIGINT": - // TODO: Add support for column types - // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 - // return AnnotationType.NUMBER; - case "VARCHAR": - case "TEXT": - default: - return AnnotationType.STRING; - } - } public async initialize(logLevel: duckdb.LogLevel = duckdb.LogLevel.INFO) { const allBundles = duckdb.getJsDelivrBundles(); @@ -42,116 +26,6 @@ export default class DatabaseServiceWeb implements DatabaseService { URL.revokeObjectURL(worker_url); } - public async createViewOfDataSources(dataSources: string[]): Promise { - const viewName = dataSources.join(", "); - - // Prevent adding the same data source multiple times by shortcutting out here - if (this.existingDataSources.has(viewName)) { - return; - } - - const columnsSoFar = new Set(); - for (const dataSource of dataSources) { - // Fetch information about this data source - const annotationsInDataSource = await this.fetchAnnotations(dataSource); - const columnsInDataSource = annotationsInDataSource.map( - (annotation) => annotation.name - ); - const newColumns = columnsInDataSource.filter((column) => !columnsSoFar.has(column)); - - // If there are no columns / data added yet we need to create the table from - // scratch so we can provide an easy shortcut around the default way of adding - // data to a table - if (columnsSoFar.size === 0) { - await this.execute(`CREATE TABLE "${viewName}" AS FROM "${dataSource}"`); - this.existingDataSources.add(viewName); - } else { - // If adding data to an existing table we will need to add any new columns - if (newColumns.length) { - await this.execute(` - ALTER TABLE "${viewName}" - ADD ${newColumns.map((c) => `"${c}" VARCHAR`).join(", ")} - `); - } - - // After we have added any new columns to the table schema we just need - // to insert the data from the new table to this table replacing any non-existent - // columns with an empty value (null) - const columnsSoFarArr = [...columnsSoFar, ...newColumns]; - await this.execute(` - INSERT INTO "${viewName}" ("${columnsSoFarArr.join('", "')}") - SELECT ${columnsSoFarArr - .map((column) => - columnsInDataSource.includes(column) ? `"${column}"` : "NULL" - ) - .join(", ")} - FROM "${dataSource}" - `); - } - - // Add the new columns from this data source to the existing columns - // to avoid adding duplicate columns - newColumns.forEach((column) => columnsSoFar.add(column)); - } - } - - private async fetchAnnotations(dataSource: string): Promise { - const sql = new SQLBuilder() - .from("information_schema.columns") - .where(`table_name = '${dataSource}'`) - .toSQL(); - const rows = (await this.query(sql)) as any[]; // TODO: so many things to do - return rows.map( - (row) => - new Annotation({ - annotationDisplayName: row["column_name"], - annotationName: row["column_name"], - description: "", - type: DatabaseServiceWeb.columnTypeToAnnotationType(row["data_type"]), - }) - ); - } - - public async addDataSource( - name: string, - type: "csv" | "json" | "parquet", - uri: File | string - ): Promise { - if (!this.database) { - throw new Error("Database failed to initialize"); - } - if (this.existingDataSources.has(name)) { - return; - } - - if (uri instanceof File) { - await this.database.registerFileHandle( - name, - uri, - duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, - true - ); - } else { - const protocol = uri.startsWith("s3") - ? duckdb.DuckDBDataProtocol.S3 - : duckdb.DuckDBDataProtocol.HTTP; - - await this.database.registerFileURL(name, uri, protocol, false); - } - - if (type === "parquet") { - await this.execute(`CREATE TABLE "${name}" AS FROM parquet_scan('${name}');`); - } else if (type === "json") { - await this.execute(`CREATE TABLE "${name}" AS FROM read_json_auto('${name}');`); - } else { - // Default to CSV - await this.execute( - `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` - ); - } - this.existingDataSources.add(name); - } - /** * Saves the result of the query to the designated location. * Returns an array representating the data from the query in the format designated @@ -199,7 +73,59 @@ export default class DatabaseServiceWeb implements DatabaseService { this.database?.detach(); } - private async execute(sql: string): Promise { + public async addDataSource(dataSource: Source): Promise { + const { name, type, uri } = dataSource; + if (!this.database) { + throw new Error("Database failed to initialize"); + } + if (this.existingDataSources.has(name)) { + return; + } + if (!type || !uri) { + throw new DataSourcePreparationError( + "Data source type and URI are missing", + dataSource.name + ); + } + + this.existingDataSources.add(name); + try { + if (uri instanceof File) { + await this.database.registerFileHandle( + name, + uri, + duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, + true + ); + } else if ((uri as any) instanceof String) { + const protocol = uri.startsWith("s3") + ? duckdb.DuckDBDataProtocol.S3 + : duckdb.DuckDBDataProtocol.HTTP; + + await this.database.registerFileURL(name, uri, protocol, false); + } else { + throw new Error( + `URI is of unexpected type, should be File instance or String: ${uri}` + ); + } + + if (type === "parquet") { + await this.execute(`CREATE TABLE "${name}" AS FROM parquet_scan('${name}');`); + } else if (type === "json") { + await this.execute(`CREATE TABLE "${name}" AS FROM read_json_auto('${name}');`); + } else { + // Default to CSV + await this.execute( + `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` + ); + } + } catch (err) { + await this.deleteDataSource(name); + throw new DataSourcePreparationError((err as Error).message, name); + } + } + + protected async execute(sql: string): Promise { if (!this.database) { throw new Error("Database failed to initialize"); } From d2adee6dd555201d6663a51fba95ceeecf9d0401 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Thu, 6 Jun 2024 14:57:28 -0700 Subject: [PATCH 08/20] Update tests --- .../test/DatabaseAnnotationService.test.ts | 23 ------------- .../test/DatabaseService.test.ts | 34 +++++++++++++++++++ packages/core/state/interaction/logics.ts | 8 ++--- .../core/state/selection/test/logics.test.ts | 1 - .../core/state/selection/test/reducer.test.ts | 4 +-- 5 files changed, 39 insertions(+), 31 deletions(-) create mode 100644 packages/core/services/DatabaseService/test/DatabaseService.test.ts diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts index 95a2fdcc7..2a2c2e0e4 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/test/DatabaseAnnotationService.test.ts @@ -1,34 +1,11 @@ import { expect } from "chai"; -import Annotation from "../../../../entity/Annotation"; import FileFilter from "../../../../entity/FileFilter"; import DatabaseServiceNoop from "../../../DatabaseService/DatabaseServiceNoop"; import DatabaseAnnotationService from ".."; describe("DatabaseAnnotationService", () => { - describe("fetchAnnotations", () => { - const annotations = ["A", "B", "Cc", "dD"].map((name) => ({ - name, - })); - class MockDatabaseService extends DatabaseServiceNoop { - public query(): Promise<{ [key: string]: string }[]> { - return Promise.resolve(annotations); - } - } - const databaseService = new MockDatabaseService(); - - it("issues request for all available Annotations", async () => { - const annotationService = new DatabaseAnnotationService({ - dataSourceNames: [], - databaseService, - }); - const actualAnnotations = await annotationService.fetchAnnotations(); - expect(actualAnnotations.length).to.equal(1); - expect(actualAnnotations[0]).to.be.instanceOf(Annotation); - }); - }); - describe("fetchAnnotationValues", () => { const annotations = ["A", "B", "Cc", "dD"].map((name, index) => ({ select_key: name.toLowerCase() + index, diff --git a/packages/core/services/DatabaseService/test/DatabaseService.test.ts b/packages/core/services/DatabaseService/test/DatabaseService.test.ts new file mode 100644 index 000000000..145bda4ba --- /dev/null +++ b/packages/core/services/DatabaseService/test/DatabaseService.test.ts @@ -0,0 +1,34 @@ +import { expect } from "chai"; + +import DatabaseService from ".."; + +describe("DatabaseService", () => { + describe("fetchAnnotations", () => { + const annotations = ["A", "B", "Cc", "dD"].map((name) => ({ + name, + })); + + // DatabaseService is abstract so we need a dummy impl to test + // implemented methods + class DatabaseServiceDummyImpl extends DatabaseService { + saveQuery(): Promise { + throw new Error("Not implemented in dummy impl"); + } + execute(): Promise { + throw new Error("Not implemented in dummy impl"); + } + addDataSource(): Promise { + throw new Error("Not implemented in dummy impl"); + } + query(): Promise { + return Promise.resolve(annotations); + } + } + + it("issues request for all available Annotations", async () => { + const service = new DatabaseServiceDummyImpl(); + const actualAnnotations = await service.fetchAnnotations(["foo"]); + expect(actualAnnotations.length).to.equal(4); + }); + }); +}); diff --git a/packages/core/state/interaction/logics.ts b/packages/core/state/interaction/logics.ts index 0954f8e1a..869472f10 100644 --- a/packages/core/state/interaction/logics.ts +++ b/packages/core/state/interaction/logics.ts @@ -476,11 +476,11 @@ const showContextMenu = createLogic({ */ const refresh = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { - const { getState } = deps; - const hierarchy = selection.selectors.getAnnotationHierarchy(getState()); - const annotationService = interactionSelectors.getAnnotationService(getState()); - try { + const { getState } = deps; + const hierarchy = selection.selectors.getAnnotationHierarchy(getState()); + const annotationService = interactionSelectors.getAnnotationService(getState()); + // Refresh list of annotations & which annotations are available const [annotations, availableAnnotations] = await Promise.all([ annotationService.fetchAnnotations(), diff --git a/packages/core/state/selection/test/logics.test.ts b/packages/core/state/selection/test/logics.test.ts index 8f3665559..31e120fae 100644 --- a/packages/core/state/selection/test/logics.test.ts +++ b/packages/core/state/selection/test/logics.test.ts @@ -931,7 +931,6 @@ describe("Selection logics", () => { name: "Test Data Source", version: 1, type: "csv", - uri: "", }, ]; diff --git a/packages/core/state/selection/test/reducer.test.ts b/packages/core/state/selection/test/reducer.test.ts index af6eaf9bb..13baa423f 100644 --- a/packages/core/state/selection/test/reducer.test.ts +++ b/packages/core/state/selection/test/reducer.test.ts @@ -46,7 +46,7 @@ describe("Selection reducer", () => { ); describe(selection.actions.CHANGE_DATA_SOURCES, () => { - it("clears hierarchy, filters, file selection, and open folders", () => { + it("clears file selection and open folders", () => { // Arrange const state = { ...selection.initialState, @@ -76,10 +76,8 @@ describe("Selection reducer", () => { ); // Assert - expect(actual.annotationHierarchy).to.be.empty; expect(actual.dataSources).to.deep.equal(dataSources); expect(actual.fileSelection.count()).to.equal(0); - expect(actual.filters).to.be.empty; expect(actual.openFileFolders).to.be.empty; }); }); From f925ce8d5964e92ed7ec674010b91a5a107579db Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Fri, 7 Jun 2024 10:16:38 -0700 Subject: [PATCH 09/20] Address PR Feedback --- .../components/QueryPart/QueryDataSource.tsx | 1 + .../core/services/DatabaseService/index.ts | 61 ++++++++++++------- .../FileService/DatabaseFileService/index.ts | 18 ++++-- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index 3ed51bab3..11339d632 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -42,6 +42,7 @@ export default function QueryDataSource(props: Props) { 1 ? (dataSource) => diff --git a/packages/core/services/DatabaseService/index.ts b/packages/core/services/DatabaseService/index.ts index 4e9275f8e..d730c20c9 100644 --- a/packages/core/services/DatabaseService/index.ts +++ b/packages/core/services/DatabaseService/index.ts @@ -11,6 +11,7 @@ export default abstract class DatabaseService { private currentAggregateSource?: string; // Initialize with AICS FMS data source name to pretend it always exists protected readonly existingDataSources = new Set([AICS_FMS_DATA_SOURCE_NAME]); + private readonly dataSourceToAnnotationsMap: Map = new Map(); public abstract saveQuery( _destination: string, @@ -59,11 +60,15 @@ export default abstract class DatabaseService { protected async deleteDataSource(dataSource: string): Promise { this.existingDataSources.delete(dataSource); + this.dataSourceToAnnotationsMap.delete(dataSource); await this.execute(`DROP TABLE IF EXISTS "${dataSource}"`); } private async aggregateDataSources(dataSources: Source[]): Promise { - const viewName = dataSources.sort().join(", "); + const viewName = dataSources + .map((source) => source.name) + .sort() + .join(", "); if (this.currentAggregateSource) { // Prevent adding the same data source multiple times by shortcutting out here @@ -88,15 +93,19 @@ export default abstract class DatabaseService { // scratch so we can provide an easy shortcut around the default way of adding // data to a table if (columnsSoFar.size === 0) { - await this.execute(`CREATE TABLE "${viewName}" AS SELECT * FROM "${dataSource}"`); + await this.execute( + `CREATE TABLE "${viewName}" AS SELECT *, '${dataSource.name}' AS "Data source" FROM "${dataSource.name}"` + ); this.currentAggregateSource = viewName; } else { // If adding data to an existing table we will need to add any new columns + // unsure why but seemingly unable to add multiple columns in one alter table + // statement so we will need to loop through and add them one by one if (newColumns.length) { - await this.execute(` - ALTER TABLE "${viewName}" - ADD ${newColumns.map((c) => `"${c}" VARCHAR`).join(", ")} - `); + const alterTableSQL = newColumns + .map((column) => `ALTER TABLE "${viewName}" ADD COLUMN "${column}" VARCHAR`) + .join("; "); + await this.execute(alterTableSQL); } // After we have added any new columns to the table schema we just need @@ -104,13 +113,13 @@ export default abstract class DatabaseService { // columns with an empty value (null) const columnsSoFarArr = [...columnsSoFar, ...newColumns]; await this.execute(` - INSERT INTO "${viewName}" ("${columnsSoFarArr.join('", "')}") + INSERT INTO "${viewName}" ("${columnsSoFarArr.join('", "')}", "Data source") SELECT ${columnsSoFarArr .map((column) => columnsInDataSource.includes(column) ? `"${column}"` : "NULL" ) - .join(", ")} - FROM "${dataSource}" + .join(", ")}, '${dataSource.name}' AS "Data source" + FROM "${dataSource.name}" `); } @@ -121,19 +130,25 @@ export default abstract class DatabaseService { } public async fetchAnnotations(dataSourceNames: string[]): Promise { - const sql = new SQLBuilder() - .from('information_schema"."columns') - .where(`table_name = '${dataSourceNames.sort().join("', '")}'`) - .toSQL(); - const rows = await this.query(sql); - return rows.map( - (row) => - new Annotation({ - annotationDisplayName: row["column_name"], - annotationName: row["column_name"], - description: "", - type: DatabaseService.columnTypeToAnnotationType(row["data_type"]), - }) - ); + const aggregateDataSourceName = dataSourceNames.sort().join(", "); + if (!this.dataSourceToAnnotationsMap.has(aggregateDataSourceName)) { + const sql = new SQLBuilder() + .from('information_schema"."columns') + .where(`table_name = '${aggregateDataSourceName}'`) + .toSQL(); + const rows = await this.query(sql); + const annotations = rows.map( + (row) => + new Annotation({ + annotationDisplayName: row["column_name"], + annotationName: row["column_name"], + description: "", + type: DatabaseService.columnTypeToAnnotationType(row["data_type"]), + }) + ); + this.dataSourceToAnnotationsMap.set(aggregateDataSourceName, annotations); + } + + return this.dataSourceToAnnotationsMap.get(aggregateDataSourceName) || []; } } diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index 4fe4d6bad..07abde0d4 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -51,10 +51,18 @@ export default class DatabaseFileService implements FileService { return new FileDetail({ annotations: [ ...annotations, - ...Object.entries(omit(row, ...annotations.keys())).map(([name, values]: any) => ({ - name, - values: `${values}`.split(",").map((value: string) => value.trim()), - })), + ...Object.entries(omit(row, ...annotations.keys())).flatMap(([name, values]: any) => + values !== null + ? [ + { + name, + values: `${values}` + .split(",") + .map((value: string) => value.trim()), + }, + ] + : [] + ), ], }); } @@ -133,7 +141,7 @@ export default class DatabaseFileService implements FileService { selections.forEach((selection) => { selection.indexRanges.forEach((indexRange) => { const subQuery = new SQLBuilder() - .select("File Path") + .select('"File Path"') .from(this.dataSourceNames) .whereOr( Object.entries(selection.filters).map(([column, values]) => { From f3b9922ef1f619edf768c7f053ada12312d13807 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Fri, 7 Jun 2024 10:40:34 -0700 Subject: [PATCH 10/20] Update from main --- .github/workflows/aws-deployment.yml | 98 +++++++++++++ LICENSE | 28 ++++ LICENSE.txt | 33 ----- .../components/AnnotationPicker/index.tsx | 54 +++++-- .../components/ListPicker/ListRow.module.css | 7 + .../core/components/ListPicker/ListRow.tsx | 6 +- packages/core/components/ListPicker/index.tsx | 3 +- .../ListPicker/test/ListPicker.test.tsx | 2 +- .../Modal/CodeSnippet/CodeSnippet.module.css | 37 ++++- .../components/Modal/CodeSnippet/index.tsx | 30 +++- .../CodeSnippet/test/CodeSnippet.test.tsx | 38 ++++- packages/core/entity/FileExplorerURL/index.ts | 104 ++++++++++++- .../test/fileexplorerurl.test.ts | 138 ++++++++++++++++++ packages/core/entity/FileSort/index.ts | 2 +- .../services/PersistentConfigService/index.ts | 2 + packages/core/state/index.ts | 4 + packages/core/state/interaction/selectors.ts | 11 +- packages/core/state/selection/reducer.ts | 19 ++- packages/core/state/selection/selectors.ts | 25 ++++ .../core/state/selection/test/reducer.test.ts | 54 ++++--- packages/desktop/src/renderer/index.tsx | 2 + .../PersistentConfigServiceElectron.ts | 6 + .../PersistentConfigServiceElectron.test.ts | 4 + 23 files changed, 613 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/aws-deployment.yml create mode 100644 LICENSE delete mode 100644 LICENSE.txt diff --git a/.github/workflows/aws-deployment.yml b/.github/workflows/aws-deployment.yml new file mode 100644 index 000000000..61fe7ce22 --- /dev/null +++ b/.github/workflows/aws-deployment.yml @@ -0,0 +1,98 @@ +--- +name: AWS Deployment + +on: + workflow_dispatch: + inputs: + environment: + description: "Environment to deploy to" + default: "staging" + options: + - staging + - production + required: true + type: choice + +env: + AWS_ACCOUNT_ID: ${{ vars.AWS_PUBLIC_DATA_RELEASES_ACCOUNT_ID }} + AWS_REGION: ${{ vars.AWS_DEFAULT_REGION }} + STAGING_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.STAGING_CLOUDFRONT_DISTRIBUTION_ID }} + STAGING_S3_BUCKET: s3://staging.biofile-finder.allencell.org + PRODUCTION_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.PRODUCTION_CLOUDFRONT_DISTRIBUTION_ID }} + PRODUCTION_S3_BUCKET: s3://biofile-finder.allencell.org + +permissions: + id-token: write # Required for requesting the JWT and OIDC + contents: write # Required for actions/checkout and OIDC tokens + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + + - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 + with: + node-version: "16" + + - name: Install Dependencies + run: npm ci + + - name: Build + run: npm run --prefix packages/web build + + - name: Upload build files + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 + with: + name: aws-deploy-files + path: ./packages/web/dist + + deploy: + needs: build + runs-on: ubuntu-latest + + # Dynamically set the environment variable based on the input above: + environment: ${{ github.event.inputs.environment }} + + steps: + + # Compute a short sha for use in the OIDC session name, which has a 64 character limit + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{ github.sha }} | cut -c1-8`" >> $GITHUB_ENV + + - name: Configure AWS credentials with OIDC + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 + with: + role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/github_biofile_finder + role-session-name: github_biofile_finder-${{ env.SHORT_SHA }} + aws-region: ${{ env.AWS_REGION }} + + # Setup variables based on the staging or production environment + - name: Set ECS variables based on environment + run: | + if [ "${{ github.event.inputs.environment }}" == "production" ]; then + echo "S3_BUCKET=${{ env.PRODUCTION_S3_BUCKET }}" >> $GITHUB_ENV + echo "CLOUDFRONT_DISTRIBUTION_ID=${{ env.PRODUCTION_CLOUDFRONT_DISTRIBUTION_ID }}" >> $GITHUB_ENV + elif [ "${{ github.event.inputs.environment }}" == "staging" ]; then + echo "S3_BUCKET=${{ env.STAGING_S3_BUCKET }}" >> $GITHUB_ENV + echo "CLOUDFRONT_DISTRIBUTION_ID=${{ env.STAGING_CLOUDFRONT_DISTRIBUTION_ID }}" >> $GITHUB_ENV + else + echo "Invalid environment specified" + exit 1 + fi + + - name: Download build artifacts + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + with: + name: aws-deploy-files + path: ./packages/web/dist + + # Note that the command below will copy the files to the root of the S3 bucket e.g., s3://biofile-finder.allencell.org/ + # If you want to copy files to a S3 prefix / subdirectory, you would want something like ${{ env.S3_BUCKET }}/your_prefix below + - name: Copy build files to S3 root + run: aws s3 sync ./packages/web/dist ${{ env.S3_BUCKET }} + + - name: Invalidate CloudFront cache + run: aws cloudfront create-invalidation --distribution-id ${{ env.CLOUDFRONT_DISTRIBUTION_ID }} --paths "/*" diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..026f61101 --- /dev/null +++ b/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2024, Allen Institute + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index a2f8d8d10..000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,33 +0,0 @@ -Allen Institute Software License – This software license is the 2-clause BSD -license plus a third clause that prohibits redistribution and use for -commercial purposes without further permission. - -Copyright © 2021. Allen Institute. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Redistributions and use for commercial purposes are not permitted without -the Allen Institute’s written permission. For purposes of this license, -commercial purposes are the incorporation of the Allen Institute's software -into anything for which you will charge fees or other compensation or use of -the software to perform a commercial service for a third party. Contact -terms@alleninstitute.org for commercial licensing opportunities. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/packages/core/components/AnnotationPicker/index.tsx b/packages/core/components/AnnotationPicker/index.tsx index 4a7df5ac5..01ac0be69 100644 --- a/packages/core/components/AnnotationPicker/index.tsx +++ b/packages/core/components/AnnotationPicker/index.tsx @@ -1,4 +1,5 @@ import * as React from "react"; +import { uniqBy } from "lodash"; import { useSelector } from "react-redux"; import ListPicker from "../ListPicker"; @@ -33,24 +34,53 @@ export default function AnnotationPicker(props: Props) { const areAvailableAnnotationLoading = useSelector( selection.selectors.getAvailableAnnotationsForHierarchyLoading ); + const recentAnnotationNames = useSelector(selection.selectors.getRecentAnnotations); - const items = annotations + const recentAnnotations = recentAnnotationNames.flatMap((name) => + annotations.filter((annotation) => annotation.name === name) + ); + + // Define buffer item + const bufferBar = { + name: "buffer", + selected: false, + disabled: false, + isBuffer: true, + value: "recent buffer", + displayValue: "", + }; + + // combine all annotation lists and buffer item objects + const nonUniqueItems = [...recentAnnotations, bufferBar, ...annotations] .filter( (annotation) => !props.disabledTopLevelAnnotations || !TOP_LEVEL_FILE_ANNOTATION_NAMES.includes(annotation.name) ) - .map((annotation) => ({ - selected: props.selections.some((selected) => selected === annotation.name), - disabled: - props.disableUnavailableAnnotations && - unavailableAnnotations.some((unavailable) => unavailable.name === annotation.name), - loading: props.disableUnavailableAnnotations && areAvailableAnnotationLoading, - description: annotation.description, - data: annotation, - value: annotation.name, - displayValue: annotation.displayName, - })); + .map((annotation) => { + // This is reached if the 'annotation' is a spacer. + if (!(annotation instanceof Annotation)) { + return annotation; + } + + const isSelected = props.selections.some((selected) => selected === annotation.name); + return { + selected: isSelected, + recent: recentAnnotationNames.includes(annotation.name) && !isSelected, + disabled: + props.disableUnavailableAnnotations && + unavailableAnnotations.some( + (unavailable) => unavailable.name === annotation.name + ), + loading: props.disableUnavailableAnnotations && areAvailableAnnotationLoading, + description: annotation.description, + data: annotation, + value: annotation.name, + displayValue: annotation.displayName, + }; + }); + + const items = uniqBy(nonUniqueItems, "value"); const removeSelection = (item: ListItem) => { props.setSelections( diff --git a/packages/core/components/ListPicker/ListRow.module.css b/packages/core/components/ListPicker/ListRow.module.css index b716bde40..748d5354e 100644 --- a/packages/core/components/ListPicker/ListRow.module.css +++ b/packages/core/components/ListPicker/ListRow.module.css @@ -31,6 +31,13 @@ .selected { background-color: var(--primary-background-color); color: var(--primary-text-color); + margin: calc(var(--spacing) / 4) 0 +} + +.isBuffer { + background-color: var(--secondary-text-color); + height: 3px; + pointer-events: none; } .item-container, .item-container label { diff --git a/packages/core/components/ListPicker/ListRow.tsx b/packages/core/components/ListPicker/ListRow.tsx index 85bf4c05e..d29d50222 100644 --- a/packages/core/components/ListPicker/ListRow.tsx +++ b/packages/core/components/ListPicker/ListRow.tsx @@ -9,6 +9,8 @@ import styles from "./ListRow.module.css"; export interface ListItem { disabled?: boolean; loading?: boolean; + recent?: boolean; + isBuffer?: boolean; selected: boolean; displayValue: AnnotationValue; value: AnnotationValue; @@ -37,9 +39,10 @@ export default function ListRow(props: Props) { className={classNames(styles.itemContainer, { [styles.selected]: item.selected, [styles.disabled]: item.disabled, + [styles.isBuffer]: item.isBuffer, })} menuIconProps={{ - iconName: props.subMenuRenderer ? "ChevronRight" : undefined, + iconName: props.subMenuRenderer && !item.isBuffer ? "ChevronRight" : undefined, }} menuProps={ props.subMenuRenderer @@ -59,6 +62,7 @@ export default function ListRow(props: Props) {
{item.selected && }
{item.displayValue} + {item.recent && } {item.loading && } ); diff --git a/packages/core/components/ListPicker/index.tsx b/packages/core/components/ListPicker/index.tsx index 3798c81a5..a621c9987 100644 --- a/packages/core/components/ListPicker/index.tsx +++ b/packages/core/components/ListPicker/index.tsx @@ -162,7 +162,8 @@ export default function ListPicker(props: ListPickerProps) {
- Displaying {filteredItems.length} of {items.length} Options + {/* (item.length -1) to account for buffer in item list. */} + Displaying {filteredItems.length - 1} of {items.length - 1} Options
diff --git a/packages/core/components/ListPicker/test/ListPicker.test.tsx b/packages/core/components/ListPicker/test/ListPicker.test.tsx index 0e98187ba..757287195 100644 --- a/packages/core/components/ListPicker/test/ListPicker.test.tsx +++ b/packages/core/components/ListPicker/test/ListPicker.test.tsx @@ -201,6 +201,6 @@ describe("", () => { ); // Act / Assert - expect(getByText(`Displaying ${items.length} of ${items.length} Options`)).to.exist; + expect(getByText(`Displaying ${items.length - 1} of ${items.length - 1} Options`)).to.exist; }); }); diff --git a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css index 77cc59fa5..df979b46e 100644 --- a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css +++ b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css @@ -25,4 +25,39 @@ .code { cursor: text; user-select: text !important; -} \ No newline at end of file + min-width: 300px; +} + +.button-menu, .button-menu button { + background-color: var(--secondary-background-color); + color: var(--secondary-text-color); +} + +.button-menu i, .button-menu li > div { + color: var(--secondary-text-color); +} + +.button-menu :is(a, button):hover, .button-menu button:hover i { + background-color: var(--highlight-background-color); + color: var(--highlight-text-color) !important; +} + +.code-actions { + max-width: 100%; + padding: 0 var(--padding) 0 var(--padding); + + /* flex parent */ + display: flex; +} + +.action-button { + background-color: var(--primary-background-color); + border: none; + border-radius: 0; + color: var(--primary-text-color); + width: 200px; +} + +.action-button i, .action-button:hover i { + color: var(--primary-text-color) !important; +} diff --git a/packages/core/components/Modal/CodeSnippet/index.tsx b/packages/core/components/Modal/CodeSnippet/index.tsx index 87f31ec53..3fd2ff906 100644 --- a/packages/core/components/Modal/CodeSnippet/index.tsx +++ b/packages/core/components/Modal/CodeSnippet/index.tsx @@ -1,4 +1,5 @@ -import { IconButton, TooltipHost } from "@fluentui/react"; +import { ActionButton, IContextualMenuItem, IconButton, TooltipHost } from "@fluentui/react"; +import classNames from "classnames"; import * as React from "react"; import { useSelector } from "react-redux"; import SyntaxHighlighter from "react-syntax-highlighter"; @@ -18,9 +19,19 @@ export default function CodeSnippet({ onDismiss }: ModalProps) { const pythonSnippet = useSelector(interaction.selectors.getPythonSnippet); const code = pythonSnippet?.code; const setup = pythonSnippet?.setup; + const languageOptions: IContextualMenuItem[] = [ + { + key: "python", + text: "Python 3.8+ (pandas)", + onClick() { + setLanguage("Python 3.8+ (pandas)"); + }, + }, + ]; const [isSetupCopied, setSetupCopied] = React.useState(false); const [isCodeCopied, setCodeCopied] = React.useState(false); + const [language, setLanguage] = React.useState(languageOptions[0].text); const onCopySetup = () => { setup && navigator.clipboard.writeText(setup); @@ -44,6 +55,19 @@ export default function CodeSnippet({ onDismiss }: ModalProps) { const body = ( <> +
+

Language

+
+ +
+

Setup

@@ -75,6 +99,10 @@ export default function CodeSnippet({ onDismiss }: ModalProps) {
", () => { interaction: { visibleModal: ModalType.CodeSnippet, }, + selection: { + dataSources: [ + { + uri: "fake-uri.test", + }, + ], + }, }); it("is visible when should not be hidden", () => { @@ -30,8 +37,8 @@ describe("", () => { it("displays snippet when present in state", async () => { // Arrange - const setup = "pip install pandas"; - const code = "TODO"; + const setup = /pip install (")?pandas/; + const code = "#No options selected"; const { store } = configureMockStore({ state: visibleDialogState }); const { findByText } = render( @@ -40,7 +47,30 @@ describe("", () => { ); // Assert - expect(await findByText(setup)).to.exist; + expect(screen.findByText((_, element) => element?.textContent?.match(setup) !== null)).to + .exist; + expect(await findByText(code)).to.exist; + }); + + it("displays temporary 'coming soon' message for internal data sources", async () => { + // Arrange + const code = "# Coming soon"; + const internalDataSourceState = { + ...visibleDialogState, + selection: { + dataSource: { + uri: undefined, + }, + }, + }; + const { store } = configureMockStore({ state: internalDataSourceState }); + const { findByText } = render( + + + + ); + + // Assert expect(await findByText(code)).to.exist; }); }); diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index 028a390a2..f61e58d96 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -1,8 +1,8 @@ -import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; import { AnnotationName } from "../Annotation"; import FileFilter from "../FileFilter"; import FileFolder from "../FileFolder"; import FileSort, { SortOrder } from "../FileSort"; +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; export interface Source { name: string; @@ -127,4 +127,106 @@ export default class FileExplorerURL { .map((parsedFolder) => new FileFolder(parsedFolder)), }; } + + public static convertToPython( + urlComponents: Partial, + userOS: string + ) { + if ( + (urlComponents?.sources?.length && urlComponents.sources.length > 1) || + urlComponents?.sources?.[0]?.name === AICS_FMS_DATA_SOURCE_NAME || + !urlComponents?.sources?.[0]?.uri + ) { + return "# Coming soon"; + } + const sourceString = this.convertDataSourceToPython(urlComponents?.sources?.[0], userOS); + const groupByQueryString = + urlComponents.hierarchy + ?.map((annotation) => this.convertGroupByToPython(annotation)) + .join("") || ""; + + // Group filters by name and use OR to concatenate same filter values + const filterGroups = new Map(); + urlComponents.filters?.forEach((filter) => { + const pythonQueryString = filterGroups.get(filter.name); + if (!pythonQueryString) { + filterGroups.set(filter.name, this.convertFilterToPython(filter)); + } else { + filterGroups.set( + filter.name, + pythonQueryString.concat(` | ${this.convertFilterToPython(filter)}`) + ); + } + }); + + // Chain the filters together + let filterQueryString = ""; + filterGroups.forEach((value) => { + filterQueryString = filterQueryString.concat(`.query('${value}')`); + }); + + const sortQueryString = urlComponents.sortColumn + ? this.convertSortToPython(urlComponents.sortColumn) + : ""; + // const fuzzy = [] // TO DO: support fuzzy filtering + + const hasQueryElements = groupByQueryString || filterQueryString || sortQueryString; + const imports = "import pandas as pd\n\n"; + const comment = hasQueryElements ? "#Query on dataframe df" : "#No options selected"; + const fullQueryString = `${comment}${ + hasQueryElements && + `\ndf_queried = df${groupByQueryString}${filterQueryString}${sortQueryString}` + }`; + return `${imports}${sourceString}${fullQueryString}`; + } + + private static convertSortToPython(sortColumn: FileSort) { + return `.sort_values(by='${sortColumn.annotationName}', ascending=${ + sortColumn.order == "ASC" ? "True" : "False" + })`; + } + + private static convertGroupByToPython(annotation: string) { + return `.groupby('${annotation}', group_keys=True).apply(lambda x: x)`; + } + + private static convertFilterToPython(filter: FileFilter) { + // TO DO: Support querying non-string types + if (filter.value.includes("RANGE")) { + return; + // let begin, end; + // return `\`${filter.name}\`>="${begin}"&\`${filter.name}\`<"${end}"` + } + return `\`${filter.name}\`=="${filter.value}"`; + } + + private static convertDataSourceToPython(source: Source | undefined, userOS: string) { + const isUsingWindowsOS = userOS === "Windows_NT" || userOS.includes("Windows NT"); + const rawFlagForWindows = isUsingWindowsOS ? "r" : ""; + + if (typeof source?.uri === "string") { + const comment = "#Convert current datasource file to a pandas dataframe"; + + // Currently suggest setting all fields to strings; otherwise pandas assumes type conversions + // TO DO: Address different non-string type conversions + const code = `df = pd.read_${source.type}(${rawFlagForWindows}'${source.uri}').astype('str')`; + // This only works if we assume that the file types will only be csv, parquet or json + + return `${comment}\n${code}\n\n`; + } else if (source?.uri) { + // Any other type, i.e., File. `instanceof` breaks testing library + // Adding strings to avoid including unwanted white space + const inputFileLineComment = + " # Unable to automatically determine " + + "local file location in the browser. Modify this variable to " + + "represent the full path to your .csv, .json, or .parquet data sources\n"; + const inputFileError = + "if not input_file:\n" + + '\traise Exception("Must supply the data source location for the query")\n'; + const inputFileCode = 'input_file = ""' + inputFileLineComment + inputFileError; + + const conversionCode = `df = pd.read_${source.type}(input_file).astype('str')`; + return `${inputFileCode}\n${conversionCode}\n\n`; + } else return ""; // Safeguard. Should not reach else + } } diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index c0973c4bb..725927b51 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -12,6 +12,13 @@ describe("FileExplorerURL", () => { type: "csv", }; + const mockSourceWithUri: Source = { + ...mockSource, + uri: "fake-uri.test", + }; + + const mockOS = "Darwin"; + describe("encode", () => { it("Encodes hierarchy, filters, open folders, and collection", () => { // Arrange @@ -150,4 +157,135 @@ describe("FileExplorerURL", () => { expect(() => FileExplorerURL.decode(encodedUrl)).to.throw(); }); }); + + describe("convert to python pandas string", () => { + it("converts groupings", () => { + // Arrange + const expectedAnnotationNames = ["Cell Line", "Donor Plasmid", "Lifting?"]; + const components: Partial = { + hierarchy: expectedAnnotationNames, + sources: [mockSourceWithUri], + }; + const expectedPandasGroups = expectedAnnotationNames.map( + (annotation) => `.groupby('${annotation}', group_keys=True).apply(lambda x: x)` + ); + const expectedResult = `df${expectedPandasGroups.join("")}`; + + // Act + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("converts filters", () => { + // Arrange + const expectedFilters = [ + { name: "Cas9", value: "spCas9" }, + { name: "Donor Plasmid", value: "ACTB-mEGFP" }, + ]; + const components: Partial = { + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + sources: [mockSourceWithUri], + }; + const expectedPandasQueries = expectedFilters.map( + (filter) => `\`${filter.name}\`=="${filter.value}"` + ); + const expectedResult = `df.query('${expectedPandasQueries[0]}').query('${expectedPandasQueries[1]}')`; + + // Act + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("converts same filter with multiple values", () => { + // Arrange + const expectedFilters = [ + { name: "Gene", value: "AAVS1" }, + { name: "Gene", value: "ACTB" }, + ]; + const components: Partial = { + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + sources: [mockSourceWithUri], + }; + const expectedPandasQueries = expectedFilters.map( + (filter) => `\`${filter.name}\`=="${filter.value}"` + ); + const expectedResult = `df.query('${expectedPandasQueries[0]} | ${expectedPandasQueries[1]}')`; + + // Act + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("converts sorts", () => { + // Arrange + const components: Partial = { + sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), + sources: [mockSourceWithUri], + }; + const expectedPandasSort = `.sort_values(by='${AnnotationName.UPLOADED}', ascending=False`; + const expectedResult = `df${expectedPandasSort}`; + + // Act + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("provides info on converting external data source to pandas dataframe", () => { + // Arrange + const components: Partial = { + sources: [mockSourceWithUri], + }; + const expectedResult = `df = pd.read_csv('${mockSourceWithUri.uri}').astype('str')`; + + // Act + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("adds raw flag in pandas conversion code for Windows OS", () => { + // Arrange + const components: Partial = { + sources: [mockSourceWithUri], + }; + const expectedResult = `df = pd.read_csv(r'${mockSourceWithUri.uri}').astype('str')`; + + // Act + const result = FileExplorerURL.convertToPython(components, "Windows_NT"); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("arranges query elements in correct order", () => { + // Arrange + const expectedAnnotationNames = ["Plate Barcode"]; + const expectedFilters = [ + { name: "Cas9", value: "spCas9" }, + { name: "Donor Plasmid", value: "ACTB-mEGFP" }, + ]; + const components: Partial = { + hierarchy: expectedAnnotationNames, + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), + sources: [mockSourceWithUri], + }; + const expectedResult = /df\.groupby\(.*\)\.query\(.*\)\.query\(.*\)\.sort_values\(.*\)/i; + + // Act + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.match(expectedResult); + }); + }); }); diff --git a/packages/core/entity/FileSort/index.ts b/packages/core/entity/FileSort/index.ts index aa7e39885..76160e0a6 100644 --- a/packages/core/entity/FileSort/index.ts +++ b/packages/core/entity/FileSort/index.ts @@ -30,7 +30,7 @@ export default class FileSort { return { annotationName: this.annotationName, order: this.order, - } + }; } public equals(other?: FileSort): boolean { diff --git a/packages/core/services/PersistentConfigService/index.ts b/packages/core/services/PersistentConfigService/index.ts index e0b2e9bad..3b4f16937 100644 --- a/packages/core/services/PersistentConfigService/index.ts +++ b/packages/core/services/PersistentConfigService/index.ts @@ -12,6 +12,7 @@ export enum PersistedConfigKeys { HasUsedApplicationBefore = "HAS_USED_APPLICATION_BEFORE", UserSelectedApplications = "USER_SELECTED_APPLICATIONS", Queries = "QUERIES", + RecentAnnotations = "RECENT_ANNOTATIONS", } export interface UserSelectedApplication { @@ -26,6 +27,7 @@ export interface PersistedConfig { [PersistedConfigKeys.ImageJExecutable]?: string; // Deprecated [PersistedConfigKeys.HasUsedApplicationBefore]?: boolean; [PersistedConfigKeys.Queries]?: Query[]; + [PersistedConfigKeys.RecentAnnotations]?: string[]; [PersistedConfigKeys.UserSelectedApplications]?: UserSelectedApplication[]; } diff --git a/packages/core/state/index.ts b/packages/core/state/index.ts index 39c0b9c23..53486a5e4 100644 --- a/packages/core/state/index.ts +++ b/packages/core/state/index.ts @@ -72,6 +72,9 @@ export function createReduxStore(options: CreateStoreOptions = {}) { const displayAnnotations = rawDisplayAnnotations ? rawDisplayAnnotations.map((annotation) => new Annotation(annotation)) : []; + const recentAnnotations = persistedConfig?.[PersistedConfigKeys.RecentAnnotations]?.length + ? persistedConfig?.[PersistedConfigKeys.RecentAnnotations] + : []; const preloadedState: State = mergeState(initialState, { interaction: { isOnWeb: !!options.isOnWeb, @@ -107,6 +110,7 @@ export function createReduxStore(options: CreateStoreOptions = {}) { ), }, })), + recentAnnotations, }, }); return configureStore({ diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index 5bdf70ac5..d4de838d3 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -2,7 +2,7 @@ import { uniqBy } from "lodash"; import { createSelector } from "reselect"; import { State } from "../"; -import { getSelectedDataSources } from "../selection/selectors"; +import { getSelectedDataSources, getPythonConversion } from "../selection/selectors"; import { AnnotationService, FileService } from "../../services"; import DatasetService, { DataSource, @@ -67,12 +67,11 @@ export const getAllDataSources = createSelector( : dataSources ); -// TODO: Implement PythonicDataAccessSnippet export const getPythonSnippet = createSelector( - [], - (): PythonicDataAccessSnippet => { - const setup = "pip install pandas"; - const code = "TODO"; + [getPythonConversion], + (pythonQuery): PythonicDataAccessSnippet => { + const setup = `pip install \"pandas>=1.5\"`; + const code = `${pythonQuery}`; return { setup, code }; } diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index 0c4fd2375..33277c6fa 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -1,5 +1,5 @@ import { makeReducer } from "@aics/redux-utils"; -import { omit, uniqBy } from "lodash"; +import { castArray, omit, uniq, uniqBy } from "lodash"; import interaction from "../interaction"; import { THUMBNAIL_SIZE_TO_NUM_COLUMNS } from "../../constants"; @@ -33,6 +33,8 @@ import { REMOVE_QUERY, RemoveQuery, ChangeDataSourcesAction, + SetSortColumnAction, + SetFileFiltersAction, } from "./actions"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; @@ -52,6 +54,7 @@ export interface SelectionStateBranch { filters: FileFilter[]; isDarkTheme: boolean; openFileFolders: FileFolder[]; + recentAnnotations: string[]; selectedQuery?: string; shouldDisplaySmallFont: boolean; shouldDisplayThumbnailView: boolean; @@ -77,6 +80,7 @@ export const initialState = { fileSelection: new FileSelection(), filters: [], openFileFolders: [], + recentAnnotations: [], shouldDisplaySmallFont: false, queries: [], shouldDisplayThumbnailView: false, @@ -100,9 +104,13 @@ export default makeReducer( ...state, fileGridColumnCount: action.payload, }), - [SET_FILE_FILTERS]: (state, action) => ({ + [SET_FILE_FILTERS]: (state, action: SetFileFiltersAction) => ({ ...state, filters: action.payload, + recentAnnotations: uniq([ + ...action.payload.map((filter) => filter.name), + ...state.recentAnnotations, + ]).slice(0, 5), // Reset file selections when file filters change fileSelection: new FileSelection(), @@ -153,8 +161,12 @@ export default makeReducer( ...state, queries: action.payload, }), - [SET_SORT_COLUMN]: (state, action) => ({ + [SET_SORT_COLUMN]: (state, action: SetSortColumnAction) => ({ ...state, + recentAnnotations: uniq([ + ...castArray(action.payload?.annotationName ?? []), + ...state.recentAnnotations, + ]).slice(0, 5), sortColumn: action.payload, }), [interaction.actions.REFRESH]: (state) => ({ @@ -185,6 +197,7 @@ export default makeReducer( ...state, annotationHierarchy: action.payload, availableAnnotationsForHierarchyLoading: true, + recentAnnotations: uniq([...action.payload, ...state.recentAnnotations]).slice(0, 5), // Reset file selections when annotation hierarchy changes fileSelection: new FileSelection(), diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index fe9ae4add..c8b49f71d 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -21,6 +21,7 @@ export const getFileFilters = (state: State) => state.selection.filters; export const getFileSelection = (state: State) => state.selection.fileSelection; export const getIsDarkTheme = (state: State) => state.selection.isDarkTheme; export const getOpenFileFolders = (state: State) => state.selection.openFileFolders; +export const getRecentAnnotations = (state: State) => state.selection.recentAnnotations; export const getSelectedDataSources = (state: State) => state.selection.dataSources; export const getSelectedQuery = (state: State) => state.selection.selectedQuery; export const getShouldDisplaySmallFont = (state: State) => state.selection.shouldDisplaySmallFont; @@ -29,6 +30,7 @@ export const getShouldDisplayThumbnailView = (state: State) => export const getSortColumn = (state: State) => state.selection.sortColumn; export const getTutorial = (state: State) => state.selection.tutorial; export const getQueries = (state: State) => state.selection.queries; +const getPlatformDependentServices = (state: State) => state.interaction.platformDependentServices; // Importing normally creates a circular dependency // COMPOSED SELECTORS export const hasQuerySelected = createSelector([getSelectedQuery], (query): boolean => !!query); @@ -60,6 +62,29 @@ export const getEncodedFileExplorerUrl = createSelector( (queryParts): string => FileExplorerURL.encode(queryParts) ); +export const getPythonConversion = createSelector( + [ + getPlatformDependentServices, + getAnnotationHierarchy, + getFileFilters, + getOpenFileFolders, + getSortColumn, + getSelectedDataSources, + ], + (platformDependentServices, hierarchy, filters, openFolders, sortColumn, sources) => { + return FileExplorerURL.convertToPython( + { + hierarchy, + filters, + openFolders, + sortColumn, + sources, + }, + platformDependentServices.executionEnvService.getOS() + ); + } +); + export const getGroupedByFilterName = createSelector( [getFileFilters, getAnnotations], (globalFilters: FileFilter[], annotations: Annotation[]) => { diff --git a/packages/core/state/selection/test/reducer.test.ts b/packages/core/state/selection/test/reducer.test.ts index 13baa423f..0ae120167 100644 --- a/packages/core/state/selection/test/reducer.test.ts +++ b/packages/core/state/selection/test/reducer.test.ts @@ -14,35 +14,31 @@ import FileFolder from "../../../entity/FileFolder"; import { DataSource } from "../../../services/DataSourceService"; describe("Selection reducer", () => { - [selection.actions.SET_ANNOTATION_HIERARCHY, interaction.actions.INITIALIZE_APP].forEach( - (actionConstant) => - it(`clears selected file state when ${actionConstant} is fired`, () => { - // arrange - const prevSelection = new FileSelection().select({ - fileSet: new FileSet(), - index: new NumericRange(1, 3), - sortOrder: 0, - }); - const initialSelectionState = { - ...selection.initialState, - fileSelection: prevSelection, - }; - - const action = { - type: actionConstant, - }; - - // act - const nextSelectionState = selection.reducer(initialSelectionState, action); - const nextSelection = selection.selectors.getFileSelection({ - ...initialState, - selection: nextSelectionState, - }); - - // assert - expect(prevSelection.count()).to.equal(3); // sanity-check - expect(nextSelection.count()).to.equal(0); - }) + [ + selection.actions.setAnnotationHierarchy([]), + interaction.actions.initializeApp("base"), + ].forEach((expectedAction) => + it(`clears selected file state when ${expectedAction.type} is fired`, () => { + // arrange + const prevSelection = new FileSelection().select({ + fileSet: new FileSet(), + index: new NumericRange(1, 3), + sortOrder: 0, + }); + const initialSelectionState = { + ...selection.initialState, + fileSelection: prevSelection, + }; + // act + const nextSelectionState = selection.reducer(initialSelectionState, expectedAction); + const nextSelection = selection.selectors.getFileSelection({ + ...initialState, + selection: nextSelectionState, + }); + // assert + expect(prevSelection.count()).to.equal(3); // consistency check + expect(nextSelection.count()).to.equal(0); + }) ); describe(selection.actions.CHANGE_DATA_SOURCES, () => { diff --git a/packages/desktop/src/renderer/index.tsx b/packages/desktop/src/renderer/index.tsx index 1785e9360..b561756f0 100644 --- a/packages/desktop/src/renderer/index.tsx +++ b/packages/desktop/src/renderer/index.tsx @@ -74,6 +74,7 @@ store.subscribe(() => { const csvColumns = interaction.selectors.getCsvColumns(state); const displayAnnotations = selection.selectors.getAnnotationsToDisplay(state); const hasUsedApplicationBefore = interaction.selectors.hasUsedApplicationBefore(state); + const recentAnnotations = selection.selectors.getRecentAnnotations(state); const userSelectedApplications = interaction.selectors.getUserSelectedApplications(state); const appState = { @@ -86,6 +87,7 @@ store.subscribe(() => { })), [PersistedConfigKeys.HasUsedApplicationBefore]: hasUsedApplicationBefore, [PersistedConfigKeys.Queries]: queries, + [PersistedConfigKeys.RecentAnnotations]: recentAnnotations, [PersistedConfigKeys.UserSelectedApplications]: userSelectedApplications, }; diff --git a/packages/desktop/src/services/PersistentConfigServiceElectron.ts b/packages/desktop/src/services/PersistentConfigServiceElectron.ts index e6e399c65..06b06429c 100644 --- a/packages/desktop/src/services/PersistentConfigServiceElectron.ts +++ b/packages/desktop/src/services/PersistentConfigServiceElectron.ts @@ -63,6 +63,12 @@ const OPTIONS: Options> = { [PersistedConfigKeys.HasUsedApplicationBefore]: { type: "boolean", }, + [PersistedConfigKeys.RecentAnnotations]: { + type: "array", + items: { + type: "string", + }, + }, [PersistedConfigKeys.UserSelectedApplications]: { type: "array", items: { diff --git a/packages/desktop/src/services/test/PersistentConfigServiceElectron.test.ts b/packages/desktop/src/services/test/PersistentConfigServiceElectron.test.ts index f0608c11a..7d1c8efa3 100644 --- a/packages/desktop/src/services/test/PersistentConfigServiceElectron.test.ts +++ b/packages/desktop/src/services/test/PersistentConfigServiceElectron.test.ts @@ -45,6 +45,7 @@ describe(`${RUN_IN_RENDERER} PersistentConfigServiceElectron`, () => { name: "ZEN", }, ]; + const expectedRecentAnnotations = ["column"]; const expectedQueries = [ { name: "foo", @@ -71,6 +72,7 @@ describe(`${RUN_IN_RENDERER} PersistentConfigServiceElectron`, () => { ); service.persist(PersistedConfigKeys.UserSelectedApplications, expectedUserSelectedApps); service.persist(PersistedConfigKeys.DisplayAnnotations, expectedDisplayAnnotations); + service.persist(PersistedConfigKeys.RecentAnnotations, expectedRecentAnnotations); const expectedConfig = { [PersistedConfigKeys.AllenMountPoint]: expectedAllenMountPoint, @@ -80,6 +82,7 @@ describe(`${RUN_IN_RENDERER} PersistentConfigServiceElectron`, () => { [PersistedConfigKeys.HasUsedApplicationBefore]: expectedHasUsedApplicationBefore, [PersistedConfigKeys.UserSelectedApplications]: expectedUserSelectedApps, [PersistedConfigKeys.DisplayAnnotations]: expectedDisplayAnnotations, + [PersistedConfigKeys.RecentAnnotations]: expectedRecentAnnotations, }; // Act @@ -100,6 +103,7 @@ describe(`${RUN_IN_RENDERER} PersistentConfigServiceElectron`, () => { [PersistedConfigKeys.ImageJExecutable]: "/my/imagej", [PersistedConfigKeys.Queries]: [], [PersistedConfigKeys.HasUsedApplicationBefore]: undefined, + [PersistedConfigKeys.RecentAnnotations]: ["column"], [PersistedConfigKeys.UserSelectedApplications]: [ { filePath: "/some/path/to/ImageJ", From 2a9560385ac1c33662b6a7e7e2cfbef21eb1d1c9 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Thu, 13 Jun 2024 13:47:31 -0700 Subject: [PATCH 11/20] Fix download over web --- .../services/FileDownloadService/FileDownloadServiceNoop.ts | 6 +++--- packages/core/services/FileDownloadService/index.ts | 2 +- packages/core/services/FileService/HttpFileService/index.ts | 4 ++-- .../desktop/src/services/FileDownloadServiceElectron.ts | 5 ++--- packages/web/src/services/FileDownloadServiceWeb.ts | 5 ++--- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts b/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts index bfcbf9b92..1b721b8f1 100644 --- a/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts +++ b/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts @@ -16,9 +16,9 @@ export default class FileDownloadServiceNoop implements FileDownloadService { }); } - prepareHttpResourceForDownload(): Promise { - return Promise.resolve( - "Triggered prepareHttpResourceForDownload on FileDownloadServiceNoop; returning without triggering a download." + prepareHttpResourceForDownload(): Promise { + return Promise.reject( + "Triggered prepareHttpResourceForDownload on FileDownloadServiceNoop" ); } diff --git a/packages/core/services/FileDownloadService/index.ts b/packages/core/services/FileDownloadService/index.ts index 0f1256943..66f468702 100644 --- a/packages/core/services/FileDownloadService/index.ts +++ b/packages/core/services/FileDownloadService/index.ts @@ -41,7 +41,7 @@ export default interface FileDownloadService { /** * Retrieve a Blob from a server over HTTP. */ - prepareHttpResourceForDownload(url: string, postBody: string): Promise; + prepareHttpResourceForDownload(url: string, postBody: string): Promise; /** * Attempt to cancel an active download request, deleting the downloaded artifact if present. diff --git a/packages/core/services/FileService/HttpFileService/index.ts b/packages/core/services/FileService/HttpFileService/index.ts index b8c79761b..b46308dfc 100644 --- a/packages/core/services/FileService/HttpFileService/index.ts +++ b/packages/core/services/FileService/HttpFileService/index.ts @@ -115,7 +115,7 @@ export default class HttpFileService extends HttpServiceBase implements FileServ const postData = JSON.stringify({ annotations, selections }); const url = `${this.baseUrl}/${HttpFileService.BASE_CSV_DOWNLOAD_URL}${this.pathSuffix}`; - const manifestAsString = await this.downloadService.prepareHttpResourceForDownload( + const manifestAsJSON = await this.downloadService.prepareHttpResourceForDownload( url, postData ); @@ -125,7 +125,7 @@ export default class HttpFileService extends HttpServiceBase implements FileServ name, id: name, path: url, - data: manifestAsString, + data: new Blob([manifestAsJSON as BlobPart], { type: "application/json" }), }, uniqueId() ); diff --git a/packages/desktop/src/services/FileDownloadServiceElectron.ts b/packages/desktop/src/services/FileDownloadServiceElectron.ts index cf6c23e03..46c25d015 100644 --- a/packages/desktop/src/services/FileDownloadServiceElectron.ts +++ b/packages/desktop/src/services/FileDownloadServiceElectron.ts @@ -317,9 +317,8 @@ export default class FileDownloadServiceElectron }); } - public async prepareHttpResourceForDownload(url: string, postBody: string): Promise { - const responseAsJSON = await this.rawPost(url, postBody); - return JSON.stringify(responseAsJSON); + public prepareHttpResourceForDownload(url: string, postBody: string): Promise { + return this.rawPost(url, postBody); } public cancelActiveRequest(downloadRequestId: string) { diff --git a/packages/web/src/services/FileDownloadServiceWeb.ts b/packages/web/src/services/FileDownloadServiceWeb.ts index 33ab5df10..1921503b9 100644 --- a/packages/web/src/services/FileDownloadServiceWeb.ts +++ b/packages/web/src/services/FileDownloadServiceWeb.ts @@ -39,9 +39,8 @@ export default class FileDownloadServiceWeb extends HttpServiceBase implements F } } - public async prepareHttpResourceForDownload(url: string, postBody: string): Promise { - const responseAsJSON = await this.rawPost(url, postBody); - return JSON.stringify(responseAsJSON); + public prepareHttpResourceForDownload(url: string, postBody: string): Promise { + return this.rawPost(url, postBody); } public getDefaultDownloadDirectory(): Promise { From a6cbd1501ddd3ce574631f1be19b952b675aed93 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Thu, 13 Jun 2024 14:16:53 -0700 Subject: [PATCH 12/20] Fix data source replacement --- .../AnnotationService/DatabaseAnnotationService/index.ts | 2 +- packages/core/services/DatabaseService/index.ts | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index 66b494081..39fa0a5a1 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -33,7 +33,7 @@ export default class DatabaseAnnotationService implements AnnotationService { /** * Fetch all annotations. */ - public async fetchAnnotations(): Promise { + public fetchAnnotations(): Promise { return this.databaseService.fetchAnnotations(this.dataSourceNames); } diff --git a/packages/core/services/DatabaseService/index.ts b/packages/core/services/DatabaseService/index.ts index d730c20c9..bb1a9fce9 100644 --- a/packages/core/services/DatabaseService/index.ts +++ b/packages/core/services/DatabaseService/index.ts @@ -1,3 +1,5 @@ +import { isEmpty } from "lodash"; + import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; import Annotation from "../../entity/Annotation"; import { AnnotationType } from "../../entity/AnnotationFormatter"; @@ -137,6 +139,9 @@ export default abstract class DatabaseService { .where(`table_name = '${aggregateDataSourceName}'`) .toSQL(); const rows = await this.query(sql); + if (isEmpty(rows)) { + throw new Error(`Unable to fetch annotations for ${aggregateDataSourceName}`); + } const annotations = rows.map( (row) => new Annotation({ From a6decd4593434af04a892e4382236a06dae7778d Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Thu, 13 Jun 2024 14:30:19 -0700 Subject: [PATCH 13/20] Fix CSV export on desktop --- .../Modal/MetadataManifest/index.tsx | 19 +++++++++++++++---- .../src/services/DatabaseServiceElectron.ts | 12 ++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/core/components/Modal/MetadataManifest/index.tsx b/packages/core/components/Modal/MetadataManifest/index.tsx index 9e5eeb8ce..ee8a9724e 100644 --- a/packages/core/components/Modal/MetadataManifest/index.tsx +++ b/packages/core/components/Modal/MetadataManifest/index.tsx @@ -6,7 +6,7 @@ import { useDispatch, useSelector } from "react-redux"; import { ModalProps } from ".."; import BaseModal from "../BaseModal"; import AnnotationPicker from "../../AnnotationPicker"; -import { interaction } from "../../../state"; +import { interaction, metadata } from "../../../state"; import styles from "./MetadataManifest.module.css"; @@ -16,12 +16,23 @@ import styles from "./MetadataManifest.module.css"; */ export default function MetadataManifest({ onDismiss }: ModalProps) { const dispatch = useDispatch(); + const annotations = useSelector(metadata.selectors.getAnnotations); const annotationsPreviouslySelected = useSelector(interaction.selectors.getCsvColumns); - const [selectedAnnotations, setSelectedAnnotations] = React.useState( - annotationsPreviouslySelected || [] - ); const fileTypeForVisibleModal = useSelector(interaction.selectors.getFileTypeForVisibleModal); + const [selectedAnnotations, setSelectedAnnotations] = React.useState([]); + + // Update the selected annotations when the previously selected annotations + // or list of all annotations change like on data source change + React.useEffect(() => { + const annotationsPreviouslySelectedAvailable = ( + annotationsPreviouslySelected || [] + ).filter((annotationName) => + annotations.some((annotation) => annotationName === annotation.name) + ); + setSelectedAnnotations(annotationsPreviouslySelectedAvailable); + }, [annotations, annotationsPreviouslySelected]); + const onDownload = () => { dispatch( interaction.actions.downloadManifest(selectedAnnotations, fileTypeForVisibleModal) diff --git a/packages/desktop/src/services/DatabaseServiceElectron.ts b/packages/desktop/src/services/DatabaseServiceElectron.ts index 604671c7f..7cdea694e 100644 --- a/packages/desktop/src/services/DatabaseServiceElectron.ts +++ b/packages/desktop/src/services/DatabaseServiceElectron.ts @@ -21,10 +21,18 @@ export default class DatabaseServiceElectron extends DatabaseService { * May return a value if the location is not a physical location but rather * a temporary database location (buffer) */ - public saveQuery(destination: string, sql: string, format: string): Promise { + public saveQuery( + destination: string, + sql: string, + format: "csv" | "json" | "parquet" + ): Promise { + const saveOptions = [`FORMAT '${format}'`]; + if (format === "csv") { + saveOptions.push("HEADER"); + } return new Promise((resolve, reject) => { this.database.run( - `COPY (${sql}) TO '${destination}.${format}' (FORMAT '${format}');`, + `COPY (${sql}) TO '${destination}.${format}' (${saveOptions.join(", ")});`, (err: any, result: any) => { if (err) { reject(err.message); From dcd5e5a09b9d009a8e41ab8caded4ec48ace9281 Mon Sep 17 00:00:00 2001 From: SeanLeRoy Date: Thu, 13 Jun 2024 14:33:25 -0700 Subject: [PATCH 14/20] Adjust test --- .../desktop/src/services/test/DatabaseServiceElectron.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts index c739aca26..b18595a37 100644 --- a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts +++ b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts @@ -79,7 +79,7 @@ describe("DatabaseServiceElectron", () => { // Assert const fileStat = await fs.promises.stat(`${destination}.${format}`); - expect(fileStat.size).to.equal(0); + expect(fileStat.size).to.equal(215); }); }); }); From 4b0d0b4d184d2c844522f1954a183d97a531651c Mon Sep 17 00:00:00 2001 From: BrianW25 Date: Fri, 31 May 2024 15:30:43 -0700 Subject: [PATCH 15/20] add inital render --- package-lock.json | 143 +++++++++++++++++- package.json | 3 +- .../FileList/LazilyRenderedThumbnail.tsx | 26 +++- packages/core/components/FileList/types.ts | 15 ++ packages/core/entity/FileDetail/index.ts | 4 + 5 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 packages/core/components/FileList/types.ts diff --git a/package-lock.json b/package-lock.json index f54ce57cf..f37338690 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,7 +37,8 @@ "redux": "4.0.x", "redux-logic": "3.x", "reselect": "4.0.x", - "string-natural-compare": "3.0.x" + "string-natural-compare": "3.0.x", + "zarrita": "^0.3.2" }, "devDependencies": { "@babel/cli": "7.x", @@ -4089,6 +4090,40 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "node_modules/@zarrita/core": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.0.3.tgz", + "integrity": "sha512-fWv51b+xbYnws1pkNDPwJQoDa76aojxplHyMup82u11UAiet3gURMsrrkhM6YbeTgSY1A8oGxDOrvar3SiZpLA==", + "dependencies": { + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1", + "numcodecs": "^0.2.2" + } + }, + "node_modules/@zarrita/indexing": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.0.3.tgz", + "integrity": "sha512-Q61d9MYX6dsK1DLltEpwx4mJWCZHj0TXiaEN4QpxNDtToa/EoyytP/pYHPypO4GXBscZooJ6eZkKT5FMx9PVfg==", + "dependencies": { + "@zarrita/core": "^0.0.3", + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1" + } + }, + "node_modules/@zarrita/storage": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.0.2.tgz", + "integrity": "sha512-uFt4abAoiOYLroalNDAnVaQxA17zGKrQ0waYKmTVm+bNonz8ggKZP+0FqMhgUZITGChqoANHuYTazbuU5AFXWA==", + "dependencies": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.3.6" + } + }, + "node_modules/@zarrita/typedarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.0.1.tgz", + "integrity": "sha512-ZdvNjYP1bEuQXoSTVkemV99w42jHYrJ3nh9golCLd4MVBlrVbfZo4wWgBslU4JZUaDikhFSH+GWMDgAq/rI32g==" + }, "node_modules/7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -13880,6 +13915,14 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/numcodecs": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.2.2.tgz", + "integrity": "sha512-Y5K8mv80yb4MgVpcElBkUeMZqeE4TrovxRit/dTZvoRl6YkB6WEjY+fiUjGCblITnt3T3fmrDg8yRWu0gOLjhQ==", + "engines": { + "node": ">=12" + } + }, "node_modules/nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -15941,6 +15984,11 @@ "redux": ">=3.5.2" } }, + "node_modules/reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "node_modules/refractor": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", @@ -18141,6 +18189,17 @@ "yaku": "^0.16.6" } }, + "node_modules/unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "dependencies": { + "uzip-module": "^1.0.2" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -18218,6 +18277,11 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "node_modules/v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -19318,6 +19382,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zarrita": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.3.2.tgz", + "integrity": "sha512-Zx9nS28C2tXZhF1BmQkgQGi0M/Z5JiM/KCMa+fEYtr/MnIzyizR4sKRA/sXjDP1iuylILWTJAWWBJD//0ONXCA==", + "dependencies": { + "@zarrita/core": "^0.0.3", + "@zarrita/indexing": "^0.0.3", + "@zarrita/storage": "^0.0.2" + } + }, "packages/desktop": { "name": "fms-file-explorer-desktop", "version": "7.0.0", @@ -22477,6 +22551,40 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "@zarrita/core": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.0.3.tgz", + "integrity": "sha512-fWv51b+xbYnws1pkNDPwJQoDa76aojxplHyMup82u11UAiet3gURMsrrkhM6YbeTgSY1A8oGxDOrvar3SiZpLA==", + "requires": { + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1", + "numcodecs": "^0.2.2" + } + }, + "@zarrita/indexing": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.0.3.tgz", + "integrity": "sha512-Q61d9MYX6dsK1DLltEpwx4mJWCZHj0TXiaEN4QpxNDtToa/EoyytP/pYHPypO4GXBscZooJ6eZkKT5FMx9PVfg==", + "requires": { + "@zarrita/core": "^0.0.3", + "@zarrita/storage": "^0.0.2", + "@zarrita/typedarray": "^0.0.1" + } + }, + "@zarrita/storage": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.0.2.tgz", + "integrity": "sha512-uFt4abAoiOYLroalNDAnVaQxA17zGKrQ0waYKmTVm+bNonz8ggKZP+0FqMhgUZITGChqoANHuYTazbuU5AFXWA==", + "requires": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.3.6" + } + }, + "@zarrita/typedarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.0.1.tgz", + "integrity": "sha512-ZdvNjYP1bEuQXoSTVkemV99w42jHYrJ3nh9golCLd4MVBlrVbfZo4wWgBslU4JZUaDikhFSH+GWMDgAq/rI32g==" + }, "7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -29949,6 +30057,11 @@ } } }, + "numcodecs": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.2.2.tgz", + "integrity": "sha512-Y5K8mv80yb4MgVpcElBkUeMZqeE4TrovxRit/dTZvoRl6YkB6WEjY+fiUjGCblITnt3T3fmrDg8yRWu0gOLjhQ==" + }, "nwsapi": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", @@ -31427,6 +31540,11 @@ "rxjs": "^6.6.6" } }, + "reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "refractor": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", @@ -33139,6 +33257,14 @@ "yaku": "^0.16.6" } }, + "unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "requires": { + "uzip-module": "^1.0.2" + } + }, "uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -33202,6 +33328,11 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, + "uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -34006,6 +34137,16 @@ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true + }, + "zarrita": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.3.2.tgz", + "integrity": "sha512-Zx9nS28C2tXZhF1BmQkgQGi0M/Z5JiM/KCMa+fEYtr/MnIzyizR4sKRA/sXjDP1iuylILWTJAWWBJD//0ONXCA==", + "requires": { + "@zarrita/core": "^0.0.3", + "@zarrita/indexing": "^0.0.3", + "@zarrita/storage": "^0.0.2" + } } } } diff --git a/package.json b/package.json index 4c97188ee..4526050e1 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "redux": "4.0.x", "redux-logic": "3.x", "reselect": "4.0.x", - "string-natural-compare": "3.0.x" + "string-natural-compare": "3.0.x", + "zarrita": "^0.3.2" } } diff --git a/packages/core/components/FileList/LazilyRenderedThumbnail.tsx b/packages/core/components/FileList/LazilyRenderedThumbnail.tsx index e6329782a..829423be2 100644 --- a/packages/core/components/FileList/LazilyRenderedThumbnail.tsx +++ b/packages/core/components/FileList/LazilyRenderedThumbnail.tsx @@ -1,6 +1,7 @@ import classNames from "classnames"; import * as React from "react"; import { useSelector } from "react-redux"; +import { open, HTTPStore } from "zarrita"; import FileSet from "../../entity/FileSet"; import FileThumbnail from "../../components/FileThumbnail"; @@ -59,6 +60,23 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr return fileSelection.isFocused(fileSet, overallIndex); }, [fileSelection, fileSet, overallIndex]); + const [zarrImage, setZarrImage] = React.useState(null); + + React.useEffect(() => { + const loadZarrImage = async () => { + if (file?.getPathToThumbnail()?.endsWith(".zarr")) { + const store = new HTTPStore(file.getPathToThumbnail()!); + const z = await open({ store, path: "" }); + const data = await z.getRaw(); + const blob = new Blob([data], { type: "image/png" }); + const url = URL.createObjectURL(blob); + setZarrImage(url); + } + }; + + loadZarrImage(); + }, [file]); + const onClick = (evt: React.MouseEvent) => { evt.preventDefault(); evt.stopPropagation(); @@ -88,7 +106,9 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr let content; if (file) { - const thumbnailPath = file.getPathToThumbnail(); + const thumbnailPath = file.getPathToThumbnail()?.endsWith(".zarr") + ? zarrImage + : file.getPathToThumbnail(); const filenameForRender = clipFileName(file?.name); content = (
; + } + + export class HTTPStore implements Store { + constructor(baseUrl: string); + getItem(key: string): Promise; + } + + export function open(params: { + store: Store; + path: string; + }): Promise<{ getRaw(): Promise }>; +} diff --git a/packages/core/entity/FileDetail/index.ts b/packages/core/entity/FileDetail/index.ts index c950572ee..9c10264d5 100644 --- a/packages/core/entity/FileDetail/index.ts +++ b/packages/core/entity/FileDetail/index.ts @@ -162,6 +162,10 @@ export default class FileDetail { public getPathToThumbnail(): string | undefined { // If no thumbnail present try to render the file itself as the thumbnail + if (this.path.endsWith(".zarr")) { + return "http://production.files.allencell.org.s3.amazonaws.com/ba8/0f7/c92/4ba/664/a78/744/91f/35b/48d/6b/multi-test-12230.zarr"; + } + if (!this.thumbnail) { const isFileRenderableImage = RENDERABLE_IMAGE_FORMATS.some((format) => this.name.toLowerCase().endsWith(format) From a190f13f368bc5fff77d4f148e37d80e4f4f16bd Mon Sep 17 00:00:00 2001 From: BrianW25 Date: Tue, 11 Jun 2024 12:44:02 -0700 Subject: [PATCH 16/20] render zarr image --- .../core/components/FileDetails/index.tsx | 8 ++- .../FileList/LazilyRenderedThumbnail.tsx | 31 +++------ .../FileDetail/RenderZarrThumbnailURL.ts | 67 +++++++++++++++++++ packages/core/entity/FileDetail/index.ts | 18 +++-- 4 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts diff --git a/packages/core/components/FileDetails/index.tsx b/packages/core/components/FileDetails/index.tsx index a93f7942f..2cc9f7801 100644 --- a/packages/core/components/FileDetails/index.tsx +++ b/packages/core/components/FileDetails/index.tsx @@ -78,6 +78,13 @@ function resizeHandleDoubleClick() { export default function FileDetails(props: Props) { const [windowState, windowDispatch] = React.useReducer(windowStateReducer, INITIAL_STATE); const [fileDetails, isLoading] = useFileDetails(); + const [thumbnailPath, setThumbnailPath] = React.useState(undefined); + + React.useEffect(() => { + if (fileDetails) { + fileDetails.getPathToThumbnail().then(setThumbnailPath); + } + }, [fileDetails]); // If FileDetails pane is minimized, set its width to the width of the WindowActionButtons. Else, let it be // defined by whatever the CSS determines (setting an inline style to undefined will prompt ReactDOM to not apply @@ -85,7 +92,6 @@ export default function FileDetails(props: Props) { const minimizedWidth = windowState.state === WindowState.MINIMIZED ? WINDOW_ACTION_BUTTON_WIDTH : undefined; - const thumbnailPath = fileDetails?.getPathToThumbnail(); let thumbnailHeight = undefined; let thumbnailWidth = undefined; if (windowState.state === WindowState.DEFAULT) { diff --git a/packages/core/components/FileList/LazilyRenderedThumbnail.tsx b/packages/core/components/FileList/LazilyRenderedThumbnail.tsx index 829423be2..05a23778c 100644 --- a/packages/core/components/FileList/LazilyRenderedThumbnail.tsx +++ b/packages/core/components/FileList/LazilyRenderedThumbnail.tsx @@ -1,7 +1,6 @@ import classNames from "classnames"; import * as React from "react"; import { useSelector } from "react-redux"; -import { open, HTTPStore } from "zarrita"; import FileSet from "../../entity/FileSet"; import FileThumbnail from "../../components/FileThumbnail"; @@ -52,6 +51,14 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr const file = fileSet.getFileByIndex(overallIndex); const thumbnailSize = measuredWidth / fileGridColCount - 2 * MARGIN; + const [thumbnailPath, setThumbnailPath] = React.useState(undefined); + + React.useEffect(() => { + if (file) { + file.getPathToThumbnail().then(setThumbnailPath); + } + }, [file]); + const isSelected = React.useMemo(() => { return fileSelection.isSelected(fileSet, overallIndex); }, [fileSelection, fileSet, overallIndex]); @@ -60,23 +67,6 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr return fileSelection.isFocused(fileSet, overallIndex); }, [fileSelection, fileSet, overallIndex]); - const [zarrImage, setZarrImage] = React.useState(null); - - React.useEffect(() => { - const loadZarrImage = async () => { - if (file?.getPathToThumbnail()?.endsWith(".zarr")) { - const store = new HTTPStore(file.getPathToThumbnail()!); - const z = await open({ store, path: "" }); - const data = await z.getRaw(); - const blob = new Blob([data], { type: "image/png" }); - const url = URL.createObjectURL(blob); - setZarrImage(url); - } - }; - - loadZarrImage(); - }, [file]); - const onClick = (evt: React.MouseEvent) => { evt.preventDefault(); evt.stopPropagation(); @@ -106,9 +96,6 @@ export default function LazilyRenderedThumbnail(props: LazilyRenderedThumbnailPr let content; if (file) { - const thumbnailPath = file.getPathToThumbnail()?.endsWith(".zarr") - ? zarrImage - : file.getPathToThumbnail(); const filenameForRender = clipFileName(file?.name); content = (
{ + try { + const store = new FetchStore(zarrUrl); + const root = zarr.root(store); + const group = await zarr.open(root, { kind: "group" }); + + if ( + group.attrs && + Array.isArray(group.attrs.multiscales) && + group.attrs.multiscales.length > 0 + ) { + const { multiscales } = group.attrs; + const datasets = multiscales[0].datasets; + const lowestResolutionDataset = datasets[datasets.length - 1]; + const lowestResolutionLocation = root.resolve(lowestResolutionDataset.path); + const lowestResolution = await zarr.open(lowestResolutionLocation, { kind: "array" }); // TODO: check the filesize before slicing + const lowestResolutionView = await zarrGet(lowestResolution, [0, 0, 21, null, null]); // TODO: make this slicing dynamic + + const data = lowestResolutionView.data as Uint16Array; + + // Normalize the Uint16Array data to 8-bit + const maxVal = Math.max(...data); + const minVal = Math.min(...data); + const normalizedData = new Uint8ClampedArray(data.length); + for (let i = 0; i < data.length; i++) { + normalizedData[i] = ((data[i] - minVal) / (maxVal - minVal)) * 255; + } + + // Create a canvas to draw the image + const canvas = document.createElement("canvas"); + const context = canvas.getContext("2d"); + const width = Math.min(100, lowestResolution.shape[3]); // Set to actual data width if less than 100 + const height = Math.min(100, lowestResolution.shape[4]); // Set to actual data height if less than 100 + canvas.width = width; + canvas.height = height; + + if (context) { + const imageData = context.createImageData(width, height); + const imageDataArray = imageData.data; + + // Populate the ImageData object with the normalized data + for (let i = 0; i < normalizedData.length; i++) { + const value = normalizedData[i]; + imageDataArray[i * 4] = value; // Red + imageDataArray[i * 4 + 1] = value; // Green + imageDataArray[i * 4 + 2] = value; // Blue + imageDataArray[i * 4 + 3] = 255; // Alpha + } + context.putImageData(imageData, 0, 0); + + const dataURL = canvas.toDataURL("image/png"); + return dataURL; + } else { + throw new Error("Unable to get 2D context from canvas"); + } + } else { + throw new Error("Invalid multiscales attribute structure"); + } + } catch (error) { + console.error("Error reading Zarr image:", error); + throw error; + } +} diff --git a/packages/core/entity/FileDetail/index.ts b/packages/core/entity/FileDetail/index.ts index 9c10264d5..4eeacb632 100644 --- a/packages/core/entity/FileDetail/index.ts +++ b/packages/core/entity/FileDetail/index.ts @@ -1,11 +1,11 @@ import { FmsFileAnnotation } from "../../services/FileService"; +import { renderZarrThumbnailURL } from "./RenderZarrThumbnailURL"; const RENDERABLE_IMAGE_FORMATS = [".jpg", ".jpeg", ".png", ".gif"]; // Should probably record this somewhere we can dynamically adjust to, or perhaps just in the file // document itself, alas for now this will do. const HARD_CODED_AICS_S3_BUCKET_PATH = "http://production.files.allencell.org.s3.amazonaws.com"; - const AICS_FMS_FILES_NGINX_SERVER = "http://aics.corp.alleninstitute.org/labkey/fmsfiles/image"; /** @@ -160,10 +160,18 @@ export default class FileDetail { return this.fileDetail.annotations.find((annotation) => annotation.name === annotationName); } - public getPathToThumbnail(): string | undefined { - // If no thumbnail present try to render the file itself as the thumbnail - if (this.path.endsWith(".zarr")) { - return "http://production.files.allencell.org.s3.amazonaws.com/ba8/0f7/c92/4ba/664/a78/744/91f/35b/48d/6b/multi-test-12230.zarr"; + public async getPathToThumbnail(): Promise { + if (this.cloudPath.endsWith(".zarr")) { + try { + // const thumbnailURL = await renderZarrThumbnailURL(this.cloudPath); + const thumbnailURL = await renderZarrThumbnailURL( + "https://animatedcell-test-data.s3.us-west-2.amazonaws.com/variance/136244.zarr" + ); + return thumbnailURL; + } catch (error) { + console.error("Error generating Zarr thumbnail:", error); + throw new Error("Unable to generate Zarr thumbnail"); + } } if (!this.thumbnail) { From b16d1673f9f92253413fc70839c6d100a44fbe1a Mon Sep 17 00:00:00 2001 From: BrianW25 Date: Wed, 12 Jun 2024 11:27:08 -0700 Subject: [PATCH 17/20] render rgb thumbnail --- .../FileDetail/RenderZarrThumbnailURL.ts | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts b/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts index be95fa728..cc6a1d0df 100644 --- a/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts +++ b/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts @@ -2,6 +2,14 @@ import * as zarr from "@zarrita/core"; import { FetchStore } from "@zarrita/storage"; import { get as zarrGet } from "@zarrita/indexing"; +// Function to map grayscale value (0-255) to RGB color +function grayscaleToRGB(value: number): [number, number, number] { + const r = value; + const g = 0 + value; + const b = 255 - value; + return [r, g, b]; +} + export async function renderZarrThumbnailURL(zarrUrl: string): Promise { try { const store = new FetchStore(zarrUrl); @@ -18,45 +26,50 @@ export async function renderZarrThumbnailURL(zarrUrl: string): Promise { const lowestResolutionDataset = datasets[datasets.length - 1]; const lowestResolutionLocation = root.resolve(lowestResolutionDataset.path); const lowestResolution = await zarr.open(lowestResolutionLocation, { kind: "array" }); // TODO: check the filesize before slicing - const lowestResolutionView = await zarrGet(lowestResolution, [0, 0, 21, null, null]); // TODO: make this slicing dynamic - const data = lowestResolutionView.data as Uint16Array; + const lowestResolutionView = await zarrGet(lowestResolution, [0, 0, 20, null, null]); // Adjusted slicing for 2D data + console.log("DATA", lowestResolutionView.data); + console.log("SHAPE", lowestResolutionView.shape); + const u16data = lowestResolutionView.data as Uint16Array; + + const min = Math.min(...u16data); + const max = Math.max(...u16data); - // Normalize the Uint16Array data to 8-bit - const maxVal = Math.max(...data); - const minVal = Math.min(...data); - const normalizedData = new Uint8ClampedArray(data.length); - for (let i = 0; i < data.length; i++) { - normalizedData[i] = ((data[i] - minVal) / (maxVal - minVal)) * 255; + const normalizedData = new Uint8Array(u16data.length); + for (let i = 0; i < u16data.length; i++) { + normalizedData[i] = Math.round((255 * (u16data[i] - min)) / (max - min)); } - // Create a canvas to draw the image + const width = lowestResolutionView.shape[1]; + const height = lowestResolutionView.shape[0]; const canvas = document.createElement("canvas"); - const context = canvas.getContext("2d"); - const width = Math.min(100, lowestResolution.shape[3]); // Set to actual data width if less than 100 - const height = Math.min(100, lowestResolution.shape[4]); // Set to actual data height if less than 100 canvas.width = width; canvas.height = height; + const context = canvas.getContext("2d"); - if (context) { - const imageData = context.createImageData(width, height); - const imageDataArray = imageData.data; + if (!context) { + throw new Error("Failed to get canvas context"); + } - // Populate the ImageData object with the normalized data - for (let i = 0; i < normalizedData.length; i++) { - const value = normalizedData[i]; - imageDataArray[i * 4] = value; // Red - imageDataArray[i * 4 + 1] = value; // Green - imageDataArray[i * 4 + 2] = value; // Blue - imageDataArray[i * 4 + 3] = 255; // Alpha - } - context.putImageData(imageData, 0, 0); + const imageData = context.createImageData(width, height); - const dataURL = canvas.toDataURL("image/png"); - return dataURL; - } else { - throw new Error("Unable to get 2D context from canvas"); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const idx = (y * width + x) * 4; + const value = normalizedData[y * width + x]; + const [r, g, b] = grayscaleToRGB(value); + imageData.data[idx] = r; // Red + imageData.data[idx + 1] = g; // Green + imageData.data[idx + 2] = b; // Blue + imageData.data[idx + 3] = 255; // Alpha + } } + + context.putImageData(imageData, 0, 0); + + const dataUrl = canvas.toDataURL("image/png"); + + return dataUrl; } else { throw new Error("Invalid multiscales attribute structure"); } From 0172366ef94b2b433dc957422af19a147403bc4e Mon Sep 17 00:00:00 2001 From: BrianW25 Date: Wed, 12 Jun 2024 13:31:11 -0700 Subject: [PATCH 18/20] create slice from dims --- .../FileDetail/RenderZarrThumbnailURL.ts | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts b/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts index cc6a1d0df..309456a78 100644 --- a/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts +++ b/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts @@ -10,6 +10,33 @@ function grayscaleToRGB(value: number): [number, number, number] { return [r, g, b]; } +interface AxisData { + name: string; + type: "time" | "channel" | "space"; + unit?: string; +} + +// Define the type for the transformed array items +interface TransformedAxes { + name: string; + value: number | null; +} + +// Function to transform the array +function transformAxes(originalArray: AxisData[]): TransformedAxes[] { + return originalArray.map((item) => { + let value: number | null; + if (item.type !== "space") { + // set non spatial to first occurence + value = 0; + } else { + value = null; + } + + return { name: item.name, value: value }; + }); +} + export async function renderZarrThumbnailURL(zarrUrl: string): Promise { try { const store = new FetchStore(zarrUrl); @@ -21,27 +48,41 @@ export async function renderZarrThumbnailURL(zarrUrl: string): Promise { Array.isArray(group.attrs.multiscales) && group.attrs.multiscales.length > 0 ) { + // Resolve file data const { multiscales } = group.attrs; const datasets = multiscales[0].datasets; const lowestResolutionDataset = datasets[datasets.length - 1]; const lowestResolutionLocation = root.resolve(lowestResolutionDataset.path); const lowestResolution = await zarr.open(lowestResolutionLocation, { kind: "array" }); // TODO: check the filesize before slicing - const lowestResolutionView = await zarrGet(lowestResolution, [0, 0, 20, null, null]); // Adjusted slicing for 2D data - console.log("DATA", lowestResolutionView.data); + // Determine Slice + const axes = transformAxes(multiscales[0].axes); + if (axes.some((item) => item.name === "z")) { + const zIndex = axes.findIndex((item) => item.name === "z"); + const zSliceIndex = Math.ceil(lowestResolution.shape[zIndex] / 2); + axes[zIndex].value = zSliceIndex; + } + + console.log(axes); + const lowestResolutionView = await zarrGet( + lowestResolution, + axes.map((item) => item.value) + ); + console.log("SHAPE", lowestResolutionView.shape); - const u16data = lowestResolutionView.data as Uint16Array; + // Normalize Data + const u16data = lowestResolutionView.data as Uint16Array; const min = Math.min(...u16data); const max = Math.max(...u16data); - const normalizedData = new Uint8Array(u16data.length); for (let i = 0; i < u16data.length; i++) { normalizedData[i] = Math.round((255 * (u16data[i] - min)) / (max - min)); } - const width = lowestResolutionView.shape[1]; - const height = lowestResolutionView.shape[0]; + // Draw Data + const width = lowestResolution.shape[axes.findIndex((item) => item.name === "x")]; + const height = lowestResolution.shape[axes.findIndex((item) => item.name === "y")]; const canvas = document.createElement("canvas"); canvas.width = width; canvas.height = height; @@ -67,8 +108,8 @@ export async function renderZarrThumbnailURL(zarrUrl: string): Promise { context.putImageData(imageData, 0, 0); + // Convert data to data URL const dataUrl = canvas.toDataURL("image/png"); - return dataUrl; } else { throw new Error("Invalid multiscales attribute structure"); From 5d9de045da0bf6bf3bf4687a0fee4cbfcf470718 Mon Sep 17 00:00:00 2001 From: BrianW25 Date: Wed, 12 Jun 2024 14:08:13 -0700 Subject: [PATCH 19/20] add grayscale image --- .../FileDetail/RenderZarrThumbnailURL.ts | 139 ++++++++---------- 1 file changed, 59 insertions(+), 80 deletions(-) diff --git a/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts b/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts index 309456a78..e3ad04523 100644 --- a/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts +++ b/packages/core/entity/FileDetail/RenderZarrThumbnailURL.ts @@ -2,14 +2,6 @@ import * as zarr from "@zarrita/core"; import { FetchStore } from "@zarrita/storage"; import { get as zarrGet } from "@zarrita/indexing"; -// Function to map grayscale value (0-255) to RGB color -function grayscaleToRGB(value: number): [number, number, number] { - const r = value; - const g = 0 + value; - const b = 255 - value; - return [r, g, b]; -} - interface AxisData { name: string; type: "time" | "channel" | "space"; @@ -24,17 +16,10 @@ interface TransformedAxes { // Function to transform the array function transformAxes(originalArray: AxisData[]): TransformedAxes[] { - return originalArray.map((item) => { - let value: number | null; - if (item.type !== "space") { - // set non spatial to first occurence - value = 0; - } else { - value = null; - } - - return { name: item.name, value: value }; - }); + return originalArray.map((item) => ({ + name: item.name, + value: item.type !== "space" ? 0 : null, + })); } export async function renderZarrThumbnailURL(zarrUrl: string): Promise { @@ -44,76 +29,70 @@ export async function renderZarrThumbnailURL(zarrUrl: string): Promise { const group = await zarr.open(root, { kind: "group" }); if ( - group.attrs && - Array.isArray(group.attrs.multiscales) && - group.attrs.multiscales.length > 0 + !group.attrs || + !Array.isArray(group.attrs.multiscales) || + group.attrs.multiscales.length === 0 ) { - // Resolve file data - const { multiscales } = group.attrs; - const datasets = multiscales[0].datasets; - const lowestResolutionDataset = datasets[datasets.length - 1]; - const lowestResolutionLocation = root.resolve(lowestResolutionDataset.path); - const lowestResolution = await zarr.open(lowestResolutionLocation, { kind: "array" }); // TODO: check the filesize before slicing - - // Determine Slice - const axes = transformAxes(multiscales[0].axes); - if (axes.some((item) => item.name === "z")) { - const zIndex = axes.findIndex((item) => item.name === "z"); - const zSliceIndex = Math.ceil(lowestResolution.shape[zIndex] / 2); - axes[zIndex].value = zSliceIndex; - } - - console.log(axes); - const lowestResolutionView = await zarrGet( - lowestResolution, - axes.map((item) => item.value) - ); - - console.log("SHAPE", lowestResolutionView.shape); + throw new Error("Invalid multiscales attribute structure"); + } - // Normalize Data - const u16data = lowestResolutionView.data as Uint16Array; - const min = Math.min(...u16data); - const max = Math.max(...u16data); - const normalizedData = new Uint8Array(u16data.length); - for (let i = 0; i < u16data.length; i++) { - normalizedData[i] = Math.round((255 * (u16data[i] - min)) / (max - min)); - } + const { multiscales } = group.attrs; + const datasets = multiscales[0].datasets; + const lowestResolutionDataset = datasets[datasets.length - 1]; + const lowestResolutionLocation = root.resolve(lowestResolutionDataset.path); + const lowestResolution = await zarr.open(lowestResolutionLocation, { kind: "array" }); + + // Determine Slice + const axes = transformAxes(multiscales[0].axes); + const zIndex = axes.findIndex((item) => item.name === "z"); + if (zIndex !== -1) { + const zSliceIndex = Math.ceil(lowestResolution.shape[zIndex] / 2); + axes[zIndex].value = zSliceIndex; + } - // Draw Data - const width = lowestResolution.shape[axes.findIndex((item) => item.name === "x")]; - const height = lowestResolution.shape[axes.findIndex((item) => item.name === "y")]; - const canvas = document.createElement("canvas"); - canvas.width = width; - canvas.height = height; - const context = canvas.getContext("2d"); + const lowestResolutionView = await zarrGet( + lowestResolution, + axes.map((item) => item.value) + ); + const u16data = lowestResolutionView.data as Uint16Array; + + // Normalize Data + const min = Math.min(...u16data); + const max = Math.max(...u16data); + const normalizedData = new Uint8Array(u16data.length); + for (let i = 0; i < u16data.length; i++) { + normalizedData[i] = Math.round((255 * (u16data[i] - min)) / (max - min)); + } - if (!context) { - throw new Error("Failed to get canvas context"); - } + // Build Canvas + const width = lowestResolution.shape[axes.findIndex((item) => item.name === "x")]; + const height = lowestResolution.shape[axes.findIndex((item) => item.name === "y")]; + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); - const imageData = context.createImageData(width, height); + if (!context) { + throw new Error("Failed to get canvas context"); + } - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const idx = (y * width + x) * 4; - const value = normalizedData[y * width + x]; - const [r, g, b] = grayscaleToRGB(value); - imageData.data[idx] = r; // Red - imageData.data[idx + 1] = g; // Green - imageData.data[idx + 2] = b; // Blue - imageData.data[idx + 3] = 255; // Alpha - } + // Draw data + const imageData = context.createImageData(width, height); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const idx = (y * width + x) * 4; + const value = normalizedData[y * width + x]; + imageData.data[idx] = value; // Red + imageData.data[idx + 1] = value; // Green + imageData.data[idx + 2] = value; // Blue + imageData.data[idx + 3] = 255; // Alpha } + } - context.putImageData(imageData, 0, 0); + context.putImageData(imageData, 0, 0); - // Convert data to data URL - const dataUrl = canvas.toDataURL("image/png"); - return dataUrl; - } else { - throw new Error("Invalid multiscales attribute structure"); - } + // Convert data to data URL + return canvas.toDataURL("image/png"); } catch (error) { console.error("Error reading Zarr image:", error); throw error; From 2bfd542472770ff9b47cc5b2ac84f798f33a1d12 Mon Sep 17 00:00:00 2001 From: BrianW25 Date: Thu, 13 Jun 2024 14:49:43 -0700 Subject: [PATCH 20/20] cloudpath --- packages/core/entity/FileDetail/index.ts | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/core/entity/FileDetail/index.ts b/packages/core/entity/FileDetail/index.ts index 4eeacb632..7484042ba 100644 --- a/packages/core/entity/FileDetail/index.ts +++ b/packages/core/entity/FileDetail/index.ts @@ -161,19 +161,6 @@ export default class FileDetail { } public async getPathToThumbnail(): Promise { - if (this.cloudPath.endsWith(".zarr")) { - try { - // const thumbnailURL = await renderZarrThumbnailURL(this.cloudPath); - const thumbnailURL = await renderZarrThumbnailURL( - "https://animatedcell-test-data.s3.us-west-2.amazonaws.com/variance/136244.zarr" - ); - return thumbnailURL; - } catch (error) { - console.error("Error generating Zarr thumbnail:", error); - throw new Error("Unable to generate Zarr thumbnail"); - } - } - if (!this.thumbnail) { const isFileRenderableImage = RENDERABLE_IMAGE_FORMATS.some((format) => this.name.toLowerCase().endsWith(format) @@ -190,6 +177,17 @@ export default class FileDetail { if (this.thumbnail?.startsWith("/allen")) { return `${AICS_FMS_FILES_NGINX_SERVER}${this.thumbnail}`; } + + if (this.cloudPath.endsWith(".zarr")) { + try { + // const thumbnailURL = await renderZarrThumbnailURL(this.cloudPath); + const thumbnailURL = await renderZarrThumbnailURL(this.cloudPath); + return thumbnailURL; + } catch (error) { + console.error("Error generating Zarr thumbnail:", error); + throw new Error("Unable to generate Zarr thumbnail"); + } + } return this.thumbnail; } }