From 4032c482e261948f0dfc5859044b0b1210338813 Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Mon, 13 May 2024 10:33:24 -0700 Subject: [PATCH 1/7] Add method to convert the file explorer url to pandas --- .../Modal/CodeSnippet/CodeSnippet.module.css | 37 ++++++++- .../components/Modal/CodeSnippet/index.tsx | 30 +++++++- packages/core/entity/FileExplorerURL/index.ts | 75 +++++++++++++++++++ packages/core/state/interaction/selectors.ts | 9 +-- packages/core/state/selection/selectors.ts | 19 +++++ 5 files changed, 163 insertions(+), 7 deletions(-) diff --git a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css index 13d15daf4..a591f6f17 100644 --- a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css +++ b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css @@ -25,4 +25,39 @@ .code { cursor: text; user-select: text; -} \ No newline at end of file +} + +.button-menu, .button-menu button { + background-color: var(--secondary-background-color); + color: var(--secondary-text-color); +} + +.button-menu i, .button-menu li > div { + color: var(--secondary-text-color); +} + +.button-menu :is(a, button):hover, .button-menu button:hover i { + background-color: var(--highlight-background-color); + color: var(--highlight-text-color) !important; +} + +.code-actions { + max-width: 100%; + padding: 0 var(--padding) 0 var(--padding); + + /* flex parent */ + display: flex; +} + +.action-button { + background-color: var(--primary-background-color); + border: none; + border-radius: 0; + color: var(--primary-text-color); + /* height: 30px;*/ + width: 150px; +} + +.action-button i, .action-button:hover i { + color: var(--primary-text-color) !important; +} diff --git a/packages/core/components/Modal/CodeSnippet/index.tsx b/packages/core/components/Modal/CodeSnippet/index.tsx index 87f31ec53..d1852c6e3 100644 --- a/packages/core/components/Modal/CodeSnippet/index.tsx +++ b/packages/core/components/Modal/CodeSnippet/index.tsx @@ -1,4 +1,5 @@ -import { IconButton, TooltipHost } from "@fluentui/react"; +import { ActionButton, IContextualMenuItem, IconButton, TooltipHost } from "@fluentui/react"; +import classNames from "classnames"; import * as React from "react"; import { useSelector } from "react-redux"; import SyntaxHighlighter from "react-syntax-highlighter"; @@ -18,9 +19,19 @@ export default function CodeSnippet({ onDismiss }: ModalProps) { const pythonSnippet = useSelector(interaction.selectors.getPythonSnippet); const code = pythonSnippet?.code; const setup = pythonSnippet?.setup; + const languageOptions: IContextualMenuItem[] = [ + { + key: "python", + text: "Python (pandas)", + onClick() { + setLanguage("Python (pandas)"); + }, + }, + ]; const [isSetupCopied, setSetupCopied] = React.useState(false); const [isCodeCopied, setCodeCopied] = React.useState(false); + const [language, setLanguage] = React.useState(languageOptions[0].text); const onCopySetup = () => { setup && navigator.clipboard.writeText(setup); @@ -44,6 +55,19 @@ export default function CodeSnippet({ onDismiss }: ModalProps) { const body = ( <> +
+

Language

+
+ +
+

Setup

@@ -75,6 +99,10 @@ export default function CodeSnippet({ onDismiss }: ModalProps) {
) { + const collectionString = this.convertCollectionToPython(urlComponents?.collection); + + const groupByQueryString = + urlComponents.hierarchy + ?.map((annotation) => this.convertGroupByToPython(annotation)) + .join("") || ""; + + // Group filters by name and use OR to concatenate same filter values + const filterGroups = new Map(); + urlComponents.filters?.forEach((filter) => { + const pythonQueryString = filterGroups.get(filter.name); + if (!pythonQueryString) { + filterGroups.set(filter.name, this.convertFilterToPython(filter)); + } else { + filterGroups.set( + filter.name, + pythonQueryString.concat(` | ${this.convertFilterToPython(filter)}`) + ); + } + }); + + // Chain the filters together + let filterQueryString = ""; + filterGroups.forEach((value) => { + filterQueryString = filterQueryString.concat(`.query('${value}')`); + }); + + const sortQueryString = urlComponents.sortColumn + ? this.convertSortToPython(urlComponents.sortColumn) + : ""; + // const fuzzy = [] // TO DO: support fuzzy filtering + + const hasQueryElements = groupByQueryString || filterQueryString || sortQueryString; + const imports = "import pandas\n"; + const comment = hasQueryElements ? "#Query on dataframe df" : "#No options selected"; + const fullQueryString = `${comment}${ + hasQueryElements && `\ndf${groupByQueryString}${filterQueryString}${sortQueryString}` + }`; + return `${imports}${collectionString}${fullQueryString}`; + } + + private static convertSortToPython(sortColumn: FileSort) { + return `.sort_values(by='${sortColumn.annotationName}', ascending=${ + sortColumn.order == "ASC" ? "True" : "False" + })`; + } + + private static convertGroupByToPython(annotation: string) { + return `.groupby('${annotation}', group_keys=True).apply(lambda x: x)`; + } + + private static convertFilterToPython(filter: FileFilter) { + // TO DO: Support querying non-string types + if (filter.value.includes("RANGE")) { + return; + // let begin, end; + // return `\`${filter.name}\`>="${begin}"&\`${filter.name}\`<"${end}"` + } + return `\`${filter.name}\`=="${filter.value}"`; + } + + private static convertCollectionToPython(collection: Collection | undefined) { + if (collection?.uri) { + const comment = "#Convert current datasource file to a pandas dataframe"; + const extension = collection.uri.substring(collection.uri.lastIndexOf(".") + 1); + // Currently suggest setting all fields to strings; otherwise pandas assumes type conversions + // TO DO: Address different non-string type conversions + const code = `df = pandas.read_${extension}('${collection.uri}').astype('str')`; + // This only works if we assume that the file types will only be csv, parquet or json + return `${comment}\n${code}\n\n`; + } + return ""; + } } diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index 5b1005b3c..0c8b293a7 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -1,7 +1,7 @@ import { createSelector } from "reselect"; import { State } from "../"; -import { getCollection } from "../selection/selectors"; +import { getCollection, getPythonConversion } from "../selection/selectors"; import { AnnotationService, FileService, HttpServiceBase } from "../../services"; import DatasetService, { PythonicDataAccessSnippet } from "../../services/DatasetService"; import DatabaseAnnotationService from "../../services/AnnotationService/DatabaseAnnotationService"; @@ -32,12 +32,11 @@ export const getUserSelectedApplications = (state: State) => export const getVisibleModal = (state: State) => state.interaction.visibleModal; // COMPOSED SELECTORS -// TODO: Implement PythonicDataAccessSnippet export const getPythonSnippet = createSelector( - [], - (): PythonicDataAccessSnippet => { + [getPythonConversion], + (pythonQuery): PythonicDataAccessSnippet => { const setup = "pip install pandas"; - const code = "TODO"; + const code = `${pythonQuery}`; return { setup, code }; } diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index 38c44e98c..e3a802698 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -52,6 +52,25 @@ export const getEncodedFileExplorerUrl = createSelector( } ); +export const getPythonConversion = createSelector( + [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getCollection], + ( + hierarchy: string[], + filters: FileFilter[], + openFolders: FileFolder[], + sortColumn?: FileSort, + collection?: Dataset + ) => { + return FileExplorerURL.convertToPython({ + hierarchy, + filters, + openFolders, + sortColumn, + collection, + }); + } +); + export const getGroupedByFilterName = createSelector( [getFileFilters, getAnnotations], (globalFilters: FileFilter[], annotations: Annotation[]) => { From b390d1d872b776d89971e63cd7a43debad8afd3f Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Mon, 13 May 2024 10:34:23 -0700 Subject: [PATCH 2/7] Add unit tests for file explorer url to pandas conversion --- .../CodeSnippet/test/CodeSnippet.test.tsx | 2 +- .../test/fileexplorerurl.test.ts | 121 ++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx index 700fe0692..51ff9069f 100644 --- a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx +++ b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx @@ -31,7 +31,7 @@ describe("", () => { it("displays snippet when present in state", async () => { // Arrange const setup = "pip install pandas"; - const code = "TODO"; + const code = "#No options selected"; const { store } = configureMockStore({ state: visibleDialogState }); const { findByText } = render( diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index 1dc2a53cd..7703cc6e5 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -18,6 +18,7 @@ describe("FileExplorerURL", () => { private: true, created: new Date(), createdBy: "test", + uri: "test/file.csv", }; describe("encode", () => { @@ -196,4 +197,124 @@ describe("FileExplorerURL", () => { expect(() => FileExplorerURL.decode(encodedUrl)).to.throw(); }); }); + + describe("convert to python pandas string", () => { + it("converts groupings", () => { + // Arrange + const expectedAnnotationNames = ["Cell Line", "Donor Plasmid", "Lifting?"]; + const components: Partial = { + hierarchy: expectedAnnotationNames, + }; + const expectedPandasGroups = expectedAnnotationNames.map( + (annotation) => `.groupby('${annotation}', group_keys=True).apply(lambda x: x)` + ); + const expectedResult = `df${expectedPandasGroups.join("")}`; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("converts filters", () => { + // Arrange + const expectedFilters = [ + { name: "Cas9", value: "spCas9" }, + { name: "Donor Plasmid", value: "ACTB-mEGFP" }, + ]; + const components: Partial = { + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + }; + const expectedPandasQueries = expectedFilters.map( + (filter) => `\`${filter.name}\`=="${filter.value}"` + ); + const expectedResult = `df.query('${expectedPandasQueries[0]}').query('${expectedPandasQueries[1]}')`; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("converts same filter with multiple values", () => { + // Arrange + const expectedFilters = [ + { name: "Gene", value: "AAVS1" }, + { name: "Gene", value: "ACTB" }, + ]; + const components: Partial = { + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + }; + const expectedPandasQueries = expectedFilters.map( + (filter) => `\`${filter.name}\`=="${filter.value}"` + ); + const expectedResult = `df.query('${expectedPandasQueries[0]} | ${expectedPandasQueries[1]}')`; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("converts sorts", () => { + // Arrange + const components: Partial = { + sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), + }; + const expectedPandasSort = `.sort_values(by='${AnnotationName.UPLOADED}', ascending=False`; + const expectedResult = `df${expectedPandasSort}`; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("provides info on converting external data source to pandas dataframe", () => { + // Arrange + const components: Partial = { + collection: { + name: mockCollection.name, + version: mockCollection.version, + uri: mockCollection.uri, + }, + }; + const expectedResult = `df = pandas.read_csv('${mockCollection.uri}').astype('str')`; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("arranges query elements in correct order", () => { + // Arrange + const expectedAnnotationNames = ["Plate Barcode"]; + const expectedFilters = [ + { name: "Cas9", value: "spCas9" }, + { name: "Donor Plasmid", value: "ACTB-mEGFP" }, + ]; + const components: Partial = { + hierarchy: expectedAnnotationNames, + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), + collection: { + name: mockCollection.name, + version: mockCollection.version, + }, + }; + const expectedResult = /df\.groupby\(.*\)\.query\(.*\)\.query\(.*\)\.sort_values\(.*\)/i; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.match(expectedResult); + }); + }); }); From 34b3d35cdd913a3da204ad30c31b42acf215b7df Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Mon, 13 May 2024 11:35:30 -0700 Subject: [PATCH 3/7] Add version floor to pandas and specify Python version --- .../components/Modal/CodeSnippet/CodeSnippet.module.css | 3 +-- packages/core/components/Modal/CodeSnippet/index.tsx | 4 ++-- .../components/Modal/CodeSnippet/test/CodeSnippet.test.tsx | 7 ++++--- packages/core/state/interaction/selectors.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css index a591f6f17..45d28374e 100644 --- a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css +++ b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css @@ -54,8 +54,7 @@ border: none; border-radius: 0; color: var(--primary-text-color); - /* height: 30px;*/ - width: 150px; + width: 200px; } .action-button i, .action-button:hover i { diff --git a/packages/core/components/Modal/CodeSnippet/index.tsx b/packages/core/components/Modal/CodeSnippet/index.tsx index d1852c6e3..3fd2ff906 100644 --- a/packages/core/components/Modal/CodeSnippet/index.tsx +++ b/packages/core/components/Modal/CodeSnippet/index.tsx @@ -22,9 +22,9 @@ export default function CodeSnippet({ onDismiss }: ModalProps) { const languageOptions: IContextualMenuItem[] = [ { key: "python", - text: "Python (pandas)", + text: "Python 3.8+ (pandas)", onClick() { - setLanguage("Python (pandas)"); + setLanguage("Python 3.8+ (pandas)"); }, }, ]; diff --git a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx index 51ff9069f..0869a92ee 100644 --- a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx +++ b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx @@ -1,5 +1,5 @@ import { configureMockStore, mergeState } from "@aics/redux-utils"; -import { render } from "@testing-library/react"; +import { render, screen } from "@testing-library/react"; import { expect } from "chai"; import * as React from "react"; import { Provider } from "react-redux"; @@ -30,7 +30,7 @@ describe("", () => { it("displays snippet when present in state", async () => { // Arrange - const setup = "pip install pandas"; + const setup = /pip install (")?pandas/; const code = "#No options selected"; const { store } = configureMockStore({ state: visibleDialogState }); const { findByText } = render( @@ -40,7 +40,8 @@ describe("", () => { ); // Assert - expect(await findByText(setup)).to.exist; + expect(screen.findByText((_, element) => element?.textContent?.match(setup) !== null)).to + .exist; expect(await findByText(code)).to.exist; }); }); diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index 0c8b293a7..5677efe1e 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -35,7 +35,7 @@ export const getVisibleModal = (state: State) => state.interaction.visibleModal; export const getPythonSnippet = createSelector( [getPythonConversion], (pythonQuery): PythonicDataAccessSnippet => { - const setup = "pip install pandas"; + const setup = `pip install \"pandas>=1.5\"`; const code = `${pythonQuery}`; return { setup, code }; From 4c46839ed84944cd4653718cf96ba1c93cd5d293 Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Tue, 21 May 2024 13:50:45 -0700 Subject: [PATCH 4/7] Refactor collection to datasource in code snippet --- .../Modal/CodeSnippet/CodeSnippet.module.css | 1 + packages/core/entity/FileExplorerURL/index.ts | 26 ++++++++++++------- packages/core/state/selection/selectors.ts | 6 ++--- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css index 3bd286663..df979b46e 100644 --- a/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css +++ b/packages/core/components/Modal/CodeSnippet/CodeSnippet.module.css @@ -25,6 +25,7 @@ .code { cursor: text; user-select: text !important; + min-width: 300px; } .button-menu, .button-menu button { diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index af61d0e01..cb1d0cf95 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -2,6 +2,7 @@ import { AnnotationName } from "../Annotation"; import FileFilter from "../FileFilter"; import FileFolder from "../FileFolder"; import FileSort, { SortOrder } from "../FileSort"; +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; export interface Source { name: string; @@ -120,8 +121,10 @@ export default class FileExplorerURL { } public static convertToPython(urlComponents: Partial) { - const collectionString = this.convertCollectionToPython(urlComponents?.collection); - + if (urlComponents?.source?.name === AICS_FMS_DATA_SOURCE_NAME) { + return "#Coming soon"; + } + const sourceString = this.convertDataSourceToPython(urlComponents?.source); const groupByQueryString = urlComponents.hierarchy ?.map((annotation) => this.convertGroupByToPython(annotation)) @@ -153,12 +156,13 @@ export default class FileExplorerURL { // const fuzzy = [] // TO DO: support fuzzy filtering const hasQueryElements = groupByQueryString || filterQueryString || sortQueryString; - const imports = "import pandas\n"; + const imports = "import pandas as pd\n\n"; const comment = hasQueryElements ? "#Query on dataframe df" : "#No options selected"; const fullQueryString = `${comment}${ - hasQueryElements && `\ndf${groupByQueryString}${filterQueryString}${sortQueryString}` + hasQueryElements && + `\ndf_queried = df${groupByQueryString}${filterQueryString}${sortQueryString}` }`; - return `${imports}${collectionString}${fullQueryString}`; + return `${imports}${sourceString}${fullQueryString}`; } private static convertSortToPython(sortColumn: FileSort) { @@ -181,13 +185,15 @@ export default class FileExplorerURL { return `\`${filter.name}\`=="${filter.value}"`; } - private static convertCollectionToPython(collection: Collection | undefined) { - if (collection?.uri) { - const comment = "#Convert current datasource file to a pandas dataframe"; - const extension = collection.uri.substring(collection.uri.lastIndexOf(".") + 1); + private static convertDataSourceToPython(source: Source | undefined) { + if (source) { + const comment = + "#Convert current datasource file to a pandas dataframe\n" + + "#You may need to manually update the path to the file"; + // Currently suggest setting all fields to strings; otherwise pandas assumes type conversions // TO DO: Address different non-string type conversions - const code = `df = pandas.read_${extension}('${collection.uri}').astype('str')`; + const code = `df = pd.read_${source.type}('${source.name}').astype('str')`; // This only works if we assume that the file types will only be csv, parquet or json return `${comment}\n${code}\n\n`; } diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index efc96bf53..97deddd44 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -62,20 +62,20 @@ export const getEncodedFileExplorerUrl = createSelector( ); export const getPythonConversion = createSelector( - [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getCollection], + [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getDataSource], ( hierarchy: string[], filters: FileFilter[], openFolders: FileFolder[], sortColumn?: FileSort, - collection?: Dataset + source?: DataSource ) => { return FileExplorerURL.convertToPython({ hierarchy, filters, openFolders, sortColumn, - collection, + source, }); } ); From 044173af972d01af995c8108b7caa82580cb36ec Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Tue, 21 May 2024 13:55:03 -0700 Subject: [PATCH 5/7] Update unit tests --- .../test/fileexplorerurl.test.ts | 77 +++++++++---------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index 23e918765..83ea6cb2e 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -224,47 +224,40 @@ describe("FileExplorerURL", () => { expect(result).to.contain(expectedResult); }); - // it("provides info on converting external data source to pandas dataframe", () => { - // // Arrange - // const components: Partial = { - // collection: { - // name: mockCollection.name, - // version: mockCollection.version, - // uri: mockCollection.uri, - // }, - // }; - // const expectedResult = `df = pandas.read_csv('${mockCollection.uri}').astype('str')`; - - // // Act - // const result = FileExplorerURL.convertToPython(components); - - // // Assert - // expect(result).to.contain(expectedResult); - // }); - - // it("arranges query elements in correct order", () => { - // // Arrange - // const expectedAnnotationNames = ["Plate Barcode"]; - // const expectedFilters = [ - // { name: "Cas9", value: "spCas9" }, - // { name: "Donor Plasmid", value: "ACTB-mEGFP" }, - // ]; - // const components: Partial = { - // hierarchy: expectedAnnotationNames, - // filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), - // sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - // collection: { - // name: mockCollection.name, - // version: mockCollection.version, - // }, - // }; - // const expectedResult = /df\.groupby\(.*\)\.query\(.*\)\.query\(.*\)\.sort_values\(.*\)/i; - - // // Act - // const result = FileExplorerURL.convertToPython(components); - - // // Assert - // expect(result).to.match(expectedResult); - // }); + it("provides info on converting external data source to pandas dataframe", () => { + // Arrange + const components: Partial = { + source: mockSource, + }; + const expectedResult = `df = pd.read_csv('${mockSource.name}').astype('str')`; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("arranges query elements in correct order", () => { + // Arrange + const expectedAnnotationNames = ["Plate Barcode"]; + const expectedFilters = [ + { name: "Cas9", value: "spCas9" }, + { name: "Donor Plasmid", value: "ACTB-mEGFP" }, + ]; + const components: Partial = { + hierarchy: expectedAnnotationNames, + filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), + source: mockSource, + }; + const expectedResult = /df\.groupby\(.*\)\.query\(.*\)\.query\(.*\)\.sort_values\(.*\)/i; + + // Act + const result = FileExplorerURL.convertToPython(components); + + // Assert + expect(result).to.match(expectedResult); + }); }); }); From 9b14578d20d689b15227e9c03bd0adba2f3fbf5e Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Tue, 21 May 2024 18:58:58 -0700 Subject: [PATCH 6/7] Update source Python conversion and account for OS --- .../CodeSnippet/test/CodeSnippet.test.tsx | 27 ++++++++++++ packages/core/entity/FileExplorerURL/index.ts | 44 ++++++++++++++----- .../test/fileexplorerurl.test.ts | 43 ++++++++++++++---- packages/core/state/selection/selectors.ts | 28 ++++++++---- 4 files changed, 113 insertions(+), 29 deletions(-) diff --git a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx index 0869a92ee..859520af3 100644 --- a/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx +++ b/packages/core/components/Modal/CodeSnippet/test/CodeSnippet.test.tsx @@ -12,6 +12,11 @@ describe("", () => { interaction: { visibleModal: ModalType.CodeSnippet, }, + selection: { + dataSource: { + uri: "fake-uri.test", + }, + }, }); it("is visible when should not be hidden", () => { @@ -44,4 +49,26 @@ describe("", () => { .exist; expect(await findByText(code)).to.exist; }); + + it("displays temporary 'coming soon' message for internal data sources", async () => { + // Arrange + const code = "# Coming soon"; + const internalDataSourceState = { + ...visibleDialogState, + selection: { + dataSource: { + uri: undefined, + }, + }, + }; + const { store } = configureMockStore({ state: internalDataSourceState }); + const { findByText } = render( + + + + ); + + // Assert + expect(await findByText(code)).to.exist; + }); }); diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index cb1d0cf95..6baaa56c1 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -120,11 +120,17 @@ export default class FileExplorerURL { }; } - public static convertToPython(urlComponents: Partial) { - if (urlComponents?.source?.name === AICS_FMS_DATA_SOURCE_NAME) { - return "#Coming soon"; + public static convertToPython( + urlComponents: Partial, + userOS: string + ) { + if ( + urlComponents?.source?.name === AICS_FMS_DATA_SOURCE_NAME || + !urlComponents?.source?.uri + ) { + return "# Coming soon"; } - const sourceString = this.convertDataSourceToPython(urlComponents?.source); + const sourceString = this.convertDataSourceToPython(urlComponents?.source, userOS); const groupByQueryString = urlComponents.hierarchy ?.map((annotation) => this.convertGroupByToPython(annotation)) @@ -185,18 +191,32 @@ export default class FileExplorerURL { return `\`${filter.name}\`=="${filter.value}"`; } - private static convertDataSourceToPython(source: Source | undefined) { - if (source) { - const comment = - "#Convert current datasource file to a pandas dataframe\n" + - "#You may need to manually update the path to the file"; + private static convertDataSourceToPython(source: Source | undefined, userOS: string) { + const rawFlagForWindows = userOS === "Windows_NT" ? "r" : ""; + + if (typeof source?.uri === "string") { + const comment = "#Convert current datasource file to a pandas dataframe"; // Currently suggest setting all fields to strings; otherwise pandas assumes type conversions // TO DO: Address different non-string type conversions - const code = `df = pd.read_${source.type}('${source.name}').astype('str')`; + const code = `df = pd.read_${source.type}(${rawFlagForWindows}'${source.uri}').astype('str')`; // This only works if we assume that the file types will only be csv, parquet or json + return `${comment}\n${code}\n\n`; - } - return ""; + } else if (source?.uri) { + // Any other type, i.e., File. `instanceof` breaks testing library + // Adding strings to avoid including unwanted white space + const inputFileLineComment = + " # Unable to automatically determine " + + "local file location in the browser. Modify this variable to " + + "represent the full path to your .csv, .json, or .parquet data sources\n"; + const inputFileError = + "if not input_file:\n" + + '\traise Exception("Must supply the data source location for the query")\n'; + const inputFileCode = 'input_file = ""' + inputFileLineComment + inputFileError; + + const conversionCode = `df = pd.read_${source.type}(input_file).astype('str')`; + return `${inputFileCode}\n${conversionCode}\n\n`; + } else return ""; // Safeguard. Should not reach else } } diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index 83ea6cb2e..86b2269e2 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -12,6 +12,13 @@ describe("FileExplorerURL", () => { type: "csv", }; + const mockSourceWithUri: Source = { + ...mockSource, + uri: "fake-uri.test", + }; + + const mockOS = "Darwin"; + describe("encode", () => { it("Encodes hierarchy, filters, open folders, and collection", () => { // Arrange @@ -154,6 +161,7 @@ describe("FileExplorerURL", () => { const expectedAnnotationNames = ["Cell Line", "Donor Plasmid", "Lifting?"]; const components: Partial = { hierarchy: expectedAnnotationNames, + source: mockSourceWithUri, }; const expectedPandasGroups = expectedAnnotationNames.map( (annotation) => `.groupby('${annotation}', group_keys=True).apply(lambda x: x)` @@ -161,7 +169,7 @@ describe("FileExplorerURL", () => { const expectedResult = `df${expectedPandasGroups.join("")}`; // Act - const result = FileExplorerURL.convertToPython(components); + const result = FileExplorerURL.convertToPython(components, mockOS); // Assert expect(result).to.contain(expectedResult); @@ -175,6 +183,7 @@ describe("FileExplorerURL", () => { ]; const components: Partial = { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + source: mockSourceWithUri, }; const expectedPandasQueries = expectedFilters.map( (filter) => `\`${filter.name}\`=="${filter.value}"` @@ -182,7 +191,7 @@ describe("FileExplorerURL", () => { const expectedResult = `df.query('${expectedPandasQueries[0]}').query('${expectedPandasQueries[1]}')`; // Act - const result = FileExplorerURL.convertToPython(components); + const result = FileExplorerURL.convertToPython(components, mockOS); // Assert expect(result).to.contain(expectedResult); @@ -196,6 +205,7 @@ describe("FileExplorerURL", () => { ]; const components: Partial = { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), + source: mockSourceWithUri, }; const expectedPandasQueries = expectedFilters.map( (filter) => `\`${filter.name}\`=="${filter.value}"` @@ -203,7 +213,7 @@ describe("FileExplorerURL", () => { const expectedResult = `df.query('${expectedPandasQueries[0]} | ${expectedPandasQueries[1]}')`; // Act - const result = FileExplorerURL.convertToPython(components); + const result = FileExplorerURL.convertToPython(components, mockOS); // Assert expect(result).to.contain(expectedResult); @@ -213,12 +223,13 @@ describe("FileExplorerURL", () => { // Arrange const components: Partial = { sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), + source: mockSourceWithUri, }; const expectedPandasSort = `.sort_values(by='${AnnotationName.UPLOADED}', ascending=False`; const expectedResult = `df${expectedPandasSort}`; // Act - const result = FileExplorerURL.convertToPython(components); + const result = FileExplorerURL.convertToPython(components, mockOS); // Assert expect(result).to.contain(expectedResult); @@ -227,12 +238,26 @@ describe("FileExplorerURL", () => { it("provides info on converting external data source to pandas dataframe", () => { // Arrange const components: Partial = { - source: mockSource, + source: mockSourceWithUri, }; - const expectedResult = `df = pd.read_csv('${mockSource.name}').astype('str')`; + const expectedResult = `df = pd.read_csv('${mockSourceWithUri.uri}').astype('str')`; // Act - const result = FileExplorerURL.convertToPython(components); + const result = FileExplorerURL.convertToPython(components, mockOS); + + // Assert + expect(result).to.contain(expectedResult); + }); + + it("adds raw flag in pandas conversion code for Windows OS", () => { + // Arrange + const components: Partial = { + source: mockSourceWithUri, + }; + const expectedResult = `df = pd.read_csv(r'${mockSourceWithUri.uri}').astype('str')`; + + // Act + const result = FileExplorerURL.convertToPython(components, "Windows_NT"); // Assert expect(result).to.contain(expectedResult); @@ -249,12 +274,12 @@ describe("FileExplorerURL", () => { hierarchy: expectedAnnotationNames, filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - source: mockSource, + source: mockSourceWithUri, }; const expectedResult = /df\.groupby\(.*\)\.query\(.*\)\.query\(.*\)\.sort_values\(.*\)/i; // Act - const result = FileExplorerURL.convertToPython(components); + const result = FileExplorerURL.convertToPython(components, mockOS); // Assert expect(result).to.match(expectedResult); diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index 97deddd44..3cd728f40 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -32,6 +32,7 @@ export const getShouldDisplayThumbnailView = (state: State) => export const getSortColumn = (state: State) => state.selection.sortColumn; export const getTutorial = (state: State) => state.selection.tutorial; export const getQueries = (state: State) => state.selection.queries; +const getPlatformDependentServices = (state: State) => state.interaction.platformDependentServices; // Importing normally creates a circular dependency // COMPOSED SELECTORS export const isQueryingAicsFms = createSelector( @@ -62,21 +63,32 @@ export const getEncodedFileExplorerUrl = createSelector( ); export const getPythonConversion = createSelector( - [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getDataSource], + [ + getPlatformDependentServices, + getAnnotationHierarchy, + getFileFilters, + getOpenFileFolders, + getSortColumn, + getDataSource, + ], ( + platformDependentServices, hierarchy: string[], filters: FileFilter[], openFolders: FileFolder[], sortColumn?: FileSort, source?: DataSource ) => { - return FileExplorerURL.convertToPython({ - hierarchy, - filters, - openFolders, - sortColumn, - source, - }); + return FileExplorerURL.convertToPython( + { + hierarchy, + filters, + openFolders, + sortColumn, + source, + }, + platformDependentServices.executionEnvService.getOS() + ); } ); From afb7f3c5be3ed9974ad5d539e4aac3e5de823ce2 Mon Sep 17 00:00:00 2001 From: Anya Wallace Date: Tue, 21 May 2024 19:13:30 -0700 Subject: [PATCH 7/7] Account for web in OS check --- packages/core/entity/FileExplorerURL/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index 6baaa56c1..679ed3016 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -192,7 +192,8 @@ export default class FileExplorerURL { } private static convertDataSourceToPython(source: Source | undefined, userOS: string) { - const rawFlagForWindows = userOS === "Windows_NT" ? "r" : ""; + const isUsingWindowsOS = userOS === "Windows_NT" || userOS.includes("Windows NT"); + const rawFlagForWindows = isUsingWindowsOS ? "r" : ""; if (typeof source?.uri === "string") { const comment = "#Convert current datasource file to a pandas dataframe";