diff --git a/package-lock.json b/package-lock.json index 0ee3b60c7..9c32643a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,12 @@ { "name": "root", + "version": "6.5.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "root", + "version": "6.5.1", "license": "2-clause BSD license plus a third clause that prohibits redistribution and use for commercial purposes without further permission.", "workspaces": [ "packages/desktop", @@ -89,6 +91,26 @@ "npm": "8.x" } }, + "node_modules/@75lb/deep-merge": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", + "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "dependencies": { + "lodash.assignwith": "^4.2.0", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/@75lb/deep-merge/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/@aics/frontend-insights": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@aics/frontend-insights/-/frontend-insights-0.2.3.tgz", @@ -2076,6 +2098,14 @@ "node": ">=10.0.0" } }, + "node_modules/@duckdb/duckdb-wasm": { + "version": "1.28.1-dev106.0", + "resolved": "https://registry.npmjs.org/@duckdb/duckdb-wasm/-/duckdb-wasm-1.28.1-dev106.0.tgz", + "integrity": "sha512-HcA9q/Yq1t8nAIg2rl8DmOTjKy1tAHSdBGHlCcWAm5StsfAjcm+f0STBEH3hmWPk0qEtOJF30OR+GfeyUOP+hA==", + "dependencies": { + "apache-arrow": "^14.0.1" + } + }, "node_modules/@electron/get": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/@electron/get/-/get-1.14.1.tgz", @@ -3152,6 +3182,16 @@ "integrity": "sha512-2koNhpWm3DgWRp5tpkiJ8JGc1xTn2q0l+jUNUE7oMKXUf5NpI9AIdC4kbjGNFBdHtcxBD18LAksoudAVhFKCjw==", "dev": true }, + "node_modules/@types/command-line-args": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", + "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==" + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz", + "integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==" + }, "node_modules/@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -3356,10 +3396,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "17.0.29", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.29.tgz", - "integrity": "sha512-tx5jMmMFwx7wBwq/V7OohKDVb/JwJU5qCVkeLMh1//xycAJ/ESuw9aJ9SEtlCZDYi2pBfe4JkisSoAtbOsBNAA==", - "dev": true + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -3367,6 +3406,11 @@ "integrity": "sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==", "dev": true }, + "node_modules/@types/pad-left": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz", + "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==" + }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -4298,6 +4342,31 @@ "node": ">= 8" } }, + "node_modules/apache-arrow": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", + "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", + "dependencies": { + "@types/command-line-args": "5.2.0", + "@types/command-line-usage": "5.0.2", + "@types/node": "20.3.0", + "@types/pad-left": "2.1.1", + "command-line-args": "5.2.1", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", + "json-bignum": "^0.0.3", + "pad-left": "^2.1.0", + "tslib": "^2.5.3" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, "node_modules/app-builder-bin": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-4.0.0.tgz", @@ -4454,6 +4523,14 @@ "node": ">=6.0" } }, + "node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "engines": { + "node": ">=6" + } + }, "node_modules/array-flatten": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", @@ -5536,6 +5613,84 @@ "node": ">=4" } }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/chalk-template/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/chalk-template/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/chalk-template/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/character-entities": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", @@ -5856,6 +6011,50 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/command-line-args": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", + "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", + "dependencies": { + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-usage": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/command-line-usage/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/command-line-usage/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -8824,6 +9023,17 @@ "node": ">=6" } }, + "node_modules/find-replace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", + "dependencies": { + "array-back": "^3.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -8862,6 +9072,11 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/flatbuffers": { + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==" + }, "node_modules/flatted": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", @@ -11137,6 +11352,14 @@ "node": ">=4" } }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/json-buffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", @@ -11671,6 +11894,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.assignwith": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", + "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "node_modules/lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", @@ -13471,6 +13704,17 @@ "node": ">=6" } }, + "node_modules/pad-left": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz", + "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==", + "dependencies": { + "repeat-string": "^1.5.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -15244,6 +15488,14 @@ "strip-ansi": "^6.0.1" } }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -16174,6 +16426,14 @@ "node": ">= 0.8" } }, + "node_modules/stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==", + "engines": { + "node": ">=10" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -16394,6 +16654,42 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "node_modules/table-layout": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", + "dependencies": { + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" + }, + "bin": { + "table-layout": "bin/cli.js" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/tapable": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", @@ -16832,6 +17128,14 @@ "node": ">=4.2.0" } }, + "node_modules/typical": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", + "engines": { + "node": ">=8" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -17747,6 +18051,14 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrapjs": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", + "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/workerpool": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", @@ -18074,7 +18386,7 @@ }, "packages/desktop": { "name": "fms-file-explorer-desktop", - "version": "6.5.1", + "version": "7.0.0", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@aics/frontend-insights": "0.2.x", @@ -18155,6 +18467,7 @@ "name": "fms-file-explorer-web", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { + "@duckdb/duckdb-wasm": "^1.28.1-dev106.0", "regenerator-runtime": "0.13.x" }, "devDependencies": { @@ -18175,6 +18488,22 @@ } }, "dependencies": { + "@75lb/deep-merge": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", + "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "requires": { + "lodash.assignwith": "^4.2.0", + "typical": "^7.1.1" + }, + "dependencies": { + "typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==" + } + } + }, "@aics/frontend-insights": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@aics/frontend-insights/-/frontend-insights-0.2.3.tgz", @@ -19528,6 +19857,14 @@ "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", "dev": true }, + "@duckdb/duckdb-wasm": { + "version": "1.28.1-dev106.0", + "resolved": "https://registry.npmjs.org/@duckdb/duckdb-wasm/-/duckdb-wasm-1.28.1-dev106.0.tgz", + "integrity": "sha512-HcA9q/Yq1t8nAIg2rl8DmOTjKy1tAHSdBGHlCcWAm5StsfAjcm+f0STBEH3hmWPk0qEtOJF30OR+GfeyUOP+hA==", + "requires": { + "apache-arrow": "^14.0.1" + } + }, "@electron/get": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/@electron/get/-/get-1.14.1.tgz", @@ -20416,6 +20753,16 @@ "integrity": "sha512-2koNhpWm3DgWRp5tpkiJ8JGc1xTn2q0l+jUNUE7oMKXUf5NpI9AIdC4kbjGNFBdHtcxBD18LAksoudAVhFKCjw==", "dev": true }, + "@types/command-line-args": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", + "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==" + }, + "@types/command-line-usage": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz", + "integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==" + }, "@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -20620,10 +20967,9 @@ "dev": true }, "@types/node": { - "version": "17.0.29", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.29.tgz", - "integrity": "sha512-tx5jMmMFwx7wBwq/V7OohKDVb/JwJU5qCVkeLMh1//xycAJ/ESuw9aJ9SEtlCZDYi2pBfe4JkisSoAtbOsBNAA==", - "dev": true + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" }, "@types/normalize-package-data": { "version": "2.4.1", @@ -20631,6 +20977,11 @@ "integrity": "sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==", "dev": true }, + "@types/pad-left": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz", + "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==" + }, "@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -21373,6 +21724,30 @@ "picomatch": "^2.0.4" } }, + "apache-arrow": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", + "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", + "requires": { + "@types/command-line-args": "5.2.0", + "@types/command-line-usage": "5.0.2", + "@types/node": "20.3.0", + "@types/pad-left": "2.1.1", + "command-line-args": "5.2.1", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", + "json-bignum": "^0.0.3", + "pad-left": "^2.1.0", + "tslib": "^2.5.3" + }, + "dependencies": { + "tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + } + } + }, "app-builder-bin": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-4.0.0.tgz", @@ -21504,6 +21879,11 @@ "integrity": "sha512-V+SM7AbUwJ+EBnB8+DXs0hPZHO0W6pqBcc0dW90OwtVG02PswOu/teuARoLQjdDOH+t9pJgGnW5/Qmouf3gPJg==", "dev": true }, + "array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==" + }, "array-flatten": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", @@ -22306,6 +22686,59 @@ "supports-color": "^5.3.0" } }, + "chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "requires": { + "chalk": "^4.1.2" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "character-entities": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", @@ -22540,6 +22973,40 @@ "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz", "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==" }, + "command-line-args": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", + "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", + "requires": { + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" + } + }, + "command-line-usage": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", + "requires": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" + }, + "dependencies": { + "array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==" + }, + "typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==" + } + } + }, "commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -24849,6 +25316,14 @@ "pkg-dir": "^3.0.0" } }, + "find-replace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", + "requires": { + "array-back": "^3.0.1" + } + }, "find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -24875,6 +25350,11 @@ "rimraf": "^3.0.2" } }, + "flatbuffers": { + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==" + }, "flatted": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", @@ -24911,7 +25391,7 @@ "webpack-bundle-analyzer": "4.x", "webpack-cli": "4.x", "webpack-dev-server": "4.x", - "webpack-node-externals": "*", + "webpack-node-externals": "^3.0.0", "xvfb-maybe": "0.2.x" }, "dependencies": { @@ -24951,6 +25431,7 @@ "fms-file-explorer-web": { "version": "file:packages/web", "requires": { + "@duckdb/duckdb-wasm": "*", "babel-loader": "8.2.x", "clean-webpack-plugin": "4.x", "css-loader": "6.x", @@ -26594,6 +27075,11 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==" + }, "json-buffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", @@ -27005,6 +27491,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.assignwith": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", + "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==" + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", @@ -28381,6 +28877,14 @@ "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, + "pad-left": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz", + "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==", + "requires": { + "repeat-string": "^1.5.4" + } + }, "pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -29668,6 +30172,11 @@ "strip-ansi": "^6.0.1" } }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==" + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -30410,6 +30919,11 @@ "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", "dev": true }, + "stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==" + }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -30572,6 +31086,32 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "table-layout": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", + "requires": { + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" + }, + "dependencies": { + "array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==" + }, + "typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==" + } + } + }, "tapable": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", @@ -30903,6 +31443,11 @@ "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw==", "dev": true }, + "typical": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==" + }, "unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -31572,6 +32117,11 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, + "wordwrapjs": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", + "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==" + }, "workerpool": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", diff --git a/packages/core/App.tsx b/packages/core/App.tsx index 1571bc33d..eec917b17 100644 --- a/packages/core/App.tsx +++ b/packages/core/App.tsx @@ -1,6 +1,7 @@ import "normalize.css"; import { initializeIcons, loadTheme } from "@fluentui/react"; import classNames from "classnames"; +import { uniqueId } from "lodash"; import * as React from "react"; import { batch, useDispatch, useSelector } from "react-redux"; @@ -14,7 +15,6 @@ import TutorialTooltip from "./components/TutorialTooltip"; import QuerySidebar from "./components/QuerySidebar"; import { FileExplorerServiceBaseUrl } from "./constants"; import { interaction, metadata, selection } from "./state"; -import { PlatformDependentServices } from "./state/interaction/actions"; import "./styles/global.css"; import styles from "./App.module.css"; @@ -36,25 +36,38 @@ interface AppProps { // Stage: "http://stg-aics-api.corp.alleninstitute.org" // From the web (behind load balancer): "/" fileExplorerServiceBaseUrl?: string; - platformDependentServices?: Partial; } -const DEFAULT_PLATFORM_DEPENDENT_SERVICES = Object.freeze({}); - export default function App(props: AppProps) { - const { - fileExplorerServiceBaseUrl = FileExplorerServiceBaseUrl.PRODUCTION, - platformDependentServices = DEFAULT_PLATFORM_DEPENDENT_SERVICES, - } = props; + const { fileExplorerServiceBaseUrl = FileExplorerServiceBaseUrl.PRODUCTION } = props; const dispatch = useDispatch(); const isDarkTheme = useSelector(selection.selectors.getIsDarkTheme); const shouldDisplaySmallFont = useSelector(selection.selectors.getShouldDisplaySmallFont); + const platformDependentServices = useSelector( + interaction.selectors.getPlatformDependentServices + ); - // Set platform-dependent services in state + // Check for updates to the application on startup React.useEffect(() => { - dispatch(interaction.actions.setPlatformDependentServices(platformDependentServices)); - }, [dispatch, platformDependentServices]); + const checkForUpdates = async () => { + try { + const isUpdateAvailable = await platformDependentServices.applicationInfoService.updateAvailable(); + if (isUpdateAvailable) { + const homepage = "https://alleninstitute.github.io/aics-fms-file-explorer-app/"; + const msg = `A new version of the application is available!
+ Visit the FMS File Explorer homepage to download.`; + dispatch(interaction.actions.promptUserToUpdateApp(uniqueId(), msg)); + } + } catch (e) { + console.error( + "Failed while checking if a newer application version is available", + e + ); + } + }; + checkForUpdates(); + }, [platformDependentServices, dispatch]); // Set data source base urls // And kick off the process of requesting metadata needed by the application. @@ -62,7 +75,7 @@ export default function App(props: AppProps) { batch(() => { dispatch(interaction.actions.setFileExplorerServiceBaseUrl(fileExplorerServiceBaseUrl)); dispatch(metadata.actions.requestAnnotations()); - dispatch(metadata.actions.requestCollections()); + dispatch(metadata.actions.requestDataSources()); dispatch(selection.actions.setAnnotationHierarchy([])); }); }, [dispatch, fileExplorerServiceBaseUrl]); diff --git a/packages/core/components/AnnotationPicker/index.tsx b/packages/core/components/AnnotationPicker/index.tsx index f4c747477..48a5d6b44 100644 --- a/packages/core/components/AnnotationPicker/index.tsx +++ b/packages/core/components/AnnotationPicker/index.tsx @@ -8,6 +8,8 @@ import Annotation from "../../entity/Annotation"; import { metadata, selection } from "../../state"; interface Props { + id?: string; + enableAllAnnotations?: boolean; disabledTopLevelAnnotations?: boolean; hasSelectAllCapability?: boolean; disableUnavailableAnnotations?: boolean; @@ -41,10 +43,10 @@ export default function AnnotationPicker(props: Props) { ) .map((annotation) => ({ selected: props.selections.some((selected) => selected.name === annotation.name), - disabled: unavailableAnnotations.some( - (unavailable) => unavailable.name === annotation.name - ), - loading: areAvailableAnnotationLoading, + disabled: + !props.enableAllAnnotations && + unavailableAnnotations.some((unavailable) => unavailable.name === annotation.name), + loading: !props.enableAllAnnotations && areAvailableAnnotationLoading, description: annotation.description, data: annotation, value: annotation.name, @@ -67,6 +69,7 @@ export default function AnnotationPicker(props: Props) { return ( ", () => { const sandbox = createSandbox(); @@ -187,7 +188,11 @@ describe("", () => { ]; const mockHttpClient = createMockHttpClient(responseStubs); const annotationService = new HttpAnnotationService({ baseUrl, httpClient: mockHttpClient }); - const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient }); + const fileService = new HttpFileService({ + baseUrl, + httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), + }); before(() => { sandbox.stub(interaction.selectors, "getAnnotationService").returns(annotationService); diff --git a/packages/core/components/FileDetails/Download.tsx b/packages/core/components/FileDetails/Download.tsx index a17e56c71..3a293d0b0 100644 --- a/packages/core/components/FileDetails/Download.tsx +++ b/packages/core/components/FileDetails/Download.tsx @@ -29,7 +29,16 @@ export default function Download(props: DownloadProps) { } return throttle(() => { - dispatch(interaction.actions.downloadFiles([fileDetails])); + dispatch( + interaction.actions.downloadFiles([ + { + id: fileDetails.id, + name: fileDetails.name, + size: fileDetails.size, + path: fileDetails.downloadPath, + }, + ]) + ); }, 1000); // 1s, in ms (arbitrary) }, [dispatch, fileDetails]); diff --git a/packages/core/components/FileDetails/FileAnnotationList.tsx b/packages/core/components/FileDetails/FileAnnotationList.tsx index aca7d277a..c0aca5145 100644 --- a/packages/core/components/FileDetails/FileAnnotationList.tsx +++ b/packages/core/components/FileDetails/FileAnnotationList.tsx @@ -84,6 +84,17 @@ export default function FileAnnotationList(props: FileAnnotationListProps) { // (i.e. POSIX path held in the database; what we have an annotation for) // as well as the path at which the file is *actually* accessible on _this_ computer ("local" file path) if (annotation.name === AnnotationName.FILE_PATH) { + if (fileDetails.path !== fileDetails.cloudPath) { + ret.push( + + ); + } + // In certain circumstances (i.e., linux), the path at which a file is accessible is === the canonical path if (localPath && localPath !== annotationValue && !localPath.startsWith("http")) { ret.splice( diff --git a/packages/core/components/FileDetails/FileDetails.module.css b/packages/core/components/FileDetails/FileDetails.module.css index 5ccbabb21..8f18e4e9d 100644 --- a/packages/core/components/FileDetails/FileDetails.module.css +++ b/packages/core/components/FileDetails/FileDetails.module.css @@ -165,7 +165,6 @@ .file-name { padding: var(--margin); text-align: center; - /* TODO: Add context menu from annotation rows onto file name too for copying */ user-select: text !important; word-break: break-all; } diff --git a/packages/core/components/FileList/ColumnPicker.tsx b/packages/core/components/FileList/ColumnPicker.tsx index 17cd5f0e0..132afe65b 100644 --- a/packages/core/components/FileList/ColumnPicker.tsx +++ b/packages/core/components/FileList/ColumnPicker.tsx @@ -13,6 +13,7 @@ export default function ColumnPicker() { return ( { // Prevent de-selecting all columns diff --git a/packages/core/components/FileList/useFileAccessContextMenu.ts b/packages/core/components/FileList/useFileAccessContextMenu.ts index e4cfd0545..37e193ac9 100644 --- a/packages/core/components/FileList/useFileAccessContextMenu.ts +++ b/packages/core/components/FileList/useFileAccessContextMenu.ts @@ -8,6 +8,138 @@ import FileFilter from "../../entity/FileFilter"; import { interaction, selection } from "../../state"; import { AnnotationName } from "../../entity/Annotation"; +export function useFileAccessMenuItems( + plateLink?: string, + filePath?: string, + filters?: FileFilter[] +): IContextualMenuItem[] { + const dispatch = useDispatch(); + const isOnWeb = useSelector(interaction.selectors.isOnWeb); + const fileSelection = useSelector(selection.selectors.getFileSelection); + const isQueryingAicsFms = useSelector(selection.selectors.isQueryingAicsFms); + const userSelectedApplications = useSelector(interaction.selectors.getUserSelectedApplications); + const { executionEnvService } = useSelector(interaction.selectors.getPlatformDependentServices); + + return React.useMemo(() => { + const savedApps: IContextualMenuItem[] = (userSelectedApplications || []).map((app) => { + const name = executionEnvService.getFilename(app.filePath); + return { + key: `open-with-${name}`, + text: name, + title: `Open files with ${name}`, + onClick() { + dispatch(interaction.actions.openWith(app, filters)); + }, + }; + }); + + savedApps.push({ + key: ContextMenuActions.OPEN_3D_WEB_VIEWER, + text: "3D Web Viewer", + title: `Open files with 3D Web Viewer`, + href: `https://allen-cell-animated.github.io/website-3d-cell-viewer/?url=${filePath}/`, + disabled: !filePath, + target: "_blank", + }); + if (plateLink) { + savedApps.push({ + key: ContextMenuActions.OPEN_PLATE_UI, + text: "LabKey Plate UI", + title: "Open this plate in the Plate UI", + href: plateLink, + target: "_blank", + }); + } + + const items = getContextMenuItems(dispatch, isQueryingAicsFms).ACCESS; + const openWithOptions = savedApps.sort((a, b) => + (a.text || "").localeCompare(b.text || "") + ); + if (!isOnWeb) { + items.splice(0, 0, { + key: ContextMenuActions.OPEN, + text: "Open", + iconProps: { + iconName: "OpenInNewWindow", + }, + onClick() { + dispatch(interaction.actions.openWithDefault(filters)); + }, + }); + // Other is a permanent option that allows the user + // to add another app for file access + openWithOptions.push( + { + key: "default-apps-border", + itemType: ContextualMenuItemType.Divider, + }, + { + key: ContextMenuActions.OPEN_WITH_OTHER, + text: "Other...", + title: "Select an application to open the selection with", + onClick() { + dispatch(interaction.actions.promptForNewExecutable(filters)); + }, + } + ); + } + + return items.map((item) => { + if (item.key === ContextMenuActions.OPEN_WITH) { + item.subMenuProps = { items: openWithOptions }; + } else if (item.key === ContextMenuActions.OPEN) { + item.onClick = () => { + dispatch(interaction.actions.openWithDefault(filters)); + }; + } else if (item.key === ContextMenuActions.SAVE_AS) { + item.subMenuProps = { + items: item.subMenuProps?.items.map((subItem) => { + if (subItem.key === ContextMenuActions.CSV) { + subItem.onClick = () => { + dispatch( + interaction.actions.showManifestDownloadDialog("csv", filters) + ); + }; + } else if (subItem.key === ContextMenuActions.PARQUET) { + subItem.onClick = () => { + dispatch( + interaction.actions.showManifestDownloadDialog( + "parquet", + filters + ) + ); + }; + } else if (subItem.key === ContextMenuActions.JSON) { + subItem.onClick = () => { + dispatch( + interaction.actions.showManifestDownloadDialog("json", filters) + ); + }; + } + + return subItem; + }) as ContextMenuItem[], + }; + } + + return { + ...item, + disabled: !filters && fileSelection.count() === 0, + }; + }); + }, [ + dispatch, + fileSelection, + executionEnvService, + isQueryingAicsFms, + isOnWeb, + userSelectedApplications, + filters, + plateLink, + filePath, + ]); +} + /** * Custom React hook for creating the file access context menu. * @@ -21,8 +153,6 @@ export default function useFileAccessContextMenu(filters?: FileFilter[], onDismi const fileExplorerServiceBaseUrl = useSelector( interaction.selectors.getFileExplorerServiceBaseUrl ); - const userSelectedApplications = useSelector(interaction.selectors.getUserSelectedApplications); - const { executionEnvService } = useSelector(interaction.selectors.getPlatformDependentServices); const [plateLink, setPlateLink] = React.useState(); const [filePath, setFilePath] = React.useState(); @@ -42,110 +172,12 @@ export default function useFileAccessContextMenu(filters?: FileFilter[], onDismi } }); + const items = useFileAccessMenuItems(plateLink, filePath, filters); return React.useCallback( (evt: React.MouseEvent) => { evt.preventDefault(); - const savedApps: IContextualMenuItem[] = (userSelectedApplications || []).map((app) => { - const name = executionEnvService.getFilename(app.filePath); - return { - key: `open-with-${name}`, - text: name, - title: `Open files with ${name}`, - onClick() { - dispatch(interaction.actions.openWith(app, filters)); - }, - }; - }); - - savedApps.push({ - key: ContextMenuActions.OPEN_3D_WEB_VIEWER, - text: "3D Web Viewer", - title: `Open files with 3D Web Viewer`, - href: `https://allen-cell-animated.github.io/website-3d-cell-viewer/?url=${filePath}/`, - disabled: !filePath, - target: "_blank", - }); - if (plateLink) { - savedApps.push({ - key: ContextMenuActions.OPEN_PLATE_UI, - text: "LabKey Plate UI", - title: "Open this plate in the Plate UI", - href: plateLink, - target: "_blank", - }); - } - - const openWithOptions = [ - ...savedApps.sort((a, b) => (a.text || "").localeCompare(b.text || "")), - { - key: "default-apps-border", - itemType: ContextualMenuItemType.Divider, - }, - // Other is a permanent option that allows the user - // to add another app for file access - { - key: ContextMenuActions.OPEN_WITH_OTHER, - text: "Other...", - title: "Select an application to open the selection with", - onClick() { - dispatch(interaction.actions.promptForNewExecutable(filters)); - }, - }, - ]; - - const items = getContextMenuItems(dispatch).ACCESS.map((item: IContextualMenuItem) => { - const disabled = !filters && fileSelection.count() === 0; - if (item.key === ContextMenuActions.OPEN_WITH) { - item.subMenuProps = { items: openWithOptions }; - } else if (item.key === ContextMenuActions.OPEN) { - item.onClick = () => { - dispatch(interaction.actions.openWithDefault(filters)); - }; - } else if (item.key === ContextMenuActions.SAVE_AS) { - item.subMenuProps = { - items: item.subMenuProps?.items.map((subItem) => { - if (subItem.key === ContextMenuActions.CSV) { - subItem.onClick = () => { - dispatch( - interaction.actions.showManifestDownloadDialog( - "csv", - filters - ) - ); - }; - } else if (subItem.key === ContextMenuActions.PARQUET) { - subItem.onClick = () => { - dispatch( - interaction.actions.showManifestDownloadDialog( - "parquet", - filters - ) - ); - }; - } - - return subItem; - }) as ContextMenuItem[], - }; - } - - return { - ...item, - disabled, - }; - }); - dispatch(interaction.actions.showContextMenu(items, evt.nativeEvent, onDismiss)); }, - [ - dispatch, - fileSelection, - executionEnvService, - userSelectedApplications, - filters, - onDismiss, - plateLink, - filePath, - ] + [dispatch, onDismiss, items] ); } diff --git a/packages/core/components/ListPicker/index.tsx b/packages/core/components/ListPicker/index.tsx index 5e23c8963..3798c81a5 100644 --- a/packages/core/components/ListPicker/index.tsx +++ b/packages/core/components/ListPicker/index.tsx @@ -8,6 +8,7 @@ import ListRow, { ListItem } from "./ListRow"; import styles from "./ListPicker.module.css"; interface ListPickerProps { + id?: string; className?: string; errorMessage?: string; items: ListItem[]; @@ -103,6 +104,7 @@ export default function ListPicker(props: ListPickerProps) {
{props.title &&

{props.title}

} setSearchValue("")} diff --git a/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css b/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css index 3962282bc..6fa93df73 100644 --- a/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css +++ b/packages/core/components/Modal/DataSourcePrompt/DataSourcePrompt.module.css @@ -71,10 +71,27 @@ display: flex; } -.text { +.text, .warning > p { + padding: 2px 0; +} + +.text, .warning { font-size: smaller; } +.warning { + background-color: var(--error-background-color); + border-radius: 4px; + color: var(--error-text-color); + margin-bottom: 8px; + padding: 4px; +} + +.warning > h4 { + margin: 0 0 4px 0; + padding: 0; +} + .url-form { width: 100%; } diff --git a/packages/core/components/Modal/DataSourcePrompt/index.tsx b/packages/core/components/Modal/DataSourcePrompt/index.tsx index 0421f2244..192e8ad9e 100644 --- a/packages/core/components/Modal/DataSourcePrompt/index.tsx +++ b/packages/core/components/Modal/DataSourcePrompt/index.tsx @@ -5,7 +5,7 @@ import { useDispatch, useSelector } from "react-redux"; import { ModalProps } from ".."; import BaseModal from "../BaseModal"; -import FileExplorerURL from "../../../entity/FileExplorerURL"; +import { Source } from "../../../entity/FileExplorerURL"; import { interaction, selection } from "../../../state"; import styles from "./DataSourcePrompt.module.css"; @@ -32,49 +32,62 @@ const DATA_SOURCE_DETAILS = [ export default function DataSourcePrompt({ onDismiss }: Props) { const dispatch = useDispatch(); + const dataSourceToReplace = useSelector(interaction.selectors.getDataSourceForVisibleModal); + const [dataSourceURL, setDataSourceURL] = React.useState(""); const [isDataSourceDetailExpanded, setIsDataSourceDetailExpanded] = React.useState(false); - const { databaseService } = useSelector(interaction.selectors.getPlatformDependentServices); + + const addOrReplaceQuery = (source: Source) => { + if (dataSourceToReplace) { + dispatch(selection.actions.replaceDataSource(source)); + } else { + dispatch( + selection.actions.addQuery({ + name: `New ${source.name} Query`, + parts: { source }, + }) + ); + } + }; const onChooseFile = (evt: React.FormEvent) => { const selectedFile = (evt.target as HTMLInputElement).files?.[0]; if (selectedFile) { // Grab name minus extension - const name = selectedFile.name.split(".").slice(0, -1).join(""); - const addDataSource = async () => { - await databaseService.addDataSource(name, selectedFile); - dispatch( - selection.actions.addQuery({ - name: `New ${name} Query`, - url: FileExplorerURL.encode({ - collection: { - name, - version: 1, - }, - }), - }) - ); + const nameAndExtension = selectedFile.name.split("."); + const name = nameAndExtension.slice(0, -1).join(""); + const extension = nameAndExtension.pop(); + if (!(extension === "csv" || extension === "json" || extension === "parquet")) { + alert("Invalid file type. Please select a .csv, .json, or .parquet file."); + return; } - addDataSource(); + addOrReplaceQuery({ name, type: extension, uri: selectedFile }); onDismiss(); } }; const onEnterURL = throttle( (evt: React.FormEvent) => { evt.preventDefault(); - const name = dataSourceURL.substring(dataSourceURL.lastIndexOf("/") + 1); - dispatch( - selection.actions.addQuery({ - name: `New ${name} Query`, - url: FileExplorerURL.encode({ - collection: { - name, - uri: dataSourceURL, - version: 1, - }, - }), - }) - ); + const uriResource = dataSourceURL + .substring(dataSourceURL.lastIndexOf("/") + 1) + .split("?")[0]; + const name = `${uriResource} (${new Date().toLocaleDateString()} ${new Date().toLocaleTimeString()})`; + let extensionGuess = uriResource.split(".").pop(); + if ( + !( + extensionGuess === "csv" || + extensionGuess === "json" || + extensionGuess === "parquet" + ) + ) { + console.warn("Guess that the source is a CSV file since no extension easily found"); + extensionGuess = "csv"; + } + addOrReplaceQuery({ + name, + type: extensionGuess as "csv" | "json" | "parquet", + uri: dataSourceURL, + }); onDismiss(); }, 10000, @@ -83,6 +96,21 @@ export default function DataSourcePrompt({ onDismiss }: Props) { const body = ( <> + {dataSourceToReplace && ( +
+

Notice

+

+ There was an error loading the data source file " + {dataSourceToReplace.name}". Please re-select the data source file or a + replacement. +

+

+ If this is a local file, the browser's permissions to access the file + may have expired since last time. If so, consider putting the file in a + cloud storage and providing the URL to avoid this issue in the future. +

+
+ )}

Please provide a ".csv", ".parquet", or ".json" file containing metadata about some files. See more details for information about what a @@ -124,7 +152,7 @@ export default function DataSourcePrompt({ onDismiss }: Props) { title="Browse for a data source file on your machine" htmlFor="data-source-selector" > - +

Choose File

); - return ; + return ; } diff --git a/packages/core/components/Modal/CsvManifest/CsvManifest.module.css b/packages/core/components/Modal/MetadataManifest/MetadataManifest.module.css similarity index 100% rename from packages/core/components/Modal/CsvManifest/CsvManifest.module.css rename to packages/core/components/Modal/MetadataManifest/MetadataManifest.module.css diff --git a/packages/core/components/Modal/CsvManifest/index.tsx b/packages/core/components/Modal/MetadataManifest/index.tsx similarity index 50% rename from packages/core/components/Modal/CsvManifest/index.tsx rename to packages/core/components/Modal/MetadataManifest/index.tsx index b75a1ff14..1c9b40c91 100644 --- a/packages/core/components/Modal/CsvManifest/index.tsx +++ b/packages/core/components/Modal/MetadataManifest/index.tsx @@ -1,53 +1,43 @@ import { PrimaryButton } from "@fluentui/react"; import classNames from "classnames"; import * as React from "react"; -import { useSelector } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import { ModalProps } from ".."; import BaseModal from "../BaseModal"; import AnnotationPicker from "../../AnnotationPicker"; import * as modalSelectors from "../selectors"; -import { interaction, selection } from "../../../state"; +import { interaction } from "../../../state"; -import styles from "./CsvManifest.module.css"; +import styles from "./MetadataManifest.module.css"; /** - * Modal overlay for selecting columns to be included in a CSV manifest download of + * Modal overlay for selecting columns to be included in a metadata manifest download of * files previously selected. */ -export default function CsvManifest({ onDismiss }: ModalProps) { +export default function MetadataManifest({ onDismiss }: ModalProps) { + const dispatch = useDispatch(); const annotationsPreviouslySelected = useSelector( modalSelectors.getAnnotationsPreviouslySelected ); const [selectedAnnotations, setSelectedAnnotations] = React.useState( annotationsPreviouslySelected ); - const csvService = useSelector(interaction.selectors.getCsvService); - const fileSelection = useSelector(selection.selectors.getFileSelection); + const fileTypeForVisibleModal = useSelector(interaction.selectors.getFileTypeForVisibleModal); const onDownload = () => { - const downloadSelection = async () => { - const selections = fileSelection.toCompactSelectionList(); - const selectedAnnotationNames = selectedAnnotations.map((annotation) => annotation.name); - const buffer = await csvService.getCsvAsBytes({ annotations: selectedAnnotationNames, selections }, ""); - - // Generate a download link (ensure to revoke the object URL after the download). - // We could use window.showSaveFilePicker() but it is only supported in Chrome. - const downloadUrl = URL.createObjectURL(new Blob([buffer])); - const a = document.createElement("a"); - a.href = downloadUrl; - a.download = `file-selection-${new Date()}.parquet`; - a.click(); - URL.revokeObjectURL(downloadUrl); - } - - downloadSelection(); + const selectedAnnotationNames = selectedAnnotations.map((annotation) => annotation.name); + dispatch( + interaction.actions.downloadManifest(selectedAnnotationNames, fileTypeForVisibleModal) + ); onDismiss(); }; const body = ( <> -

Select which annotations you would like included as columns in the downloaded CSV

+

+ Select which annotations you would like included as columns in the downloaded file +

} onDismiss={onDismiss} - title="Download CSV Manifest" + title="Download Metadata Manifest" /> ); } diff --git a/packages/core/components/Modal/CsvManifest/test/CsvManifest.test.tsx b/packages/core/components/Modal/MetadataManifest/test/MetadataManifest.test.tsx similarity index 73% rename from packages/core/components/Modal/CsvManifest/test/CsvManifest.test.tsx rename to packages/core/components/Modal/MetadataManifest/test/MetadataManifest.test.tsx index 26efa2448..006e4860d 100644 --- a/packages/core/components/Modal/CsvManifest/test/CsvManifest.test.tsx +++ b/packages/core/components/Modal/MetadataManifest/test/MetadataManifest.test.tsx @@ -14,16 +14,16 @@ import { createSandbox } from "sinon"; import Modal, { ModalType } from "../.."; import Annotation from "../../../../entity/Annotation"; import FileFilter from "../../../../entity/FileFilter"; -import FileDownloadService, { DownloadResolution } from "../../../../services/FileDownloadService"; import { initialState, interaction, reduxLogics } from "../../../../state"; import HttpFileService from "../../../../services/FileService/HttpFileService"; +import FileDownloadServiceNoop from "../../../../services/FileDownloadService/FileDownloadServiceNoop"; -describe("", () => { +describe("", () => { const baseUrl = "test"; const visibleDialogState = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: baseUrl, - visibleModal: ModalType.CsvManifest, + visibleModal: ModalType.MetadataManifest, }, }); @@ -34,7 +34,11 @@ describe("", () => { }, }; const mockHttpClient = createMockHttpClient(responseStub); - const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient }); + const fileService = new HttpFileService({ + baseUrl, + httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), + }); const sandbox = createSandbox(); @@ -60,49 +64,15 @@ describe("", () => { ); // Assert - expect(getByText("Download CSV Manifest")).to.exist; + expect(getByText("Download Metadata Manifest")).to.exist; }); - it("starts download and saves columns when download button is clicked", async () => { + it("starts download when download button is clicked", async () => { // Arrange - let downloadTriggered = false; - class TestDownloadService implements FileDownloadService { - downloadCsvManifest(_url: string, _data: string, downloadRequestId: string) { - downloadTriggered = true; - return Promise.resolve({ - downloadRequestId, - resolution: DownloadResolution.SUCCESS, - }); - } - - downloadFile() { - return Promise.reject(); - } - - getDefaultDownloadDirectory(): Promise { - return Promise.reject(); - } - - promptForSaveLocation() { - return Promise.reject(); - } - - promptForDownloadDirectory() { - return Promise.reject(); - } - - cancelActiveRequest() { - return Promise.reject(); - } - } - const state = mergeState(visibleDialogState, { interaction: { csvColumns: ["Cell Line"], fileFiltersForVisibleModal: [new FileFilter("Cell Line", "AICS-11")], - platformDependentServices: { - fileDownloadService: new TestDownloadService(), - }, }, metadata: { annotations: [ @@ -138,7 +108,6 @@ describe("", () => { type: interaction.actions.DOWNLOAD_MANIFEST, }) ).to.be.true; - expect(downloadTriggered).to.be.true; }); describe("column list", () => { diff --git a/packages/core/components/Modal/index.tsx b/packages/core/components/Modal/index.tsx index 049f205fe..c982f2472 100644 --- a/packages/core/components/Modal/index.tsx +++ b/packages/core/components/Modal/index.tsx @@ -3,17 +3,17 @@ import { useDispatch, useSelector } from "react-redux"; import { interaction } from "../../state"; import CodeSnippet from "./CodeSnippet"; -import CsvManifest from "./CsvManifest"; import DataSourcePrompt from "./DataSourcePrompt"; +import MetadataManifest from "./MetadataManifest"; export interface ModalProps { onDismiss: () => void; } export enum ModalType { - CsvManifest = 1, + CodeSnippet = 1, DataSourcePrompt = 2, - CodeSnippet = 3, + MetadataManifest = 3, } /** @@ -28,12 +28,12 @@ export default function Modal() { }; switch (visibleModal) { - case ModalType.CsvManifest: - return ; - case ModalType.DataSourcePrompt: - return ; case ModalType.CodeSnippet: return ; + case ModalType.DataSourcePrompt: + return ; + case ModalType.MetadataManifest: + return ; default: return null; } diff --git a/packages/core/components/QueryPart/QueryDataSource.tsx b/packages/core/components/QueryPart/QueryDataSource.tsx index 9bbdb8650..dc819fe2e 100644 --- a/packages/core/components/QueryPart/QueryDataSource.tsx +++ b/packages/core/components/QueryPart/QueryDataSource.tsx @@ -1,10 +1,11 @@ import * as React from "react"; import QueryPart from "."; -import { Collection } from "../../entity/FileExplorerURL"; +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; +import { Source } from "../../entity/FileExplorerURL"; interface Props { - dataSources: (Collection | undefined)[]; + dataSources: (Source | undefined)[]; } /** @@ -17,17 +18,19 @@ export default function QueryDataSource(props: Props) { addButtonIconName="Folder" onRenderAddMenuList={() =>
TODO: To be implemented in another ticket
} rows={props.dataSources.map((dataSource) => { - // Undefined data source must mean we are querying AICS FMS - // we should have a more sentinal value for this + // TODO: This should change when we move towards + // having a blank data source only possible + // on an empty load + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/105 if (!dataSource) { return { - id: "AICS FMS", - title: "AICS FMS", + id: AICS_FMS_DATA_SOURCE_NAME, + title: AICS_FMS_DATA_SOURCE_NAME, }; } return { - id: `${dataSource.name} ${dataSource.version}`, + id: dataSource.name, title: dataSource.name, }; })} diff --git a/packages/core/components/QueryPart/QueryFilter.tsx b/packages/core/components/QueryPart/QueryFilter.tsx index e24ca4a50..8f2497154 100644 --- a/packages/core/components/QueryPart/QueryFilter.tsx +++ b/packages/core/components/QueryPart/QueryFilter.tsx @@ -37,6 +37,7 @@ export default function QueryFilter(props: Props) { } onRenderAddMenuList={() => ( ( diff --git a/packages/core/components/QuerySidebar/Query.tsx b/packages/core/components/QuerySidebar/Query.tsx index dc29b46c0..f7ff4c04e 100644 --- a/packages/core/components/QuerySidebar/Query.tsx +++ b/packages/core/components/QuerySidebar/Query.tsx @@ -8,7 +8,6 @@ import QueryDataSource from "../QueryPart/QueryDataSource"; import QueryFilter from "../QueryPart/QueryFilter"; import QueryGroup from "../QueryPart/QueryGroup"; import QuerySort from "../QueryPart/QuerySort"; -import FileExplorerURL from "../../entity/FileExplorerURL"; import { metadata, selection } from "../../state"; import { Query as QueryType } from "../../state/selection/actions"; @@ -27,7 +26,7 @@ export default function Query(props: QueryProps) { const dispatch = useDispatch(); const queries = useSelector(selection.selectors.getQueries); const annotations = useSelector(metadata.selectors.getSortedAnnotations); - const currentGlobalURL = useSelector(selection.selectors.getEncodedFileExplorerUrl); + const currentQueryParts = useSelector(selection.selectors.getCurrentQueryParts); const [isExpanded, setIsExpanded] = React.useState(false); React.useEffect(() => { @@ -35,11 +34,8 @@ export default function Query(props: QueryProps) { }, [props.isSelected]); const decodedURL = React.useMemo( - () => - props.isSelected - ? FileExplorerURL.decode(currentGlobalURL) - : FileExplorerURL.decode(props.query.url), - [props.query.url, currentGlobalURL, props.isSelected] + () => (props.isSelected ? currentQueryParts : props.query.parts), + [props.query.parts, currentQueryParts, props.isSelected] ); const onQueryUpdate = (updatedQuery: QueryType) => { @@ -51,12 +47,9 @@ export default function Query(props: QueryProps) { }; const onQueryDelete = () => { - const filteredQueries = queries.filter((query) => query.name !== props.query.name); - dispatch(selection.actions.changeQuery(filteredQueries[0])); - dispatch(selection.actions.setQueries(filteredQueries)); + dispatch(selection.actions.removeQuery(props.query.name)); }; - const dataSourceName = decodedURL.collection?.name || "AICS FMS"; if (!isExpanded) { return (
@@ -83,7 +76,7 @@ export default function Query(props: QueryProps) {
{props.isSelected &&
}

- Data Source: {dataSourceName} + Data Source: {decodedURL.source?.name}

{!!decodedURL.hierarchy.length && (

@@ -141,7 +134,7 @@ export default function Query(props: QueryProps) { />

- + diff --git a/packages/core/components/QuerySidebar/QueryFooter.tsx b/packages/core/components/QuerySidebar/QueryFooter.tsx index 2b1a56e58..ff2178dd7 100644 --- a/packages/core/components/QuerySidebar/QueryFooter.tsx +++ b/packages/core/components/QuerySidebar/QueryFooter.tsx @@ -27,7 +27,7 @@ export default function QueryFooter(props: Props) { const onCopy = async () => { try { - navigator.clipboard.writeText(url); + navigator.clipboard.writeText(`https://biofile-finder.allencell.org/app?${url}`); window.alert("Link copied to clipboard!"); } catch (error) { window.alert("Failed to copy shareable link to clipboard"); @@ -64,7 +64,7 @@ export default function QueryFooter(props: Props) { return (
- { + if (!selectedQuery && queries.length) { + dispatch(selection.actions.changeQuery(queries[0])); + } + }, [selectedQuery, queries, dispatch]); - const [isExpanded, setIsExpanded] = React.useState(true); + // Determine a default query to render or prompt the user for a data source + // if no default is accessible + React.useEffect(() => { + if (!queries.length) { + if (!window.location.search) { + if (isAicsEmployee === true) { + // If the user is an AICS employee and there is no query in the URL, add a default query + dispatch( + selection.actions.addQuery({ + name: "New AICS Query", + parts: DEFAULT_AICS_FMS_QUERY, + }) + ); + } else if (isAicsEmployee === false) { + // If no query is selected and there is no query in the URL, prompt the user to select a data source + dispatch(interaction.actions.setVisibleModal(ModalType.DataSourcePrompt)); + } + } else if (isAicsEmployee === undefined) { + dispatch( + selection.actions.addQuery({ + name: "New Query", + parts: FileExplorerURL.decode(window.location.search), + }) + ); + } + } + }, [isAicsEmployee, queries, dispatch]); - // Default to first query in array if none selected yet some available - // this is primarily useful for when loading queries from persisted state React.useEffect(() => { - if (queries.length && !selectedQuery) { - dispatch(selection.actions.changeQuery(queries[0])); + if (selectedQuery) { + const newurl = + window.location.protocol + + "//" + + window.location.host + + window.location.pathname + + "?" + + currentGlobalURL; + window.history.pushState({ path: newurl }, "", newurl); } - }, [queries, selectedQuery, dispatch]); + }, [currentGlobalURL, selectedQuery]); + + const [isExpanded, setIsExpanded] = React.useState(true); const helpMenuOptions = React.useMemo(() => HELP_OPTIONS(dispatch), [dispatch]); - const addQueryOptions = React.useMemo(() => ([ - ...(isAICSEmployee - ? [ - { - key: "AICS FMS", - text: "AICS FMS", - iconProps: { iconName: "Database" }, - onClick: () => { - dispatch( - selection.actions.addQuery({ - name: "New AICS Query", - url: FileExplorerURL.DEFAULT_FMS_URL, - }) - ); - }, - secondaryText: "Data Source", - }, - ] - : []), - ...collections - .filter((collection) => !!collection.uri) - .map((collection) => ({ - key: collection.id, - text: `${ - collection.name - } (${collection.created.toLocaleDateString()})`, + const addQueryOptions = React.useMemo( + () => [ + ...dataSources.map((source) => ({ + key: source.id, + text: source.name, iconProps: { iconName: "Folder" }, onClick: () => { dispatch( selection.actions.addQuery({ - name: `New ${collection.name} query`, - url: collection.uri as string, + name: `New ${source.name} query`, + parts: { source }, }) ); }, secondaryText: "Data Source", })), - { - key: "New Data Source...", - text: "New Data Source...", - iconProps: { iconName: "NewFolder" }, - onClick: () => { - dispatch( - interaction.actions.setVisibleModal(ModalType.DataSourcePrompt) - ); + { + key: "New Data Source...", + text: "New Data Source...", + iconProps: { iconName: "NewFolder" }, + onClick: () => { + dispatch(interaction.actions.setVisibleModal(ModalType.DataSourcePrompt)); + }, }, - }, - ]), [dispatch, collections, isAICSEmployee]); + ], + [dispatch, dataSources] + ); if (!isExpanded) { return ( @@ -107,7 +121,7 @@ export default function QuerySidebar(props: QuerySidebarProps) { />

- {selectedQuery?.name} + {selectedQuery}

); @@ -145,7 +159,7 @@ export default function QuerySidebar(props: QuerySidebarProps) { {queries.map((query) => ( ))} diff --git a/packages/core/components/QuerySidebar/tutorials/FilterFiles.tsx b/packages/core/components/QuerySidebar/tutorials/FilterFiles.tsx index 7ac811854..8c94ebb37 100644 --- a/packages/core/components/QuerySidebar/tutorials/FilterFiles.tsx +++ b/packages/core/components/QuerySidebar/tutorials/FilterFiles.tsx @@ -13,13 +13,12 @@ export const FILTER_FILES_TUTORIAL = new Tutorial("Filtering") ), }) - // TODO: Do this .addStep({ targetId: Tutorial.FILE_ATTRIBUTE_FILTER_ID, - message: "All files have the attributes listed in this dropdown.", + message: "All metadata tags present in the data sources are listed in this dropdown.", }) .addStep({ targetId: Tutorial.FILE_ATTRIBUTE_FILTER_ID, message: - 'You can filter on the selected attribute by entering a value here. These values do not have to be exact, e.g. entering a File Name of "ZSD1" would return all files with names starting with or containing "ZSD1".', + 'You can filter on the selected metadata tag by entering a value here. These values do not have to be exact, e.g. entering a File Name of "ZSD1" would return all files with names starting with or containing "ZSD1".', }); diff --git a/packages/core/components/QuerySidebar/tutorials/index.ts b/packages/core/components/QuerySidebar/tutorials/index.ts index 092030cc1..335821609 100644 --- a/packages/core/components/QuerySidebar/tutorials/index.ts +++ b/packages/core/components/QuerySidebar/tutorials/index.ts @@ -12,7 +12,6 @@ import { selection } from "../../../state"; export const HELP_OPTIONS = (dispatch: Dispatch): IContextualMenuItem[] => { return [ - // TODO: Remove if on web { key: "download-newest-version", text: "Download Newest Version", diff --git a/packages/core/components/StatusMessage/StatusMessage.module.css b/packages/core/components/StatusMessage/StatusMessage.module.css index 2e8cfd815..a225de419 100644 --- a/packages/core/components/StatusMessage/StatusMessage.module.css +++ b/packages/core/components/StatusMessage/StatusMessage.module.css @@ -39,8 +39,8 @@ } .error { - background-color: var(--error-color); - color: white; + background-color: var(--error-background-color); + color: var(--error-text-color); } .error button i { diff --git a/packages/core/constants/index.ts b/packages/core/constants/index.ts index 15a7f9a93..50152c70a 100644 --- a/packages/core/constants/index.ts +++ b/packages/core/constants/index.ts @@ -56,3 +56,5 @@ export const THUMBNAIL_SIZE_TO_NUM_COLUMNS = { LARGE: 5, SMALL: 10, }; + +export const AICS_FMS_DATA_SOURCE_NAME = "AICS FMS"; diff --git a/packages/core/entity/FileDetail/index.ts b/packages/core/entity/FileDetail/index.ts index 0b3794b9f..c950572ee 100644 --- a/packages/core/entity/FileDetail/index.ts +++ b/packages/core/entity/FileDetail/index.ts @@ -2,6 +2,12 @@ import { FmsFileAnnotation } from "../../services/FileService"; const RENDERABLE_IMAGE_FORMATS = [".jpg", ".jpeg", ".png", ".gif"]; +// Should probably record this somewhere we can dynamically adjust to, or perhaps just in the file +// document itself, alas for now this will do. +const HARD_CODED_AICS_S3_BUCKET_PATH = "http://production.files.allencell.org.s3.amazonaws.com"; + +const AICS_FMS_FILES_NGINX_SERVER = "http://aics.corp.alleninstitute.org/labkey/fmsfiles/image"; + /** * Expected JSON response of a file detail returned from the query service. Example: * { @@ -104,12 +110,35 @@ export default class FileDetail { return path as string; } - public get size(): number { + public get cloudPath(): string { + // Can retrieve a cloud like path for AICS FMS files + if (this.path.startsWith("/allen")) { + const pathWithoutDrive = this.path.replace("/allen/programs/allencell/data/proj0", ""); + return `${HARD_CODED_AICS_S3_BUCKET_PATH}${pathWithoutDrive}`; + } + + return this.path; + } + + public get downloadPath(): string { + if (!this.path.startsWith("/allen")) { + return this.path; + } + + // For AICS files we don't have permission to the bucket nor do we expect to have the /allen + // drive mounted on the client machine. So we use the NGINX server to serve the file. + return `${AICS_FMS_FILES_NGINX_SERVER}${this.path}`; + } + + public get size(): number | undefined { const size = this.fileDetail.file_size || this.getFirstAnnotationValue("File Size"); if (size === undefined) { - throw new Error("File Size is not defined"); + return 0; // Default to 0 if size is not defined for now, need better system } - return size as number; + if (typeof size === "number") { + return size; + } + return parseInt(size as string, 10); } public get thumbnail(): string | undefined { @@ -132,23 +161,23 @@ export default class FileDetail { } public getPathToThumbnail(): string | undefined { - let thumbnailPath = this.thumbnail; - // If no thumbnail present try to render the file itself as the thumbnail - if (!thumbnailPath) { + if (!this.thumbnail) { const isFileRenderableImage = RENDERABLE_IMAGE_FORMATS.some((format) => this.name.toLowerCase().endsWith(format) ); - if (isFileRenderableImage) { - thumbnailPath = this.path; + if (!isFileRenderableImage) { + return undefined; } + + return this.downloadPath; } // If the thumbnail is a relative path on the allen drive then preprend it to // the AICS FMS NGINX server path - if (thumbnailPath?.startsWith("/allen")) { - return `http://aics.corp.alleninstitute.org/labkey/fmsfiles/image${thumbnailPath}`; + if (this.thumbnail?.startsWith("/allen")) { + return `${AICS_FMS_FILES_NGINX_SERVER}${this.thumbnail}`; } - return thumbnailPath; + return this.thumbnail; } } diff --git a/packages/core/entity/FileExplorerURL/index.ts b/packages/core/entity/FileExplorerURL/index.ts index 06ce0da0b..f2f1936f3 100644 --- a/packages/core/entity/FileExplorerURL/index.ts +++ b/packages/core/entity/FileExplorerURL/index.ts @@ -1,38 +1,28 @@ -import { isObject } from "lodash"; - import { AnnotationName } from "../Annotation"; -import FileFilter, { FileFilterJson } from "../FileFilter"; +import FileFilter from "../FileFilter"; import FileFolder from "../FileFolder"; -import { AnnotationValue } from "../../services/AnnotationService"; -import { ValueError } from "../../errors"; import FileSort, { SortOrder } from "../FileSort"; -export interface Collection { +export interface Source { name: string; - version?: number; - uri?: string; + type: "csv" | "json" | "parquet"; + uri?: string | File; } // Components of the application state this captures export interface FileExplorerURLComponents { hierarchy: string[]; - collection?: Collection; + source?: Source; filters: FileFilter[]; openFolders: FileFolder[]; sortColumn?: FileSort; } -// JSON format this outputs & expects to receive back from the user -interface FileExplorerURLJson { - groupBy: string[]; - collection?: Collection; - filters: FileFilterJson[]; - openFolders: AnnotationValue[][]; - sort?: { - annotationName: string; - order: SortOrder; - }; -} +export const EMPTY_QUERY_COMPONENTS: FileExplorerURLComponents = { + hierarchy: [], + filters: [], + openFolders: [], +}; const BEGINNING_OF_TODAY = new Date(); BEGINNING_OF_TODAY.setHours(0, 0, 0, 0); @@ -50,7 +40,7 @@ export const PAST_YEAR_FILTER = new FileFilter( AnnotationName.UPLOADED, `RANGE(${DATE_LAST_YEAR.toISOString()},${END_OF_TODAY.toISOString()})` ); -const DEFAULT_FMS_URL_COMPONENTS = { +export const DEFAULT_AICS_FMS_QUERY: FileExplorerURLComponents = { hierarchy: [], openFolders: [], filters: [PAST_YEAR_FILTER], @@ -63,9 +53,6 @@ const DEFAULT_FMS_URL_COMPONENTS = { * URL decoded & rehydrated back in. */ export default class FileExplorerURL { - public static readonly PROTOCOL = "fms-file-explorer://"; - public static readonly DEFAULT_FMS_URL = FileExplorerURL.encode(DEFAULT_FMS_URL_COMPONENTS); - /** * Encode this FileExplorerURL into a format easily transferable between users * that can be decoded back into the data used to create this FileExplorerURL. @@ -73,31 +60,25 @@ export default class FileExplorerURL { * of our application state. As in, the names / system we track data in can change * without breaking an existing FileExplorerURL. * */ - public static encode(urlComponents: Partial) { - const groupBy = urlComponents.hierarchy?.map((annotation) => annotation) || []; - const filters = urlComponents.filters?.map((filter) => filter.toJSON()) || []; - const openFolders = urlComponents.openFolders?.map((folder) => folder.fileFolder) || []; - const sort = urlComponents.sortColumn - ? { - annotationName: urlComponents.sortColumn.annotationName, - order: urlComponents.sortColumn.order, - } - : undefined; + public static encode(urlComponents: Partial): string { + const params = new URLSearchParams(); + urlComponents.hierarchy?.forEach((annotation) => { + params.append("group", annotation); + }); + urlComponents.filters?.forEach((filter) => { + params.append("filter", JSON.stringify(filter.toJSON())); + }); + urlComponents.openFolders?.map((folder) => { + params.append("openFolder", JSON.stringify(folder.fileFolder)); + }); + if (urlComponents.sortColumn) { + params.append("sort", JSON.stringify(urlComponents.sortColumn.toJSON())); + } + if (urlComponents.source) { + params.append("source", JSON.stringify(urlComponents.source)); + } - const dataToEncode: FileExplorerURLJson = { - groupBy, - filters, - openFolders, - sort, - collection: urlComponents.collection - ? { - name: urlComponents.collection.name, - version: urlComponents.collection.version, - uri: urlComponents.collection.uri, - } - : undefined, - }; - return `${FileExplorerURL.PROTOCOL}${JSON.stringify(dataToEncode)}`; + return params.toString(); } /** @@ -105,56 +86,36 @@ export default class FileExplorerURL { * application state */ public static decode(encodedURL: string): FileExplorerURLComponents { - const trimmedEncodedURL = encodedURL.trim(); - if (!trimmedEncodedURL.startsWith(FileExplorerURL.PROTOCOL)) { - throw new ValueError( - "This does not look like an FMS File Explorer URL, invalid protocol." - ); - } + const params = new URLSearchParams(encodedURL.trim()); - const parsedURL: FileExplorerURLJson = JSON.parse( - trimmedEncodedURL.substring(FileExplorerURL.PROTOCOL.length) - ); - - let sortColumn = undefined; - if (parsedURL.sort) { - if (!Object.values(SortOrder).includes(parsedURL.sort.order)) { - throw new Error( - `Unable to decode FileExplorerURL, sort order must be one of ${Object.values( - SortOrder - )}` - ); - } - sortColumn = new FileSort(parsedURL.sort.annotationName, parsedURL.sort.order); - } + const unparsedOpenFolders = params.getAll("openFolder"); + const unparsedFilters = params.getAll("filter"); + const unparsedSource = params.get("source"); + const hierarchy = params.getAll("group"); + const unparsedSort = params.get("sort"); + const hierarchyDepth = hierarchy.length; + const parsedSort = unparsedSort ? JSON.parse(unparsedSort) : undefined; if ( - parsedURL.collection && - (!isObject(parsedURL.collection) || - !parsedURL.collection.name || - !parsedURL.collection.version) + parsedSort && + parsedSort?.order !== SortOrder.ASC && + parsedSort?.order !== SortOrder.DESC ) { - throw new ValueError( - `Unable to decode FileExplorerURL, unexpected format (${parsedURL.collection})` - ); + throw new Error("Sort order must be ASC or DESC"); } - - const hierarchyDepth = parsedURL.groupBy.length; return { - hierarchy: parsedURL.groupBy, - collection: parsedURL.collection, - filters: parsedURL.filters.map((filter) => { - return new FileFilter(filter.name, filter.value); - }), - openFolders: parsedURL.openFolders.map((folder) => { - if (folder.length > hierarchyDepth) { - throw new Error( - "Unable to decode FileExplorerURL, opened folder depth is greater than hierarchy depth" - ); - } - return new FileFolder(folder); - }), - sortColumn, + hierarchy, + sortColumn: parsedSort + ? new FileSort(parsedSort.annotationName, parsedSort.order || SortOrder.ASC) + : undefined, + filters: unparsedFilters + .map((unparsedFilter) => JSON.parse(unparsedFilter)) + .map((parsedFilter) => new FileFilter(parsedFilter.name, parsedFilter.value)), + source: unparsedSource ? JSON.parse(unparsedSource) : undefined, + openFolders: unparsedOpenFolders + .map((unparsedFolder) => JSON.parse(unparsedFolder)) + .filter((parsedFolder) => parsedFolder.length <= hierarchyDepth) + .map((parsedFolder) => new FileFolder(parsedFolder)), }; } } diff --git a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts index 1dc2a53cd..cd626fa50 100644 --- a/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts +++ b/packages/core/entity/FileExplorerURL/test/fileexplorerurl.test.ts @@ -1,23 +1,15 @@ import { expect } from "chai"; -import FileExplorerURL, { FileExplorerURLComponents } from ".."; -import { Dataset } from "../../../services/DatasetService"; +import FileExplorerURL, { FileExplorerURLComponents, Source } from ".."; import { AnnotationName } from "../../Annotation"; import FileFilter from "../../FileFilter"; import FileFolder from "../../FileFolder"; import FileSort, { SortOrder } from "../../FileSort"; describe("FileExplorerURL", () => { - const mockCollection: Dataset = { - id: "12341", + const mockSource: Source = { name: "Fake Collection", - version: 1, - query: "test", - client: "test", - fixed: true, - private: true, - created: new Date(), - createdBy: "test", + type: "csv", }; describe("encode", () => { @@ -34,38 +26,21 @@ describe("FileExplorerURL", () => { ["AICS-0", "ACTB-mEGFP", false], ["AICS-0", "ACTB-mEGFP", true], ]; - const expectedSort = { - annotationName: AnnotationName.FILE_SIZE, - order: SortOrder.DESC, - }; const components: FileExplorerURLComponents = { hierarchy: expectedAnnotationNames, filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.FILE_SIZE, SortOrder.DESC), - collection: { - name: mockCollection.name, - version: mockCollection.version, - }, + source: mockSource, }; - const expectedResult = - FileExplorerURL.PROTOCOL + - JSON.stringify({ - groupBy: expectedAnnotationNames, - filters: expectedFilters, - openFolders: expectedOpenFolders, - sort: expectedSort, - collection: { - name: mockCollection.name, - version: mockCollection.version, - }, - }); // Act const result = FileExplorerURL.encode(components); // Assert - expect(result).to.be.equal(expectedResult); + expect(result).to.be.equal( + "group=Cell+Line&group=Donor+Plasmid&group=Lifting%3F&filter=%7B%22name%22%3A%22Cas9%22%2C%22value%22%3A%22spCas9%22%7D&filter=%7B%22name%22%3A%22Donor+Plasmid%22%2C%22value%22%3A%22ACTB-mEGFP%22%7D&openFolder=%5B%22AICS-0%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Cfalse%5D&openFolder=%5B%22AICS-0%22%2C%22ACTB-mEGFP%22%2Ctrue%5D&sort=%7B%22annotationName%22%3A%22file_size%22%2C%22order%22%3A%22DESC%22%7D&source=%7B%22name%22%3A%22Fake+Collection%22%2C%22type%22%3A%22csv%22%7D" + ); }); it("Encodes empty state", () => { @@ -75,19 +50,12 @@ describe("FileExplorerURL", () => { filters: [], openFolders: [], }; - const expectedResult = - FileExplorerURL.PROTOCOL + - JSON.stringify({ - groupBy: [], - filters: [], - openFolders: [], - }); // Act const result = FileExplorerURL.encode(components); // Assert - expect(result).to.be.equal(expectedResult); + expect(result).to.be.equal(""); }); }); @@ -110,10 +78,7 @@ describe("FileExplorerURL", () => { filters: expectedFilters.map(({ name, value }) => new FileFilter(name, value)), openFolders: expectedOpenFolders.map((folder) => new FileFolder(folder)), sortColumn: new FileSort(AnnotationName.UPLOADED, SortOrder.DESC), - collection: { - name: mockCollection.name, - version: mockCollection.version, - }, + source: mockSource, }; const encodedUrl = FileExplorerURL.encode(components); const encodedUrlWithWhitespace = " " + encodedUrl + " "; @@ -132,7 +97,7 @@ describe("FileExplorerURL", () => { filters: [], openFolders: [], sortColumn: undefined, - collection: undefined, + source: undefined, }; const encodedUrl = FileExplorerURL.encode(components); @@ -143,22 +108,7 @@ describe("FileExplorerURL", () => { expect(result).to.be.deep.equal(components); }); - it("Throws error for urls without protocol at beginning", () => { - // Arrange - const components: FileExplorerURLComponents = { - hierarchy: [], - filters: [], - openFolders: [], - }; - const encodedUrl = FileExplorerURL.encode(components).substring( - FileExplorerURL.PROTOCOL.length - ); - - // Act / Assert - expect(() => FileExplorerURL.decode(encodedUrl)).to.throw(); - }); - - it("Throws error when folder depth is greater than hierarchy depth", () => { + it("Removes folders that are too deep for hierachy", () => { // Arrange const components: FileExplorerURLComponents = { hierarchy: ["Cell Line"], @@ -168,7 +118,8 @@ describe("FileExplorerURL", () => { const encodedUrl = FileExplorerURL.encode(components); // Act / Assert - expect(() => FileExplorerURL.decode(encodedUrl)).to.throw(); + const { openFolders } = FileExplorerURL.decode(encodedUrl); + expect(openFolders).to.be.deep.equal([new FileFolder(["AICS-0"])]); }); it("Throws error when sort order is not DESC or ASC", () => { diff --git a/packages/core/entity/FileFolder/index.ts b/packages/core/entity/FileFolder/index.ts index f46a2403e..93c1ffb4b 100644 --- a/packages/core/entity/FileFolder/index.ts +++ b/packages/core/entity/FileFolder/index.ts @@ -57,6 +57,10 @@ export default class FileFolder { return this.fileFolderPath.length; } + public toJSON(): string { + return this.fileFolderPath.join("."); + } + /** * This returns true if the open file folder given is the same * as this open file folder. diff --git a/packages/core/entity/FileSelection/index.ts b/packages/core/entity/FileSelection/index.ts index 7c257ceba..2ec8de15b 100644 --- a/packages/core/entity/FileSelection/index.ts +++ b/packages/core/entity/FileSelection/index.ts @@ -1,12 +1,12 @@ import { find, isArray, reject } from "lodash"; -import { IndexError, ValueError } from "../../errors"; -import { Selection } from "../../services/FileService/HttpFileService"; import FileFilter from "../FileFilter"; import FileSet from "../FileSet"; import { SortOrder } from "../FileSort"; import NumericRange from "../NumericRange"; import FileDetail from "../FileDetail"; +import { IndexError, ValueError } from "../../errors"; +import { Selection } from "../../services/FileService"; /** * Enumeration of directives that can be used to change the focus of the FileSelection. diff --git a/packages/core/entity/FileSelection/test/FileSelection.test.ts b/packages/core/entity/FileSelection/test/FileSelection.test.ts index 83581517e..7d5a88634 100644 --- a/packages/core/entity/FileSelection/test/FileSelection.test.ts +++ b/packages/core/entity/FileSelection/test/FileSelection.test.ts @@ -6,10 +6,11 @@ import FileSet from "../../FileSet"; import NumericRange from "../../NumericRange"; import FileSelection, { FocusDirective } from ".."; +import FileDetail from "../../FileDetail"; import FileFilter from "../../FileFilter"; import { IndexError, ValueError } from "../../../errors"; import HttpFileService from "../../../services/FileService/HttpFileService"; -import FileDetail from "../../FileDetail"; +import FileDownloadServiceNoop from "../../../services/FileDownloadService/FileDownloadServiceNoop"; describe("FileSelection", () => { describe("select", () => { @@ -355,7 +356,11 @@ describe("FileSelection", () => { data: { data: queryResult }, }, }); - const fileService = new HttpFileService({ baseUrl, httpClient }); + const fileService = new HttpFileService({ + baseUrl, + httpClient, + downloadService: new FileDownloadServiceNoop(), + }); const selection = new FileSelection().select({ fileSet: new FileSet({ fileService }), index: new NumericRange(1, 30), diff --git a/packages/core/entity/FileSet/test/FileSet.test.ts b/packages/core/entity/FileSet/test/FileSet.test.ts index fd90d2059..77f859c53 100644 --- a/packages/core/entity/FileSet/test/FileSet.test.ts +++ b/packages/core/entity/FileSet/test/FileSet.test.ts @@ -7,6 +7,7 @@ import FileFilter from "../../FileFilter"; import FileSort, { SortOrder } from "../../FileSort"; import { makeFileDetailMock } from "../../FileDetail/mocks"; import HttpFileService from "../../../services/FileService/HttpFileService"; +import FileDownloadServiceNoop from "../../../services/FileDownloadService/FileDownloadServiceNoop"; describe("FileSet", () => { const scientistEqualsJane = new FileFilter("scientist", "jane"); @@ -110,7 +111,11 @@ describe("FileSet", () => { const getSpy = sandbox.spy(httpClient, "get"); const fileSet = new FileSet({ - fileService: new HttpFileService({ httpClient, baseUrl }), + fileService: new HttpFileService({ + httpClient, + baseUrl, + downloadService: new FileDownloadServiceNoop(), + }), }); await fileSet.fetchFileRange(testCase.start, testCase.end); diff --git a/packages/core/entity/FileSort/index.ts b/packages/core/entity/FileSort/index.ts index 2ea50b0d4..aa7e39885 100644 --- a/packages/core/entity/FileSort/index.ts +++ b/packages/core/entity/FileSort/index.ts @@ -26,6 +26,13 @@ export default class FileSort { return new SQLBuilder().orderBy(`"${this.annotationName}" ${this.order}`); } + public toJSON(): Record { + return { + annotationName: this.annotationName, + order: this.order, + } + } + public equals(other?: FileSort): boolean { return ( !!other && this.annotationName === other.annotationName && this.order === other.order diff --git a/packages/core/entity/SQLBuilder/index.ts b/packages/core/entity/SQLBuilder/index.ts index 4cf4fd322..5caaa0324 100644 --- a/packages/core/entity/SQLBuilder/index.ts +++ b/packages/core/entity/SQLBuilder/index.ts @@ -63,6 +63,10 @@ export default class SQLBuilder { return this; } + public toString(): string { + return this.toSQL(); + } + public toSQL(): string { if (!this.fromStatement) { throw new Error("Unable to build SLQ without a FROM statement"); diff --git a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts index 2c60f5031..e5443eb50 100644 --- a/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/DatabaseAnnotationService/index.ts @@ -30,7 +30,9 @@ export default class DatabaseAnnotationService implements AnnotationService { private readonly databaseService: DatabaseService; private readonly dataSourceName: string; - constructor(config: Config = { dataSourceName: "Unknown", databaseService: new DatabaseServiceNoop() }) { + constructor( + config: Config = { dataSourceName: "Unknown", databaseService: new DatabaseServiceNoop() } + ) { this.dataSourceName = config.dataSourceName; this.databaseService = config.databaseService; } @@ -40,6 +42,7 @@ export default class DatabaseAnnotationService implements AnnotationService { case "INTEGER": case "BIGINT": // TODO: Add support for column types + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/60 // return AnnotationType.NUMBER; case "VARCHAR": case "TEXT": diff --git a/packages/core/services/AnnotationService/HttpAnnotationService/index.ts b/packages/core/services/AnnotationService/HttpAnnotationService/index.ts index 039635762..ff235cad5 100644 --- a/packages/core/services/AnnotationService/HttpAnnotationService/index.ts +++ b/packages/core/services/AnnotationService/HttpAnnotationService/index.ts @@ -28,7 +28,6 @@ export default class HttpAnnotationService extends HttpServiceBase implements An */ public async fetchAnnotations(): Promise { const requestUrl = `${this.baseUrl}/${HttpAnnotationService.BASE_ANNOTATION_URL}${this.pathSuffix}`; - console.log(`Requesting annotation values from ${requestUrl}`); const response = await this.get(requestUrl); return [ @@ -44,7 +43,6 @@ export default class HttpAnnotationService extends HttpServiceBase implements An // Encode any special characters in the annotation as necessary const encodedAnnotation = HttpServiceBase.encodeURISection(annotation); const requestUrl = `${this.baseUrl}/${HttpAnnotationService.BASE_ANNOTATION_URL}/${encodedAnnotation}/values${this.pathSuffix}`; - console.log(`Requesting annotation values from ${requestUrl}`); const response = await this.get(requestUrl); return response.data; @@ -70,7 +68,6 @@ export default class HttpAnnotationService extends HttpServiceBase implements An .join("&"); const requestUrl = `${this.baseUrl}/${HttpAnnotationService.BASE_ANNOTATION_HIERARCHY_ROOT_URL}${this.pathSuffix}?${queryParams}`; - console.log(`Requesting root hierarchy values: ${requestUrl}`); const response = await this.get(requestUrl); return response.data; @@ -92,7 +89,6 @@ export default class HttpAnnotationService extends HttpServiceBase implements An .filter((param) => !!param) .join("&"); const requestUrl = `${this.baseUrl}/${HttpAnnotationService.BASE_ANNOTATION_HIERARCHY_UNDER_PATH_URL}${this.pathSuffix}?${queryParams}`; - console.log(`Requesting hierarchy values under path: ${requestUrl}`); const response = await this.get(requestUrl); return response.data; @@ -105,7 +101,6 @@ export default class HttpAnnotationService extends HttpServiceBase implements An public async fetchAvailableAnnotationsForHierarchy(annotations: string[]): Promise { const queryParams = this.buildQueryParams(QueryParam.HIERARCHY, [...annotations].sort()); const requestUrl = `${this.baseUrl}/${HttpAnnotationService.BASE_AVAILABLE_ANNOTATIONS_UNDER_HIERARCHY}${this.pathSuffix}?${queryParams}`; - console.log(`Requesting available annotations with current hierarchy: ${requestUrl}`); const response = await this.get(requestUrl); if (!response.data) { diff --git a/packages/core/services/CsvService/CsvServiceNoop.ts b/packages/core/services/CsvService/CsvServiceNoop.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/core/services/CsvService/DatabaseCsvService/index.ts b/packages/core/services/CsvService/DatabaseCsvService/index.ts deleted file mode 100644 index 3d31db8c7..000000000 --- a/packages/core/services/CsvService/DatabaseCsvService/index.ts +++ /dev/null @@ -1,145 +0,0 @@ -import CsvService, { CsvManifestRequest } from ".."; -import SQLBuilder from "../../../entity/SQLBuilder"; -import DatabaseService from "../../DatabaseService"; -import { ConnectionConfig } from "../../HttpServiceBase"; - -interface Config extends ConnectionConfig { - dataSourceName?: string; - databaseService: DatabaseService; -} - -/** - * Service responsible for requesting a CSV manifest of metadata for selected files from the - * DuckDB database. - */ -export default class DatabaseCsvService implements CsvService { - private readonly databaseService: DatabaseService; - private readonly dataSourceName?: string; - - public constructor(config: Config) { - this.dataSourceName = config.dataSourceName; - this.databaseService = config.databaseService; - } - - public async getCsvAsBytes(selectionRequest: CsvManifestRequest): Promise { - const { annotations, selections } = selectionRequest; - if (!this.dataSourceName) { - throw new Error("blah") - } - // TODO: use file id - // TODO: Automatically generate file id on startup of db if not present in source - const sqlBuilder = new SQLBuilder() - .select(annotations.map((annotation) => `'${annotation}'`).join(', ')) - .from(this.dataSourceName) - - if (!this.dataSourceName) { - throw new Error("blah") - } - selections.forEach((selection) => { - selection.indexRanges.forEach((indexRange) => { - const subQuery = new SQLBuilder() - .select("'File Path'") - .from(this.dataSourceName as string) - .whereOr(Object.entries(selection.filters).map(([column, values]) => { - const commaSeperatedValues = values.map(v => `'${v}'`).join(", "); - return `'${column}' IN (${commaSeperatedValues}}`; - })) - .offset(indexRange.start) - .limit(indexRange.end - indexRange.start); - - if (selection.sort) { - subQuery.orderBy(`'${selection.sort.annotationName}' (${selection.sort.ascending ? "ASC" : "DESC"})`); - } - - sqlBuilder.whereOr(`'File Path' IN (${subQuery})`) - }); - }); - return this.databaseService.saveQueryAsBuffer( - sqlBuilder.toSQL() - ); - } - - // public async download( - // selectionRequest: CsvManifestRequest, - // manifestDownloadId: string - // ): Promise { - // if (!this.dataSourceName) { - // throw new Error("blah") - // } - // const saveLocation = await this.downloadService.promptForSaveLocation( - // "Save CSV", - // "fms-explorer-selections.csv", - // "Save CSV", - // [{ name: "CSV files", extensions: ["csv"] }] - // ); - // if (saveLocation === FileDownloadCancellationToken) { - // return { - // downloadRequestId: manifestDownloadId, - // msg: `Cancelled download`, - // resolution: DownloadResolution.CANCELLED, - // }; - // } - - // const annotationsAsSelect = selectionRequest.annotations - // .map((annotation) => `"${annotation}"`) - // .join(", "); - - // const rowNumberKey = "row_number"; - // const subQueries = selectionRequest.selections.map((selection) => { - // const numberRangeAsWhereConditions = selection.indexRanges.map( - // (indexRange) => - // `"${rowNumberKey}" BETWEEN ${indexRange.start} AND ${indexRange.end}` - // ); - - // let filtersAsWhereClause = ""; - // const columnNames = Object.keys(selection.filters); - // if (!!columnNames.length) { - // const filtersAsWhereConditions = columnNames.map((columnName) => - // selection.filters[columnName] - // .map((columnValue) => `"${columnName}" = '${columnValue}'`) - // .join(" OR ") - // ); - // filtersAsWhereClause = `WHERE (${filtersAsWhereConditions.join(") AND (")})`; - // } - - // let orderByClause = ""; - // if (selection.sort) { - // orderByClause = `ORDER BY "${selection.sort.annotationName}" ${ - // selection.sort.ascending ? "ASC" : "DESC" - // }`; - // } - - // return ` - // SELECT "File Path" - // FROM ( - // SELECT ROW_NUMBER() OVER (${orderByClause}) AS "${rowNumberKey}", "File Path" - // FROM ${this.dataSourceName} - // ${filtersAsWhereClause} - // ) AS Row - // WHERE (${numberRangeAsWhereConditions.join(") OR (")}) - // ${orderByClause} - // `; - // }, [] as string[]); - - // // TODO: This should be cancellable, but moving this to be - // // web compatible would by default make that true - // // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/62 - // const sql = ` - // COPY ( - // SELECT ${annotationsAsSelect} - // FROM ${this.dataSourceName} - // WHERE "File Path" IN ( - // ${subQueries.join(") OR (")} - // ) - // ) - // TO '${saveLocation}' (HEADER, DELIMITER ','); - // `; - - // await this.databaseService.query(sql); - // return { - // downloadRequestId: manifestDownloadId, - // msg: `CSV downloaded to ${saveLocation}`, - // resolution: DownloadResolution.SUCCESS, - // }; - // } -} diff --git a/packages/core/services/CsvService/HttpCsvService/index.ts b/packages/core/services/CsvService/HttpCsvService/index.ts deleted file mode 100644 index 16b9b2532..000000000 --- a/packages/core/services/CsvService/HttpCsvService/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -import CsvService, { CsvManifestRequest } from ".."; -import FileDownloadService, { - DownloadResult, -} from "../../FileDownloadService"; -import HttpServiceBase, { ConnectionConfig } from "../../HttpServiceBase"; - -interface Config extends ConnectionConfig { - downloadService: FileDownloadService; -} - -/** - * Service responsible for requesting a CSV manifest of metadata for selected files. Delegates - * heavy-lifting of the downloading to a platform-dependent implementation of the FileDownloadService. - */ -export default class HttpCsvService extends HttpServiceBase implements CsvService { - private static readonly ENDPOINT_VERSION = "2.0"; - public static readonly BASE_CSV_DOWNLOAD_URL = `file-explorer-service/${HttpCsvService.ENDPOINT_VERSION}/files/selection/manifest`; - private readonly downloadService: FileDownloadService; - - public constructor(config: Config) { - super(config); - this.downloadService = config.downloadService; - } - - public async getCsvAsBytes( - selectionRequest: CsvManifestRequest, - manifestDownloadId: string - ): Promise { - const stringifiedPostBody = JSON.stringify(selectionRequest); - const url = `${this.baseUrl}/${HttpCsvService.BASE_CSV_DOWNLOAD_URL}${this.pathSuffix}`; - return Promise.reject("blah") - // return this.downloadService.downloadCsvManifest( - // url, - // stringifiedPostBody, - // manifestDownloadId - // ); - } -} diff --git a/packages/core/services/CsvService/index.ts b/packages/core/services/CsvService/index.ts deleted file mode 100644 index c1603ac40..000000000 --- a/packages/core/services/CsvService/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { - DownloadResult, -} from "../FileDownloadService"; -import { Selection } from "../FileService/HttpFileService"; - -export interface CsvManifestRequest { - annotations: string[]; - selections: Selection[]; -} - -/** - * Service responsible for requesting a CSV manifest of metadata for selected files - */ -export default interface CsvService { - getCsvAsBytes( - selectionRequest: CsvManifestRequest, - manifestDownloadId: string - ): Promise -} diff --git a/packages/core/services/DataSourceService/index.ts b/packages/core/services/DataSourceService/index.ts new file mode 100644 index 000000000..ef38d93f9 --- /dev/null +++ b/packages/core/services/DataSourceService/index.ts @@ -0,0 +1,30 @@ +import { Source } from "../../entity/FileExplorerURL"; +import HttpServiceBase, { ConnectionConfig } from "../HttpServiceBase"; + +export interface DataSource extends Source { + id: string; + version?: number; +} + +export interface PythonicDataAccessSnippet { + code: string; + setup: string; +} + +/** + * Service responsible for fetching dataset related metadata. + */ +export default class DataSourceService extends HttpServiceBase { + constructor(config: ConnectionConfig = {}) { + super(config); + } + + /** + * Requests for all available data sources. + */ + public async getAll(): Promise { + // TODO: Placeholder until infra S3 bucket is ready + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/66 + return []; + } +} diff --git a/packages/core/services/DataSourceService/test/DataSourceService.test.ts b/packages/core/services/DataSourceService/test/DataSourceService.test.ts new file mode 100644 index 000000000..765ca8f48 --- /dev/null +++ b/packages/core/services/DataSourceService/test/DataSourceService.test.ts @@ -0,0 +1,18 @@ +import { expect } from "chai"; + +import DatasetService from ".."; + +describe("DataSourceService", () => { + describe("getAll", () => { + it("issues request for datasets", async () => { + // Arrange + const service = new DatasetService(); + + // Act + const datasets = await service.getAll(); + + // Assert + expect(datasets).to.deep.equal([]); + }); + }); +}); diff --git a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts index 0778865e7..84243cf85 100644 --- a/packages/core/services/DatabaseService/DatabaseServiceNoop.ts +++ b/packages/core/services/DatabaseService/DatabaseServiceNoop.ts @@ -1,18 +1,12 @@ import DatabaseService from "."; export default class DatabaseServiceNoop implements DatabaseService { - public table = "noop"; - public addDataSource() { return Promise.reject("DatabaseServiceNoop:addDataSource"); } - public getDataSource() { - return Promise.reject("DatabaseServiceNoop:getDataSource"); - } - - public saveQueryAsBuffer(): Promise { - return Promise.reject("DatabaseServiceNoop:saveQueryAsBuffer"); + public saveQuery(): Promise { + return Promise.reject("DatabaseServiceNoop:saveQuery"); } public query(): Promise<{ [key: string]: string }[]> { diff --git a/packages/core/services/DatabaseService/index.ts b/packages/core/services/DatabaseService/index.ts index 69a51b4df..98bec99a5 100644 --- a/packages/core/services/DatabaseService/index.ts +++ b/packages/core/services/DatabaseService/index.ts @@ -1,17 +1,18 @@ -export interface DataSource { - name: string; - created: Date; -} - /** * Service reponsible for querying against a database */ export default interface DatabaseService { - addDataSource(name: string, uri: File): Promise; - - getDataSource(uri: string): Promise; + addDataSource( + name: string, + type: "csv" | "json" | "parquet", + uri: File | string + ): Promise; - saveQueryAsBuffer(sql: string): Promise; + saveQuery( + destination: string, + sql: string, + format: "csv" | "parquet" | "json" + ): Promise; query(sql: string): Promise<{ [key: string]: string }[]>; } diff --git a/packages/core/services/DatasetService/index.ts b/packages/core/services/DatasetService/index.ts deleted file mode 100644 index 93d6ce675..000000000 --- a/packages/core/services/DatasetService/index.ts +++ /dev/null @@ -1,103 +0,0 @@ -import HttpServiceBase, { ConnectionConfig } from "../HttpServiceBase"; -import DatabaseService from "../DatabaseService"; -import DatabaseServiceNoop from "../DatabaseService/DatabaseServiceNoop"; - -export interface Dataset { - id: string; - name: string; - annotations?: string[]; - version?: number; - expiration?: Date; - collection?: string; // When fixed Dataset should not point to a collection - data?: ArrayBuffer; - query?: string; - client?: string; - fixed?: boolean; - uri?: string; - private?: boolean; - created: Date; - createdBy: string; -} - -export interface PythonicDataAccessSnippet { - code: string; - setup: string; -} - -interface DatasetConnectionConfig extends ConnectionConfig { - database: DatabaseService; -} - -/** - * Service responsible for fetching dataset related metadata. - */ -export default class DatasetService extends HttpServiceBase { - private static readonly ENDPOINT_VERSION = "2.0"; - public static readonly BASE_DATASET_URL = `file-explorer-service/${DatasetService.ENDPOINT_VERSION}/dataset`; - private readonly database: DatabaseService; - - constructor(config: DatasetConnectionConfig = { database: new DatabaseServiceNoop() }) { - super(config); - this.database = config.database; - } - - /** - * Requests for all available (e.g., non-expired) datasets. - */ - public async getDatasets(): Promise { - const requestUrl = `${this.baseUrl}/${DatasetService.BASE_DATASET_URL}`; - console.log(`Requesting all datasets from the following url: ${requestUrl}`); - - // This data should never be stale, so, avoid using a response cache - const response = await this.getWithoutCaching(requestUrl); - - return response.data; - } - - /** - * Request for a specific dataset. - */ - public async getDataset(collection: { - name: string; - version?: number; - uri?: string; - }): Promise { - if (collection.uri) { - const info = await this.database.getDataSource(collection.uri); - return { - id: info.name, - name: info.name, - version: 1, - uri: collection.uri, - created: info.created, - createdBy: "Unknown", - }; - } - - return { - id: collection.name, - name: collection.name, - version: collection.version, - created: new Date(), - createdBy: "Unknown", - }; - } - - public async getPythonicDataAccessSnippet( - datasetName: string, - datasetVersion?: number - ): Promise { - const requestUrl = `${this.baseUrl}/${DatasetService.BASE_DATASET_URL}/${encodeURIComponent( - datasetName - )}/${datasetVersion}/pythonSnippet`; - console.log(`Requesting Python snippet for accessing dataset at: ${requestUrl}`); - - const response = await this.get(requestUrl); - - if (response.data.length !== 1) { - throw new Error(`Unexpected number of Python snippets received from ${requestUrl}`); - } - - return response.data[0]; - } -} diff --git a/packages/core/services/DatasetService/test/DatasetService.test.ts b/packages/core/services/DatasetService/test/DatasetService.test.ts deleted file mode 100644 index 5715270ad..000000000 --- a/packages/core/services/DatasetService/test/DatasetService.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { createMockHttpClient } from "@aics/redux-utils"; -import { expect } from "chai"; -import sinon from "sinon"; - -import DatabaseServiceNoop from "../../DatabaseService/DatabaseServiceNoop"; - -import DatasetService, { PythonicDataAccessSnippet } from ".."; - -describe("DatasetService", () => { - const baseUrl = "test"; - const expectedDatasetId = "abc123"; - const expectedDatasets = [expectedDatasetId, "def456", "ghi789"].map((id) => ({ - id, - name: "Something" + id, - version: 1, - })); - - describe("getDatasets", () => { - const httpClient = createMockHttpClient({ - when: `${baseUrl}/${DatasetService.BASE_DATASET_URL}`, - respondWith: { - data: { - data: expectedDatasets, - }, - }, - }); - - it("issues request for datasets", async () => { - // Arrange - const service = new DatasetService({ - baseUrl, - httpClient, - userName: "test", - database: new DatabaseServiceNoop(), - }); - - // Act - const datasets = await service.getDatasets(); - - // Assert - expect(datasets).to.deep.equal(expectedDatasets); - }); - }); - - describe("getPythonicDataAccessSnippet", () => { - it("returns requested Pythonic dataset access snippet", async () => { - // Arrange - const datasetName = "foo"; - const datasetVersion = 3; - - const expected: PythonicDataAccessSnippet = { - setup: "pip install foobar", - code: "import foobar; foobar.baz()", - }; - - const httpClient = createMockHttpClient({ - when: `${baseUrl}/${DatasetService.BASE_DATASET_URL}/${datasetName}/${datasetVersion}/pythonSnippet`, - respondWith: { - data: { - data: [expected], - }, - }, - }); - const service = new DatasetService({ - baseUrl, - httpClient, - database: new DatabaseServiceNoop(), - }); - - // Act - const snippet = await service.getPythonicDataAccessSnippet(datasetName, datasetVersion); - - // Assert - expect(snippet).to.deep.equal(expected); - }); - - it("encodes dataset name in URL path", async () => { - // Arrange - const datasetName = "files & more?"; - const encodedDatasetName = "files%20%26%20more%3F"; - const datasetVersion = 3; - const expectedUrl = `${baseUrl}/${DatasetService.BASE_DATASET_URL}/files%20%26%20more%3F/${encodedDatasetName}/pythonSnippet`; - - const httpClient = createMockHttpClient(); - const getStub = sinon.stub(httpClient, "get").resolves({ - data: { - data: [{}], - }, - status: 200, - statusText: "OK", - }); - const service = new DatasetService({ - baseUrl, - httpClient, - database: new DatabaseServiceNoop(), - }); - - // Act - await service.getPythonicDataAccessSnippet(datasetName, datasetVersion); - - // Assert - getStub.calledOnceWith(expectedUrl); - }); - }); -}); diff --git a/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts b/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts index 45478a205..bfcbf9b92 100644 --- a/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts +++ b/packages/core/services/FileDownloadService/FileDownloadServiceNoop.ts @@ -1,44 +1,29 @@ -import { uniqueId } from "lodash"; - -import FileDownloadService, { DownloadResolution, FileInfo } from "."; +import FileDownloadService, { DownloadResolution, DownloadResult, FileInfo } from "."; export default class FileDownloadServiceNoop implements FileDownloadService { - getDefaultDownloadDirectory() { - return Promise.resolve("Default directory request from FileDownloadServiceNoop"); - } - - downloadCsvManifest() { - return Promise.resolve({ - downloadRequestId: uniqueId(), - msg: - "Download of CSV manifest triggered on FileDownloadServiceNoop; returning without triggering a download.", - resolution: DownloadResolution.SUCCESS, - }); - } + isFileSystemAccessible = false; - downloadFile( + download( fileInfo: FileInfo, - _: string, - destination: string, - onProgress?: (bytesDownloaded: number) => void - ) { + downloadRequestId: string, + onProgress?: (progress: number) => void + ): Promise { return Promise.resolve({ - downloadRequestId: uniqueId(), - destination, + downloadRequestId, onProgress, msg: `Download of ${fileInfo.path} triggered on FileDownloadServiceNoop; returning without triggering a download.`, resolution: DownloadResolution.SUCCESS, }); } - promptForDownloadDirectory() { + prepareHttpResourceForDownload(): Promise { return Promise.resolve( - "Prompt for download directory triggered on FileDownloadServiceNoop" + "Triggered prepareHttpResourceForDownload on FileDownloadServiceNoop; returning without triggering a download." ); } - promptForSaveLocation(): Promise { - return Promise.resolve("Prompt for save location triggered on FileDownloadServiceNoop"); + getDefaultDownloadDirectory(): Promise { + return Promise.resolve("Triggered getDefaultDownloadDirectory on FileDownloadServiceNoop"); } cancelActiveRequest() { diff --git a/packages/core/services/FileDownloadService/index.ts b/packages/core/services/FileDownloadService/index.ts index 0a5b5900f..0f1256943 100644 --- a/packages/core/services/FileDownloadService/index.ts +++ b/packages/core/services/FileDownloadService/index.ts @@ -14,52 +14,34 @@ export interface FileInfo { name: string; path: string; size?: number; + data?: Uint8Array | Blob | string; } /** * Interface that defines a platform-dependent service for implementing download functionality. */ export default interface FileDownloadService { - /** - * Get default download directory for the OS - */ - getDefaultDownloadDirectory(): Promise; + isFileSystemAccessible: boolean; /** - * Download a CSV manifest from `url` of selected files described by `data` (POST data). + * Retrieves the file system's default download location. */ - downloadCsvManifest( - url: string, - data: string, - downloadRequestId: string - ): Promise; + getDefaultDownloadDirectory(): Promise; /** * Download a file described by `fileInfo`. Optionally provide an "onProgress" callback that will be * called repeatedly over the course of the file download with the number of bytes downloaded so far. */ - downloadFile( + download( fileInfo: FileInfo, downloadRequestId: string, - destination: string, onProgress?: (bytesDownloaded: number) => void ): Promise; /** - * Prompts the user for a download directory. - * Will first notify the user of this request. - */ - promptForDownloadDirectory(): Promise; - - /** - * Prompts the user for a save location. + * Retrieve a Blob from a server over HTTP. */ - promptForSaveLocation( - title: string, - defaultFileName: string, - buttonLabel: string, - filters?: Record[] - ): Promise; + prepareHttpResourceForDownload(url: string, postBody: string): Promise; /** * Attempt to cancel an active download request, deleting the downloaded artifact if present. diff --git a/packages/core/services/FileService/DatabaseFileService/index.ts b/packages/core/services/FileService/DatabaseFileService/index.ts index c976643c5..598f419c0 100644 --- a/packages/core/services/FileService/DatabaseFileService/index.ts +++ b/packages/core/services/FileService/DatabaseFileService/index.ts @@ -1,15 +1,19 @@ -import { isNil, omit } from "lodash"; +import { isNil, omit, uniqueId } from "lodash"; -import FileService, { GetFilesRequest, SelectionAggregationResult } from ".."; +import FileService, { GetFilesRequest, SelectionAggregationResult, Selection } from ".."; import DatabaseService from "../../DatabaseService"; import DatabaseServiceNoop from "../../DatabaseService/DatabaseServiceNoop"; +import FileDownloadService, { DownloadResolution, DownloadResult } from "../../FileDownloadService"; +import FileDownloadServiceNoop from "../../FileDownloadService/FileDownloadServiceNoop"; import FileSelection from "../../../entity/FileSelection"; import FileSet from "../../../entity/FileSet"; import FileDetail from "../../../entity/FileDetail"; +import SQLBuilder from "../../../entity/SQLBuilder"; interface Config { databaseService: DatabaseService; dataSourceName: string; + downloadService: FileDownloadService; } /** @@ -17,6 +21,7 @@ interface Config { */ export default class DatabaseFileService implements FileService { private readonly databaseService: DatabaseService; + private readonly downloadService: FileDownloadService; private readonly dataSourceName: string; private static convertDatabaseRowToFileDetail( @@ -54,8 +59,15 @@ export default class DatabaseFileService implements FileService { }); } - constructor(config: Config = { dataSourceName: "Unknown", databaseService: new DatabaseServiceNoop() }) { + constructor( + config: Config = { + dataSourceName: "Unknown", + databaseService: new DatabaseServiceNoop(), + downloadService: new FileDownloadServiceNoop(), + } + ) { this.databaseService = config.databaseService; + this.downloadService = config.downloadService; this.dataSourceName = config.dataSourceName; } @@ -80,11 +92,7 @@ export default class DatabaseFileService implements FileService { if (allFiles.length && allFiles[0].size === undefined) { return { count }; } - // TODO: Should have file size return as number not a string - const size = allFiles.reduce( - (acc, file) => acc + parseInt((file.size as any) || "0", 10), - 0 - ); + const size = allFiles.reduce((acc, file) => acc + (file.size || 0), 0); return { count, size }; } @@ -107,4 +115,72 @@ export default class DatabaseFileService implements FileService { ) ); } + + /** + * Download file selection as a file in the specified format. + */ + public async download( + annotations: string[], + selections: Selection[], + format: "csv" | "json" | "parquet" + ): Promise { + const sqlBuilder = new SQLBuilder() + .select(annotations.map((annotation) => `"${annotation}"`).join(", ")) + .from(this.dataSourceName); + + selections.forEach((selection) => { + selection.indexRanges.forEach((indexRange) => { + const subQuery = new SQLBuilder() + .select('"File Path"') + .from(this.dataSourceName as string) + .whereOr( + Object.entries(selection.filters).map(([column, values]) => { + const commaSeperatedValues = values.map((v) => `'${v}'`).join(", "); + return `"${column}" IN (${commaSeperatedValues}}`; + }) + ) + .offset(indexRange.start) + .limit(indexRange.end - indexRange.start + 1); + + if (selection.sort) { + subQuery.orderBy( + `"${selection.sort.annotationName}" ${ + selection.sort.ascending ? "ASC" : "DESC" + }` + ); + } + + sqlBuilder.whereOr(`"File Path" IN (${subQuery})`); + }); + }); + + // If the file system is accessible we can just have DuckDB write the + // output query directly to the system rather than to a buffer then the file + if (this.downloadService.isFileSystemAccessible) { + const downloadDir = await this.downloadService.getDefaultDownloadDirectory(); + const lowerCaseUserAgent = navigator.userAgent.toLowerCase(); + const separator = lowerCaseUserAgent.includes("Windows") ? "\\" : "/"; + const destination = `${downloadDir}${separator}file-selection-${Date.now().toLocaleString( + "en-us" + )}`; + await this.databaseService.saveQuery(destination, sqlBuilder.toSQL(), format); + return { + downloadRequestId: uniqueId(), + msg: `File downloaded to ${destination}.${format}`, + resolution: DownloadResolution.SUCCESS, + }; + } + + const buffer = await this.databaseService.saveQuery(uniqueId(), sqlBuilder.toSQL(), format); + const name = `file-selection-${new Date()}.${format}`; + return this.downloadService.download( + { + id: name, + name: name, + path: name, + data: buffer, + }, + uniqueId() + ); + } } diff --git a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts index 6ed79839b..2fdb2cb7d 100644 --- a/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts +++ b/packages/core/services/FileService/DatabaseFileService/test/DatabaseFileService.test.ts @@ -4,6 +4,7 @@ import FileSelection from "../../../../entity/FileSelection"; import FileSet from "../../../../entity/FileSet"; import NumericRange from "../../../../entity/NumericRange"; import DatabaseServiceNoop from "../../../DatabaseService/DatabaseServiceNoop"; +import FileDownloadServiceNoop from "../../../FileDownloadService/FileDownloadServiceNoop"; import DatabaseFileService from ".."; @@ -26,7 +27,11 @@ describe("DatabaseFileService", () => { describe("getFiles", () => { it("issues request for files that match given parameters", async () => { - const databaseFileService = new DatabaseFileService({ dataSourceName: "Unknown", databaseService }); + const databaseFileService = new DatabaseFileService({ + dataSourceName: "Unknown", + databaseService, + downloadService: new FileDownloadServiceNoop(), + }); const fileSet = new FileSet(); const response = await databaseFileService.getFiles({ from: 0, @@ -77,7 +82,11 @@ describe("DatabaseFileService", () => { describe("getAggregateInformation", () => { it("issues request for aggregated information about given files", async () => { // Arrange - const fileService = new DatabaseFileService({ dataSourceName: "Unknown", databaseService }); + const fileService = new DatabaseFileService({ + dataSourceName: "Unknown", + databaseService, + downloadService: new FileDownloadServiceNoop(), + }); const selection = new FileSelection().select({ fileSet: new FileSet({ fileService }), index: new NumericRange(0, 1), @@ -95,7 +104,11 @@ describe("DatabaseFileService", () => { describe("getCountOfMatchingFiles", () => { it("issues request for count of files matching given parameters", async () => { - const fileService = new DatabaseFileService({ dataSourceName: "Unknown", databaseService }); + const fileService = new DatabaseFileService({ + dataSourceName: "Unknown", + databaseService, + downloadService: new FileDownloadServiceNoop(), + }); const fileSet = new FileSet(); const count = await fileService.getCountOfMatchingFiles(fileSet); expect(count).to.equal(6); diff --git a/packages/core/services/FileService/FileServiceNoop.ts b/packages/core/services/FileService/FileServiceNoop.ts index f8878662f..7063fa214 100644 --- a/packages/core/services/FileService/FileServiceNoop.ts +++ b/packages/core/services/FileService/FileServiceNoop.ts @@ -1,16 +1,25 @@ import FileService, { SelectionAggregationResult } from "."; +import { DownloadResolution, DownloadResult } from "../FileDownloadService"; import FileDetail from "../../entity/FileDetail"; export default class FileServiceNoop implements FileService { - public async getCountOfMatchingFiles(): Promise { - return 0; + public getCountOfMatchingFiles(): Promise { + return Promise.resolve(0); } - public async getAggregateInformation(): Promise { - return { count: 0, size: 0 }; + public getAggregateInformation(): Promise { + return Promise.resolve({ count: 0, size: 0 }); } - public async getFiles(): Promise { - return []; + public getFiles(): Promise { + return Promise.resolve([]); + } + + public getFilesAsBuffer(): Promise { + return Promise.resolve(new Uint8Array()); + } + + public download(): Promise { + return Promise.resolve({ downloadRequestId: "", resolution: DownloadResolution.CANCELLED }); } } diff --git a/packages/core/services/FileService/HttpFileService/index.ts b/packages/core/services/FileService/HttpFileService/index.ts index 422700c82..b8c79761b 100644 --- a/packages/core/services/FileService/HttpFileService/index.ts +++ b/packages/core/services/FileService/HttpFileService/index.ts @@ -1,27 +1,21 @@ -import { compact, join } from "lodash"; +import { compact, join, uniqueId } from "lodash"; -import FileService, { GetFilesRequest, SelectionAggregationResult } from ".."; -import HttpServiceBase from "../../HttpServiceBase"; +import FileService, { GetFilesRequest, SelectionAggregationResult, Selection } from ".."; +import FileDownloadService, { DownloadResult } from "../../FileDownloadService"; +import FileDownloadServiceNoop from "../../FileDownloadService/FileDownloadServiceNoop"; +import HttpServiceBase, { ConnectionConfig } from "../../HttpServiceBase"; import FileSelection from "../../../entity/FileSelection"; -import { JSONReadyRange } from "../../../entity/NumericRange"; import FileSet from "../../../entity/FileSet"; import FileDetail, { FmsFile } from "../../../entity/FileDetail"; -export interface Selection { - filters: { - [index: string]: (string | number | boolean)[]; - }; - indexRanges: JSONReadyRange[]; - sort?: { - annotationName: string; - ascending: boolean; - }; -} - interface SelectionAggregationRequest { selections: Selection[]; } +interface Config extends ConnectionConfig { + downloadService: FileDownloadService; +} + /** * Service responsible for fetching file related metadata. */ @@ -30,6 +24,27 @@ export default class HttpFileService extends HttpServiceBase implements FileServ public static readonly BASE_FILES_URL = `file-explorer-service/${HttpFileService.ENDPOINT_VERSION}/files`; public static readonly BASE_FILE_COUNT_URL = `${HttpFileService.BASE_FILES_URL}/count`; public static readonly SELECTION_AGGREGATE_URL = `${HttpFileService.BASE_FILES_URL}/selection/aggregate`; + private static readonly CSV_ENDPOINT_VERSION = "2.0"; + public static readonly BASE_CSV_DOWNLOAD_URL = `file-explorer-service/${HttpFileService.CSV_ENDPOINT_VERSION}/files/selection/manifest`; + private readonly downloadService: FileDownloadService; + + constructor(config: Config = { downloadService: new FileDownloadServiceNoop() }) { + super(config); + this.downloadService = config.downloadService; + } + + /** + * Basic check to see if the network is accessible by attempting to fetch the file explorer service base url + */ + public async isNetworkAccessible(): Promise { + try { + await this.get(`${this.baseUrl}/${HttpFileService.BASE_FILE_COUNT_URL}`); + return true; + } catch (error) { + console.error(`Unable to access AICS network ${error}`); + return false; + } + } public async getCountOfMatchingFiles(fileSet: FileSet): Promise { const requestUrl = join( @@ -39,7 +54,6 @@ export default class HttpFileService extends HttpServiceBase implements FileServ ]), "?" ); - console.log(`Requesting count of matching files from ${requestUrl}`); const response = await this.get(requestUrl); @@ -57,7 +71,6 @@ export default class HttpFileService extends HttpServiceBase implements FileServ const selections = fileSelection.toCompactSelectionList(); const postBody: SelectionAggregationRequest = { selections }; const requestUrl = `${this.baseUrl}/${HttpFileService.SELECTION_AGGREGATE_URL}${this.pathSuffix}`; - console.log(`Requesting aggregate results of matching files ${postBody}`); const response = await this.post( requestUrl, @@ -80,9 +93,41 @@ export default class HttpFileService extends HttpServiceBase implements FileServ const base = `${this.baseUrl}/${HttpFileService.BASE_FILES_URL}${this.pathSuffix}?from=${from}&limit=${limit}`; const requestUrl = join(compact([base, fileSet.toQueryString()]), "&"); - console.log(`Requesting files from ${requestUrl}`); const response = await this.get(requestUrl); return response.data.map((file) => new FileDetail(file)); } + + /** + * Download the given file selection query to local storage in the given format + */ + public async download( + annotations: string[], + selections: Selection[], + format: "csv" | "json" | "parquet" + ): Promise { + if (format !== "csv") { + throw new Error( + "Only CSV download is supported at this time for downloading from AICS FMS" + ); + } + + const postData = JSON.stringify({ annotations, selections }); + const url = `${this.baseUrl}/${HttpFileService.BASE_CSV_DOWNLOAD_URL}${this.pathSuffix}`; + + const manifestAsString = await this.downloadService.prepareHttpResourceForDownload( + url, + postData + ); + const name = `file-manifest-${new Date()}.csv`; + return this.downloadService.download( + { + name, + id: name, + path: url, + data: manifestAsString, + }, + uniqueId() + ); + } } diff --git a/packages/core/services/FileService/HttpFileService/test/HttpFileService.test.ts b/packages/core/services/FileService/HttpFileService/test/HttpFileService.test.ts index a87d4d41f..4f5691804 100644 --- a/packages/core/services/FileService/HttpFileService/test/HttpFileService.test.ts +++ b/packages/core/services/FileService/HttpFileService/test/HttpFileService.test.ts @@ -5,6 +5,7 @@ import HttpFileService from ".."; import FileSelection from "../../../../entity/FileSelection"; import FileSet from "../../../../entity/FileSet"; import NumericRange from "../../../../entity/NumericRange"; +import FileDownloadServiceNoop from "../../../FileDownloadService/FileDownloadServiceNoop"; describe("HttpFileService", () => { const baseUrl = "test"; @@ -26,7 +27,11 @@ describe("HttpFileService", () => { ]); it("issues request for files that match given parameters", async () => { - const httpFileService = new HttpFileService({ baseUrl, httpClient }); + const httpFileService = new HttpFileService({ + baseUrl, + httpClient, + downloadService: new FileDownloadServiceNoop(), + }); const fileSet = new FileSet(); const response = await httpFileService.getFiles({ from: 0, @@ -53,7 +58,11 @@ describe("HttpFileService", () => { it("issues request for aggregated information about given files", async () => { // Arrange - const fileService = new HttpFileService({ baseUrl, httpClient }); + const fileService = new HttpFileService({ + baseUrl, + httpClient, + downloadService: new FileDownloadServiceNoop(), + }); const selection = new FileSelection().select({ fileSet: new FileSet(), index: new NumericRange(0, 1), @@ -80,7 +89,11 @@ describe("HttpFileService", () => { }); it("issues request for count of files matching given parameters", async () => { - const fileService = new HttpFileService({ baseUrl, httpClient }); + const fileService = new HttpFileService({ + baseUrl, + httpClient, + downloadService: new FileDownloadServiceNoop(), + }); const fileSet = new FileSet(); const count = await fileService.getCountOfMatchingFiles(fileSet); expect(count).to.equal(2); diff --git a/packages/core/services/FileService/index.ts b/packages/core/services/FileService/index.ts index 748d83508..e06b4aa3d 100644 --- a/packages/core/services/FileService/index.ts +++ b/packages/core/services/FileService/index.ts @@ -1,6 +1,8 @@ +import { DownloadResult } from "../FileDownloadService"; import FileDetail from "../../entity/FileDetail"; import FileSelection from "../../entity/FileSelection"; import FileSet from "../../entity/FileSet"; +import { JSONReadyRange } from "../../entity/NumericRange"; /** * Represents a sub-document that can be found within an FmsFile's `annotations` list. @@ -22,8 +24,24 @@ export interface SelectionAggregationResult { size?: number; } +export interface Selection { + filters: { + [index: string]: (string | number | boolean)[]; + }; + indexRanges: JSONReadyRange[]; + sort?: { + annotationName: string; + ascending: boolean; + }; +} + export default interface FileService { baseUrl?: string; + download( + annotations: string[], + selections: Selection[], + format: "csv" | "json" | "parquet" + ): Promise; getCountOfMatchingFiles(fileSet: FileSet): Promise; getAggregateInformation(fileSelection: FileSelection): Promise; getFiles(request: GetFilesRequest): Promise; diff --git a/packages/core/services/HttpServiceBase/index.ts b/packages/core/services/HttpServiceBase/index.ts index bdf84959e..e152a0ff1 100644 --- a/packages/core/services/HttpServiceBase/index.ts +++ b/packages/core/services/HttpServiceBase/index.ts @@ -129,7 +129,6 @@ export default class HttpServiceBase { public async get(url: string): Promise> { const encodedUrl = HttpServiceBase.encodeURI(url); - console.log(`Sanitized ${url} to ${encodedUrl}`); if (!this.urlToResponseDataCache.has(encodedUrl)) { // if this fails, bubble up exception @@ -158,15 +157,37 @@ export default class HttpServiceBase { */ public async getWithoutCaching(url: string): Promise> { const encodedUrl = HttpServiceBase.encodeURI(url); - console.log(`Sanitized ${url} to ${encodedUrl}`); const response = await retry.execute(() => this.httpClient.get(encodedUrl)); return new RestServiceResponse(response.data); } + public async rawPost(url: string, body: string): Promise { + const encodedUrl = HttpServiceBase.encodeURI(url); + const config = { headers: { "Content-Type": "application/json" } }; + + let response; + try { + // if this fails, bubble up exception + response = await retry.execute(() => this.httpClient.post(encodedUrl, body, config)); + } catch (err) { + // Specific errors about the failure from services will be in this path + if (axios.isAxiosError(err) && err?.response?.data?.message) { + throw new Error(JSON.stringify(err.response.data.message)); + } + throw err; + } + + if (response.status >= 400 || response.data === undefined) { + // by default axios will reject if does not satisfy: status >= 200 && status < 300 + throw new Error(`Request for ${encodedUrl} failed`); + } + + return response.data; + } + public async post(url: string, body: string): Promise> { const encodedUrl = HttpServiceBase.encodeURI(url); - console.log(`Sanitized ${url} to ${encodedUrl}`); const config = { headers: { "Content-Type": "application/json" } }; let response; @@ -191,7 +212,6 @@ export default class HttpServiceBase { public async patch(url: string, body: string): Promise> { const encodedUrl = HttpServiceBase.encodeURI(url); - console.log(`Sanitized ${url} to ${encodedUrl}`); const config = { headers: { "Content-Type": "application/json" } }; let response; diff --git a/packages/core/services/PersistentConfigService/PersistentConfigServiceNoop.ts b/packages/core/services/PersistentConfigService/PersistentConfigServiceNoop.ts deleted file mode 100644 index 58728cfd1..000000000 --- a/packages/core/services/PersistentConfigService/PersistentConfigServiceNoop.ts +++ /dev/null @@ -1,15 +0,0 @@ -import PersistentConfigService from "."; - -export default class PersistentConfigServiceNoop implements PersistentConfigService { - public get() { - return undefined; - } - - public getAll() { - return {}; - } - - public persist() { - return; - } -} diff --git a/packages/core/services/index.ts b/packages/core/services/index.ts index 2c7c09ceb..aaf71aa59 100644 --- a/packages/core/services/index.ts +++ b/packages/core/services/index.ts @@ -1,9 +1,15 @@ +import FrontendInsights from "@aics/frontend-insights"; +import ApplicationInfoService from "./ApplicationInfoService"; +import DatabaseService from "./DatabaseService"; +import ExecutionEnvService from "./ExecutionEnvService"; +import FileDownloadService from "./FileDownloadService"; +import FileViewerService from "./FileViewerService"; +import NotificationService from "./NotificationService"; + export { default as AnnotationService } from "./AnnotationService"; export type { default as ApplicationInfoService } from "./ApplicationInfoService"; -export { default as CsvService } from "./CsvService"; export { default as DatabaseService } from "./DatabaseService"; -export { DataSource } from "./DatabaseService"; -export { default as DatasetService } from "./DatasetService"; +export { default as DatasetService } from "./DataSourceService"; export type { default as ExecutionEnvService } from "./ExecutionEnvService"; export { ExecutableEnvCancellationToken, SystemDefaultAppLocation } from "./ExecutionEnvService"; export type { SaveLocationResolution } from "./ExecutionEnvService"; @@ -24,3 +30,13 @@ export type { UserSelectedApplication, } from "./PersistentConfigService"; export { PersistedConfigKeys } from "./PersistentConfigService"; + +export interface PlatformDependentServices { + applicationInfoService: ApplicationInfoService; + databaseService: DatabaseService; + fileDownloadService: FileDownloadService; + fileViewerService: FileViewerService; + frontendInsights: FrontendInsights; + executionEnvService: ExecutionEnvService; + notificationService: NotificationService; +} diff --git a/packages/core/state/index.ts b/packages/core/state/index.ts index 9d335adda..39c0b9c23 100644 --- a/packages/core/state/index.ts +++ b/packages/core/state/index.ts @@ -6,9 +6,13 @@ import { createLogicMiddleware } from "redux-logic"; import interaction, { InteractionStateBranch } from "./interaction"; import metadata, { MetadataStateBranch } from "./metadata"; import selection, { SelectionStateBranch } from "./selection"; -import FileExplorerURL from "../entity/FileExplorerURL"; +import { PlatformDependentServices } from "../services"; import { PersistedConfig, PersistedConfigKeys } from "../services/PersistentConfigService"; import Annotation from "../entity/Annotation"; +import FileSort from "../entity/FileSort"; +import FileFilter from "../entity/FileFilter"; +import FileFolder from "../entity/FileFolder"; +import { Query } from "./selection/actions"; export { interaction, metadata, selection }; @@ -53,14 +57,16 @@ logicMiddleware.addDeps(reduxLogicDependencies); export const middleware = [logicMiddleware]; interface CreateStoreOptions { + isOnWeb?: boolean; middleware?: Middleware[]; persistedConfig?: PersistedConfig; + platformDependentServices?: Partial; } export function createReduxStore(options: CreateStoreOptions = {}) { const { persistedConfig } = options; const queries = persistedConfig?.[PersistedConfigKeys.Queries]?.length - ? persistedConfig?.[PersistedConfigKeys.Queries] - : [{ url: FileExplorerURL.DEFAULT_FMS_URL, name: "New AICS Query" }]; + ? (persistedConfig[PersistedConfigKeys.Queries] as Query[]) + : []; const rawDisplayAnnotations = persistedConfig && persistedConfig[PersistedConfigKeys.DisplayAnnotations]; const displayAnnotations = rawDisplayAnnotations @@ -68,6 +74,11 @@ export function createReduxStore(options: CreateStoreOptions = {}) { : []; const preloadedState: State = mergeState(initialState, { interaction: { + isOnWeb: !!options.isOnWeb, + platformDependentServices: { + ...initialState.interaction.platformDependentServices, + ...options.platformDependentServices, + }, csvColumns: persistedConfig?.[PersistedConfigKeys.CsvColumns], hasUsedApplicationBefore: persistedConfig?.[PersistedConfigKeys.HasUsedApplicationBefore], @@ -76,7 +87,26 @@ export function createReduxStore(options: CreateStoreOptions = {}) { }, selection: { displayAnnotations, - queries, + queries: queries.map((query) => ({ + ...query, + parts: { + ...query.parts, + // These are persisted to the store in JSON format so when we rehydrated when creating the + // store we have to convert back into their class instances + sortColumn: query.parts.sortColumn + ? new FileSort( + query.parts.sortColumn.annotationName, + query.parts.sortColumn.order + ) + : undefined, + filters: query.parts.filters.map( + (filter) => new FileFilter(filter.name, filter.value) + ), + openFolders: query.parts.openFolders.map( + (folder) => new FileFolder(((folder as unknown) as string).split(".")) + ), + }, + })), }, }); return configureStore({ diff --git a/packages/core/state/interaction/actions.ts b/packages/core/state/interaction/actions.ts index ccc80ca9d..f568de9d8 100644 --- a/packages/core/state/interaction/actions.ts +++ b/packages/core/state/interaction/actions.ts @@ -1,23 +1,38 @@ -import FrontendInsights from "@aics/frontend-insights"; import { makeConstant } from "@aics/redux-utils"; import { uniqueId } from "lodash"; -import Annotation from "../../entity/Annotation"; -import ApplicationInfoService from "../../services/ApplicationInfoService"; import { ContextMenuItem, PositionReference } from "../../components/ContextMenu"; -import ExecutionEnvService from "../../services/ExecutionEnvService"; -import FileDownloadService from "../../services/FileDownloadService"; import FileFilter from "../../entity/FileFilter"; -import FileViewerService from "../../services/FileViewerService"; import { ModalType } from "../../components/Modal"; -import PersistentConfigService, { - UserSelectedApplication, -} from "../../services/PersistentConfigService"; -import { DatabaseService, NotificationService } from "../../services"; +import { UserSelectedApplication } from "../../services/PersistentConfigService"; import FileDetail from "../../entity/FileDetail"; +import { Source } from "../../entity/FileExplorerURL"; +import { FileInfo } from "../../services"; const STATE_BRANCH_NAME = "interaction"; +/** + * PROMPT_FOR_DATA_SOURCE + * + * Intention to prompt the user for a data source; this is largely necessarily for replacing a data source + * that has expired or is otherwise no longer available. + */ +export const PROMPT_FOR_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "prompt-for-data-source"); + +type PartialSource = Omit; + +export interface PromptForDataSource { + type: string; + payload: PartialSource; +} + +export function promptForDataSource(dataSource: PartialSource): PromptForDataSource { + return { + type: PROMPT_FOR_DATA_SOURCE, + payload: dataSource, + }; +} + /** * DOWNLOAD_MANIFEST */ @@ -25,15 +40,20 @@ export const DOWNLOAD_MANIFEST = makeConstant(STATE_BRANCH_NAME, "download-manif export interface DownloadManifestAction { payload: { - annotations: Annotation[]; + annotations: string[]; + type: "csv" | "parquet" | "json"; }; type: string; } -export function downloadManifest(annotations: Annotation[]): DownloadManifestAction { +export function downloadManifest( + annotations: string[], + type: "csv" | "json" | "parquet" +): DownloadManifestAction { return { payload: { annotations, + type, }, type: DOWNLOAD_MANIFEST, }; @@ -70,22 +90,13 @@ export function cancelFileDownload(id: string): CancelFileDownloadAction { export const DOWNLOAD_FILES = makeConstant(STATE_BRANCH_NAME, "download-files"); export interface DownloadFilesAction { - payload: { - files?: FileDetail[]; - shouldPromptForDownloadDirectory: boolean; - }; + payload?: FileInfo[]; type: string; } -export function downloadFiles( - files?: FileDetail[], - shouldPromptForDownloadDirectory = false -): DownloadFilesAction { +export function downloadFiles(files?: FileInfo[]): DownloadFilesAction { return { - payload: { - files, - shouldPromptForDownloadDirectory, - }, + payload: files, type: DOWNLOAD_FILES, }; } @@ -111,25 +122,6 @@ export function markAsUsedApplicationBefore(): MarkAsUsedApplicationBefore { }; } -/** - * SET_CSV_COLUMNS - * - * Intention to set the csv columns - */ -export const SET_CSV_COLUMNS = makeConstant(STATE_BRANCH_NAME, "set-csv-columns"); - -export interface SetCsvColumnsAction { - payload: string[]; - type: string; -} - -export function setCsvColumns(csvColumns: string[]): SetCsvColumnsAction { - return { - payload: csvColumns, - type: SET_CSV_COLUMNS, - }; -} - /** * SHOW_CONTEXT_MENU * @@ -196,57 +188,39 @@ export function hideVisibleModal(): HideVisibleModalAction { } /** - * SET CONNECTION CONFIGURATION FOR THE FILE EXPLORER SERVICE + * Intention is to set whether the current user is an AICS employee */ -export const SET_FILE_EXPLORER_SERVICE_BASE_URL = makeConstant( - STATE_BRANCH_NAME, - "set-file-explorer-service-connection-config" -); +export const SET_IS_AICS_EMPLOYEE = makeConstant(STATE_BRANCH_NAME, "set-is-aics-employee"); -export interface SetFileExplorerServiceBaseUrl { +export interface SetIsAicsEmployee { type: string; - payload: string; + payload: boolean; } -export function setFileExplorerServiceBaseUrl(baseUrl: string): SetFileExplorerServiceBaseUrl { +export function setIsAicsEmployee(isAicsEmployee: boolean): SetIsAicsEmployee { return { - type: SET_FILE_EXPLORER_SERVICE_BASE_URL, - payload: baseUrl, + type: SET_IS_AICS_EMPLOYEE, + payload: isAicsEmployee, }; } /** - * SET PLATFORM-DEPENDENT SERVICES - * - * These services provide platform-dependent functionality and are expected to be injected once on application load. + * SET CONNECTION CONFIGURATION FOR THE FILE EXPLORER SERVICE */ -export const SET_PLATFORM_DEPENDENT_SERVICES = makeConstant( +export const SET_FILE_EXPLORER_SERVICE_BASE_URL = makeConstant( STATE_BRANCH_NAME, - "set-platform-dependent-services" + "set-file-explorer-service-connection-config" ); -export interface PlatformDependentServices { - applicationInfoService: ApplicationInfoService; - databaseService: DatabaseService; - fileDownloadService: FileDownloadService; - fileViewerService: FileViewerService; - frontendInsights: FrontendInsights; - executionEnvService: ExecutionEnvService; - notificationService: NotificationService; - persistentConfigService: PersistentConfigService; -} - -export interface SetPlatformDependentServices { +export interface SetFileExplorerServiceBaseUrl { type: string; - payload: Partial; + payload: string; } -export function setPlatformDependentServices( - services: Partial -): SetPlatformDependentServices { +export function setFileExplorerServiceBaseUrl(baseUrl: string): SetFileExplorerServiceBaseUrl { return { - type: SET_PLATFORM_DEPENDENT_SERVICES, - payload: services, + type: SET_FILE_EXPLORER_SERVICE_BASE_URL, + payload: baseUrl, }; } @@ -556,26 +530,6 @@ export function openWith( }; } -/** - * BROWSE_FOR_NEW_DATA_SOURCE - * - * Intention to prompt the user to browse for a new data source. - */ -export const BROWSE_FOR_NEW_DATA_SOURCE = makeConstant( - STATE_BRANCH_NAME, - "browse-for-new-data-source" -); - -export interface BrowseForNewDataSourceAction { - type: string; -} - -export function browseForNewDataSource(): BrowseForNewDataSourceAction { - return { - type: BROWSE_FOR_NEW_DATA_SOURCE, - }; -} - /** * SET_VISIBLE_MODAL * diff --git a/packages/core/state/interaction/logics.ts b/packages/core/state/interaction/logics.ts index 05943dc62..74d86c3f8 100644 --- a/packages/core/state/interaction/logics.ts +++ b/packages/core/state/interaction/logics.ts @@ -1,4 +1,5 @@ -import { isEmpty, sumBy, throttle, uniqueId } from "lodash"; +import { isEmpty, sumBy, throttle, uniq, uniqueId } from "lodash"; +import { AnyAction } from "redux"; import { createLogic } from "redux-logic"; import { metadata, ReduxLogicDeps, selection } from "../"; @@ -13,10 +14,7 @@ import { CANCEL_FILE_DOWNLOAD, cancelFileDownload, CancelFileDownloadAction, - setCsvColumns, REFRESH, - SET_PLATFORM_DEPENDENT_SERVICES, - promptUserToUpdateApp, OPEN_WITH, openWith, OpenWithAction, @@ -25,49 +23,35 @@ import { DOWNLOAD_FILES, DownloadFilesAction, OpenWithDefaultAction, - BROWSE_FOR_NEW_DATA_SOURCE, + PROMPT_FOR_NEW_EXECUTABLE, + setUserSelectedApplication, + SET_FILE_EXPLORER_SERVICE_BASE_URL, + setIsAicsEmployee, } from "./actions"; import * as interactionSelectors from "./selectors"; -import CsvService, { CsvManifestRequest } from "../../services/CsvService"; -import { - DownloadResolution, - FileDownloadCancellationToken, -} from "../../services/FileDownloadService"; +import { DownloadResolution, FileInfo } from "../../services/FileDownloadService"; import annotationFormatterFactory, { AnnotationType } from "../../entity/AnnotationFormatter"; import FileSet from "../../entity/FileSet"; -import NumericRange from "../../entity/NumericRange"; import { ExecutableEnvCancellationToken, SystemDefaultAppLocation, } from "../../services/ExecutionEnvService"; import { UserSelectedApplication } from "../../services/PersistentConfigService"; -import FileSelection from "../../entity/FileSelection"; -import FileExplorerURL from "../../entity/FileExplorerURL"; import FileDetail from "../../entity/FileDetail"; import { AnnotationName } from "../../entity/Annotation"; +import FileSelection from "../../entity/FileSelection"; +import NumericRange from "../../entity/NumericRange"; /** - * Interceptor responsible for responding to a SET_PLATFORM_DEPENDENT_SERVICES action and - * determining if an application update is available. + * Interceptor responsible for checking if the user is able to access the AICS network */ -const checkForUpdates = createLogic({ - type: SET_PLATFORM_DEPENDENT_SERVICES, +const checkAicsEmployee = createLogic({ + type: SET_FILE_EXPLORER_SERVICE_BASE_URL, async process(deps: ReduxLogicDeps, dispatch, done) { - const platformDependentServices = interactionSelectors.getPlatformDependentServices( - deps.getState() - ); - try { - if (await platformDependentServices.applicationInfoService.updateAvailable()) { - const homepage = "https://alleninstitute.github.io/aics-fms-file-explorer-app/"; - const msg = `A new version of the application is available!
- Visit the FMS File Explorer homepage to download.`; - dispatch(promptUserToUpdateApp(uniqueId(), msg)); - } - } catch (e) { - console.error("Failed while checking if a newer application version is available", e); - } finally { - done(); - } + const fileService = interactionSelectors.getHttpFileService(deps.getState()); + const isAicsEmployee = await fileService.isNetworkAccessible(); + dispatch(setIsAicsEmployee(isAicsEmployee) as AnyAction); + done(); }, }); @@ -79,85 +63,68 @@ const downloadManifest = createLogic({ warnTimeout: 0, // no way to know how long this will take--don't print console warning if it takes a while async process(deps: ReduxLogicDeps, dispatch, done) { const { - payload: { annotations }, + payload: { annotations, type }, } = deps.action as DownloadManifestAction; const manifestDownloadProcessId = uniqueId(); + const sortColumn = selection.selectors.getSortColumn(deps.getState()); + const fileService = interactionSelectors.getFileService(deps.getState()); + let fileSelection = selection.selectors.getFileSelection(deps.getState()); + const filters = interactionSelectors.getFileFiltersForVisibleModal(deps.getState()); - try { - const state = deps.getState(); - let fileSelection = selection.selectors.getFileSelection(state); - const filters = interactionSelectors.getFileFiltersForVisibleModal(state); - const csvService = interactionSelectors.getCsvService(state); - const fileService = interactionSelectors.getFileService(state); - const sortColumn = selection.selectors.getSortColumn(state); - const selectedCollection = selection.selectors.getCollection(state); - - // If we have a specific path to get files from ignore selected files - if (filters.length) { - const fileSet = new FileSet({ - filters, - fileService, - sort: sortColumn, - }); - const count = await fileSet.fetchTotalCount(); - fileSelection = new FileSelection([ - { - selection: new NumericRange(0, count - 1), - fileSet, - sortOrder: 0, - }, - ]); - } + // If we have a specific path to get files from ignore selected files + if (filters.length) { + const fileSet = new FileSet({ + filters, + fileService, + sort: sortColumn, + }); + const count = await fileSet.fetchTotalCount(); + fileSelection = new FileSelection([ + { + selection: new NumericRange(0, count - 1), + fileSet, + sortOrder: 0, + }, + ]); + } - const selections = fileSelection.toCompactSelectionList(); + const selections = fileSelection.toCompactSelectionList(); - if (isEmpty(selections)) { - return; - } + if (isEmpty(selections)) { + done(); + return; + } - const onManifestDownloadCancel = () => { - dispatch(cancelFileDownload(manifestDownloadProcessId)); - }; - dispatch( - processStart( - manifestDownloadProcessId, - "Download of CSV manifest in progress.", - onManifestDownloadCancel - ) - ); + const onManifestDownloadCancel = () => { + dispatch(cancelFileDownload(manifestDownloadProcessId)); + }; + dispatch( + processStart( + manifestDownloadProcessId, + "Download of metadata manifest in progress.", + onManifestDownloadCancel + ) + ); - const request: CsvManifestRequest = { - annotations: annotations.map((annotation) => annotation.name), - selections, - }; - const shouldDownloadFromDatabase = !!selectedCollection?.uri; - let result; - // if (shouldDownloadFromDatabase) { - // result = await csvService.downloadCsvFromDatabase( - // request, - // manifestDownloadProcessId - // ); - // } else { - // result = await csvService.downloadCsvFromServer(request, manifestDownloadProcessId); - // } - - // if (result.resolution === DownloadResolution.CANCELLED) { - // dispatch(removeStatus(manifestDownloadProcessId)); - // return; - // } else { - // const successMsg = `Download of CSV manifest successfully finished.
${result.msg}`; - // dispatch(processSuccess(manifestDownloadProcessId, successMsg)); - // return; - // } + try { + const result = await fileService.download(annotations, selections, type); + + if (result.resolution === DownloadResolution.CANCELLED) { + dispatch(removeStatus(manifestDownloadProcessId)); + } else { + const successMsg = `Download of metadata manifest finished.
${ + result.msg || "" + }`; + dispatch(processSuccess(manifestDownloadProcessId, successMsg)); + } } catch (err) { - const errorMsg = `Download of CSV manifest failed. Details: ${ + const errorMsg = `Download of metadata manifest failed. Details: ${ err instanceof Error ? err.message : err }`; dispatch(processFailure(manifestDownloadProcessId, errorMsg)); - } finally { - dispatch(setCsvColumns(annotations.map((annotation) => annotation.displayName))); - done(); } + + done(); }, }); @@ -194,7 +161,7 @@ const cancelFileDownloadLogic = createLogic({ * Interceptor responsible for responding to a DOWNLOAD_FILES action and * initiating the downloads of the files, showing notifications of process status along the way. */ -const downloadFiles = createLogic({ +const downloadFilesLogic = createLogic({ type: DOWNLOAD_FILES, warnTimeout: 0, // no way to know how long this will take--don't print console warning if it takes a while async process(deps: ReduxLogicDeps, dispatch, done) { @@ -204,87 +171,141 @@ const downloadFiles = createLogic({ ); const numberFormatter = annotationFormatterFactory(AnnotationType.NUMBER); - const { - payload: { files, shouldPromptForDownloadDirectory }, - } = deps.action as DownloadFilesAction; - const destination = shouldPromptForDownloadDirectory - ? await fileDownloadService.promptForDownloadDirectory() - : await fileDownloadService.getDefaultDownloadDirectory(); - - if (destination !== FileDownloadCancellationToken) { - let filesToDownload: FileDetail[]; - if (files !== undefined) { - filesToDownload = files; - } else { - filesToDownload = await fileSelection.fetchAllDetails(); - } + const { payload: files } = deps.action as DownloadFilesAction; - const totalBytesToDownload = sumBy(filesToDownload, "size"); - const totalBytesDisplay = numberFormatter.displayValue(totalBytesToDownload, "bytes"); - await Promise.all( - filesToDownload.map(async (file) => { - const downloadRequestId = uniqueId(); - // TODO: The byte display should be fixed automatically when moving to downloading using browser - // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/62 - const fileByteDisplay = numberFormatter.displayValue(file.size || 0, "bytes"); - const msg = `Downloading ${file.name}, ${fileByteDisplay} out of the total of ${totalBytesDisplay} set to download`; - - const onCancel = () => { - dispatch(cancelFileDownload(downloadRequestId)); - }; - - let totalBytesDownloaded = 0; - // A function that dispatches progress events, throttled - // to only be invokable at most once/second - const throttledProgressDispatcher = throttle(() => { - dispatch( - processProgress( - downloadRequestId, - file.size ? totalBytesDownloaded / file.size : 0, - msg, - onCancel, - [file.id] - ) - ); - }, 1000); - const onProgress = (transferredBytes: number) => { - totalBytesDownloaded += transferredBytes; - throttledProgressDispatcher(); - }; - - try { - dispatch(processStart(downloadRequestId, msg, onCancel, [file.id])); + let filesToDownload: FileInfo[]; + if (files !== undefined) { + filesToDownload = files; + } else { + const selectedFilesDetails = await fileSelection.fetchAllDetails(); + filesToDownload = selectedFilesDetails.map((file) => ({ + id: file.id, + name: file.name, + size: file.size, + path: file.downloadPath, + })); + } - const result = await fileDownloadService.downloadFile( - { - id: file.id, - name: file.name, - path: file.path, - size: file.size, - }, - destination, + const totalBytesToDownload = sumBy(filesToDownload, "size"); + const totalBytesDisplay = numberFormatter.displayValue(totalBytesToDownload, "bytes"); + await Promise.all( + filesToDownload.map(async (file) => { + const downloadRequestId = uniqueId(); + // TODO: The byte display should be fixed automatically when moving to downloading using browser + // https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/62 + const fileByteDisplay = numberFormatter.displayValue(file.size || 0, "bytes"); + const msg = `Downloading ${file.name}, ${fileByteDisplay} out of the total of ${totalBytesDisplay} set to download`; + + const onCancel = () => { + dispatch(cancelFileDownload(downloadRequestId)); + }; + + let totalBytesDownloaded = 0; + // A function that dispatches progress events, throttled + // to only be invokable at most once/second + const throttledProgressDispatcher = throttle(() => { + dispatch( + processProgress( downloadRequestId, - onProgress - ); + file.size ? totalBytesDownloaded / file.size : 0, + msg, + onCancel, + [file.id] + ) + ); + }, 1000); + const onProgress = (transferredBytes: number) => { + totalBytesDownloaded += transferredBytes; + throttledProgressDispatcher(); + }; + + try { + if (totalBytesToDownload) { + dispatch(processStart(downloadRequestId, msg, onCancel, [file.id])); + } + + const result = await fileDownloadService.download( + file, + downloadRequestId, + onProgress + ); + if (totalBytesToDownload) { if (result.resolution === DownloadResolution.CANCELLED) { // Clear status if request was cancelled dispatch(removeStatus(downloadRequestId)); } else { dispatch(processSuccess(downloadRequestId, result.msg || "")); } - } catch (err) { - const errorMsg = `File download failed for file ${ - file.name - }. Details:
${err instanceof Error ? err.message : err}`; - dispatch(processFailure(downloadRequestId, errorMsg)); } - }) + } catch (err) { + const errorMsg = `File download failed for file ${file.name}. Details:
${ + err instanceof Error ? err.message : err + }`; + dispatch(processFailure(downloadRequestId, errorMsg)); + } + }) + ); + + done(); + }, +}); + +const promptForNewExecutable = createLogic({ + async process(deps: ReduxLogicDeps, dispatch, done) { + const { + executionEnvService, + notificationService, + } = interactionSelectors.getPlatformDependentServices(deps.getState()); + const fileSelection = selection.selectors.getFileSelection(deps.getState()); + const userSelectedApplications = interactionSelectors.getUserSelectedApplications( + deps.getState() + ); + + const executableLocation = await executionEnvService.promptForExecutable( + "Select Application to Open Selected Files With" + ); + + // Continue unless the user cancelled the prompt + if (executableLocation !== ExecutableEnvCancellationToken) { + // Determine the kinds of files currently selected + const selectedFilesDetails = await fileSelection.fetchAllDetails(); + const fileKinds = uniq( + selectedFilesDetails.flatMap( + (file) => + file.annotations.find((a) => a.name === AnnotationName.KIND) + ?.values as string[] + ) ); - } + // Ask whether this app should be the default for + // the file kinds selected + const filename = await executionEnvService.getFilename(executableLocation); + const shouldSetAsDefault = await notificationService.showQuestion( + `${filename}`, + `Set ${filename} as the default for ${fileKinds} files?` + ); + const defaultFileKinds = shouldSetAsDefault ? fileKinds : []; + + // Update previously saved apps if necessary & add this one + const newApp = { filePath: executableLocation, defaultFileKinds }; + const existingApps = (userSelectedApplications || []) + .filter((app) => app.filePath !== executableLocation) + .map((app) => ({ + ...app, + defaultFileKinds: app.defaultFileKinds.filter( + (kind: string) => !defaultFileKinds.includes(kind) + ), + })); + const apps = [...existingApps, newApp]; + + // Save app configuration & open files in new app + dispatch(setUserSelectedApplication(apps)); + dispatch(openWith(newApp, deps.action.payload)); + } done(); }, + type: PROMPT_FOR_NEW_EXECUTABLE, }); const SYSTEM_DEFAULT_APP: UserSelectedApplication = Object.freeze({ @@ -401,30 +422,6 @@ const openWithLogic = createLogic({ type: OPEN_WITH, }); -const browseForNewDataSource = createLogic({ - async process(deps: ReduxLogicDeps, dispatch, done) { - const { executionEnvService } = interactionSelectors.getPlatformDependentServices( - deps.getState() - ); - - const filePath = await executionEnvService.promptForFile(["csv", "parquet", "json"]); - if (filePath !== ExecutableEnvCancellationToken) { - const dataSourceName = await executionEnvService.getFilename(filePath); - dispatch( - selection.actions.addQuery({ - name: `New ${dataSourceName} Query`, - url: FileExplorerURL.encode({ - collection: { name: dataSourceName, uri: filePath, version: 1 }, - }), - }) - ); - } - - done(); - }, - type: BROWSE_FOR_NEW_DATA_SOURCE, -}); - /** * Interceptor responsible for responding to a SHOW_CONTEXT_MENU action and ensuring the previous * context menu is dismissed gracefully. @@ -473,13 +470,13 @@ const refresh = createLogic({ }); export default [ - checkForUpdates, + checkAicsEmployee, downloadManifest, cancelFileDownloadLogic, - browseForNewDataSource, + promptForNewExecutable, openWithDefault, openWithLogic, - downloadFiles, + downloadFilesLogic, showContextMenu, refresh, ]; diff --git a/packages/core/state/interaction/reducer.ts b/packages/core/state/interaction/reducer.ts index 36826dd8b..c126ff36e 100644 --- a/packages/core/state/interaction/reducer.ts +++ b/packages/core/state/interaction/reducer.ts @@ -5,13 +5,10 @@ import { filter, sortBy } from "lodash"; import { HIDE_CONTEXT_MENU, HIDE_VISIBLE_MODAL, - PlatformDependentServices, REFRESH, REMOVE_STATUS, SET_USER_SELECTED_APPLICATIONS, - SET_CSV_COLUMNS, SET_FILE_EXPLORER_SERVICE_BASE_URL, - SET_PLATFORM_DEPENDENT_SERVICES, SET_STATUS, SET_VISIBLE_MODAL, SHOW_CONTEXT_MENU, @@ -19,15 +16,20 @@ import { StatusUpdate, MARK_AS_USED_APPLICATION_BEFORE, ShowManifestDownloadDialogAction, + SET_IS_AICS_EMPLOYEE, + PROMPT_FOR_DATA_SOURCE, + DownloadManifestAction, + DOWNLOAD_MANIFEST, } from "./actions"; import { ContextMenuItem, PositionReference } from "../../components/ContextMenu"; import { ModalType } from "../../components/Modal"; +import { Source } from "../../entity/FileExplorerURL"; +import FileFilter from "../../entity/FileFilter"; +import { PlatformDependentServices } from "../../services"; import ApplicationInfoServiceNoop from "../../services/ApplicationInfoService/ApplicationInfoServiceNoop"; import FileDownloadServiceNoop from "../../services/FileDownloadService/FileDownloadServiceNoop"; import FileViewerServiceNoop from "../../services/FileViewerService/FileViewerServiceNoop"; -import PersistentConfigServiceNoop from "../../services/PersistentConfigService/PersistentConfigServiceNoop"; import { DEFAULT_CONNECTION_CONFIG } from "../../services/HttpServiceBase"; -import FileFilter from "../../entity/FileFilter"; import ExecutionEnvServiceNoop from "../../services/ExecutionEnvService/ExecutionEnvServiceNoop"; import { UserSelectedApplication } from "../../services/PersistentConfigService"; import NotificationServiceNoop from "../../services/NotificationService/NotificationServiceNoop"; @@ -35,15 +37,18 @@ import DatabaseServiceNoop from "../../services/DatabaseService/DatabaseServiceN export interface InteractionStateBranch { applicationVersion?: string; + dataSourceForVisibleModal?: Source; contextMenuIsVisible: boolean; contextMenuItems: ContextMenuItem[]; contextMenuPositionReference: PositionReference; contextMenuOnDismiss?: () => void; csvColumns?: string[]; fileExplorerServiceBaseUrl: string; - fileTypeForVisibleModal?: "csv" | "json" | "parquet"; + fileTypeForVisibleModal: "csv" | "json" | "parquet"; fileFiltersForVisibleModal: FileFilter[]; hasUsedApplicationBefore: boolean; + isAicsEmployee?: boolean; + isOnWeb: boolean; platformDependentServices: PlatformDependentServices; refreshKey?: string; status: StatusUpdate[]; @@ -61,7 +66,9 @@ export const initialState: InteractionStateBranch = { contextMenuPositionReference: null, fileExplorerServiceBaseUrl: DEFAULT_CONNECTION_CONFIG.baseUrl, fileFiltersForVisibleModal: [], + fileTypeForVisibleModal: "csv", hasUsedApplicationBefore: false, + isOnWeb: false, platformDependentServices: { applicationInfoService: new ApplicationInfoServiceNoop(), databaseService: new DatabaseServiceNoop(), @@ -75,7 +82,6 @@ export const initialState: InteractionStateBranch = { }), executionEnvService: new ExecutionEnvServiceNoop(), notificationService: new NotificationServiceNoop(), - persistentConfigService: new PersistentConfigServiceNoop(), }, status: [], }; @@ -109,6 +115,7 @@ export default makeReducer( }), [HIDE_VISIBLE_MODAL]: (state) => ({ ...state, + dataSourceForVisibleModal: undefined, visibleModal: undefined, }), [REFRESH]: (state, action) => ({ @@ -119,6 +126,10 @@ export default makeReducer( ...state, userSelectedApplications: action.payload, }), + [SET_IS_AICS_EMPLOYEE]: (state, action) => ({ + ...state, + isAicsEmployee: action.payload, + }), [SET_STATUS]: (state, action) => ({ ...state, status: sortBy( @@ -132,26 +143,14 @@ export default makeReducer( "processId" ), }), - [SET_CSV_COLUMNS]: (state, action) => ({ + [DOWNLOAD_MANIFEST]: (state, action: DownloadManifestAction) => ({ ...state, - csvColumns: action.payload, + csvColumns: action.payload.annotations, }), [SET_FILE_EXPLORER_SERVICE_BASE_URL]: (state, action) => ({ ...state, fileExplorerServiceBaseUrl: action.payload, }), - [SET_PLATFORM_DEPENDENT_SERVICES]: (state, action) => { - const platformDependentServices: PlatformDependentServices = { - ...state.platformDependentServices, - ...action.payload, - }; - - return { - ...state, - applicationVersion: platformDependentServices.applicationInfoService.getApplicationVersion(), - platformDependentServices, - }; - }, [SET_VISIBLE_MODAL]: (state, action) => ({ ...state, ...action.payload, @@ -159,10 +158,15 @@ export default makeReducer( }), [SHOW_MANIFEST_DOWNLOAD_DIALOG]: (state, action: ShowManifestDownloadDialogAction) => ({ ...state, - visibleModal: ModalType.CsvManifest, + visibleModal: ModalType.MetadataManifest, fileTypeForVisibleModal: action.payload.fileType, fileFiltersForVisibleModal: action.payload.fileFilters, }), + [PROMPT_FOR_DATA_SOURCE]: (state, action) => ({ + ...state, + visibleModal: ModalType.DataSourcePrompt, + dataSourceForVisibleModal: action.payload, + }), }, initialState ); diff --git a/packages/core/state/interaction/selectors.ts b/packages/core/state/interaction/selectors.ts index 98b1bacb7..bd7412994 100644 --- a/packages/core/state/interaction/selectors.ts +++ b/packages/core/state/interaction/selectors.ts @@ -1,30 +1,38 @@ +import { uniqBy } from "lodash"; import { createSelector } from "reselect"; import { State } from "../"; -import { getCollection } from "../selection/selectors"; -import { AnnotationService, CsvService, FileService } from "../../services"; -import DatasetService, { PythonicDataAccessSnippet } from "../../services/DatasetService"; +import { getDataSource } from "../selection/selectors"; +import { AnnotationService, FileService } from "../../services"; +import DatasetService, { + DataSource, + PythonicDataAccessSnippet, +} from "../../services/DataSourceService"; import DatabaseAnnotationService from "../../services/AnnotationService/DatabaseAnnotationService"; import DatabaseFileService from "../../services/FileService/DatabaseFileService"; import HttpAnnotationService from "../../services/AnnotationService/HttpAnnotationService"; import HttpFileService from "../../services/FileService/HttpFileService"; -import HttpCsvService from "../../services/CsvService/HttpCsvService"; -import DatabaseCsvService from "../../services/CsvService/DatabaseCsvService"; +import { getDataSources } from "../metadata/selectors"; +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; // BASIC SELECTORS -export const getApplicationVersion = (state: State) => state.interaction.applicationVersion; export const getContextMenuVisibility = (state: State) => state.interaction.contextMenuIsVisible; export const getContextMenuItems = (state: State) => state.interaction.contextMenuItems; export const getContextMenuPositionReference = (state: State) => state.interaction.contextMenuPositionReference; export const getContextMenuOnDismiss = (state: State) => state.interaction.contextMenuOnDismiss; export const getCsvColumns = (state: State) => state.interaction.csvColumns; +export const getDataSourceForVisibleModal = (state: State) => + state.interaction.dataSourceForVisibleModal; export const getFileExplorerServiceBaseUrl = (state: State) => state.interaction.fileExplorerServiceBaseUrl; export const getFileFiltersForVisibleModal = (state: State) => state.interaction.fileFiltersForVisibleModal; +export const getFileTypeForVisibleModal = (state: State) => + state.interaction.fileTypeForVisibleModal; export const hasUsedApplicationBefore = (state: State) => state.interaction.hasUsedApplicationBefore; +export const isOnWeb = (state: State) => state.interaction.isOnWeb; export const getPlatformDependentServices = (state: State) => state.interaction.platformDependentServices; export const getProcessStatuses = (state: State) => state.interaction.status; @@ -32,8 +40,33 @@ export const getRefreshKey = (state: State) => state.interaction.refreshKey; export const getUserSelectedApplications = (state: State) => state.interaction.userSelectedApplications; export const getVisibleModal = (state: State) => state.interaction.visibleModal; +export const isAicsEmployee = (state: State) => state.interaction.isAicsEmployee; // COMPOSED SELECTORS +export const getApplicationVersion = createSelector( + [getPlatformDependentServices], + ({ applicationInfoService }): string => applicationInfoService.getApplicationVersion() +); + +export const getAllDataSources = createSelector( + [getDataSources, isAicsEmployee], + (dataSources, isAicsEmployee): DataSource[] => + isAicsEmployee + ? uniqBy( + [ + ...dataSources, + { + id: AICS_FMS_DATA_SOURCE_NAME, + name: AICS_FMS_DATA_SOURCE_NAME, + type: "csv", + version: 1, + }, + ], + "id" + ) + : dataSources +); + // TODO: Implement PythonicDataAccessSnippet export const getPythonSnippet = createSelector( [], @@ -55,33 +88,35 @@ export const getUserName = createSelector( } ); -export const getFileService = createSelector( +export const getHttpFileService = createSelector( [ getApplicationVersion, getUserName, getFileExplorerServiceBaseUrl, - getCollection, getPlatformDependentServices, getRefreshKey, ], - ( - applicationVersion, - userName, - fileExplorerBaseUrl, - collection, - platformDependentServices - ): FileService => { - if (collection) { + (applicationVersion, userName, fileExplorerBaseUrl, platformDependentServices) => + new HttpFileService({ + applicationVersion, + userName, + baseUrl: fileExplorerBaseUrl, + downloadService: platformDependentServices.fileDownloadService, + }) +); + +export const getFileService = createSelector( + [getHttpFileService, getDataSource, getPlatformDependentServices, getRefreshKey], + (httpFileService, dataSource, platformDependentServices): FileService => { + if (dataSource && dataSource?.name !== AICS_FMS_DATA_SOURCE_NAME) { return new DatabaseFileService({ databaseService: platformDependentServices.databaseService, - dataSourceName: collection.name, + dataSourceName: dataSource.name, + downloadService: platformDependentServices.fileDownloadService, }); } - return new HttpFileService({ - applicationVersion, - userName, - baseUrl: fileExplorerBaseUrl, - }); + + return httpFileService; } ); @@ -90,7 +125,7 @@ export const getAnnotationService = createSelector( getApplicationVersion, getUserName, getFileExplorerServiceBaseUrl, - getCollection, + getDataSource, getPlatformDependentServices, getRefreshKey, ], @@ -98,13 +133,13 @@ export const getAnnotationService = createSelector( applicationVersion, userName, fileExplorerBaseUrl, - collection, + dataSource, platformDependentServices ): AnnotationService => { - if (collection) { + if (dataSource && dataSource?.name !== AICS_FMS_DATA_SOURCE_NAME) { return new DatabaseAnnotationService({ databaseService: platformDependentServices.databaseService, - dataSourceName: collection.name, + dataSourceName: dataSource.name, }); } return new HttpAnnotationService({ @@ -115,42 +150,14 @@ export const getAnnotationService = createSelector( } ); -export const getCsvService = createSelector( - [ - getCollection, - getPlatformDependentServices - ], - (collection, platformDependentServices): CsvService => { - if (collection) { - return new DatabaseCsvService({ - dataSourceName: collection?.name, - databaseService: platformDependentServices.databaseService, - }) - } - - return new HttpCsvService({ - downloadService: platformDependentServices.fileDownloadService, - }) - } -) - export const getDatasetService = createSelector( - [ - getApplicationVersion, - getUserName, - getFileExplorerServiceBaseUrl, - getPlatformDependentServices, - getRefreshKey, - ], - (applicationVersion, userName, fileExplorerBaseUrl, platformDependentServices) => { - const { databaseService } = platformDependentServices; - return new DatasetService({ + [getApplicationVersion, getUserName, getFileExplorerServiceBaseUrl, getRefreshKey], + (applicationVersion, userName, fileExplorerBaseUrl) => + new DatasetService({ applicationVersion, userName, baseUrl: fileExplorerBaseUrl, - database: databaseService, - }); - } + }) ); /** diff --git a/packages/core/state/interaction/test/logics.test.ts b/packages/core/state/interaction/test/logics.test.ts index a9f2b0027..e3e4ef4eb 100644 --- a/packages/core/state/interaction/test/logics.test.ts +++ b/packages/core/state/interaction/test/logics.test.ts @@ -10,7 +10,6 @@ import { REMOVE_STATUS, SET_STATUS, cancelFileDownload, - SET_CSV_COLUMNS, refresh, OPEN_WITH, openWith, @@ -83,7 +82,7 @@ describe("Interaction logics", () => { }); // act - store.dispatch(downloadManifest([])); + store.dispatch(downloadManifest([], "csv")); await logicMiddleware.whenComplete(); // assert @@ -117,7 +116,7 @@ describe("Interaction logics", () => { }); // act - store.dispatch(downloadManifest([])); + store.dispatch(downloadManifest([], "csv")); await logicMiddleware.whenComplete(); // assert @@ -136,19 +135,14 @@ describe("Interaction logics", () => { it("marks the failure of a manifest download with a status update", async () => { // arrange class FailingDownloadSerivce implements FileDownloadService { + isFileSystemAccessible = false; getDefaultDownloadDirectory() { return Promise.reject(); } - downloadCsvManifest() { + download() { return Promise.reject(); } - downloadFile() { - return Promise.reject(); - } - promptForSaveLocation() { - return Promise.reject(); - } - promptForDownloadDirectory() { + prepareHttpResourceForDownload() { return Promise.reject(); } cancelActiveRequest() { @@ -172,7 +166,7 @@ describe("Interaction logics", () => { }); // act - store.dispatch(downloadManifest([])); + store.dispatch(downloadManifest([], "csv")); await logicMiddleware.whenComplete(); // assert @@ -230,6 +224,7 @@ describe("Interaction logics", () => { const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), }); sandbox.stub(interaction.selectors, "getFileService").returns(fileService); @@ -241,7 +236,7 @@ describe("Interaction logics", () => { }); // act - store.dispatch(downloadManifest([])); + store.dispatch(downloadManifest([], "csv")); await logicMiddleware.whenComplete(); // assert @@ -257,35 +252,6 @@ describe("Interaction logics", () => { }) ).to.be.true; }); - - it("updates annotations to persist for the next time a user opens a selection action modal", async () => { - // arrange - const state = mergeState(initialState, { - interaction: { - platformDependentServices: { - fileDownloadService: new FileDownloadServiceNoop(), - }, - }, - selection: { - fileSelection, - }, - }); - const { store, logicMiddleware, actions } = configureMockStore({ - state, - logics: interactionLogics, - }); - - // act - store.dispatch(downloadManifest([])); - await logicMiddleware.whenComplete(); - - // assert - expect( - actions.includesMatch({ - type: SET_CSV_COLUMNS, - }) - ).to.equal(true); - }); }); describe("downloadFiles", () => { @@ -435,16 +401,16 @@ describe("Interaction logics", () => { it("dispatches progress events", async () => { // Arrange class TestDownloadSerivce implements FileDownloadService { + isFileSystemAccessible = true; getDefaultDownloadDirectory() { return Promise.resolve("wherever"); } - downloadCsvManifest() { + prepareHttpResourceForDownload() { return Promise.reject(); } - downloadFile( + download( _fileInfo: FileInfo, downloadRequestId: string, - _?: string, onProgress?: (bytesDownloaded: number) => void ) { onProgress?.(1); @@ -454,9 +420,6 @@ describe("Interaction logics", () => { resolution: DownloadResolution.SUCCESS, }); } - promptForSaveLocation() { - return Promise.reject(); - } promptForDownloadDirectory() { return Promise.reject(); } @@ -504,19 +467,14 @@ describe("Interaction logics", () => { it("marks the failure of a file download with a status update", async () => { // Arrange class TestDownloadSerivce implements FileDownloadService { + isFileSystemAccessible = true; getDefaultDownloadDirectory() { return Promise.resolve("wherever"); } - downloadCsvManifest() { - return Promise.reject(); - } - downloadFile() { + prepareHttpResourceForDownload() { return Promise.reject(); } - promptForSaveLocation() { - return Promise.reject(); - } - promptForDownloadDirectory() { + download() { return Promise.reject(); } cancelActiveRequest() { @@ -575,25 +533,19 @@ describe("Interaction logics", () => { it("clears status for download request if request was cancelled", async () => { // Arrange class TestDownloadSerivce implements FileDownloadService { + isFileSystemAccessible = true; getDefaultDownloadDirectory() { return Promise.resolve("wherever"); } - downloadCsvManifest() { + prepareHttpResourceForDownload() { return Promise.reject(); } - downloadFile(_fileInfo: FileInfo, destination: string, downloadRequestId: string) { + download(_fileInfo: FileInfo, downloadRequestId: string) { return Promise.resolve({ - destination, downloadRequestId, resolution: DownloadResolution.CANCELLED, }); } - promptForSaveLocation() { - return Promise.reject(); - } - promptForDownloadDirectory() { - return Promise.reject(); - } cancelActiveRequest() { return Promise.reject(); } @@ -630,32 +582,34 @@ describe("Interaction logics", () => { ).to.equal(true); }); - it("downloads files to prompted location", async () => { + it("triggers platform specific download", async () => { // Arrange - let actualDestination = "never got set"; + let isDownloading = false; const expectedDestination = "yay real destination"; - class UselessFileDownloadService extends FileDownloadServiceNoop { - public downloadFile( + class UselessFileDownloadService implements FileDownloadService { + isFileSystemAccessible = false; + public prepareHttpResourceForDownload() { + return Promise.reject(); + } + public download( _: FileInfo, - destination: string, downloadRequestId: string, onProgress?: (bytesDownloaded: number) => void ) { - actualDestination = destination; + isDownloading = true; return Promise.resolve({ downloadRequestId, - destination, onProgress, msg: "", resolution: DownloadResolution.SUCCESS, }); } - public promptForSaveLocation() { - return Promise.reject(); - } - public promptForDownloadDirectory(): Promise { + public getDefaultDownloadDirectory(): Promise { return Promise.resolve(expectedDestination); } + public cancelActiveRequest() { + // no-op + } } const state = mergeState(initialState, { interaction: { @@ -678,11 +632,11 @@ describe("Interaction logics", () => { }); // Act - store.dispatch(downloadFiles([file], true)); + store.dispatch(downloadFiles([file])); await logicMiddleware.whenComplete(); // Assert - expect(actualDestination).to.equal(expectedDestination); + expect(isDownloading).to.be.true; }); }); @@ -690,28 +644,24 @@ describe("Interaction logics", () => { it("marks the failure of a download cancellation (on error)", async () => { // arrange class TestDownloadService implements FileDownloadService { + isFileSystemAccessible = false; getDefaultDownloadDirectory() { return Promise.reject(); } - downloadCsvManifest(_url: string, _data: string, downloadRequestId: string) { - return Promise.resolve({ - downloadRequestId, - resolution: DownloadResolution.CANCELLED, - }); + prepareHttpResourceForDownload() { + return Promise.reject(); } - downloadFile(_fileInfo: FileInfo, destination: string, downloadRequestId: string) { + download( + _fileInfo: FileInfo, + downloadRequestId: string, + onProgress?: (bytesDownloaded: number) => void + ) { return Promise.resolve({ - destination, downloadRequestId, + onProgress, resolution: DownloadResolution.CANCELLED, }); } - promptForSaveLocation() { - return Promise.reject(); - } - promptForDownloadDirectory() { - return Promise.reject(); - } cancelActiveRequest() { throw new Error("KABOOM"); } @@ -753,22 +703,17 @@ describe("Interaction logics", () => { // arrange const downloadRequestId = "beepbop"; class TestDownloadService implements FileDownloadService { + isFileSystemAccessible = false; getDefaultDownloadDirectory() { return Promise.reject(); } - downloadCsvManifest() { - return Promise.resolve({ - downloadRequestId, - resolution: DownloadResolution.CANCELLED, - }); - } - downloadFile() { + download() { return Promise.resolve({ downloadRequestId, resolution: DownloadResolution.CANCELLED, }); } - promptForSaveLocation() { + prepareHttpResourceForDownload() { return Promise.reject(); } promptForDownloadDirectory() { @@ -939,6 +884,7 @@ describe("Interaction logics", () => { const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), }); const fakeSelection = new FileSelection().select({ fileSet: new FileSet({ fileService }), @@ -1154,6 +1100,7 @@ describe("Interaction logics", () => { const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), }); const fakeSelection = new FileSelection().select({ fileSet: new FileSet({ fileService }), @@ -1268,6 +1215,7 @@ describe("Interaction logics", () => { const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), }); it("attempts to open selected files", async () => { diff --git a/packages/core/state/metadata/actions.ts b/packages/core/state/metadata/actions.ts index 59799bae5..ed7083357 100644 --- a/packages/core/state/metadata/actions.ts +++ b/packages/core/state/metadata/actions.ts @@ -1,7 +1,7 @@ import { makeConstant } from "@aics/redux-utils"; import Annotation from "../../entity/Annotation"; -import { Dataset } from "../../services/DatasetService"; +import { DataSource } from "../../services/DataSourceService"; const STATE_BRANCH_NAME = "metadata"; @@ -43,38 +43,38 @@ export function requestAnnotations(): RequestAnnotationAction { } /** - * RECEIVE_DATASETS + * RECEIVE_DATA_SOURCES * - * Intention to store listing of collections returned from data service. These are sets of file metadata + * Intention to store listing of data sources returned from data service. These are sets of file metadata * that can be used to narrow the set of explorable files down. */ -export const RECEIVE_COLLECTIONS = makeConstant(STATE_BRANCH_NAME, "receive-collections"); +export const RECEIVE_DATA_SOURCES = makeConstant(STATE_BRANCH_NAME, "receive-data-sources"); -export interface ReceiveCollectionsAction { - payload: Dataset[]; +export interface ReceiveDataSourcesAction { + payload: DataSource[]; type: string; } -export function receiveCollections(payload: Dataset[]): ReceiveCollectionsAction { +export function receiveDataSources(payload: DataSource[]): ReceiveDataSourcesAction { return { payload, - type: RECEIVE_COLLECTIONS, + type: RECEIVE_DATA_SOURCES, }; } /** - * REQUEST_COLLECTIONS + * REQUEST_DATA_SOURCES * - * Intention to request listing of available collections usable as a data source for files. + * Intention to request listing of available data sources usable as a data source for files. */ -export const REQUEST_COLLECTIONS = makeConstant(STATE_BRANCH_NAME, "request-collections"); +export const REQUEST_DATA_SOURCES = makeConstant(STATE_BRANCH_NAME, "request-data-sources"); -export interface RequestCollectionsAction { +export interface RequestDataSourcesAction { type: string; } -export function requestCollections(): RequestCollectionsAction { +export function requestDataSources(): RequestDataSourcesAction { return { - type: REQUEST_COLLECTIONS, + type: REQUEST_DATA_SOURCES, }; } diff --git a/packages/core/state/metadata/logics.ts b/packages/core/state/metadata/logics.ts index cac370190..818574124 100644 --- a/packages/core/state/metadata/logics.ts +++ b/packages/core/state/metadata/logics.ts @@ -1,3 +1,4 @@ +import { uniqBy } from "lodash"; import { createLogic } from "redux-logic"; import { interaction, ReduxLogicDeps, selection } from ".."; @@ -5,10 +6,11 @@ import { RECEIVE_ANNOTATIONS, ReceiveAnnotationAction, receiveAnnotations, - receiveCollections, + receiveDataSources, REQUEST_ANNOTATIONS, - REQUEST_COLLECTIONS, + REQUEST_DATA_SOURCES, } from "./actions"; +import * as metadataSelectors from "./selectors"; import Annotation, { AnnotationName } from "../../entity/Annotation"; import FileSort, { SortOrder } from "../../entity/FileSort"; import HttpAnnotationService from "../../services/AnnotationService/HttpAnnotationService"; @@ -43,7 +45,7 @@ const requestAnnotations = createLogic({ }); /** - * Interceptor responsible for turning REQUEST_COLLECTIONS action into selecting default + * Interceptor responsible for turning REQUEST_DATA_SOURCES action into selecting default * display annotations */ const receiveAnnotationsLogic = createLogic({ @@ -61,8 +63,10 @@ const receiveAnnotationsLogic = createLogic({ ); // Filter out any annotations that were selected for display that no longer // exist as annotations in the state - const displayAnnotationsThatStillExist = currentDisplayAnnotations.filter( - (annotation) => annotation.name in annotationNameToAnnotationMap + const displayAnnotationsThatStillExist = currentDisplayAnnotations.flatMap((annotation) => + annotation.name in annotationNameToAnnotationMap + ? [annotationNameToAnnotationMap[annotation.name]] + : [] ); // These are the default annotations we want to display so this will @@ -122,23 +126,24 @@ const receiveAnnotationsLogic = createLogic({ }); /** - * Interceptor responsible for turning REQUEST_COLLECTIONS action into a network call for collections. Outputs - * RECEIVE_COLLECTIONS action. + * Interceptor responsible for turning REQUEST_DATA_SOURCES action into a network call for data source. Outputs + * RECEIVE_DATA_SOURCES action. */ -const requestCollections = createLogic({ +const requestDataSources = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const datasetService = interaction.selectors.getDatasetService(deps.getState()); + const existingDataSources = metadataSelectors.getDataSources(deps.getState()); try { - const collections = await datasetService.getDatasets(); - dispatch(receiveCollections(collections)); + const dataSources = await datasetService.getAll(); + dispatch(receiveDataSources(uniqBy([...existingDataSources, ...dataSources], "id"))); } catch (err) { console.error("Failed to fetch datasets", err); } finally { done(); } }, - type: [REQUEST_COLLECTIONS, interaction.actions.REFRESH], + type: [REQUEST_DATA_SOURCES, interaction.actions.REFRESH], }); -export default [requestAnnotations, receiveAnnotationsLogic, requestCollections]; +export default [requestAnnotations, receiveAnnotationsLogic, requestDataSources]; diff --git a/packages/core/state/metadata/reducer.ts b/packages/core/state/metadata/reducer.ts index 9e3aa02d7..963a07fb6 100644 --- a/packages/core/state/metadata/reducer.ts +++ b/packages/core/state/metadata/reducer.ts @@ -1,18 +1,18 @@ import { makeReducer } from "@aics/redux-utils"; import Annotation from "../../entity/Annotation"; -import { Dataset } from "../../services/DatasetService"; +import { DataSource } from "../../services/DataSourceService"; -import { RECEIVE_ANNOTATIONS, RECEIVE_COLLECTIONS } from "./actions"; +import { RECEIVE_ANNOTATIONS, RECEIVE_DATA_SOURCES } from "./actions"; export interface MetadataStateBranch { annotations: Annotation[]; - collections: Dataset[]; + dataSources: DataSource[]; } export const initialState = { annotations: [], - collections: [], + dataSources: [], }; export default makeReducer( @@ -21,9 +21,9 @@ export default makeReducer( ...state, annotations: action.payload, }), - [RECEIVE_COLLECTIONS]: (state, action) => ({ + [RECEIVE_DATA_SOURCES]: (state, action) => ({ ...state, - collections: action.payload, + dataSources: action.payload, }), }, initialState diff --git a/packages/core/state/metadata/selectors.ts b/packages/core/state/metadata/selectors.ts index aaa092713..fe2f4fe19 100644 --- a/packages/core/state/metadata/selectors.ts +++ b/packages/core/state/metadata/selectors.ts @@ -5,7 +5,7 @@ import Annotation, { AnnotationName } from "../../entity/Annotation"; // BASIC SELECTORS export const getAnnotations = (state: State) => state.metadata.annotations; -export const getCollections = (state: State) => state.metadata.collections; +export const getDataSources = (state: State) => state.metadata.dataSources; // COMPOSED SELECTORS export const getSortedAnnotations = createSelector(getAnnotations, (annotations: Annotation[]) => { diff --git a/packages/core/state/metadata/test/logics.test.ts b/packages/core/state/metadata/test/logics.test.ts index cd0e52764..8adca5881 100644 --- a/packages/core/state/metadata/test/logics.test.ts +++ b/packages/core/state/metadata/test/logics.test.ts @@ -3,14 +3,14 @@ import { expect } from "chai"; import { createSandbox } from "sinon"; import { - receiveCollections, + receiveDataSources, RECEIVE_ANNOTATIONS, requestAnnotations, - requestCollections, + requestDataSources, } from "../actions"; import metadataLogics from "../logics"; import { initialState, interaction } from "../../"; -import DatasetService, { Dataset } from "../../../services/DatasetService"; +import DatasetService, { DataSource } from "../../../services/DataSourceService"; describe("Metadata logics", () => { describe("requestAnnotations", () => { @@ -53,26 +53,22 @@ describe("Metadata logics", () => { }); }); - describe("requestCollections", () => { + describe("requestDataSources", () => { const sandbox = createSandbox(); - const collections: Dataset[] = [ + const dataSources: DataSource[] = [ { id: "123414", name: "Microscopy Set", + type: "csv", version: 1, - query: "", - client: "", - fixed: false, - private: false, - created: new Date(), - createdBy: "test", + uri: "", }, ]; before(() => { const datasetService = new DatasetService(); sandbox.stub(interaction.selectors, "getDatasetService").returns(datasetService); - sandbox.stub(datasetService, "getDatasets").resolves(collections); + sandbox.stub(datasetService, "getAll").resolves(dataSources); }); afterEach(() => { @@ -83,8 +79,8 @@ describe("Metadata logics", () => { sandbox.restore(); }); - [requestCollections, interaction.actions.refresh].forEach((action) => { - it(`Processes ${action().type} into RECEIVE_COLLECTIONS action`, async () => { + [requestDataSources, interaction.actions.refresh].forEach((action) => { + it(`Processes ${action().type} into RECEIVE_DATA_SOURCES action`, async () => { // Arrange const { actions, logicMiddleware, store } = configureMockStore({ state: initialState, @@ -96,7 +92,7 @@ describe("Metadata logics", () => { await logicMiddleware.whenComplete(); // Assert - expect(actions.includesMatch(receiveCollections(collections))).to.be.true; + expect(actions.includesMatch(receiveDataSources(dataSources))).to.be.true; }); }); }); diff --git a/packages/core/state/selection/actions.ts b/packages/core/state/selection/actions.ts index f4d71fda2..642f19270 100644 --- a/packages/core/state/selection/actions.ts +++ b/packages/core/state/selection/actions.ts @@ -8,7 +8,12 @@ import FileSet from "../../entity/FileSet"; import FileSort from "../../entity/FileSort"; import NumericRange from "../../entity/NumericRange"; import Tutorial from "../../entity/Tutorial"; -import { Dataset } from "../../services/DatasetService"; +import { DataSource } from "../../services/DataSourceService"; +import { + EMPTY_QUERY_COMPONENTS, + FileExplorerURLComponents, + Source, +} from "../../entity/FileExplorerURL"; const STATE_BRANCH_NAME = "selection"; @@ -277,7 +282,12 @@ export const ADD_QUERY = makeConstant(STATE_BRANCH_NAME, "add-query"); export interface Query { name: string; - url: string; + parts: FileExplorerURLComponents; +} + +interface PartialQuery { + name: string; + parts: Partial; } export interface AddQuery { @@ -285,9 +295,15 @@ export interface AddQuery { type: string; } -export function addQuery(view: Query): AddQuery { +export function addQuery(query: PartialQuery): AddQuery { return { - payload: view, + payload: { + ...query, + parts: { + ...EMPTY_QUERY_COMPONENTS, + ...query.parts, + }, + }, type: ADD_QUERY, }; } @@ -299,23 +315,62 @@ export function addQuery(view: Query): AddQuery { */ export const CHANGE_QUERY = makeConstant(STATE_BRANCH_NAME, "change-query"); -export interface Query { - name: string; - url: string; -} - export interface ChangeQuery { payload: Query; type: string; } -export function changeQuery(view: Query): ChangeQuery { +export function changeQuery(query: PartialQuery): ChangeQuery { return { - payload: view, + payload: { + ...query, + parts: { + ...EMPTY_QUERY_COMPONENTS, + ...query.parts, + }, + }, type: CHANGE_QUERY, }; } +/** + * REMOVE_QUERY + * + * Intention is to remove a query from the list of queries available to switch to in the file explorer. + */ +export const REMOVE_QUERY = makeConstant(STATE_BRANCH_NAME, "remove-query"); + +export interface RemoveQuery { + payload: string; + type: string; +} + +export function removeQuery(queryName: string): RemoveQuery { + return { + payload: queryName, + type: REMOVE_QUERY, + }; +} + +/** + * REPLACE_DATA_SOURCE + * + * Intention to replace the current data source with a new one. + */ +export const REPLACE_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "replace-data-source"); + +export interface ReplaceDataSource { + payload: Source; + type: string; +} + +export function replaceDataSource(dataSource: Source): ReplaceDataSource { + return { + payload: dataSource, + type: REPLACE_DATA_SOURCE, + }; +} + /** * SET_FILE_SELECTION * @@ -496,21 +551,21 @@ export function decodeFileExplorerURL(decodedFileExplorerURL: string): DecodeFil } /** - * CHANGE_COLLECTION + * CHANGE_DATA_SOURCE * - * Intention to update the collection queries are run against. + * Intention to update the data source queries are run against. */ -export const CHANGE_COLLECTION = makeConstant(STATE_BRANCH_NAME, "change-collection"); +export const CHANGE_DATA_SOURCE = makeConstant(STATE_BRANCH_NAME, "change-data-source"); -export interface ChangeCollectionAction { - payload?: Dataset; +export interface ChangeDataSourceAction { + payload?: DataSource; type: string; } -export function changeCollection(collection?: Dataset): ChangeCollectionAction { +export function changeDataSource(dataSource?: DataSource): ChangeDataSourceAction { return { - payload: collection, - type: CHANGE_COLLECTION, + payload: dataSource, + type: CHANGE_DATA_SOURCE, }; } diff --git a/packages/core/state/selection/logics.ts b/packages/core/state/selection/logics.ts index 66d12dac4..9c13195b9 100644 --- a/packages/core/state/selection/logics.ts +++ b/packages/core/state/selection/logics.ts @@ -20,10 +20,10 @@ import { SET_ANNOTATION_HIERARCHY, SELECT_NEARBY_FILE, setSortColumn, - changeCollection, - CHANGE_COLLECTION, + changeDataSource, + CHANGE_DATA_SOURCE, CHANGE_QUERY, - ChangeCollectionAction, + ChangeDataSourceAction, SetAnnotationHierarchyAction, RemoveFromAnnotationHierarchyAction, ReorderAnnotationHierarchyAction, @@ -33,6 +33,9 @@ import { changeQuery, ChangeQuery, setQueries, + REPLACE_DATA_SOURCE, + ReplaceDataSource, + REMOVE_QUERY, } from "./actions"; import { interaction, metadata, ReduxLogicDeps, selection } from "../"; import * as selectionSelectors from "./selectors"; @@ -291,26 +294,25 @@ const toggleFileFolderCollapse = createLogic({ const decodeFileExplorerURLLogics = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const encodedURL = deps.action.payload; - const collections = metadata.selectors.getCollections(deps.getState()); - const { hierarchy, filters, openFolders, sortColumn, collection } = FileExplorerURL.decode( + const dataSources = interaction.selectors.getAllDataSources(deps.getState()); + const { hierarchy, filters, openFolders, sortColumn, source } = FileExplorerURL.decode( encodedURL ); - let selectedCollection = collections.find( - (c) => c.name === collection?.name && c.version === collection?.version - ); - // It is possible the user was sent a private collection, in that event the collection is likely not stored - // in the state's collection set yet & should be loaded in. - if (collection && !selectedCollection) { - const datasetService = interaction.selectors.getDatasetService(deps.getState()); - // TODO: Good spot to verify dataset...? - const newCollection = await datasetService.getDataset(collection); - dispatch(metadata.actions.receiveCollections([...collections, newCollection])); - selectedCollection = newCollection; + let selectedDataSource = dataSources.find((c) => c.name === source?.name); + // It is possible the user was sent a novel data source in the URL + if (source && !selectedDataSource) { + const newDataSource = { + ...source, + id: source.name, + version: 1, + }; + dispatch(metadata.actions.receiveDataSources([...dataSources, newDataSource])); + selectedDataSource = newDataSource; } batch(() => { - dispatch(changeCollection(selectedCollection)); + dispatch(changeDataSource(selectedDataSource)); dispatch(setAnnotationHierarchy(hierarchy)); dispatch(setFileFilters(filters)); dispatch(setOpenFileFolders(openFolders)); @@ -439,22 +441,22 @@ const selectNearbyFile = createLogic({ }); /** - * Interceptor responsible for processing a new collection into - * a refresh action so that the resources pertain to the current collection + * Interceptor responsible for processing a new data source into + * a refresh action so that the resources pertain to the current data source */ -const changeCollectionLogic = createLogic({ +const changeDataSourceLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { - const action: ChangeCollectionAction = deps.action; - const collection = action.payload; - const collections = metadata.selectors.getCollections(deps.getState()); - if (collection && !collections.find((collection) => collection.id === collection.id)) { - dispatch(metadata.actions.receiveCollections([...collections, collection])); + const action: ChangeDataSourceAction = deps.action; + const dataSource = action.payload; + const dataSources = interaction.selectors.getAllDataSources(deps.getState()); + if (dataSource && !dataSources.some((dataSource) => dataSource.id === dataSource.id)) { + dispatch(metadata.actions.receiveDataSources([...dataSources, dataSource])); } dispatch(interaction.actions.refresh() as AnyAction); done(); }, - type: CHANGE_COLLECTION, + type: CHANGE_DATA_SOURCE, }); /** @@ -468,6 +470,8 @@ const addQueryLogic = createLogic({ async transform(deps: ReduxLogicDeps, next) { const queries = selectionSelectors.getQueries(deps.getState()); const { payload: newQuery } = deps.action as AddQuery; + // Map the query names to their occurrences so that queries with the same name + // have their occurences appended to their name to make them unique const queryNameToOccurrence = queries.reduce((acc, query) => { const nameWithoutOccurence = query.name.replace(/ \(\d+\)$/, ""); return { ...acc, [nameWithoutOccurence]: (acc[nameWithoutOccurence] || 0) + 1 }; @@ -494,25 +498,102 @@ const addQueryLogic = createLogic({ const changeQueryLogic = createLogic({ async process(deps: ReduxLogicDeps, dispatch, done) { const { payload: newlySelectedQuery } = deps.action as ChangeQuery; + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); const currentQueries = selectionSelectors.getQueries(deps.getState()); - const currentURL = selectionSelectors.getEncodedFileExplorerUrl(deps.getState()); + const currentQueryParts = selectionSelectors.getCurrentQueryParts(deps.getState()); const updatedQueries = currentQueries.map((query) => ({ ...query, - url: query.name === deps.ctx.previouslySelectedQuerywName ? currentURL : query.url, + parts: + query.name === deps.ctx.previouslySelectedQueryName + ? currentQueryParts + : query.parts, })); - dispatch(decodeFileExplorerURL(newlySelectedQuery.url) as AnyAction); + + if (newlySelectedQuery.parts.source?.uri) { + try { + await databaseService.addDataSource( + newlySelectedQuery.parts.source.name, + newlySelectedQuery.parts.source.type, + newlySelectedQuery.parts.source.uri + ); + } catch (error) { + console.error("Failed to add data source, prompting for replacement", error); + dispatch(interaction.actions.promptForDataSource(newlySelectedQuery.parts.source)); + } + } + + dispatch( + decodeFileExplorerURL(FileExplorerURL.encode(newlySelectedQuery.parts)) as AnyAction + ); dispatch(setQueries(updatedQueries)); done(); }, - async transform(deps: ReduxLogicDeps, next) { - deps.ctx.previouslySelectedQuerywName = selectionSelectors.getSelectedQuery( - deps.getState() - )?.name; + transform(deps: ReduxLogicDeps, next) { + deps.ctx.previouslySelectedQueryName = selectionSelectors.getSelectedQuery(deps.getState()); next(deps.action); }, type: CHANGE_QUERY, }); +const removeQueryLogic = createLogic({ + type: REMOVE_QUERY, + async process(deps: ReduxLogicDeps, dispatch, done) { + const queries = selectionSelectors.getQueries(deps.getState()); + dispatch(changeQuery(queries[0])); + done(); + }, +}); + +const replaceDataSourceLogic = createLogic({ + type: REPLACE_DATA_SOURCE, + async process(deps: ReduxLogicDeps, dispatch, done) { + const { + payload: { name, type, uri }, + } = deps.ctx.replaceDataSourceAction as ReplaceDataSource; + const { databaseService } = interaction.selectors.getPlatformDependentServices( + deps.getState() + ); + + try { + if (uri) { + await databaseService.addDataSource(name, type, uri); + } + } catch (error) { + console.error("Failed to add data source, prompting for replacement", error); + dispatch( + interaction.actions.promptForDataSource({ + name, + uri, + }) + ); + } + + dispatch(interaction.actions.refresh() as AnyAction); + done(); + }, + transform(deps: ReduxLogicDeps, next) { + const { payload: replacementDataSource } = deps.action as ReplaceDataSource; + deps.ctx.replaceDataSourceAction = deps.action; + const queries = selectionSelectors.getQueries(deps.getState()); + const updatedQueries = queries.map((query) => { + if (query.parts.source?.name !== replacementDataSource.name) { + return query; + } + + return { + ...query, + parts: { + ...query.parts, + source: replacementDataSource, + }, + }; + }); + next(selection.actions.setQueries(updatedQueries)); + }, +}); + export default [ selectFile, modifyAnnotationHierarchy, @@ -521,7 +602,9 @@ export default [ decodeFileExplorerURLLogics, selectNearbyFile, setAvailableAnnotationsLogic, - changeCollectionLogic, + changeDataSourceLogic, addQueryLogic, + replaceDataSourceLogic, changeQueryLogic, + removeQueryLogic, ]; diff --git a/packages/core/state/selection/reducer.ts b/packages/core/state/selection/reducer.ts index c570be5c1..e43a187e0 100644 --- a/packages/core/state/selection/reducer.ts +++ b/packages/core/state/selection/reducer.ts @@ -19,7 +19,7 @@ import { RESET_COLUMN_WIDTH, SORT_COLUMN, SET_SORT_COLUMN, - CHANGE_COLLECTION, + CHANGE_DATA_SOURCE, SELECT_TUTORIAL, ADJUST_GLOBAL_FONT_SIZE, Query, @@ -30,26 +30,28 @@ import { ChangeQuery, SET_FILE_THUMBNAIL_VIEW, SET_FILE_GRID_COLUMN_COUNT, + REMOVE_QUERY, + RemoveQuery, } from "./actions"; import FileSort, { SortOrder } from "../../entity/FileSort"; import Tutorial from "../../entity/Tutorial"; -import { Dataset } from "../../services/DatasetService"; +import { DataSource } from "../../services/DataSourceService"; export interface SelectionStateBranch { annotationHierarchy: string[]; availableAnnotationsForHierarchy: string[]; availableAnnotationsForHierarchyLoading: boolean; - collection?: Dataset; columnWidths: { [index: string]: number; // columnName to widthPercent mapping }; + dataSource?: DataSource; displayAnnotations: Annotation[]; fileGridColumnCount: number; fileSelection: FileSelection; filters: FileFilter[]; isDarkTheme: boolean; openFileFolders: FileFolder[]; - selectedQuery?: Query; + selectedQuery?: string; shouldDisplaySmallFont: boolean; shouldDisplayThumbnailView: boolean; sortColumn?: FileSort; @@ -127,10 +129,10 @@ export default makeReducer( sortColumn: new FileSort(action.payload, SortOrder.DESC), }; }, - [CHANGE_COLLECTION]: (state, action) => ({ + [CHANGE_DATA_SOURCE]: (state, action) => ({ ...state, annotationHierarchy: [], - collection: action.payload, + dataSource: action.payload, filters: [], fileSelection: new FileSelection(), openFileFolders: [], @@ -141,7 +143,11 @@ export default makeReducer( }), [CHANGE_QUERY]: (state, action: ChangeQuery) => ({ ...state, - selectedQuery: action.payload, + selectedQuery: action.payload.name, + }), + [REMOVE_QUERY]: (state, action: RemoveQuery) => ({ + ...state, + queries: state.queries.filter((query) => query.name !== action.payload), }), [SET_QUERIES]: (state, action: SetQueries) => ({ ...state, diff --git a/packages/core/state/selection/selectors.ts b/packages/core/state/selection/selectors.ts index 38c44e98c..ec484a4d6 100644 --- a/packages/core/state/selection/selectors.ts +++ b/packages/core/state/selection/selectors.ts @@ -3,12 +3,13 @@ import { createSelector } from "reselect"; import { State } from "../"; import Annotation from "../../entity/Annotation"; -import FileExplorerURL from "../../entity/FileExplorerURL"; +import FileExplorerURL, { FileExplorerURLComponents } from "../../entity/FileExplorerURL"; import FileFilter from "../../entity/FileFilter"; import FileFolder from "../../entity/FileFolder"; import FileSort from "../../entity/FileSort"; -import { Dataset } from "../../services/DatasetService"; +import { DataSource } from "../../services/DataSourceService"; import { getAnnotations } from "../metadata/selectors"; +import { AICS_FMS_DATA_SOURCE_NAME } from "../../constants"; // BASIC SELECTORS export const getAnnotationHierarchy = (state: State) => state.selection.annotationHierarchy; @@ -18,9 +19,9 @@ export const getAvailableAnnotationsForHierarchy = (state: State) => export const getAvailableAnnotationsForHierarchyLoading = (state: State) => state.selection.availableAnnotationsForHierarchyLoading; export const getColumnWidths = (state: State) => state.selection.columnWidths; +export const getDataSource = (state: State) => state.selection.dataSource; export const getFileGridColumnCount = (state: State) => state.selection.fileGridColumnCount; export const getFileFilters = (state: State) => state.selection.filters; -export const getCollection = (state: State) => state.selection.collection; export const getFileSelection = (state: State) => state.selection.fileSelection; export const getIsDarkTheme = (state: State) => state.selection.isDarkTheme; export const getOpenFileFolders = (state: State) => state.selection.openFileFolders; @@ -33,23 +34,31 @@ export const getTutorial = (state: State) => state.selection.tutorial; export const getQueries = (state: State) => state.selection.queries; // COMPOSED SELECTORS -export const getEncodedFileExplorerUrl = createSelector( - [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getCollection], +export const isQueryingAicsFms = createSelector( + [getDataSource], + (dataSource): boolean => !dataSource || dataSource.name === AICS_FMS_DATA_SOURCE_NAME +); + +export const getCurrentQueryParts = createSelector( + [getAnnotationHierarchy, getFileFilters, getOpenFileFolders, getSortColumn, getDataSource], ( hierarchy: string[], filters: FileFilter[], openFolders: FileFolder[], sortColumn?: FileSort, - collection?: Dataset - ) => { - return FileExplorerURL.encode({ - hierarchy, - filters, - openFolders, - sortColumn, - collection, - }); - } + source?: DataSource + ): FileExplorerURLComponents => ({ + hierarchy, + filters, + openFolders, + sortColumn, + source, + }) +); + +export const getEncodedFileExplorerUrl = createSelector( + [getCurrentQueryParts], + (queryParts): string => FileExplorerURL.encode(queryParts) ); export const getGroupedByFilterName = createSelector( diff --git a/packages/core/state/selection/test/logics.test.ts b/packages/core/state/selection/test/logics.test.ts index 6e4236895..83f673245 100644 --- a/packages/core/state/selection/test/logics.test.ts +++ b/packages/core/state/selection/test/logics.test.ts @@ -23,7 +23,7 @@ import { setAnnotationHierarchy, selectNearbyFile, SET_SORT_COLUMN, - changeCollection, + changeDataSource, } from "../actions"; import { initialState, interaction } from "../../"; import Annotation, { AnnotationName } from "../../../entity/Annotation"; @@ -31,15 +31,15 @@ import FileFilter from "../../../entity/FileFilter"; import selectionLogics from "../logics"; import { annotationsJson } from "../../../entity/Annotation/mocks"; import NumericRange from "../../../entity/NumericRange"; -import FileExplorerURL from "../../../entity/FileExplorerURL"; +import FileExplorerURL, { Source } from "../../../entity/FileExplorerURL"; import FileFolder from "../../../entity/FileFolder"; import FileSet from "../../../entity/FileSet"; import FileSelection from "../../../entity/FileSelection"; import FileSort, { SortOrder } from "../../../entity/FileSort"; import { DatasetService } from "../../../services"; -import { Dataset } from "../../../services/DatasetService"; -import { receiveCollections } from "../../metadata/actions"; +import { DataSource } from "../../../services/DataSourceService"; import HttpFileService from "../../../services/FileService/HttpFileService"; +import FileDownloadServiceNoop from "../../../services/FileDownloadService/FileDownloadServiceNoop"; describe("Selection logics", () => { describe("selectFile", () => { @@ -329,7 +329,11 @@ describe("Selection logics", () => { ]; const baseUrl = "test"; const mockHttpClient = createMockHttpClient(responseStubs); - const fileService = new HttpFileService({ baseUrl, httpClient: mockHttpClient }); + const fileService = new HttpFileService({ + baseUrl, + httpClient: mockHttpClient, + downloadService: new FileDownloadServiceNoop(), + }); const fileSet = new FileSet({ fileService: fileService }); before(() => { @@ -454,7 +458,7 @@ describe("Selection logics", () => { it("adds a new annotation to the end of the hierarchy", async () => { // setup - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -465,7 +469,7 @@ describe("Selection logics", () => { annotationHierarchy: annotations.slice(0, 2).map((a) => a.name), openFileFolders: [], }, - }; + }); const { store, logicMiddleware, actions } = configureMockStore({ logics: selectionLogics, state, @@ -486,7 +490,7 @@ describe("Selection logics", () => { it("moves an annotation within the hierarchy to a new position", async () => { // setup - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -502,7 +506,7 @@ describe("Selection logics", () => { ], openFileFolders: [], }, - }; + }); const { store, logicMiddleware, actions } = configureMockStore({ logics: selectionLogics, state, @@ -532,7 +536,7 @@ describe("Selection logics", () => { // Create new Annotation entities rather than re-use existing // ones to test proper comparison using annotationName const annotationHierarchy = annotations.slice(0, 4).map((a) => a.name); - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -543,7 +547,7 @@ describe("Selection logics", () => { annotationHierarchy, openFileFolders: [], }, - }; + }); const { store, logicMiddleware, actions } = configureMockStore({ logics: selectionLogics, state, @@ -572,7 +576,7 @@ describe("Selection logics", () => { new FileFolder(["AICS-0"]), new FileFolder(["AICS-0", "false"]), ]; - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -580,10 +584,10 @@ describe("Selection logics", () => { annotations: [...annotations], }, selection: { - annotationHierarchy: annotations.slice(0, 3), + annotationHierarchy: annotations.slice(0, 3).map((a) => a.name), openFileFolders, }, - }; + }); const { store, logicMiddleware, actions } = configureMockStore({ logics: selectionLogics, state, @@ -604,7 +608,7 @@ describe("Selection logics", () => { it("determines which paths can still be opened after annotation is added", async () => { // setup - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -618,7 +622,7 @@ describe("Selection logics", () => { new FileFolder(["AICS-0", "false"]), ], }, - }; + }); const { store, logicMiddleware, actions } = configureMockStore({ logics: selectionLogics, state, @@ -639,7 +643,7 @@ describe("Selection logics", () => { it("determines which paths can still be opened after annotation is removed", async () => { // setup - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -653,7 +657,7 @@ describe("Selection logics", () => { new FileFolder(["AICS-0", "false"]), ], }, - }; + }); const { store, logicMiddleware, actions } = configureMockStore({ logics: selectionLogics, state, @@ -673,7 +677,7 @@ describe("Selection logics", () => { }); }); - describe("changeCollectionLogic", () => { + describe("changeDataSourceLogic", () => { it("dispatches refresh action", async () => { // Arrange const { store, logicMiddleware, actions } = configureMockStore({ @@ -682,7 +686,7 @@ describe("Selection logics", () => { }); // Act - store.dispatch(changeCollection({} as any)); + store.dispatch(changeDataSource({} as any)); await logicMiddleware.whenComplete(); // Assert @@ -703,7 +707,7 @@ describe("Selection logics", () => { it("sets available annotations", async () => { // Arrange - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -711,9 +715,9 @@ describe("Selection logics", () => { annotations: [...annotations], }, selection: { - annotationHierarchy: annotations.slice(0, 3), + annotationHierarchy: annotations.slice(0, 3).map((a) => a.name), }, - }; + }); const responseStub = { when: () => true, respondWith: { @@ -744,7 +748,7 @@ describe("Selection logics", () => { it("sets all annotations as available when actual cannot be found", async () => { // Arrange - const state = { + const state = mergeState(initialState, { interaction: { fileExplorerServiceBaseUrl: "test", }, @@ -752,9 +756,9 @@ describe("Selection logics", () => { annotations: [...annotations], }, selection: { - annotationHierarchy: annotations.slice(0, 3), + annotationHierarchy: annotations.slice(0, 3).map((a) => a.name), }, - }; + }); const responseStub = { when: "test/file-explorer-service/1.0/annotations/hierarchy/available?hierarchy=date_created&hierarchy=cell_line", @@ -921,35 +925,30 @@ describe("Selection logics", () => { }); describe("decodeFileExplorerURL", () => { - const mockCollection: Dataset = { + const mockDataSource: DataSource = { id: "1234148", - name: "Test Collection", + name: "Test Data Source", version: 1, - query: "", - client: "", - fixed: false, - private: true, - created: new Date(), - createdBy: "test", + type: "csv", + uri: "", }; beforeEach(() => { const datasetService = new DatasetService(); sinon.stub(interaction.selectors, "getDatasetService").returns(datasetService); - sinon.stub(datasetService, "getDataset").resolves(mockCollection); }); afterEach(() => { sinon.restore(); }); - it("dispatches new hierarchy, filters, sort, collection, & opened folders from given URL", async () => { + it("dispatches new hierarchy, filters, sort, source, & opened folders from given URL", async () => { // Arrange const annotations = annotationsJson.map((annotation) => new Annotation(annotation)); const state = mergeState(initialState, { metadata: { annotations, - collections: [mockCollection], + dataSources: [mockDataSource], }, }); const { store, logicMiddleware, actions } = configureMockStore({ @@ -960,16 +959,17 @@ describe("Selection logics", () => { const filters = [new FileFilter(annotations[3].name, "20x")]; const openFolders = [["a"], ["a", false]].map((folder) => new FileFolder(folder)); const sortColumn = new FileSort(AnnotationName.UPLOADED, SortOrder.DESC); - const collection = { - name: mockCollection.name, - version: mockCollection.version, + const source: Source = { + name: mockDataSource.name, + uri: "", + type: "csv", }; const encodedURL = FileExplorerURL.encode({ hierarchy, filters, openFolders, sortColumn, - collection, + source, }); // Act @@ -1001,34 +1001,7 @@ describe("Selection logics", () => { payload: sortColumn, }) ).to.be.true; - expect(actions.includesMatch(changeCollection(mockCollection))).to.be.true; - }); - - it("validates unknown collection against dataset service", async () => { - // Arrange - const { store, logicMiddleware, actions } = configureMockStore({ - logics: selectionLogics, - state: initialState, - }); - const collection = { - name: mockCollection.name, - version: mockCollection.version, - }; - const encodedURL = FileExplorerURL.encode({ - hierarchy: [], - filters: [], - openFolders: [], - sortColumn: undefined, - collection, - }); - - // Act - store.dispatch(decodeFileExplorerURL(encodedURL)); - await logicMiddleware.whenComplete(); - - // Assert - expect(actions.includesMatch(changeCollection(mockCollection))).to.be.true; - expect(actions.includesMatch(receiveCollections([mockCollection]))).to.be.true; + expect(actions.includesMatch(changeDataSource(mockDataSource))).to.be.true; }); }); }); diff --git a/packages/core/state/selection/test/reducer.test.ts b/packages/core/state/selection/test/reducer.test.ts index 0ead3a24c..2e421c376 100644 --- a/packages/core/state/selection/test/reducer.test.ts +++ b/packages/core/state/selection/test/reducer.test.ts @@ -11,6 +11,7 @@ import NumericRange from "../../../entity/NumericRange"; import FileSort, { SortOrder } from "../../../entity/FileSort"; import { AnnotationName } from "../../../entity/Annotation"; import FileFolder from "../../../entity/FileFolder"; +import { DataSource } from "../../../services/DataSourceService"; describe("Selection reducer", () => { [ @@ -46,7 +47,7 @@ describe("Selection reducer", () => { }) ); - describe(selection.actions.CHANGE_COLLECTION, () => { + describe(selection.actions.CHANGE_DATA_SOURCE, () => { it("clears hierarchy, filters, file selection, and open folders", () => { // Arrange const state = { @@ -60,24 +61,20 @@ describe("Selection reducer", () => { filters: [new FileFilter("file_id", "1238401234")], openFileFolders: [new FileFolder(["AICS-11"])], }; - const collection = { + const dataSource: DataSource = { name: "My Tiffs", version: 2, + type: "csv", id: "13123019", - query: "", - fixed: false, - private: true, - client: "explorer", - created: new Date(), - createdBy: "test", + uri: "", }; // Act - const actual = selection.reducer(state, selection.actions.changeCollection(collection)); + const actual = selection.reducer(state, selection.actions.changeDataSource(dataSource)); // Assert expect(actual.annotationHierarchy).to.be.empty; - expect(actual.collection).to.deep.equal(collection); + expect(actual.dataSource).to.deep.equal(dataSource); expect(actual.fileSelection.count()).to.equal(0); expect(actual.filters).to.be.empty; expect(actual.openFileFolders).to.be.empty; diff --git a/packages/core/styles/global.css b/packages/core/styles/global.css index ec806fbfd..1256b1c56 100644 --- a/packages/core/styles/global.css +++ b/packages/core/styles/global.css @@ -41,7 +41,8 @@ body { --secondary-text-color: var(--light-grey); --highlight-background-color: var(--dark-purple); --highlight-text-color: var(--white); - --error-color: var(--red); + --error-background-color: var(--red); + --error-text-color: var(--white); --query-sidebar-max-width: 350px; --url-bar-height: 40px; @@ -57,7 +58,8 @@ body:has(> main:has(.light-theme)) { --secondary-text-color: var(--light-grey); --highlight-background-color: var(--dark-purple); --highlight-text-color: var(--white); - --error-color: var(--red); + --error-background-color: var(--red); + --error-text-color: var(--white); } h3 { diff --git a/packages/desktop/src/renderer/index.tsx b/packages/desktop/src/renderer/index.tsx index 2f17d35cf..1785e9360 100644 --- a/packages/desktop/src/renderer/index.tsx +++ b/packages/desktop/src/renderer/index.tsx @@ -19,7 +19,7 @@ import FileDownloadServiceElectron from "../services/FileDownloadServiceElectron import FileViewerServiceElectron from "../services/FileViewerServiceElectron"; import PersistentConfigServiceElectron from "../services/PersistentConfigServiceElectron"; import NotificationServiceElectron from "../services/NotificationServiceElectron"; -import { GlobalVariableChannels, FileDownloadServiceBaseUrl } from "../util/constants"; +import { GlobalVariableChannels } from "../util/constants"; const APP_ID = "fms-file-explorer"; @@ -50,20 +50,14 @@ frontendInsights.dispatchUserEvent({ type: "SESSION_START" }); // Memoized to make sure the object that collects these services doesn't // unnecessarily change with regard to referential equality between re-renders of the application -const collectPlatformDependentServices = memoize( - (downloadServiceBaseUrl: FileDownloadServiceBaseUrl) => ({ - applicationInfoService, - databaseService, - executionEnvService, - fileDownloadService: new FileDownloadServiceElectron( - notificationService, - downloadServiceBaseUrl - ), - fileViewerService: new FileViewerServiceElectron(notificationService), - frontendInsights, - persistentConfigService, - }) -); +const collectPlatformDependentServices = memoize(() => ({ + applicationInfoService, + databaseService, + executionEnvService, + fileDownloadService: new FileDownloadServiceElectron(), + fileViewerService: new FileViewerServiceElectron(notificationService), + frontendInsights, +})); const frontendInsightsMiddleware = reduxMiddleware(frontendInsights, { useActionAsProperties: true, @@ -71,6 +65,7 @@ const frontendInsightsMiddleware = reduxMiddleware(frontendInsights, { const store = createReduxStore({ middleware: [frontendInsightsMiddleware], persistedConfig: persistentConfigService.getAll(), + platformDependentServices: collectPlatformDependentServices(), }); // https://redux.js.org/api/store#subscribelistener store.subscribe(() => { @@ -109,12 +104,7 @@ store.subscribe(() => { function renderFmsFileExplorer() { render( - + , document.getElementById(APP_ID) ); diff --git a/packages/desktop/src/services/DatabaseServiceElectron.ts b/packages/desktop/src/services/DatabaseServiceElectron.ts index d64cc664c..1373da2b2 100644 --- a/packages/desktop/src/services/DatabaseServiceElectron.ts +++ b/packages/desktop/src/services/DatabaseServiceElectron.ts @@ -1,79 +1,105 @@ import * as fs from "fs"; +import * as os from "os"; import * as path from "path"; -import * as url from "url"; -import axios from "axios"; -const httpAdapter = require("axios/lib/adapters/http"); // exported from lib, but not typed (can't be fixed through typing augmentation) import duckdb from "duckdb"; -import { DatabaseService, DataSource } from "../../../core/services"; +import { DatabaseService } from "../../../core/services"; export default class DatabaseServiceElectron implements DatabaseService { private database: duckdb.Database; + private readonly existingDataSources = new Set(); constructor() { this.database = new duckdb.Database(":memory:"); } - public async addDataSource(name: string, fileURI: File): Promise { - this.database = new duckdb.Database(":memory:"); - // const extension = path.extname(fileURI); - // let sql; - // switch (extension) { - // case ".json": - // sql = `CREATE TABLE ${name} AS FROM read_json_auto('${fileURI}')`; - // break; - // case ".parquet": - // sql = `CREATE TABLE ${name} AS FROM read_parquet('${fileURI}')`; - // break; - // case ".csv": - // sql = `CREATE TABLE ${name} AS FROM read_csv_auto('${fileURI}')`; - // break; - // default: - // throw new Error(`Unsupport data source type ${extension} of ${fileURI}`); - // } - // await this.query(sql); - } - - public async saveQueryAsBuffer(sql: string): Promise { - throw new Error("Not yet implemented (saveQueryAsBuffer)") - } - - public async getDataSource(csvUri: string): Promise { - if (csvUri.startsWith("http")) { - const response = await axios.get(csvUri, { - // Ensure this runs with the NodeJS http/https client so that testing across code that makes use of Electron/NodeJS APIs - // can be done with consistent patterns. - // Requires the Electron renderer process to be run with `nodeIntegration: true`. - adapter: httpAdapter, - }); + public async addDataSource( + name: string, + type: "csv" | "json" | "parquet", + uri: File | string + ): Promise { + if (this.existingDataSources.has(name)) { + return; // no-op + } - // TODO: Can we make sure this doesn't just request 30GB suddenly for example? - if (response.status >= 400 || response.data === undefined) { - throw new Error( - `Failed to fetch CSV from ${csvUri}. Response status text: ${response.statusText}` - ); + let source: string; + let tempLocation; + try { + if (typeof uri === "string") { + source = uri; + } else { + source = path.resolve(os.tmpdir(), name); + const arrayBuffer = await uri.arrayBuffer(); + const writeStream = fs.createWriteStream(source); + await new Promise((resolve, reject) => { + writeStream.write(Buffer.from(arrayBuffer), (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + tempLocation = source; } - const urlObj = new url.URL(csvUri); - return { - name: urlObj.pathname.split("/").pop() || "Unknown", - created: new Date(), - }; - } else { - try { - await fs.promises.access(csvUri, fs.constants.R_OK); - const stats = await fs.promises.stat(csvUri); - return { - name: path.parse(csvUri).name, - created: stats.birthtime, + await new Promise((resolve, reject) => { + const callback = (err: any) => { + if (err) { + reject(err.message); + } else { + resolve(); + } }; - } catch (err) { - throw new Error(`Failed to access file at ${csvUri}. Exact error: ${err}`); + + if (type === "parquet") { + this.database.run( + `CREATE TABLE "${name}" AS FROM parquet_scan('${source}');`, + callback + ); + } else if (type === "json") { + this.database.run( + `CREATE TABLE "${name}" AS FROM read_json_auto('${source}');`, + callback + ); + } else { + // Default to CSV + this.database.exec( + `CREATE TABLE "${name}" AS FROM read_csv_auto('${source}', header=true);`, + callback + ); + } + }); + + this.existingDataSources.add(name); + } finally { + if (tempLocation) { + await fs.promises.unlink(tempLocation); } } } + /** + * Saves the result of the query to the designated location. + * May return a value if the location is not a physical location but rather + * a temporary database location (buffer) + */ + public saveQuery(destination: string, sql: string, format: string): Promise { + return new Promise((resolve, reject) => { + this.database.run( + `COPY (${sql}) TO '${destination}.${format}' (FORMAT '${format}');`, + (err: any, result: any) => { + if (err) { + reject(err.message); + } else { + resolve(result); + } + } + ); + }); + } + public query(sql: string): Promise { return new Promise((resolve, reject) => { try { @@ -89,4 +115,21 @@ export default class DatabaseServiceElectron implements DatabaseService { } }); } + + public async reset(): Promise { + await this.close(); + this.database = new duckdb.Database(":memory:"); + } + + public close(): Promise { + return new Promise((resolve, reject) => { + this.database.close((err) => { + if (err) { + reject(err.message); + } else { + resolve(); + } + }); + }); + } } diff --git a/packages/desktop/src/services/FileDownloadServiceElectron.ts b/packages/desktop/src/services/FileDownloadServiceElectron.ts index ad419334c..dd895ce92 100644 --- a/packages/desktop/src/services/FileDownloadServiceElectron.ts +++ b/packages/desktop/src/services/FileDownloadServiceElectron.ts @@ -4,18 +4,17 @@ import * as https from "https"; import * as path from "path"; import { Policy } from "cockatiel"; -import { app, dialog, FileFilter, ipcMain, IpcMainInvokeEvent, ipcRenderer } from "electron"; +import { app, ipcMain, ipcRenderer } from "electron"; -import { DownloadFailure } from "../../../core/errors"; import { FileDownloadService, - FileDownloadCancellationToken, - DownloadResolution, DownloadResult, FileInfo, + DownloadResolution, + FileDownloadCancellationToken, + HttpServiceBase, } from "../../../core/services"; -import { FileDownloadServiceBaseUrl } from "../util/constants"; -import NotificationServiceElectron from "./NotificationServiceElectron"; +import { DownloadFailure } from "../../../core/errors"; // Maps active request ids (uuids) to request download info interface ActiveRequestMap { @@ -26,13 +25,6 @@ interface ActiveRequestMap { }; } -interface ShowSaveDialogParams { - title: string; - defaultFileName: string; - buttonLabel: string; - filters?: FileFilter[]; -} - interface WriteStreamOptions { flags: string; start?: number; @@ -48,42 +40,19 @@ interface DownloadOptions { writeStreamOptions: WriteStreamOptions; } -export default class FileDownloadServiceElectron implements FileDownloadService { +export default class FileDownloadServiceElectron + extends HttpServiceBase + implements FileDownloadService { // IPC events registered both within the main and renderer processes public static GET_FILE_SAVE_PATH = "get-file-save-path"; public static GET_DOWNLOADS_DIR = "get-downloads-dir"; public static SHOW_OPEN_DIALOG = "show-open-dialog-for-download"; - private activeRequestMap: ActiveRequestMap = {}; - private cancellationRequests: Set = new Set(); - private fileDownloadServiceBaseUrl: FileDownloadServiceBaseUrl; - private notificationService: NotificationServiceElectron; + private readonly activeRequestMap: ActiveRequestMap = {}; + private readonly cancellationRequests: Set = new Set(); + public readonly isFileSystemAccessible = true; public static registerIpcHandlers() { - // Handler for displaying "Save as" prompt - async function getSavePathHandler(_: IpcMainInvokeEvent, params: ShowSaveDialogParams) { - return await dialog.showSaveDialog({ - title: params.title, - defaultPath: path.resolve(app.getPath("downloads"), params.defaultFileName), - buttonLabel: params.buttonLabel, - filters: params.filters || [], - }); - } - ipcMain.handle(FileDownloadServiceElectron.GET_FILE_SAVE_PATH, getSavePathHandler); - - // Handler for opening a native file browser dialog - async function getOpenDialogHandler( - _: IpcMainInvokeEvent, - dialogOptions: Electron.OpenDialogOptions - ) { - return dialog.showOpenDialog({ - defaultPath: path.resolve("/"), - buttonLabel: "Select", - ...dialogOptions, - }); - } - ipcMain.handle(FileDownloadServiceElectron.SHOW_OPEN_DIALOG, getOpenDialogHandler); - // Handler for returning where the downloads directory lives on this computer async function getDownloadsDirHandler() { return app.getPath("downloads"); @@ -91,55 +60,67 @@ export default class FileDownloadServiceElectron implements FileDownloadService ipcMain.handle(FileDownloadServiceElectron.GET_DOWNLOADS_DIR, getDownloadsDirHandler); } - private static async isDirectory(directoryPath: string): Promise { - try { - // Check if path actually leads to a directory - const pathStat = await fs.promises.stat(directoryPath); - return pathStat.isDirectory(); - } catch (_) { - return false; + public async download( + fileInfo: FileInfo, + downloadRequestId: string, + onProgress?: (transferredBytes: number) => void, + destination?: string + ): Promise { + let downloadUrl; + if (fileInfo.data instanceof Uint8Array) { + downloadUrl = URL.createObjectURL(new Blob([fileInfo.data])); + } else if (fileInfo.data instanceof Blob) { + downloadUrl = URL.createObjectURL(fileInfo.data); + } else if (typeof fileInfo.data === "string") { + const dataAsBlob = new Blob([fileInfo.data], { type: "application/json" }); + downloadUrl = URL.createObjectURL(dataAsBlob); + } else { + return this.downloadHttpFile(fileInfo, downloadRequestId, onProgress, destination); } - } - private static async isWriteable(path: string): Promise { try { - // Ensure folder is writeable by this user - await fs.promises.access(path, fs.constants.W_OK); - return true; - } catch (_) { - return false; + const a = document.createElement("a"); + a.href = downloadUrl; + a.download = fileInfo.name; + a.target = "_blank"; + a.click(); + a.remove(); + return { + downloadRequestId: fileInfo.id, + resolution: DownloadResolution.SUCCESS, + }; + } catch (err) { + console.error(`Failed to download file: ${err}`); + throw err; + } finally { + URL.revokeObjectURL(downloadUrl); } } - constructor( - notificationService: NotificationServiceElectron, - fileDownloadServiceBaseUrl: FileDownloadServiceBaseUrl = FileDownloadServiceBaseUrl.PRODUCTION - ) { - this.notificationService = notificationService; - this.fileDownloadServiceBaseUrl = fileDownloadServiceBaseUrl; - } - - public async downloadFile( + private async downloadHttpFile( fileInfo: FileInfo, - destination: string, downloadRequestId: string, - onProgress?: (transferredBytes: number) => void - ): Promise { - const url = `${this.fileDownloadServiceBaseUrl}${fileInfo.path}`; + onProgress?: (transferredBytes: number) => void, + destination?: string + ) { + const fileSize = fileInfo.size || 0; + destination = destination || (await this.getDefaultDownloadDirectory()); + if (destination === FileDownloadCancellationToken) { + return { + downloadRequestId, + resolution: DownloadResolution.CANCELLED, + }; + } const outFilePath = path.join(destination, fileInfo.name); const chunkSize = 1024 * 1024 * 5; // 5MB; arbitrary // retry policy: 3 times no matter the exception, with randomized exponential backoff between attempts const retry = Policy.handleAll().retry().attempts(3).exponential(); let bytesDownloaded = -1; - if (!fileInfo.size) { - // TODO: INCLUDE IN TICKET - seems like this could just be handled by browser - throw new Error("Unable to handle download without knowing file size"); - } - while (bytesDownloaded < fileInfo.size) { + while (bytesDownloaded < fileSize) { const startByte = bytesDownloaded + 1; - const endByte = Math.min(startByte + chunkSize - 1, fileInfo.size); + const endByte = Math.min(startByte + chunkSize - 1, fileSize); let writeStreamOptions: WriteStreamOptions; if (startByte === 0) { @@ -168,7 +149,7 @@ export default class FileDownloadServiceElectron implements FileDownloadService }; } const result = await retry.execute(() => - this.download({ + this.downloadOverHttp({ downloadRequestId, outFilePath, requestOptions: { @@ -177,7 +158,7 @@ export default class FileDownloadServiceElectron implements FileDownloadService Range: `bytes=${startByte}-${endByte}`, }, }, - url, + url: fileInfo.path, writeStreamOptions, }) ); @@ -197,129 +178,22 @@ export default class FileDownloadServiceElectron implements FileDownloadService }; } - public async promptForSaveLocation( - title: string, - defaultFileName: string, - buttonLabel: string, - filters?: Record[] - ): Promise { - const result = await ipcRenderer.invoke(FileDownloadServiceElectron.GET_FILE_SAVE_PATH, { - title, - defaultFileName, - buttonLabel, - filters, - }); - - if (result.canceled) { - return FileDownloadCancellationToken; - } - - return result.filePath; - } - - public async downloadCsvManifest( - url: string, - postData: string, - downloadRequestId: string - ): Promise { - const result = await this.promptForSaveLocation( - "Save CSV manifest", - "fms-explorer-selections.csv", - "Save manifest", - [{ name: "CSV files", extensions: ["csv"] }] - ); - - if (result === FileDownloadCancellationToken) { - return Promise.resolve({ - downloadRequestId, - resolution: DownloadResolution.CANCELLED, - }); - } - - const requestOptions = { - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Length": Buffer.byteLength(postData), - }, - }; - - // On Windows (at least) you have to self-append the file extension when overwriting the name - // I imagine this is not something a lot of people think to do (and is kind of inconvenient) - // - Sean M 08/20/20 - const outFilePath = result.endsWith(".csv") ? result : result + ".csv"; - - return this.download({ - downloadRequestId, - encoding: "utf-8", - outFilePath, - postData, - requestOptions, - url, - writeStreamOptions: { flags: "w" }, // The file is created (if it does not exist) or truncated (if it exists). - }); - } - - public async promptForDownloadDirectory(): Promise { - const title = "Select download directory"; - - // Continuously try to set a valid directory location until the user cancels - while (true) { - const defaultDownloadDirectory = await this.getDefaultDownloadDirectory(); - const directoryPath = await this.promptUserWithDialog({ - title, - properties: ["openDirectory"], - defaultPath: defaultDownloadDirectory, - }); - - if (directoryPath === FileDownloadCancellationToken) { - return FileDownloadCancellationToken; - } - - const isDirectory = await FileDownloadServiceElectron.isDirectory(directoryPath); - const isWriteable = - isDirectory && (await FileDownloadServiceElectron.isWriteable(directoryPath)); - - // If the directory has passed validation, return - if (isDirectory && isWriteable) { - return directoryPath; - } - - // Otherwise if the directory failed validation, alert - // user to error with executable location - let errorMessage = `Whoops! ${directoryPath} is not verifiably a directory on your computer.`; - if (isDirectory && !isWriteable) { - errorMessage += ` Directory does not appear to be writeable by the current user.`; - } - await this.notificationService.showError(title, errorMessage); - } - } - - public getDefaultDownloadDirectory(): Promise { - return ipcRenderer.invoke(FileDownloadServiceElectron.GET_DOWNLOADS_DIR); - } - - public cancelActiveRequest(downloadRequestId: string) { - this.cancellationRequests.add(downloadRequestId); - if (!this.activeRequestMap.hasOwnProperty(downloadRequestId)) { - return; - } - - const { cancel } = this.activeRequestMap[downloadRequestId]; - cancel(); - delete this.activeRequestMap[downloadRequestId]; - } - - private download(options: DownloadOptions): Promise { + private async downloadOverHttp(options: DownloadOptions): Promise { const { downloadRequestId, encoding, - outFilePath, postData, + outFilePath, requestOptions, url, writeStreamOptions, } = options; + // let outFilePath = options.outFilePath; + // if (outFilePath === undefined) { + // const defaultDirectory = await this.getDefaultDownloadDirectory(); + // outFilePath = path.join(defaultDirectory, path.basename(url)); + // } + return new Promise((resolve, reject) => { // HTTP requests are made when pointed at localhost, HTTPS otherwise. If that ever changes, // this logic can be safely removed. @@ -449,6 +323,26 @@ export default class FileDownloadServiceElectron implements FileDownloadService }); } + public async prepareHttpResourceForDownload(url: string, postBody: string): Promise { + const responseAsJSON = await this.rawPost(url, postBody); + return JSON.stringify(responseAsJSON); + } + + public cancelActiveRequest(downloadRequestId: string) { + this.cancellationRequests.add(downloadRequestId); + if (!this.activeRequestMap.hasOwnProperty(downloadRequestId)) { + return; + } + + const { cancel } = this.activeRequestMap[downloadRequestId]; + cancel(); + delete this.activeRequestMap[downloadRequestId]; + } + + public getDefaultDownloadDirectory(): Promise { + return ipcRenderer.invoke(FileDownloadServiceElectron.GET_DOWNLOADS_DIR); + } + /** * If a downloaded artifact (partial or otherwise) exists, delete it */ @@ -464,16 +358,4 @@ export default class FileDownloadServiceElectron implements FileDownloadService }); }); } - - // Prompts user using native file browser for a file path - private async promptUserWithDialog(dialogOptions: Electron.OpenDialogOptions): Promise { - const result = await ipcRenderer.invoke( - FileDownloadServiceElectron.SHOW_OPEN_DIALOG, - dialogOptions - ); - if (result.canceled || !result.filePaths.length) { - return FileDownloadCancellationToken; - } - return result.filePaths[0]; - } } diff --git a/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts new file mode 100644 index 000000000..b29aaba91 --- /dev/null +++ b/packages/desktop/src/services/test/DatabaseServiceElectron.test.ts @@ -0,0 +1,86 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { expect } from "chai"; + +import DatabaseServiceElectron from "../DatabaseServiceElectron"; + +describe("DatabaseServiceElectron", () => { + const service = new DatabaseServiceElectron(); + const tempDir = path.join(os.tmpdir(), "DatabaseServiceElectronTest"); + + before(async () => { + await fs.promises.mkdir(tempDir); + }); + + beforeEach(async () => { + await service.reset(); + }); + + after(async () => { + await fs.promises.rm(tempDir, { recursive: true }); + await service.close(); + }); + + describe("addDataSource", () => { + it("creates table from file of type csv", async () => { + // Arrange + const tempFileName = "test.csv"; + const tempFile = path.resolve(tempDir, tempFileName); + await fs.promises.writeFile(tempFile, "color\nblue\ngreen\norange"); + + // Act + await service.addDataSource(tempFileName, "csv", tempFile); + + // Assert + const result = await service.query(`SELECT * FROM "${tempFileName}"`); + expect(result).to.be.lengthOf(3); + }); + + it("creates table from file of type json", async () => { + // Arrange + const tempFileName = "test.json"; + const tempFile = path.resolve(tempDir, tempFileName); + await fs.promises.writeFile( + tempFile, + JSON.stringify([{ color: "blue" }, { color: "green" }]) + ); + + // Act + await service.addDataSource(tempFileName, "json", tempFile); + + // Assert + const result = await service.query(`SELECT * FROM "${tempFileName}"`); + expect(result).to.be.lengthOf(2); + }); + }); + + describe("query", () => { + it("executes a query", async () => { + // Act + const result = await service.query("SELECT * FROM INFORMATION_SCHEMA.TABLES"); + + // Assert + expect(result).to.be.lengthOf(0); + }); + }); + + describe("saveQuery", () => { + ["csv", "json", "parquet"].forEach((type: any) => { + it(`saves query out to a ${type} file`, async () => { + // Arrange + const destination = path.join(tempDir, "saveQueryTest"); + const sql = "SELECT * FROM INFORMATION_SCHEMA.TABLES"; + const format = "csv"; + + // Act + await service.saveQuery(destination, sql, format); + + // Assert + const fileStat = await fs.promises.stat(`${destination}.${format}`); + expect(fileStat.size).to.equal(0); + }); + }); + }); +}); diff --git a/packages/desktop/src/services/test/FileDownloadServiceElectron.test.ts b/packages/desktop/src/services/test/FileDownloadServiceElectron.test.ts index 2b4d7fdca..79c564550 100644 --- a/packages/desktop/src/services/test/FileDownloadServiceElectron.test.ts +++ b/packages/desktop/src/services/test/FileDownloadServiceElectron.test.ts @@ -11,10 +11,10 @@ import nock from "nock"; import sinon from "sinon"; import { DownloadFailure } from "../../../../core/errors"; -import { DownloadResolution, FileDownloadCancellationToken } from "../../../../core/services"; -import { FileDownloadServiceBaseUrl, RUN_IN_RENDERER } from "../../util/constants"; +import { DownloadResolution } from "../../../../core/services"; +import { RUN_IN_RENDERER } from "../../util/constants"; import FileDownloadServiceElectron from "../FileDownloadServiceElectron"; -import NotificationServiceElectron from "../NotificationServiceElectron"; +import { noop } from "lodash"; function parseRangeHeader(rangeHeader: string): { start: number; end: number } { const [, range] = rangeHeader.split("="); @@ -35,146 +35,6 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { nock.enableNetConnect(); }); - describe("downloadCsvManifest", () => { - const tempfile = `${os.tmpdir()}/manifest.csv`; - - beforeEach(async () => { - if (!nock.isActive()) { - nock.activate(); - } - - // ensure tempfile exists - const fd = await fs.promises.open(tempfile, "w+"); - await fd.close(); - }); - - afterEach(async () => { - nock.restore(); - - try { - await fs.promises.unlink(tempfile); - } catch (err) { - // if the file doesn't exist (e.g., because it was already cleaned up), ignore. else, re-raise. - const typedErr = err as NodeJS.ErrnoException; - if (typedErr.code !== "ENOENT") { - throw err; - } - } - - sinon.restore(); - }); - - it("saves CSV to a file", async () => { - // Arrange - const DOWNLOAD_HOST = "https://aics-test.corp.alleninstitute.org"; - const DOWNLOAD_PATH = "/file-explorer-service/2.0/files/selection/manifest"; - const CSV_BODY = - "Hello, it's me, I was wondering if after all these years you'd like to meet"; - - // intercept request for download and return canned response - nock(DOWNLOAD_HOST).post(DOWNLOAD_PATH).reply(200, CSV_BODY, { - "Content-Type": "text/csv;charset=UTF-8", - "Content-Disposition": "attachment;filename=manifest.csv", - }); - - sinon - .stub(ipcRenderer, "invoke") - .withArgs(FileDownloadServiceElectron.GET_FILE_SAVE_PATH) - .resolves({ - filePath: tempfile, - }); - - const service = new FileDownloadServiceElectron(new NotificationServiceElectron()); - - // Act - await service.downloadCsvManifest( - `${DOWNLOAD_HOST}${DOWNLOAD_PATH}`, - JSON.stringify({ some: "data" }), - "beepbop" - ); - - // Assert - expect(await fs.promises.readFile(tempfile, "utf-8")).to.equal(CSV_BODY); - }); - - it("resolves meaningfully if user cancels download when prompted for save path", async () => { - // Arrange - sinon - .stub(ipcRenderer, "invoke") - .withArgs(FileDownloadServiceElectron.GET_FILE_SAVE_PATH) - .resolves({ - canceled: true, - }); - - const service = new FileDownloadServiceElectron(new NotificationServiceElectron()); - const downloadRequestId = "beepbop"; - - // Act - const result = await service.downloadCsvManifest( - "/some/url", - JSON.stringify({ some: "data" }), - downloadRequestId - ); - - // Assert - expect(result).to.have.property("downloadRequestId", downloadRequestId); - expect(result).to.have.property("resolution", DownloadResolution.CANCELLED); - }); - - it("rejects with error message and clears partial artifact if request for CSV is unsuccessful", async () => { - // Arrange - const DOWNLOAD_HOST = "https://aics-test.corp.alleninstitute.org"; - const DOWNLOAD_PATH = "/file-explorer-service/2.0/files/selection/manifest"; - const ERROR_MSG = "Something went wrong and nobody knows why"; - - // intercept request for download and return canned error - nock(DOWNLOAD_HOST).post(DOWNLOAD_PATH).reply(500, ERROR_MSG); - - sinon - .stub(ipcRenderer, "invoke") - .withArgs(FileDownloadServiceElectron.GET_FILE_SAVE_PATH) - .resolves({ - filePath: tempfile, - }); - - const service = new FileDownloadServiceElectron(new NotificationServiceElectron()); - const downloadRequestId = "beepbop"; - - // Write a partial CSV manifest to enable testing that it is cleaned up on error - await fs.promises.writeFile(tempfile, "This, That, The Other"); - - try { - // Act - await service.downloadCsvManifest( - `${DOWNLOAD_HOST}${DOWNLOAD_PATH}`, - JSON.stringify({ some: "data" }), - downloadRequestId - ); - - // Evergreen detector - throw new assert.AssertionError({ - message: - "FileDownloadServiceElectron::downloadCsvManifest expected to throw on failure", - }); - } catch (err) { - // Assert - expect(err).to.be.instanceOf(DownloadFailure); - expect((err as DownloadFailure).message).to.include(ERROR_MSG); - expect((err as DownloadFailure).downloadIdentifier).to.equal(downloadRequestId); - } finally { - // Assert that any partial file is cleaned up - try { - await fs.promises.access(tempfile); - throw new assert.AssertionError({ message: `${tempfile} not cleaned up` }); - } catch (err) { - // Expect the file to be missing - const typedErr = err as NodeJS.ErrnoException; - expect(typedErr.code).to.equal("ENOENT", typedErr.message); - } - } - }); - }); - describe("getDefaultDownloadDirectory", () => { afterEach(() => { sinon.restore(); @@ -182,17 +42,13 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { it("returns default directory", async () => { // Arrange - const downloadHost = "https://aics-test.corp.alleninstitute.org/labkey/fmsfiles/image"; const expectedDirectory = "somewhere/that/is/a/dir"; sinon .stub(ipcRenderer, "invoke") .withArgs(FileDownloadServiceElectron.GET_DOWNLOADS_DIR) .resolves(expectedDirectory); - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); + const service = new FileDownloadServiceElectron(); // Act const actualDirectory = await service.getDefaultDownloadDirectory(); @@ -202,69 +58,7 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { }); }); - describe("promptForDownloadDirectory", () => { - afterEach(() => { - sinon.restore(); - }); - - it("returns user selected directory", async () => { - // Arrange - const expectedDirectory = os.tmpdir(); - class DialogResult { - public readonly canceled = false; - public readonly filePaths = [expectedDirectory]; - } - - const downloadHost = "https://aics-test.corp.alleninstitute.org/labkey/fmsfiles/image"; - const invokeStub = sinon.stub(ipcRenderer, "invoke"); - invokeStub.onFirstCall().resolves("anything"); - invokeStub.onSecondCall().resolves(new DialogResult()); - - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); - - // Act - const actualDirectory = await service.promptForDownloadDirectory(); - - // Assert - expect(actualDirectory).to.equal(expectedDirectory); - }); - - it("complains about non-writeable directory when given", async () => { - // Arrange - const expectedDirectory = "somewhere/over/here"; - class DialogResult { - public readonly canceled: boolean; - public readonly filePaths = [expectedDirectory]; - - constructor(canceled: boolean) { - this.canceled = canceled; - } - } - - const downloadHost = "https://aics-test.corp.alleninstitute.org/labkey/fmsfiles/image"; - const invokeStub = sinon.stub(ipcRenderer, "invoke"); - invokeStub.onFirstCall().resolves("anything"); - invokeStub.onSecondCall().resolves(new DialogResult(false)); - invokeStub.onSecondCall().resolves("anything"); - invokeStub.onSecondCall().resolves(new DialogResult(true)); - - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); - - // Act - const actualDirectory = await service.promptForDownloadDirectory(); - - // Assert - expect(actualDirectory).to.equal(FileDownloadCancellationToken); - }); - }); - - describe("downloadFile", () => { + describe("download", () => { let tempdir: string; let sourceFile: string; @@ -307,10 +101,7 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { return fs.createReadStream(sourceFile, { start, end }); }); - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); + const service = new FileDownloadServiceElectron(); const downloadRequestId = "beepbop"; const expectedDownloadPath = path.join(tempdir, fileName); @@ -321,12 +112,12 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { const fileInfo = { id: "abc123", name: fileName, - path: filePath, + path: path.join(downloadHost, filePath), size: (await fs.promises.stat(sourceFile)).size, }; // Act - const result = await service.downloadFile(fileInfo, tempdir, downloadRequestId); + const result = await service.download(fileInfo, downloadRequestId, noop, tempdir); // Assert expect(result.resolution).to.equal(DownloadResolution.SUCCESS); @@ -354,25 +145,22 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { return fs.createReadStream(sourceFile, { start, end }); }); - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); + const service = new FileDownloadServiceElectron(); const downloadRequestId = "beepbop"; const onProgressSpy = sinon.spy(); const fileInfo = { id: "abc123", name: fileName, - path: filePath, + path: path.join(downloadHost, filePath), size: (await fs.promises.stat(sourceFile)).size, }; // Act - const result = await service.downloadFile( + const result = await service.download( fileInfo, - tempdir, downloadRequestId, - onProgressSpy + onProgressSpy, + tempdir ); // Assert @@ -381,61 +169,6 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { expect(onProgressSpy.callCount).to.equal(callCount); }); - it("cancels a download and cleans up after itself", async () => { - // Arrange - const downloadHost = "https://aics-test.corp.alleninstitute.org/labkey/fmsfiles/image"; - const fileName = "image.czi"; - const filePath = `/some/path/${fileName}`; - - // intercept request for download and return canned response - nock(downloadHost) - .persist() - .get(filePath) - .reply(206, function () { - const { range } = this.req.headers; - const { start, end } = parseRangeHeader(range); - return fs.createReadStream(sourceFile, { start, end }); - }); - - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); - const downloadRequestId = "beepbop"; - const onProgress = () => { - service.cancelActiveRequest(downloadRequestId); - }; - const expectedDownloadPath = path.join(tempdir, fileName); - const fileInfo = { - id: "abc123", - name: fileName, - path: filePath, - size: (await fs.promises.stat(sourceFile)).size, - }; - - // Act - const result = await service.downloadFile( - fileInfo, - tempdir, - downloadRequestId, - onProgress - ); - - // Assert - expect(result.resolution).to.equal(DownloadResolution.CANCELLED); - - try { - await fs.promises.access(expectedDownloadPath); - throw new assert.AssertionError({ - message: `${expectedDownloadPath} not cleaned up`, - }); - } catch (err) { - // Expect the file to be missing - const typedErr = err as NodeJS.ErrnoException; - expect(typedErr.code).to.equal("ENOENT", typedErr.message); - } - }); - it("returns meaningful resolution and cleans up after itself if download fails", async () => { // Arrange const downloadHost = "https://aics-test.corp.alleninstitute.org/labkey/fmsfiles/image"; @@ -452,22 +185,19 @@ describe(`${RUN_IN_RENDERER} FileDownloadServiceElectron`, () => { return fs.createReadStream(sourceFile, { start, end }); }); - const service = new FileDownloadServiceElectron( - new NotificationServiceElectron(), - downloadHost as FileDownloadServiceBaseUrl - ); + const service = new FileDownloadServiceElectron(); const downloadRequestId = "beepbop"; const expectedDownloadPath = path.join(tempdir, fileName); const fileInfo = { id: "abc123", name: fileName, - path: filePath, + path: path.join(downloadHost, filePath), size: (await fs.promises.stat(sourceFile)).size, }; try { // Act - await service.downloadFile(fileInfo, tempdir, downloadRequestId); + await service.download(fileInfo, downloadRequestId, noop, tempdir); // Shouldn't hit, but here to ensure test isn't evergreen throw new assert.AssertionError({ message: `Expected exception to be thrown` }); diff --git a/packages/web/package.json b/packages/web/package.json index 270d211ac..d49040554 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@duckdb/duckdb-wasm": "1.28.1-dev106.0", + "react-router-dom": "^6.23.1", "regenerator-runtime": "0.13.x", "semver": "7.3.x" } diff --git a/packages/web/src/index.tsx b/packages/web/src/index.tsx index 64e31d811..6b6909303 100644 --- a/packages/web/src/index.tsx +++ b/packages/web/src/index.tsx @@ -4,9 +4,9 @@ import { memoize } from "lodash"; import * as React from "react"; import { render } from "react-dom"; import { Provider } from "react-redux"; +import { createBrowserRouter, RouterProvider } from "react-router-dom"; import NotificationServiceWeb from "./services/NotificationServiceWeb"; -import PersistentConfigServiceWeb from "./services/PersistentConfigServiceWeb"; import ApplicationInfoServiceWeb from "./services/ApplicationInfoServiceWeb"; import ExecutionEnvServiceWeb from "./services/ExecutionEnvServiceWeb"; import DatabaseServiceWeb from "./services/DatabaseServiceWeb"; @@ -14,49 +14,45 @@ import FmsFileExplorer from "../../core/App"; import { createReduxStore } from "../../core/state"; import FileViewerServiceWeb from "./services/FileViewerServiceWeb"; import FileDownloadServiceWeb from "./services/FileDownloadServiceWeb"; +import Root from "./routes/Root"; const APP_ID = "fms-file-explorer-web"; -const applicationInfoService = new ApplicationInfoServiceWeb(); -const databaseService = new DatabaseServiceWeb(); -const notificationService = new NotificationServiceWeb(); -const persistentConfigService = new PersistentConfigServiceWeb(); -const executionEnvService = new ExecutionEnvServiceWeb(); -// application analytics/metrics -// const frontendInsights = new FrontendInsights( -// { -// application: { -// name: APP_ID, -// version: applicationInfoService.getApplicationVersion(), -// }, -// userInfo: { -// userId: applicationInfoService.getUserName(), -// }, -// session: { -// platform: "Web", -// deviceId: `${applicationInfoService.getUserName()}-${executionEnvService.getOS()}`, -// }, -// loglevel: process.env.NODE_ENV === "production" ? LogLevel.Error : LogLevel.Debug, -// }, -// [new AmplitudeNodePlugin({ apiKey: process.env.AMPLITUDE_API_KEY })] -// ); -// frontendInsights.dispatchUserEvent({ type: "SESSION_START" }); +const router = createBrowserRouter([ + { + path: "/", + element: , // Splash page placeholder + }, + { + path: "app", + element: , + }, +]); -// Memoized to make sure the object that collects these services doesn't -// unnecessarily change with regard to referential equality between re-renders of the application -const collectPlatformDependentServices = memoize(() => ({ - applicationInfoService, - databaseService, - executionEnvService, - fileDownloadService: new FileDownloadServiceWeb(notificationService), - fileViewerService: new FileViewerServiceWeb(notificationService), - // frontendInsights, - persistentConfigService, -})); +async function asyncRender() { + const databaseService = new DatabaseServiceWeb(); + await databaseService.initialize(); -render( - - - , - document.getElementById(APP_ID) -); + // Memoized to make sure the object that collects these services doesn't + // unnecessarily change with regard to referential equality between re-renders of the application + const collectPlatformDependentServices = memoize(() => ({ + databaseService, + notificationService: new NotificationServiceWeb(), + executionEnvService: new ExecutionEnvServiceWeb(), + applicationInfoService: new ApplicationInfoServiceWeb(), + fileViewerService: new FileViewerServiceWeb(), + fileDownloadService: new FileDownloadServiceWeb(), + })); + const store = createReduxStore({ + isOnWeb: true, + platformDependentServices: collectPlatformDependentServices(), + }); + render( + + + , + document.getElementById(APP_ID) + ); +} + +asyncRender(); diff --git a/packages/web/src/routes/Root/index.tsx b/packages/web/src/routes/Root/index.tsx new file mode 100644 index 000000000..08b46822d --- /dev/null +++ b/packages/web/src/routes/Root/index.tsx @@ -0,0 +1,10 @@ +import * as React from "react"; + +// Placeholder for the splash page +export default function Root() { + return ( +
+

Coming soon

+
+ ); +} diff --git a/packages/web/src/services/DatabaseServiceWeb.ts b/packages/web/src/services/DatabaseServiceWeb.ts index 6a8942ac5..d65b03782 100644 --- a/packages/web/src/services/DatabaseServiceWeb.ts +++ b/packages/web/src/services/DatabaseServiceWeb.ts @@ -1,18 +1,16 @@ import * as duckdb from "@duckdb/duckdb-wasm"; -import axios from "axios"; -const httpAdapter = require("axios/lib/adapters/http"); // exported from lib, but not typed (can't be fixed through typing augmentation) -import { DatabaseService, DataSource } from "../../../core/services"; +import { DatabaseService } from "../../../core/services"; export default class DatabaseServiceWeb implements DatabaseService { private database: duckdb.AsyncDuckDB | undefined; + private readonly existingDataSources = new Set(); - private async initializeDatabaseWorker() { - // this.database = new duckdb.Database(":memory:"); - const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles(); + public async initialize(logLevel: duckdb.LogLevel = duckdb.LogLevel.INFO) { + const allBundles = duckdb.getJsDelivrBundles(); - // Select a bundle based on browser checks - const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES); + // Selects the best bundle based on browser checks + const bundle = await duckdb.selectBundle(allBundles); const worker_url = URL.createObjectURL( new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }) @@ -20,111 +18,87 @@ export default class DatabaseServiceWeb implements DatabaseService { // Instantiate the asynchronus version of DuckDB-wasm const worker = new Worker(worker_url); - const logger = new duckdb.ConsoleLogger(); + const logger = new duckdb.ConsoleLogger(logLevel); this.database = new duckdb.AsyncDuckDB(logger, worker); await this.database.instantiate(bundle.mainModule, bundle.pthreadWorker); URL.revokeObjectURL(worker_url); } - constructor() { - this.initializeDatabaseWorker(); - } - - public async addDataSource(name: string, uri: File): Promise { + public async addDataSource( + name: string, + type: "csv" | "json" | "parquet", + uri: File | string + ): Promise { if (!this.database) { - throw new Error("Database has not yet been initialized"); + throw new Error("Database failed to initialize"); } - await this.database.registerFileHandle( - name, - uri, - duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, - true - ); - // await this.database.registerFileBuffer(this.table, pickedFile as any); - // } else { - // throw new Error("yo yo yoooooooooooooooooooooooooo") - // await this.database.registerFileURL(this.table, pickedFile, duckdb.DuckDBDataProtocol.HTTP, false); - // } - const connection = await this.database.connect(); - try { - // TODO: Other file types... - await connection.insertCSVFromPath(name, { - name, - schema: "main", - detect: true, - header: true, - // detect: false, - // header: false, - delimiter: ",", - // columns: { - // col1: new arrow.Int32(), - // col2: new arrow.Utf8(), - // }, - }); - } finally { - await connection.close(); + if (!this.existingDataSources.has(name)) { + if (uri instanceof File) { + await this.database.registerFileHandle( + name, + uri, + duckdb.DuckDBDataProtocol.BROWSER_FILEREADER, + true + ); + } else { + const protocol = uri.startsWith("s3") + ? duckdb.DuckDBDataProtocol.S3 + : duckdb.DuckDBDataProtocol.HTTP; + + await this.database.registerFileURL(name, uri, protocol, false); + } + + const connection = await this.database.connect(); + try { + if (type === "parquet") { + await connection.query( + `CREATE TABLE "${name}" AS FROM parquet_scan('${name}');` + ); + } else if (type === "json") { + await connection.query( + `CREATE TABLE "${name}" AS FROM read_json_auto('${name}');` + ); + } else { + // Default to CSV + await connection.query( + `CREATE TABLE "${name}" AS FROM read_csv_auto('${name}', header=true);` + ); + } + this.existingDataSources.add(name); + } finally { + await connection.close(); + } } } - public async saveQueryAsBuffer(sql: string): Promise { + /** + * Saves the result of the query to the designated location. + * Returns an array representating the data from the query in the format designated + */ + public async saveQuery( + destination: string, + sql: string, + format: "parquet" | "csv" | "json" + ): Promise { if (!this.database) { - throw new Error("Database has not yet been initialized"); + throw new Error("Database failed to initialize"); } + const resultName = `${destination}.${format}`; + const finalSQL = `COPY (${sql}) TO '${resultName}' (FORMAT '${format}');`; const connection = await this.database.connect(); try { - await connection.send(`COPY (${sql}) TO 'result-example.parquet' (FORMAT 'parquet');`); - return await this.database.copyFileToBuffer('result-snappy.parquet'); - // const link = URL.createObjectURL(new Blob([parquet_buffer])); + await connection.send(finalSQL); + return await this.database.copyFileToBuffer(resultName); } finally { await connection.close(); } - - // this.database?.copyFileToBuffer() - // conn.send(`COPY (SELECT * FROM tbl) TO 'result-snappy.parquet' (FORMAT 'parquet');`); - // const parquet_buffer = await this._db.copyFileToBuffer('result-snappy.parquet'); - - // // Generate a download link - // const link = URL.createObjectURL(new Blob([parquet_buffer])); - - // // Close the connection to release memory - // await conn.close(); - } - - public async getDataSource(csvUri: string): Promise { - if (csvUri.startsWith("http")) { - const response = await axios.get(csvUri, { - // Ensure this runs with the NodeJS http/https client so that testing across code that makes use of Electron/NodeJS APIs - // can be done with consistent patterns. - // Requires the Electron renderer process to be run with `nodeIntegration: true`. - adapter: httpAdapter, - }); - - // TODO: Can we make sure this doesn't just request 30GB suddenly for example? - if (response.status >= 400 || response.data === undefined) { - throw new Error( - `Failed to fetch CSV from ${csvUri}. Response status text: ${response.statusText}` - ); - } - const urlObj = new URL(csvUri); - - return { - name: urlObj.pathname.split("/").pop() || "Unknown", - created: new Date(), - }; - } - - // TODO: Hmmmm.... - return { - name: csvUri, - created: new Date(), - }; } public async query(sql: string): Promise { if (!this.database) { - throw new Error("Database has not yet been initialized"); + throw new Error("Database failed to initialize"); } const connection = await this.database.connect(); @@ -140,4 +114,8 @@ export default class DatabaseServiceWeb implements DatabaseService { await connection.close(); } } + + public async close(): Promise { + this.database?.detach(); + } } diff --git a/packages/web/src/services/ExecutionEnvServiceWeb.ts b/packages/web/src/services/ExecutionEnvServiceWeb.ts index fffe6aaa2..112ca7cc9 100644 --- a/packages/web/src/services/ExecutionEnvServiceWeb.ts +++ b/packages/web/src/services/ExecutionEnvServiceWeb.ts @@ -2,13 +2,11 @@ import { ExecutionEnvService, SaveLocationResolution } from "../../../core/servi export default class ExecutionEnvServiceWeb implements ExecutionEnvService { public async formatPathForHost(posixPath: string): Promise { - console.log("formatPathForHost", posixPath); - return ""; + return posixPath; } public getFilename(filePath: string): string { - console.log("getFilename", filePath); - return ""; + return filePath.replace(/^.*[\\/]/, ""); } public getOS(): string { @@ -23,21 +21,15 @@ export default class ExecutionEnvServiceWeb implements ExecutionEnvService { return navigator.userAgent; } - public async promptForExecutable( - promptTitle: string, - reasonForPrompt?: string - ): Promise { - console.log(promptTitle, reasonForPrompt); - throw Error("blah"); + public async promptForExecutable(): Promise { + throw Error("ExecutionEnvServiceWeb::promptForExecutable not yet implemented"); } - public async promptForFile(extensions?: string[], reasonForPrompt?: string): Promise { - console.log(extensions, reasonForPrompt); - throw Error("blah"); + public async promptForFile(): Promise { + throw Error("ExecutionEnvServiceWeb::promptForFile not yet implemented"); } - public async promptForSaveLocation(promptTitle?: string): Promise { - console.log(promptTitle); - throw Error("blah"); + public async promptForSaveLocation(): Promise { + throw Error("ExecutionEnvServiceWeb::promptForSaveLocation not yet implemented"); } } diff --git a/packages/web/src/services/FileDownloadServiceWeb.ts b/packages/web/src/services/FileDownloadServiceWeb.ts index fb6342f19..33ab5df10 100644 --- a/packages/web/src/services/FileDownloadServiceWeb.ts +++ b/packages/web/src/services/FileDownloadServiceWeb.ts @@ -1,439 +1,56 @@ import { FileDownloadService, - FileDownloadCancellationToken, DownloadResult, FileInfo, + DownloadResolution, + HttpServiceBase, } from "../../../core/services"; -import NotificationServiceWeb from "./NotificationServiceWeb"; -// Maps active request ids (uuids) to request download info -interface ActiveRequestMap { - [id: string]: { - filePath: string; - cancel: () => void; - onProgress?: (bytes: number) => void; - }; -} - -interface WriteStreamOptions { - flags: string; - start?: number; -} - -interface DownloadOptions { - downloadRequestId: string; - encoding?: BufferEncoding; - outFilePath: string; - postData?: string; - // requestOptions: http.RequestOptions | https.RequestOptions; - url: string; - writeStreamOptions: WriteStreamOptions; -} - -export default class FileDownloadServiceWeb implements FileDownloadService { - // IPC events registered both within the main and renderer processes - public static GET_FILE_SAVE_PATH = "get-file-save-path"; - public static GET_DOWNLOADS_DIR = "get-downloads-dir"; - public static SHOW_OPEN_DIALOG = "show-open-dialog-for-download"; - - private activeRequestMap: ActiveRequestMap = {}; - private cancellationRequests: Set = new Set(); - private readonly fileDownloadServiceBaseUrl = - "http://aics.corp.alleninstitute.org/labkey/fmsfiles/image"; - private notificationService: NotificationServiceWeb; - - public static registerIpcHandlers() { - // Handler for displaying "Save as" prompt - // async function getSavePathHandler(_: IpcMainInvokeEvent, params: ShowSaveDialogParams) { - // return await dialog.showSaveDialog({ - // title: params.title, - // defaultPath: path.resolve(app.getPath("downloads"), params.defaultFileName), - // buttonLabel: params.buttonLabel, - // filters: params.filters || [], - // }); - // } - // ipcMain.handle(FileDownloadServiceWeb.GET_FILE_SAVE_PATH, getSavePathHandler); - // // Handler for opening a native file browser dialog - // async function getOpenDialogHandler( - // _: IpcMainInvokeEvent, - // dialogOptions: Electron.OpenDialogOptions - // ) { - // return dialog.showOpenDialog({ - // defaultPath: path.resolve("/"), - // buttonLabel: "Select", - // ...dialogOptions, - // }); - // } - // ipcMain.handle(FileDownloadServiceWeb.SHOW_OPEN_DIALOG, getOpenDialogHandler); - // // Handler for returning where the downloads directory lives on this computer - // async function getDownloadsDirHandler() { - // return app.getPath("downloads"); - // } - // ipcMain.handle(FileDownloadServiceWeb.GET_DOWNLOADS_DIR, getDownloadsDirHandler); - } - - private static async isDirectory(directoryPath: string): Promise { - console.log(directoryPath); - throw Error("blah"); - // try { - // // Check if path actually leads to a directory - // const pathStat = await fs.promises.stat(directoryPath); - // return pathStat.isDirectory(); - // } catch (_) { - // return false; - // } - } - - private static async isWriteable(path: string): Promise { - console.log(path); - throw Error("blah"); - // try { - // // Ensure folder is writeable by this user - // await fs.promises.access(path, fs.constants.W_OK); - // return true; - // } catch (_) { - // return false; - // } - } - - constructor(notificationService: NotificationServiceWeb) { - this.notificationService = notificationService; - } - - public async downloadFile( - fileInfo: FileInfo, - destination: string, - downloadRequestId: string, - onProgress?: (transferredBytes: number) => void - ): Promise { - console.log(fileInfo, destination, downloadRequestId, onProgress); - throw new Error("blah"); - // const url = `${this.fileDownloadServiceBaseUrl}${fileInfo.path}`; - - // const outFilePath = path.join(destination, fileInfo.name); - // const chunkSize = 1024 * 1024 * 5; // 5MB; arbitrary - - // // retry policy: 3 times no matter the exception, with randomized exponential backoff between attempts - // const retry = Policy.handleAll().retry().attempts(3).exponential(); - // let bytesDownloaded = -1; - // while (bytesDownloaded < fileInfo.size) { - // const startByte = bytesDownloaded + 1; - // const endByte = Math.min(startByte + chunkSize - 1, fileInfo.size); - - // let writeStreamOptions: WriteStreamOptions; - // if (startByte === 0) { - // // First request: ensure outfile is created if doesn't exist or truncated if it does - // writeStreamOptions = { - // flags: "w", - // }; - // } else { - // // Handle edge-case in which cancellation requested in-between range requests - // if (this.cancellationRequests.has(downloadRequestId)) { - // this.cancellationRequests.delete(downloadRequestId); - // await this.deleteArtifact(outFilePath); - // return { - // downloadRequestId, - // resolution: DownloadResolution.CANCELLED, - // }; - // } - - // writeStreamOptions = { - // // Open file for reading and writing. Required with use of `start` param. - // flags: "r+", - - // // Start writing at this offset. Enables retrying chunks that may have failed - // // part of the way through. - // start: startByte, - // }; - // } - // const result = await retry.execute(() => - // this.download({ - // downloadRequestId, - // outFilePath, - // requestOptions: { - // method: "GET", - // headers: { - // Range: `bytes=${startByte}-${endByte}`, - // }, - // }, - // url, - // writeStreamOptions, - // }) - // ); - // if (result.resolution !== DownloadResolution.SUCCESS) { - // return result; - // } - // if (onProgress) { - // onProgress(endByte - startByte + 1); - // } - // bytesDownloaded = endByte; - // } - - // return { - // downloadRequestId, - // msg: `Successfully downloaded ${outFilePath}`, - // resolution: DownloadResolution.SUCCESS, - // }; - } - - public async promptForSaveLocation(): Promise { - return Promise.resolve("blah"); - } - - public async downloadCsvManifest( - url: string, - postData: string, - downloadRequestId: string - ): Promise { - console.log(url, postData, downloadRequestId); - throw new Error("blah"); - // const saveDialogParams = { - // title: "Save CSV manifest", - // defaultFileName: "fms-explorer-selections.csv", - // buttonLabel: "Save manifest", - // filters: [{ name: "CSV files", extensions: ["csv"] }], - // }; - // const result = await ipcRenderer.invoke( - // FileDownloadServiceWeb.GET_FILE_SAVE_PATH, - // saveDialogParams - // ); - - // if (result.canceled) { - // return Promise.resolve({ - // downloadRequestId, - // resolution: DownloadResolution.CANCELLED, - // }); - // } - - // const requestOptions = { - // method: "POST", - // headers: { - // "Content-Type": "application/json", - // "Content-Length": Buffer.byteLength(postData), - // }, - // }; - - // // On Windows (at least) you have to self-append the file extension when overwriting the name - // // I imagine this is not something a lot of people think to do (and is kind of inconvenient) - // // - Sean M 08/20/20 - // const outFilePath = result.filePath.endsWith(".csv") - // ? result.filePath - // : result.filePath + ".csv"; - - // return this.download({ - // downloadRequestId, - // encoding: "utf-8", - // outFilePath, - // postData, - // requestOptions, - // url, - // writeStreamOptions: { flags: "w" }, // The file is created (if it does not exist) or truncated (if it exists). - // }); - } - - public async promptForDownloadDirectory(): Promise { - const title = "Select download directory"; - - // Continuously try to set a valid directory location until the user cancels - while (true) { - const defaultDownloadDirectory = await this.getDefaultDownloadDirectory(); - const directoryPath = await this.promptUserWithDialog({ - title, - properties: ["openDirectory"], - defaultPath: defaultDownloadDirectory, - }); - - if (directoryPath === FileDownloadCancellationToken) { - return FileDownloadCancellationToken; - } - - const isDirectory = await FileDownloadServiceWeb.isDirectory(directoryPath); - const isWriteable = - isDirectory && (await FileDownloadServiceWeb.isWriteable(directoryPath)); - - // If the directory has passed validation, return - if (isDirectory && isWriteable) { - return directoryPath; - } - - // Otherwise if the directory failed validation, alert - // user to error with executable location - let errorMessage = `Whoops! ${directoryPath} is not verifiably a directory on your computer.`; - if (isDirectory && !isWriteable) { - errorMessage += ` Directory does not appear to be writeable by the current user.`; - } - await this.notificationService.showError(title, errorMessage); +export default class FileDownloadServiceWeb extends HttpServiceBase implements FileDownloadService { + isFileSystemAccessible = false; + + public async download(fileInfo: FileInfo): Promise { + const data = fileInfo.data || fileInfo.path; + let downloadUrl: string; + if (data instanceof Uint8Array) { + downloadUrl = URL.createObjectURL(new Blob([data])); + } else if (data instanceof Blob) { + downloadUrl = URL.createObjectURL(data); + } else { + downloadUrl = data; } - } - - public getDefaultDownloadDirectory(): Promise { - throw new Error("blah"); - // return ipcRenderer.invoke(FileDownloadServiceWeb.GET_DOWNLOADS_DIR); - } - public cancelActiveRequest(downloadRequestId: string) { - this.cancellationRequests.add(downloadRequestId); - if (!this.activeRequestMap.hasOwnProperty(downloadRequestId)) { - return; + try { + const a = document.createElement("a"); + a.href = downloadUrl; + a.download = fileInfo.name; + a.target = "_blank"; + a.click(); + a.remove(); + return { + downloadRequestId: fileInfo.id, + resolution: DownloadResolution.SUCCESS, + }; + } catch (err) { + console.error(`Failed to download file: ${err}`); + throw err; + } finally { + URL.revokeObjectURL(downloadUrl); } - - const { cancel } = this.activeRequestMap[downloadRequestId]; - cancel(); - delete this.activeRequestMap[downloadRequestId]; } - private download(options: DownloadOptions): Promise { - console.log(options); - throw new Error("blah"); - // const { - // downloadRequestId, - // encoding, - // outFilePath, - // postData, - // requestOptions, - // url, - // writeStreamOptions, - // } = options; - // return new Promise((resolve, reject) => { - // // HTTP requests are made when pointed at localhost, HTTPS otherwise. If that ever changes, - // // this logic can be safely removed. - // const requestor = new URL(url).protocol === "http:" ? http : https; - - // const req = requestor.request(url, requestOptions, (incomingMsg) => { - // if (encoding) { - // incomingMsg.setEncoding(encoding); - // } - - // const outFileStream = fs.createWriteStream(outFilePath, writeStreamOptions); - - // if (incomingMsg.statusCode !== undefined && incomingMsg.statusCode >= 400) { - // const errorChunks: string[] = []; - // incomingMsg.on("data", (chunk: string) => { - // errorChunks.push(chunk); - // }); - // incomingMsg.on("end", async () => { - // try { - // delete this.activeRequestMap[downloadRequestId]; - // await this.deleteArtifact(outFilePath); - // } finally { - // const error = errorChunks.join(""); - // const msg = `Failed to download ${outFilePath}. Error details: ${error}`; - // reject(new DownloadFailure(msg, downloadRequestId)); - // } - // }); - // } else { - // incomingMsg.on("end", async () => { - // delete this.activeRequestMap[downloadRequestId]; - // if (incomingMsg.aborted) { - // try { - // await this.deleteArtifact(outFilePath); - // } finally { - // resolve({ - // downloadRequestId, - // resolution: DownloadResolution.CANCELLED, - // }); - // } - // } else { - // resolve({ - // downloadRequestId, - // msg: `Successfully downloaded ${outFilePath}`, - // resolution: DownloadResolution.SUCCESS, - // }); - // } - // }); - - // incomingMsg.pipe(outFileStream); - // } - - // const cleanUp = async (sourceErrorMessage: string) => { - // const errors = [sourceErrorMessage]; - // try { - // delete this.activeRequestMap[downloadRequestId]; - - // // Need to manually close outFileStream if attached read stream - // // (i.e., `incomingMsg`) ends with error - // await new Promise((resolve, reject) => - // outFileStream.end((err: Error) => { - // if (err) { - // reject(err); - // } else { - // resolve(); - // } - // }) - // ); - // await this.deleteArtifact(outFilePath); - // } catch (err) { - // if (err instanceof Error) { - // const formatted = `${err.name}: ${err.message}`; - // errors.push(formatted); - // } - // } finally { - // reject(new DownloadFailure(errors.join("
"), downloadRequestId)); - // } - // }; - - // incomingMsg.on("error", (err) => { - // cleanUp(err.message); - // }); - - // incomingMsg.on("aborted", () => { - // cleanUp(`Download of ${outFilePath} aborted.`); - // }); - - // incomingMsg.on("timeout", () => { - // cleanUp(`Download of ${outFilePath} timed out.`); - // }); - // }); - - // req.on("error", async (err) => { - // delete this.activeRequestMap[downloadRequestId]; - // // This first branch applies when the download has been explicitly cancelled - // if (err.message === FileDownloadCancellationToken) { - // resolve({ - // downloadRequestId, - // resolution: DownloadResolution.CANCELLED, - // }); - // } else { - // try { - // await this.deleteArtifact(outFilePath); - // } finally { - // reject( - // new DownloadFailure( - // `Failed to download file: ${err.message}`, - // downloadRequestId - // ) - // ); - // } - // } - // }); + public async prepareHttpResourceForDownload(url: string, postBody: string): Promise { + const responseAsJSON = await this.rawPost(url, postBody); + return JSON.stringify(responseAsJSON); + } - // this.activeRequestMap[downloadRequestId] = { - // cancel: () => { - // if (this.cancellationRequests.has(downloadRequestId)) { - // this.cancellationRequests.delete(downloadRequestId); - // } - // req.destroy(new Error(FileDownloadCancellationToken)); - // }, - // filePath: outFilePath, - // }; - // if (postData) { - // req.write(postData); - // } - // req.end(); - // }); + public getDefaultDownloadDirectory(): Promise { + throw new Error( + "FileDownloadServiceWeb:getDefaultDownloadDirectory not implemented for web" + ); } - // Prompts user using native file browser for a file path - private async promptUserWithDialog(dialogOptions: Electron.OpenDialogOptions): Promise { - console.log(dialogOptions); - throw new Error("blah"); - // const result = await ipcRenderer.invoke( - // FileDownloadServiceWeb.SHOW_OPEN_DIALOG, - // dialogOptions - // ); - // if (result.canceled || !result.filePaths.length) { - // return FileDownloadCancellationToken; - // } - // return result.filePaths[0]; + public cancelActiveRequest() { + /** noop: Browser will handle cancellation */ } } diff --git a/packages/web/src/services/FileViewerServiceWeb.ts b/packages/web/src/services/FileViewerServiceWeb.ts index 8acb40d85..c5fc2d18e 100644 --- a/packages/web/src/services/FileViewerServiceWeb.ts +++ b/packages/web/src/services/FileViewerServiceWeb.ts @@ -1,25 +1,7 @@ import { FileViewerService } from "../../../core/services"; -import NotificationServiceWeb from "./NotificationServiceWeb"; -export default class FileViewerServiceElectron implements FileViewerService { - private notificationService: NotificationServiceWeb; - - public constructor(notificationService: NotificationServiceWeb) { - this.notificationService = notificationService; - } - - public async open(executable: string, filePaths: string[]): Promise { - const reportErrorToUser = async (error: unknown) => { - await this.notificationService.showError( - `Opening executable ${executable}`, - `Failure reported while attempting to open files: Files: ${filePaths}, Error: ${error}` - ); - }; - - try { - // TODO - } catch (error) { - await reportErrorToUser(error); - } +export default class FileViewerServiceWeb implements FileViewerService { + public async open(): Promise { + throw new Error("FileViewerServiceWeb::open is not yet implemented"); } } diff --git a/packages/web/src/services/NotificationServiceWeb.ts b/packages/web/src/services/NotificationServiceWeb.ts index 438a4cdfb..8cfed8329 100644 --- a/packages/web/src/services/NotificationServiceWeb.ts +++ b/packages/web/src/services/NotificationServiceWeb.ts @@ -1,17 +1,15 @@ import { NotificationService } from "../../../core/services"; export default class NotificationServiceWeb implements NotificationService { - public showMessage(title: string, message: string): Promise { - const x = prompt(`${title}: ${message}`, "No"); - console.log(x); - return Promise.resolve(x === "No"); + public showMessage(): Promise { + throw new Error("NotificationServiceWeb::showMessage is not yet implemented"); } - public async showError(title: string, message: string): Promise { - alert(`${title}: ${message}`); + public async showError(): Promise { + throw new Error("NotificationServiceWeb::showError is not yet implemented"); } - public async showQuestion(title: string, message: string): Promise { - return this.showMessage(title, message); + public async showQuestion(): Promise { + throw new Error("NotificationServiceWeb::showQuestion is not yet implemented"); } } diff --git a/packages/web/src/services/PersistentConfigServiceWeb.ts b/packages/web/src/services/PersistentConfigServiceWeb.ts deleted file mode 100644 index cf5bbba14..000000000 --- a/packages/web/src/services/PersistentConfigServiceWeb.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - PersistentConfigService, - PersistedConfig, - PersistedConfigKeys, -} from "../../../core/services"; - -export default class PersistentConfigServiceElectron implements PersistentConfigService { - public get(): any { - return undefined; - } - - public getAll(): PersistedConfig { - return {}; - } - - public persist(config: PersistedConfig): void; - public persist(key: PersistedConfigKeys, value: any): void; - public persist() { - // No-op - } -} diff --git a/packages/web/src/services/test/ApplicationInfoServiceWeb.test.ts b/packages/web/src/services/test/ApplicationInfoServiceWeb.test.ts new file mode 100644 index 000000000..281807435 --- /dev/null +++ b/packages/web/src/services/test/ApplicationInfoServiceWeb.test.ts @@ -0,0 +1,44 @@ +import { expect } from "chai"; + +import ApplicationInfoServiceWeb from "../ApplicationInfoServiceWeb"; + +describe(`ApplicationInfoServiceWeb`, () => { + describe("updateAvailable", () => { + it("returns false", async () => { + // Arrange + const service = new ApplicationInfoServiceWeb(); + + // Act + const result = await service.updateAvailable(); + + // Assert + expect(result).to.be.false; + }); + }); + + describe("getUserName", () => { + it("returns 'Anonymous Web User'", async () => { + // Arrange + const service = new ApplicationInfoServiceWeb(); + + // Act + const result = await service.getUserName(); + + // Assert + expect(result).to.equal("Anonymous Web User"); + }); + }); + + describe("getApplicationVersion", () => { + it("returns '999.999.999'", async () => { + // Arrange + const service = new ApplicationInfoServiceWeb(); + + // Act + const version = await service.getApplicationVersion(); + + // Assert + expect(version).to.equal("999.999.999"); + }); + }); +}); diff --git a/packages/web/src/services/test/DatabaseServiceWeb.test.ts b/packages/web/src/services/test/DatabaseServiceWeb.test.ts new file mode 100644 index 000000000..e9c233d4c --- /dev/null +++ b/packages/web/src/services/test/DatabaseServiceWeb.test.ts @@ -0,0 +1,47 @@ +// TODO: Need a publically available data source to properly test this +// https://github.com/AllenInstitute/aics-fms-file-explorer-app/issues/66 +describe("DatabaseServiceWeb", () => { + // const service = new DatabaseServiceWeb(); + // before(async () => { + // await service.initialize(); + // }) + // after(() => { + // service.close(); + // }) + // describe("addDataSource", () => { + // ["csv", "json", "parquet"].forEach((type: any) => { + // it(`creates table from file of type ${type}`, async () => { + // // Arrange + // const tempFileName = `test.${type}`; + // const tempFile = path.resolve(tempDir, tempFileName); + // await fs.promises.writeFile(tempFile, "a,b,c\n1,2,3\n4,5,6\n"); + // // Act + // service.addDataSource(tempFileName, type, tempFile); + // // Assert + // const result = await service.query(`SELECT * FROM ${tempFileName}`); + // expect(result).to.be.lengthOf(2); + // }); + // }); + // }); + // describe("query", () => { + // it("executes a query", async () => { + // // Act + // const result = service.query("SELECT * FROM INFORMATION_SCHEMA.TABLES"); + // // Assert + // expect(result).to.be.lengthOf(0); + // }); + // }); + // describe("saveQuery", () => { + // ["csv", "json", "parquet"].forEach((type: any) => { + // it(`saves query out to a ${type} file`, async () => { + // // Arrange + // const sql = "SELECT * FROM INFORMATION_SCHEMA.TABLES"; + // const format = "csv"; + // // Act + // const result = await service.saveQuery("test.csv", sql, format); + // // Assert + // expect(result).to.not.be.undefined; + // }); + // }); + // }); +}); diff --git a/packages/web/src/services/test/ExecutionEnvServiceWeb.test.ts b/packages/web/src/services/test/ExecutionEnvServiceWeb.test.ts new file mode 100644 index 000000000..7e40f042c --- /dev/null +++ b/packages/web/src/services/test/ExecutionEnvServiceWeb.test.ts @@ -0,0 +1,58 @@ +import { expect } from "chai"; +import ExecutionEnvServiceWeb from "../ExecutionEnvServiceWeb"; + +describe("ExecutionEnvServiceWeb", () => { + const service = new ExecutionEnvServiceWeb(); + + describe("formatPathForHost", () => { + it("is just an identity function", async () => { + // Arrange + const input = "/Volumes/programs/object/path.ext"; + + // Act + const actual = await service.formatPathForHost(input); + + // Assert + expect(actual).to.equal(input); + }); + }); + + describe("promptForExecutable", () => { + it("throw error", async () => { + // Act / Assert + try { + await service.promptForExecutable(); + } catch (error) { + expect((error as Error).message).to.equal( + "ExecutionEnvServiceWeb::promptForExecutable not yet implemented" + ); + } + }); + }); + + describe("promptForFile", () => { + it("throw error", async () => { + // Act / Assert + try { + await service.promptForFile(); + } catch (error) { + expect((error as Error).message).to.equal( + "ExecutionEnvServiceWeb::promptForFile not yet implemented" + ); + } + }); + }); + + describe("promptForSaveLocation", () => { + it("throw error", async () => { + // Act / Assert + try { + await service.promptForSaveLocation(); + } catch (error) { + expect((error as Error).message).to.equal( + "ExecutionEnvServiceWeb::promptForSaveLocation not yet implemented" + ); + } + }); + }); +}); diff --git a/packages/web/src/services/test/FileViewerServiceWeb.test.ts b/packages/web/src/services/test/FileViewerServiceWeb.test.ts new file mode 100644 index 000000000..a6d86b562 --- /dev/null +++ b/packages/web/src/services/test/FileViewerServiceWeb.test.ts @@ -0,0 +1,21 @@ +import { expect } from "chai"; + +import FileViewerServiceWeb from "../FileViewerServiceWeb"; + +describe("FileViewerServiceWeb", () => { + describe("open", () => { + it("throws error (not implemented)", async () => { + // Arrange + const service = new FileViewerServiceWeb(); + + // Act / Assert + try { + await service.open(); + } catch (error) { + expect((error as Error).message).to.equal( + "FileViewerServiceWeb::open is not yet implemented" + ); + } + }); + }); +}); diff --git a/packages/web/webpack/webpack.config.js b/packages/web/webpack/webpack.config.js index 5234c3377..d9dc42f81 100644 --- a/packages/web/webpack/webpack.config.js +++ b/packages/web/webpack/webpack.config.js @@ -10,6 +10,7 @@ module.exports = ({ analyze, production } = {}) => ({ devServer: { host: devServer.host, port: devServer.port, + historyApiFallback: true, }, entry: { app: "./src/index.tsx",