Perspective viewer (#2576)

Co-authored-by: Sergey Fedoseev <[email protected]>

catalog/app/components/Preview/index.js

@@ -1,4 +1,4 @@
 export { default as Display, bind as display } from './Display'
 export { default as render } from './render'
 export { default as load, getRenderProps } from './load'
-export { PreviewData, PreviewError } from './types'
+export { PreviewData, PreviewError, CONTEXT } from './types'

catalog/app/components/Preview/load.js

@@ -1,39 +1,35 @@
 import * as React from 'react'
 
 import * as Audio from './loaders/Audio'
-import * as Csv from './loaders/Csv'
 import * as Echarts from './loaders/Echarts'
-import * as Excel from './loaders/Excel'
 import * as Fcs from './loaders/Fcs'
 import * as Html from './loaders/Html'
 import * as Image from './loaders/Image'
 import * as Json from './loaders/Json'
 import * as Markdown from './loaders/Markdown'
 import * as Notebook from './loaders/Notebook'
-import * as Parquet from './loaders/Parquet'
 import * as Pdf from './loaders/Pdf'
+import * as Tabular from './loaders/Tabular'
 import * as Text from './loaders/Text'
 import * as Vcf from './loaders/Vcf'
 import * as Video from './loaders/Video'
 import * as Voila from './loaders/Voila'
 import * as fallback from './loaders/fallback'
 
 const loaderChain = [
-  Csv,
-  Excel,
   Fcs,
   Echarts, // should be before Json, or TODO: add "type is not 'echarts'" to Json.detect
   Json,
   Markdown,
   Voila, // should be before Notebook, or TODO: add "type is not 'voila'" to Notebook.detect
   Notebook,
-  Parquet,
   Pdf,
   Vcf,
   Html,
   Image,
   Video,
   Audio,
+  Tabular,
   Text,
   fallback,
 ]

catalog/app/components/Preview/loaders/Csv.js

@@ -3,9 +3,11 @@ import * as R from 'ramda'
 import { PreviewData } from '../types'
 import * as utils from './utils'
 
-export const detect = R.pipe(utils.stripCompression, utils.extIn(['.csv', '.tsv']))
+export const isCsv = R.pipe(utils.stripCompression, utils.extIs('.csv'))
 
-const isTsv = R.pipe(utils.stripCompression, utils.extIs('.tsv'))
+export const isTsv = R.pipe(utils.stripCompression, utils.extIs('.tsv'))
+
+export const detect = R.anyPass([isCsv, isTsv])
 
 export const Loader = function CsvLoader({ handle, children }) {
   const data = utils.usePreview({

catalog/app/components/Preview/loaders/Json.js

@@ -161,10 +161,10 @@ function JsonLoader({ gated, handle, children }) {
 
 export const detect = R.either(utils.extIs('.json'), R.startsWith('.quilt/'))
 
-export const Loader = function GatedJsonLoader({ handle, children }) {
+export const Loader = function GatedJsonLoader({ handle, children, options }) {
   return utils.useFirstBytes({ bytes: BYTES_TO_SCAN, handle }).case({
     Ok: ({ firstBytes, contentLength }) =>
-      detectSchema(firstBytes) && handle.mode !== 'json' ? (
+      detectSchema(firstBytes) && options.mode !== 'json' ? (
         <VegaLoader {...{ handle, children, gated: contentLength > MAX_SIZE }} />
       ) : (
         <JsonLoader {...{ handle, children, gated: contentLength > MAX_SIZE }} />

catalog/app/components/Preview/loaders/Notebook.js

@@ -20,8 +20,8 @@ function NotebookLoader({ handle, children }) {
   return children(utils.useErrorHandling(processed, { handle, retry: data.fetch }))
 }
 
-export const Loader = function WrappedNotebookLoader({ handle, children }) {
-  switch (handle.mode) {
+export const Loader = function WrappedNotebookLoader({ handle, children, options }) {
+  switch (options.mode) {
     case 'voila':
       return <Voila.Loader {...{ handle, children }} />
     case 'json':

catalog/app/components/Preview/loaders/Tabular.tsx

@@ -0,0 +1,168 @@
+import * as R from 'ramda'
+import * as React from 'react'
+
+import { HTTPError } from 'utils/APIConnector'
+import * as AWS from 'utils/AWS'
+import * as Config from 'utils/Config'
+import * as Data from 'utils/Data'
+import mkSearch from 'utils/mkSearch'
+import type { S3HandleBase } from 'utils/s3paths'
+
+import { CONTEXT, PreviewData } from '../types'
+
+import * as Csv from './Csv'
+import * as Excel from './Excel'
+import * as Parquet from './Parquet'
+import * as utils from './utils'
+
+const isJsonl = R.pipe(utils.stripCompression, utils.extIs('.jsonl'))
+
+export const detect = R.anyPass([Csv.detect, Excel.detect, Parquet.detect, isJsonl])
+
+type TabularType = 'csv' | 'jsonl' | 'excel' | 'parquet' | 'tsv' | 'txt'
+
+const detectTabularType: (type: string) => TabularType = R.cond([
+  [Csv.isCsv, R.always('csv')],
+  [Csv.isTsv, R.always('tsv')],
+  [Excel.detect, R.always('excel')],
+  [Parquet.detect, R.always('parquet')],
+  [isJsonl, R.always('jsonl')],
+  [R.T, R.always('txt')],
+])
+
+function getQuiltInfo(headers: Headers): { truncated: boolean } | null {
+  try {
+    const header = headers.get('x-quilt-info')
+    return header ? JSON.parse(header) : null
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.error(error)
+    return null
+  }
+}
+
+function getContentLength(headers: Headers): number | null {
+  try {
+    const header = headers.get('content-length')
+    return header ? Number(header) : null
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.error(error)
+    return null
+  }
+}
+
+async function getCsvFromResponse(r: Response): Promise<ArrayBuffer | string> {
+  const isArrow = r.headers.get('content-type') === 'application/vnd.apache.arrow.file'
+  return isArrow ? r.arrayBuffer() : r.text()
+}
+
+interface LoadTabularDataArgs {
+  compression?: 'gz' | 'bz2'
+  endpoint: string
+  handle: S3HandleBase
+  sign: (h: S3HandleBase) => string
+  type: TabularType
+  size: 'small' | 'medium' | 'large'
+}
+
+interface TabularDataOutput {
+  csv: ArrayBuffer | string
+  size: number | null
+  truncated: boolean
+}
+
+const loadTabularData = async ({
+  compression,
+  endpoint,
+  size,
+  handle,
+  sign,
+  type,
+}: LoadTabularDataArgs): Promise<TabularDataOutput> => {
+  const url = sign(handle)
+  const r = await fetch(
+    `${endpoint}/tabular-preview${mkSearch({
+      compression,
+      input: type,
+      size,
+      url,
+    })}`,
+  )
+  try {
+    if (r.status >= 400) {
+      throw new HTTPError(r)
+    }
+
+    const csv = await getCsvFromResponse(r)
+
+    const quiltInfo = getQuiltInfo(r.headers)
+    const contentLength = getContentLength(r.headers)
+
+    return {
+      csv,
+      size: contentLength,
+      truncated: !!quiltInfo?.truncated,
+    }
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn('Error loading tabular preview', e)
+    // eslint-disable-next-line no-console
+    console.error(e)
+    throw e
+  }
+}
+
+function getNeededSize(context: string, gated: boolean) {
+  switch (context) {
+    case CONTEXT.FILE:
+      return gated ? 'medium' : 'large'
+    case CONTEXT.LISTING:
+      return gated ? 'small' : 'large'
+    // no default
+  }
+}
+
+interface TabularLoaderProps {
+  children: (result: $TSFixMe) => React.ReactNode
+  handle: S3HandleBase
+  options: { context: string } // TODO: restrict type
+}
+
+export const Loader = function TabularLoader({
+  handle,
+  children,
+  options,
+}: TabularLoaderProps) {
+  const [gated, setGated] = React.useState(true)
+  const endpoint = Config.use().binaryApiGatewayEndpoint
+  const sign = AWS.Signer.useS3Signer()
+  const type = React.useMemo(() => detectTabularType(handle.key), [handle.key])
+  const onLoadMore = React.useCallback(() => setGated(false), [setGated])
+  const size = React.useMemo(
+    () => getNeededSize(options.context, gated),
+    [options.context, gated],
+  )
+  const compression = utils.getCompression(handle.key)
+  const data = Data.use(loadTabularData, {
+    compression,
+    endpoint,
+    size,
+    handle,
+    sign,
+    type,
+  })
+  // TODO: get correct sises from API
+  const processed = utils.useProcessing(
+    data.result,
+    ({ csv, truncated }: TabularDataOutput) =>
+      PreviewData.Perspective({
+        context: options.context,
+        data: csv,
+        handle,
+        onLoadMore: truncated && size !== 'large' ? onLoadMore : null,
+        truncated,
+      }),
+  )
+  return children(utils.useErrorHandling(processed, { handle, retry: data.fetch }))
+}

catalog/app/components/Preview/renderers/Perspective.tsx

@@ -0,0 +1,103 @@
+import cx from 'classnames'
+import * as React from 'react'
+import * as M from '@material-ui/core'
+
+import * as perspective from 'utils/perspective'
+import type { S3HandleBase } from 'utils/s3paths'
+
+import { CONTEXT } from '../types'
+
+const useTruncatedWarningStyles = M.makeStyles((t) => ({
+  root: {
+    alignItems: 'center',
+    display: 'flex',
+  },
+  message: {
+    color: t.palette.text.secondary,
+    marginRight: t.spacing(2),
+  },
+  icon: {
+    display: 'inline-block',
+    fontSize: '1.25rem',
+    marginRight: t.spacing(0.5),
+    verticalAlign: '-5px',
+  },
+}))
+
+interface TruncatedWarningProps {
+  className: string
+  onLoadMore: () => void
+}
+
+function TruncatedWarning({ className, onLoadMore }: TruncatedWarningProps) {
+  const classes = useTruncatedWarningStyles()
+  return (
+    <div className={cx(classes.root, className)}>
+      <span className={classes.message}>
+        <M.Icon fontSize="small" color="inherit" className={classes.icon}>
+          info_outlined
+        </M.Icon>
+        Partial preview
+      </span>
+
+      {!!onLoadMore && (
+        <M.Button startIcon={<M.Icon>refresh</M.Icon>} size="small" onClick={onLoadMore}>
+          Load more
+        </M.Button>
+      )}
+    </div>
+  )
+}
+
+const useStyles = M.makeStyles((t) => ({
+  root: {
+    width: '100%',
+  },
+  viewer: {
+    height: ({ context }: { context: 'file' | 'listing' }) =>
+      context === CONTEXT.LISTING ? t.spacing(30) : t.spacing(50),
+    overflow: 'auto',
+    resize: 'vertical',
+  },
+  warning: {
+    marginBottom: t.spacing(1),
+  },
+}))
+
+interface PerspectiveProps extends React.HTMLAttributes<HTMLDivElement> {
+  context: 'file' | 'listing'
+  data: string | ArrayBuffer
+  handle: S3HandleBase
+  onLoadMore: () => void
+  truncated: boolean
+}
+
+function Perspective({
+  children,
+  className,
+  context,
+  data,
+  handle,
+  onLoadMore,
+  truncated,
+  ...props
+}: PerspectiveProps) {
+  const classes = useStyles({ context })
+
+  const [root, setRoot] = React.useState<HTMLDivElement | null>(null)
+
+  const attrs = React.useMemo(() => ({ className: classes.viewer }), [classes])
+  perspective.use(root, data, attrs)
+
+  return (
+    <div className={cx(className, classes.root)} ref={setRoot} {...props}>
+      {truncated && (
+        <TruncatedWarning className={classes.warning} onLoadMore={onLoadMore} />
+      )}
+    </div>
+  )
+}
+
+export default (data: PerspectiveProps, props: PerspectiveProps) => (
+  <Perspective {...data} {...props} />
+)

catalog/app/components/Preview/renderers/index.js

@@ -9,6 +9,7 @@ export { default as Markdown } from './Markdown'
 export { default as Notebook } from './Notebook'
 export { default as Parquet } from './Parquet'
 export { default as Pdf } from './Pdf'
+export { default as Perspective } from './Perspective'
 export { default as Text } from './Text'
 export { default as Vcf } from './Vcf'
 export { default as Vega } from './Vega'

catalog/app/components/Preview/types.js

@@ -24,6 +24,7 @@ PreviewStatus: {
 */
 
 export const PreviewData = tagged([
+  'Audio', // { src: string }
   'DataFrame', // { preview: string, ...PreviewStatus }
   'ECharts', // { option: object }
   'Fcs', // { preview: string, metadata: object, ...PreviewStatus }
@@ -34,11 +35,11 @@ export const PreviewData = tagged([
   'Notebook', // { preview: string, ...PreviewStatus }
   'Parquet', // { preview: string, ...ParquetMeta, ...PreviewStatus }
   'Pdf', // { handle: object, pages: number, firstPageBlob: Blob, type: 'pdf' | 'pptx' }
+  'Perspective', // { context: CONTEXT, data: string | ArrayBuffer, handle: S3Handle, onLoadMore: () => void, truncated: boolean }
   'Text', // { head: string, tail: string, lang: string, highlighted: { head: string, tail: string }, ...PreviewStatus }
   'Vcf', // { meta: string[], header: string[], body: string[][], variants: string[], ...PreviewStatus }
   'Vega', // { spec: Object }
   'Video', // { src: string }
-  'Audio', // { src: string }
   'Voila', // { src: string }
 ])
 
@@ -55,3 +56,8 @@ export const PreviewError = tagged([
   'MalformedJson', // { handle, message }
   'Unexpected', // { handle, retry, originalError: any }
 ])
+
+export const CONTEXT = {
+  FILE: 'file',
+  LISTING: 'listing',
+}