Release v1.31.2 (#827)

## Changes
1. Incremental row loading for tiles
2. Tracking tiles count and load time in DD

packages/backend/package.json

@@ -105,5 +105,5 @@
     "tsconfig-paths": "^4.2.0",
     "type-fest": "4.10.3"
   },
-  "version": "1.31.1"
+  "version": "1.31.2"
 }

packages/backend/src/apps/tiles/actions/find-single-row/index.ts

@@ -176,12 +176,15 @@ const action: IRawAction = {
       )
     }
 
-    const result = await getTableRows({
+    /**
+     * When columnIds are not provided, we only return rowId
+     */
+    const { rows } = await getTableRows({
       tableId,
       filters,
     })
 
-    if (!result || !result.length) {
+    if (!rows || !rows.length) {
       $.setActionItem({
         raw: {
           rowsFound: 0,
@@ -190,8 +193,8 @@ const action: IRawAction = {
       return
     }
     const rowIdToUse = returnLastRow
-      ? result[result.length - 1].rowId
-      : result[0].rowId
+      ? rows[rows.length - 1].rowId
+      : rows[0].rowId
 
     /**
      * We use raw row data instead of mapped column names as we want them to
@@ -205,7 +208,7 @@ const action: IRawAction = {
 
     $.setActionItem({
       raw: {
-        rowsFound: result.length,
+        rowsFound: rows.length,
         rowId: rowIdToUse,
         row: rowToReturn.data,
       } satisfies FindSingleRowOutput,

packages/backend/src/db/dynamodb_seeds/seed_rows.ts

@@ -25,7 +25,7 @@ const COUNT_ONLY = argv[3]
 const ROW_COUNT = 10000
 async function seedRows(tableId: string) {
   // delete existing rows
-  const existingRows = await getTableRows({ tableId })
+  const { rows: existingRows } = await getTableRows({ tableId })
   console.log('count', existingRows.length)
 
   if (COUNT_ONLY === 'count') {

packages/backend/src/graphql/__tests__/mutations/tiles/create-rows.itest.ts

@@ -56,7 +56,7 @@ describe('create row mutation', () => {
       context,
     )
 
-    const rows = await getTableRows({
+    const { rows } = await getTableRows({
       tableId: dummyTable.id,
       columnIds: dummyColumnIds,
     })
@@ -81,7 +81,7 @@ describe('create row mutation', () => {
       context,
     )
 
-    const rows = await getTableRows({
+    const { rows } = await getTableRows({
       tableId: dummyTable.id,
       columnIds: dummyColumnIds,
     })

packages/backend/src/graphql/__tests__/queries/tiles/get-all-rows.itest.ts

@@ -46,7 +46,7 @@ describe('get all rows query', () => {
     const numRowsToInsert = 100
     await insertMockTableRows(dummyTable.id, numRowsToInsert, dummyColumnIds)
 
-    const rows = await getAllRows(
+    const { rows } = await getAllRows(
       null,
       {
         tableId: dummyTable.id,
@@ -69,7 +69,7 @@ describe('get all rows query', () => {
       rowIdsInserted.push(rowId)
     }
 
-    const rows = await getAllRows(
+    const { rows } = await getAllRows(
       null,
       {
         tableId: dummyTable.id,
@@ -79,25 +79,32 @@ describe('get all rows query', () => {
     expect(rows.map((r: ITableRow) => r.rowId)).toEqual(rowIdsInserted)
   })
 
-  it('should fetch all rows even if more than 1MB', async () => {
+  it('should fetch all rows even if more than 1MB by pagination', async () => {
     // 1 randomly generated row is about 470 bytes
-    // 4000 rows will be about about 1.8MB
-    const numRowsToInsert = 4000
+    // 10000 rows will be about 4.7MB
+    const numRowsToInsert = 10000
     await insertMockTableRows(dummyTable.id, numRowsToInsert, dummyColumnIds)
 
-    const rows = await getAllRows(
-      null,
-      {
-        tableId: dummyTable.id,
-      },
-      context,
-    )
-
-    expect(rows).toHaveLength(numRowsToInsert)
+    let cursor: string | null = null
+    let rows: ITableRow[] = []
+    do {
+      const { rows: pageRows, stringifiedCursor } = await getAllRows(
+        null,
+        {
+          tableId: dummyTable.id,
+          stringifiedCursor: cursor,
+        },
+        context,
+      )
+      cursor = stringifiedCursor
+      expect(pageRows.length).toBeLessThan(numRowsToInsert)
+      rows = rows.concat(pageRows)
+    } while (cursor)
+    expect(rows.length).toBe(numRowsToInsert)
   }, 100000)
 
   it('should return empty array if no rows', async () => {
-    const rows = await getAllRows(
+    const { rows } = await getAllRows(
       null,
       {
         tableId: dummyTable.id,
@@ -120,7 +127,7 @@ describe('get all rows query', () => {
 
     await createTableRow(rowToInsert)
 
-    const returnedRows = await getAllRows(
+    const { rows: returnedRows } = await getAllRows(
       null,
       {
         tableId: dummyTable.id,

packages/backend/src/graphql/queries/tiles/get-all-rows.ts

@@ -12,7 +12,7 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
   params,
   context,
 ) => {
-  const { tableId } = params
+  const { tableId, stringifiedCursor } = params
 
   try {
     const table = context.tilesViewKey
@@ -37,7 +37,12 @@ const getAllRows: QueryResolvers['getAllRows'] = async (
     }
 
     const columnIds = table.columns.map((column) => column.id)
-    return getTableRows({ tableId, columnIds })
+    return await getTableRows({
+      tableId,
+      columnIds,
+      stringifiedCursor: stringifiedCursor ?? 'start',
+    })
+    // TODO: remove keys from rows to reduce payload size
   } catch (e) {
     logger.error(e)
     if (e instanceof NotFoundError) {

packages/backend/src/graphql/schema.graphql

@@ -39,7 +39,7 @@ type Query {
   getTableConnections(tableIds: [String!]!): JSONObject
   getTables(limit: Int!, offset: Int!, name: String): PaginatedTables!
   # Tiles rows
-  getAllRows(tableId: String!): Any!
+ getAllRows(tableId: String!, stringifiedCursor: String): GetTableRowsResult!
   getCurrentUser: User
   healthcheck: AppHealth
   getPlumberStats: Stats
@@ -765,6 +765,11 @@ type PaginatedTables {
   pageInfo: PageInfo!
 }
 
+type GetTableRowsResult {
+  rows: Any!
+  stringifiedCursor: String
+}
+
 # End Tiles types
 
 # Start Tiles row types

packages/backend/src/models/dynamodb/__tests__/table-row/functions.itest.ts

@@ -80,7 +80,7 @@ describe('dynamodb table row functions', () => {
         })
       }
       await createTableRows({ tableId: dummyTable.id, dataArray })
-      const rows = await getTableRows({
+      const { rows } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b', 'randomcol'],
       })
@@ -100,7 +100,7 @@ describe('dynamodb table row functions', () => {
       await createTableRows({ tableId: dummyTable.id, dataArray })
 
       // LTE
-      const rows1 = await getTableRows({
+      const { rows: rows1 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -114,7 +114,7 @@ describe('dynamodb table row functions', () => {
       expect(rows1.length).toEqual(501)
 
       // LT
-      const rows2 = await getTableRows({
+      const { rows: rows2 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -128,7 +128,7 @@ describe('dynamodb table row functions', () => {
       expect(rows2.length).toEqual(500)
 
       // GT
-      const rows3 = await getTableRows({
+      const { rows: rows3 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -142,7 +142,7 @@ describe('dynamodb table row functions', () => {
       expect(rows3.length).toEqual(499)
 
       // GTE
-      const rows4 = await getTableRows({
+      const { rows: rows4 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -156,7 +156,7 @@ describe('dynamodb table row functions', () => {
       expect(rows4.length).toEqual(500)
 
       // EQUALS
-      const rows5 = await getTableRows({
+      const { rows: rows5 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -170,7 +170,7 @@ describe('dynamodb table row functions', () => {
       expect(rows5.length).toEqual(1)
 
       // Contains
-      const rows6 = await getTableRows({
+      const { rows: rows6 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -184,7 +184,7 @@ describe('dynamodb table row functions', () => {
       expect(rows6.length).toEqual(19)
 
       // Contains
-      const rows7 = await getTableRows({
+      const { rows: rows7 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -198,7 +198,7 @@ describe('dynamodb table row functions', () => {
       expect(rows7.length).toEqual(111)
 
       // Empty
-      const rows8 = await getTableRows({
+      const { rows: rows8 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -223,7 +223,7 @@ describe('dynamodb table row functions', () => {
       await createTableRows({ tableId: dummyTable.id, dataArray })
 
       // LTE & GTE
-      const rows1 = await getTableRows({
+      const { rows: rows1 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -243,7 +243,7 @@ describe('dynamodb table row functions', () => {
       expect(rows1.length).toEqual(301)
 
       // CONTAINS & BEGINS WITH
-      const rows2 = await getTableRows({
+      const { rows: rows2 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [
@@ -262,7 +262,7 @@ describe('dynamodb table row functions', () => {
       expect(rows2.length).toEqual(11)
 
       // IS EMPTY & EQUALS
-      const rows3 = await getTableRows({
+      const { rows: rows3 } = await getTableRows({
         tableId: dummyTable.id,
         columnIds: ['a', 'b'],
         filters: [

packages/backend/src/models/dynamodb/table-row/functions.ts

@@ -261,11 +261,20 @@ export const getTableRows = async ({
   tableId,
   columnIds,
   filters,
+  stringifiedCursor,
 }: {
   tableId: string
   columnIds?: string[]
   filters?: TableRowFilter[]
-}): Promise<TableRowOutput[]> => {
+  /**
+   * if stringifiedCursor is 'start', we will fetch the first page of results
+   * if undefined, we will auto-paginate
+   */
+  stringifiedCursor?: string | 'start'
+}): Promise<{
+  rows: TableRowOutput[]
+  stringifiedCursor?: string
+}> => {
   try {
     // need to use ProjectionExpression to select nested attributes
     const { ProjectionExpression, ExpressionAttributeNames } =
@@ -275,15 +284,18 @@ export const getTableRows = async ({
         indexUsed: 'byCreatedAt',
       })
     const tableRows = []
-    let cursor: any = null
+    let cursor: any =
+      stringifiedCursor && stringifiedCursor !== 'start'
+        ? JSON.parse(stringifiedCursor)
+        : null
     do {
       const query = TableRow.query.byCreatedAt({ tableId })
       if (filters?.length) {
         addFiltersToQuery(query, filters)
       }
       const response = await query.go({
         order: 'asc',
-        pages: 'all',
+        pages: 'all', // this is ignored, we need to paginate manually
         cursor,
         params: {
           ProjectionExpression,
@@ -301,11 +313,16 @@ export const getTableRows = async ({
       }
       tableRows.push(...data.Items)
       cursor = data.LastEvaluatedKey
-    } while (cursor)
-    return tableRows.map((row) => ({
-      ...row,
-      data: row.data || {}, // data can be undefined if values are empty
-    }))
+      // loop only if cursor is
+    } while (cursor && !stringifiedCursor)
+
+    return {
+      rows: tableRows.map((row) => ({
+        ...row,
+        data: row.data || {}, // data can be undefined if values are empty
+      })),
+      stringifiedCursor: cursor ? JSON.stringify(cursor) : undefined,
+    }
   } catch (e: unknown) {
     handleDynamoDBError(e)
   }

packages/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "frontend",
-  "version": "1.31.1",
+  "version": "1.31.2",
   "scripts": {
     "dev": "wait-on tcp:3000 && vite --host --force",
     "build": "tsc && vite build --mode=${VITE_MODE:-prod}",