diff --git a/package-lock.json b/package-lock.json index dfde0d7a7..4553fdf23 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23451,7 +23451,7 @@ } }, "packages/backend": { - "version": "1.31.1", + "version": "1.31.2", "dependencies": { "@apollo/server": "4.9.4", "@aws-sdk/client-dynamodb": "3.460.0", @@ -23551,7 +23551,7 @@ } }, "packages/frontend": { - "version": "1.31.1", + "version": "1.31.2", "hasInstallScript": true, "dependencies": { "@datadog/browser-rum": "^5.2.0", @@ -23762,7 +23762,7 @@ }, "packages/types": { "name": "@plumber/types", - "version": "1.31.1" + "version": "1.31.2" } } } diff --git a/packages/backend/package.json b/packages/backend/package.json index 55097c99c..446518ea7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -105,5 +105,5 @@ "tsconfig-paths": "^4.2.0", "type-fest": "4.10.3" }, - "version": "1.31.1" + "version": "1.31.2" } diff --git a/packages/backend/src/apps/tiles/actions/find-single-row/index.ts b/packages/backend/src/apps/tiles/actions/find-single-row/index.ts index fea8b38f6..0b2c82356 100644 --- a/packages/backend/src/apps/tiles/actions/find-single-row/index.ts +++ b/packages/backend/src/apps/tiles/actions/find-single-row/index.ts @@ -176,12 +176,15 @@ const action: IRawAction = { ) } - const result = await getTableRows({ + /** + * When columnIds are not provided, we only return rowId + */ + const { rows } = await getTableRows({ tableId, filters, }) - if (!result || !result.length) { + if (!rows || !rows.length) { $.setActionItem({ raw: { rowsFound: 0, @@ -190,8 +193,8 @@ const action: IRawAction = { return } const rowIdToUse = returnLastRow - ? result[result.length - 1].rowId - : result[0].rowId + ? rows[rows.length - 1].rowId + : rows[0].rowId /** * We use raw row data instead of mapped column names as we want them to @@ -205,7 +208,7 @@ const action: IRawAction = { $.setActionItem({ raw: { - rowsFound: result.length, + rowsFound: rows.length, rowId: rowIdToUse, row: rowToReturn.data, } satisfies FindSingleRowOutput, diff --git a/packages/backend/src/db/dynamodb_seeds/seed_rows.ts b/packages/backend/src/db/dynamodb_seeds/seed_rows.ts index 14f1470a5..772429111 100644 --- a/packages/backend/src/db/dynamodb_seeds/seed_rows.ts +++ b/packages/backend/src/db/dynamodb_seeds/seed_rows.ts @@ -25,7 +25,7 @@ const COUNT_ONLY = argv[3] const ROW_COUNT = 10000 async function seedRows(tableId: string) { // delete existing rows - const existingRows = await getTableRows({ tableId }) + const { rows: existingRows } = await getTableRows({ tableId }) console.log('count', existingRows.length) if (COUNT_ONLY === 'count') { diff --git a/packages/backend/src/graphql/__tests__/mutations/tiles/create-rows.itest.ts b/packages/backend/src/graphql/__tests__/mutations/tiles/create-rows.itest.ts index 1326d0809..0637b0e2f 100644 --- a/packages/backend/src/graphql/__tests__/mutations/tiles/create-rows.itest.ts +++ b/packages/backend/src/graphql/__tests__/mutations/tiles/create-rows.itest.ts @@ -56,7 +56,7 @@ describe('create row mutation', () => { context, ) - const rows = await getTableRows({ + const { rows } = await getTableRows({ tableId: dummyTable.id, columnIds: dummyColumnIds, }) @@ -81,7 +81,7 @@ describe('create row mutation', () => { context, ) - const rows = await getTableRows({ + const { rows } = await getTableRows({ tableId: dummyTable.id, columnIds: dummyColumnIds, }) diff --git a/packages/backend/src/graphql/__tests__/queries/tiles/get-all-rows.itest.ts b/packages/backend/src/graphql/__tests__/queries/tiles/get-all-rows.itest.ts index 08400183f..6ed1c8bce 100644 --- a/packages/backend/src/graphql/__tests__/queries/tiles/get-all-rows.itest.ts +++ b/packages/backend/src/graphql/__tests__/queries/tiles/get-all-rows.itest.ts @@ -46,7 +46,7 @@ describe('get all rows query', () => { const numRowsToInsert = 100 await insertMockTableRows(dummyTable.id, numRowsToInsert, dummyColumnIds) - const rows = await getAllRows( + const { rows } = await getAllRows( null, { tableId: dummyTable.id, @@ -69,7 +69,7 @@ describe('get all rows query', () => { rowIdsInserted.push(rowId) } - const rows = await getAllRows( + const { rows } = await getAllRows( null, { tableId: dummyTable.id, @@ -79,25 +79,32 @@ describe('get all rows query', () => { expect(rows.map((r: ITableRow) => r.rowId)).toEqual(rowIdsInserted) }) - it('should fetch all rows even if more than 1MB', async () => { + it('should fetch all rows even if more than 1MB by pagination', async () => { // 1 randomly generated row is about 470 bytes - // 4000 rows will be about about 1.8MB - const numRowsToInsert = 4000 + // 10000 rows will be about 4.7MB + const numRowsToInsert = 10000 await insertMockTableRows(dummyTable.id, numRowsToInsert, dummyColumnIds) - const rows = await getAllRows( - null, - { - tableId: dummyTable.id, - }, - context, - ) - - expect(rows).toHaveLength(numRowsToInsert) + let cursor: string | null = null + let rows: ITableRow[] = [] + do { + const { rows: pageRows, stringifiedCursor } = await getAllRows( + null, + { + tableId: dummyTable.id, + stringifiedCursor: cursor, + }, + context, + ) + cursor = stringifiedCursor + expect(pageRows.length).toBeLessThan(numRowsToInsert) + rows = rows.concat(pageRows) + } while (cursor) + expect(rows.length).toBe(numRowsToInsert) }, 100000) it('should return empty array if no rows', async () => { - const rows = await getAllRows( + const { rows } = await getAllRows( null, { tableId: dummyTable.id, @@ -120,7 +127,7 @@ describe('get all rows query', () => { await createTableRow(rowToInsert) - const returnedRows = await getAllRows( + const { rows: returnedRows } = await getAllRows( null, { tableId: dummyTable.id, diff --git a/packages/backend/src/graphql/queries/tiles/get-all-rows.ts b/packages/backend/src/graphql/queries/tiles/get-all-rows.ts index c67136baf..de308dd59 100644 --- a/packages/backend/src/graphql/queries/tiles/get-all-rows.ts +++ b/packages/backend/src/graphql/queries/tiles/get-all-rows.ts @@ -12,7 +12,7 @@ const getAllRows: QueryResolvers['getAllRows'] = async ( params, context, ) => { - const { tableId } = params + const { tableId, stringifiedCursor } = params try { const table = context.tilesViewKey @@ -37,7 +37,12 @@ const getAllRows: QueryResolvers['getAllRows'] = async ( } const columnIds = table.columns.map((column) => column.id) - return getTableRows({ tableId, columnIds }) + return await getTableRows({ + tableId, + columnIds, + stringifiedCursor: stringifiedCursor ?? 'start', + }) + // TODO: remove keys from rows to reduce payload size } catch (e) { logger.error(e) if (e instanceof NotFoundError) { diff --git a/packages/backend/src/graphql/schema.graphql b/packages/backend/src/graphql/schema.graphql index 231048be5..7df812f2f 100644 --- a/packages/backend/src/graphql/schema.graphql +++ b/packages/backend/src/graphql/schema.graphql @@ -39,7 +39,7 @@ type Query { getTableConnections(tableIds: [String!]!): JSONObject getTables(limit: Int!, offset: Int!, name: String): PaginatedTables! # Tiles rows - getAllRows(tableId: String!): Any! + getAllRows(tableId: String!, stringifiedCursor: String): GetTableRowsResult! getCurrentUser: User healthcheck: AppHealth getPlumberStats: Stats @@ -765,6 +765,11 @@ type PaginatedTables { pageInfo: PageInfo! } +type GetTableRowsResult { + rows: Any! + stringifiedCursor: String +} + # End Tiles types # Start Tiles row types diff --git a/packages/backend/src/models/dynamodb/__tests__/table-row/functions.itest.ts b/packages/backend/src/models/dynamodb/__tests__/table-row/functions.itest.ts index 39e0dc14f..a8323b1f1 100644 --- a/packages/backend/src/models/dynamodb/__tests__/table-row/functions.itest.ts +++ b/packages/backend/src/models/dynamodb/__tests__/table-row/functions.itest.ts @@ -80,7 +80,7 @@ describe('dynamodb table row functions', () => { }) } await createTableRows({ tableId: dummyTable.id, dataArray }) - const rows = await getTableRows({ + const { rows } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b', 'randomcol'], }) @@ -100,7 +100,7 @@ describe('dynamodb table row functions', () => { await createTableRows({ tableId: dummyTable.id, dataArray }) // LTE - const rows1 = await getTableRows({ + const { rows: rows1 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -114,7 +114,7 @@ describe('dynamodb table row functions', () => { expect(rows1.length).toEqual(501) // LT - const rows2 = await getTableRows({ + const { rows: rows2 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -128,7 +128,7 @@ describe('dynamodb table row functions', () => { expect(rows2.length).toEqual(500) // GT - const rows3 = await getTableRows({ + const { rows: rows3 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -142,7 +142,7 @@ describe('dynamodb table row functions', () => { expect(rows3.length).toEqual(499) // GTE - const rows4 = await getTableRows({ + const { rows: rows4 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -156,7 +156,7 @@ describe('dynamodb table row functions', () => { expect(rows4.length).toEqual(500) // EQUALS - const rows5 = await getTableRows({ + const { rows: rows5 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -170,7 +170,7 @@ describe('dynamodb table row functions', () => { expect(rows5.length).toEqual(1) // Contains - const rows6 = await getTableRows({ + const { rows: rows6 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -184,7 +184,7 @@ describe('dynamodb table row functions', () => { expect(rows6.length).toEqual(19) // Contains - const rows7 = await getTableRows({ + const { rows: rows7 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -198,7 +198,7 @@ describe('dynamodb table row functions', () => { expect(rows7.length).toEqual(111) // Empty - const rows8 = await getTableRows({ + const { rows: rows8 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -223,7 +223,7 @@ describe('dynamodb table row functions', () => { await createTableRows({ tableId: dummyTable.id, dataArray }) // LTE & GTE - const rows1 = await getTableRows({ + const { rows: rows1 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -243,7 +243,7 @@ describe('dynamodb table row functions', () => { expect(rows1.length).toEqual(301) // CONTAINS & BEGINS WITH - const rows2 = await getTableRows({ + const { rows: rows2 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ @@ -262,7 +262,7 @@ describe('dynamodb table row functions', () => { expect(rows2.length).toEqual(11) // IS EMPTY & EQUALS - const rows3 = await getTableRows({ + const { rows: rows3 } = await getTableRows({ tableId: dummyTable.id, columnIds: ['a', 'b'], filters: [ diff --git a/packages/backend/src/models/dynamodb/table-row/functions.ts b/packages/backend/src/models/dynamodb/table-row/functions.ts index 9946d9f87..6105e7bfd 100644 --- a/packages/backend/src/models/dynamodb/table-row/functions.ts +++ b/packages/backend/src/models/dynamodb/table-row/functions.ts @@ -261,11 +261,20 @@ export const getTableRows = async ({ tableId, columnIds, filters, + stringifiedCursor, }: { tableId: string columnIds?: string[] filters?: TableRowFilter[] -}): Promise => { + /** + * if stringifiedCursor is 'start', we will fetch the first page of results + * if undefined, we will auto-paginate + */ + stringifiedCursor?: string | 'start' +}): Promise<{ + rows: TableRowOutput[] + stringifiedCursor?: string +}> => { try { // need to use ProjectionExpression to select nested attributes const { ProjectionExpression, ExpressionAttributeNames } = @@ -275,7 +284,10 @@ export const getTableRows = async ({ indexUsed: 'byCreatedAt', }) const tableRows = [] - let cursor: any = null + let cursor: any = + stringifiedCursor && stringifiedCursor !== 'start' + ? JSON.parse(stringifiedCursor) + : null do { const query = TableRow.query.byCreatedAt({ tableId }) if (filters?.length) { @@ -283,7 +295,7 @@ export const getTableRows = async ({ } const response = await query.go({ order: 'asc', - pages: 'all', + pages: 'all', // this is ignored, we need to paginate manually cursor, params: { ProjectionExpression, @@ -301,11 +313,16 @@ export const getTableRows = async ({ } tableRows.push(...data.Items) cursor = data.LastEvaluatedKey - } while (cursor) - return tableRows.map((row) => ({ - ...row, - data: row.data || {}, // data can be undefined if values are empty - })) + // loop only if cursor is + } while (cursor && !stringifiedCursor) + + return { + rows: tableRows.map((row) => ({ + ...row, + data: row.data || {}, // data can be undefined if values are empty + })), + stringifiedCursor: cursor ? JSON.stringify(cursor) : undefined, + } } catch (e: unknown) { handleDynamoDBError(e) } diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 8f3876f03..d54c6dedb 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "1.31.1", + "version": "1.31.2", "scripts": { "dev": "wait-on tcp:3000 && vite --host --force", "build": "tsc && vite build --mode=${VITE_MODE:-prod}", diff --git a/packages/frontend/src/graphql/queries/tiles/get-all-rows.ts b/packages/frontend/src/graphql/queries/tiles/get-all-rows.ts index 4b8669a74..1394734ee 100644 --- a/packages/frontend/src/graphql/queries/tiles/get-all-rows.ts +++ b/packages/frontend/src/graphql/queries/tiles/get-all-rows.ts @@ -1,7 +1,10 @@ -import { gql } from '@apollo/client' +import { graphql } from '@/graphql/__generated__' -export const GET_ALL_ROWS = gql` - query GetAllRows($tableId: String!) { - getAllRows(tableId: $tableId) +export const GET_ALL_ROWS = graphql(` + query GetAllRows($tableId: String!, $stringifiedCursor: String) { + getAllRows(tableId: $tableId, stringifiedCursor: $stringifiedCursor) { + rows + stringifiedCursor + } } -` +`) diff --git a/packages/frontend/src/pages/Tile/components/TableBanner/RefreshButton.tsx b/packages/frontend/src/pages/Tile/components/TableBanner/RefreshButton.tsx new file mode 100644 index 000000000..e357850c6 --- /dev/null +++ b/packages/frontend/src/pages/Tile/components/TableBanner/RefreshButton.tsx @@ -0,0 +1,25 @@ +import { Flex } from '@chakra-ui/react' +import { Spinner } from '@opengovsg/design-system-react' + +import { useTableContext } from '../../contexts/TableContext' + +export default function RefreshButton() { + const { isFetching } = useTableContext() + + if (isFetching) { + return ( + + Fetching more rows... + + ) + } + + // Will add refresh button in future + return null +} diff --git a/packages/frontend/src/pages/Tile/components/TableBanner/index.tsx b/packages/frontend/src/pages/Tile/components/TableBanner/index.tsx index 02260dba9..57bb269e4 100644 --- a/packages/frontend/src/pages/Tile/components/TableBanner/index.tsx +++ b/packages/frontend/src/pages/Tile/components/TableBanner/index.tsx @@ -6,9 +6,10 @@ import { useTableContext } from '../../contexts/TableContext' import BreadCrumb from './BreadCrumb' import EditMode from './EditMode' import ImportExportToolbar from './ImportExportToolbar' +import RefreshButton from './RefreshButton' function TableBanner() { - const { tableName, role } = useTableContext() + const { tableName, role, isFetching } = useTableContext() return ( : {tableName}} - + + {isFetching && } + + ) } diff --git a/packages/frontend/src/pages/Tile/components/TableFooter/DeleteRowsButton.tsx b/packages/frontend/src/pages/Tile/components/TableFooter/DeleteRowsButton.tsx index 19d9407d5..ea902f2ac 100644 --- a/packages/frontend/src/pages/Tile/components/TableFooter/DeleteRowsButton.tsx +++ b/packages/frontend/src/pages/Tile/components/TableFooter/DeleteRowsButton.tsx @@ -24,7 +24,7 @@ export default function DeleteRowsButton({ const [isDialogOpen, setIsDialogOpen] = useState(false) - if (isViewMode) { + if (isViewMode || !rowsSelected.length) { return null } @@ -34,7 +34,6 @@ export default function DeleteRowsButton({ height: ROW_HEIGHT.FOOTER, maxHeight: ROW_HEIGHT.FOOTER, overflow: 'visible', - visibility: rowsSelected.length ? 'visible' : 'hidden', }} >