diff --git a/backend/cloneStories.sh b/backend/cloneStories.sh new file mode 100755 index 00000000..8eb02c34 --- /dev/null +++ b/backend/cloneStories.sh @@ -0,0 +1,14 @@ +#!/bin/bash +PGPASSWORD=$(aws ssm get-parameter --name fourtiesnyc-production-db-password --query Parameter.Value --output text --with-decryption) \ +pg_dump \ +--clean --if-exists --table stories \ + --host $(aws ssm get-parameter --name fourtiesnyc-production-db-host --query Parameter.Value --output text --with-decryption) \ + --port $(aws ssm get-parameter --name fourtiesnyc-production-db-port --query Parameter.Value --output text --with-decryption) \ + --username $(aws ssm get-parameter --name fourtiesnyc-production-db-username --query Parameter.Value --output text --with-decryption) \ + --dbname $(aws ssm get-parameter --name fourtiesnyc-production-db-database --query Parameter.Value --output text --with-decryption) \ + | \ +PGPASSWORD=$(aws ssm get-parameter --name fourtiesnyc-staging-db-password --query Parameter.Value --output text --with-decryption) \ +psql --host $(aws ssm get-parameter --name fourtiesnyc-staging-db-host --query Parameter.Value --output text --with-decryption) \ + --port $(aws ssm get-parameter --name fourtiesnyc-staging-db-port --query Parameter.Value --output text --with-decryption) \ + --username $(aws ssm get-parameter --name fourtiesnyc-staging-db-username --query Parameter.Value --output text --with-decryption) \ + --dbname $(aws ssm get-parameter --name fourtiesnyc-staging-db-database --query Parameter.Value --output text --with-decryption) \ No newline at end of file diff --git a/backend/src/api/photos/PhotosController.ts b/backend/src/api/photos/PhotosController.ts index 7fbe65d8..25faac52 100644 --- a/backend/src/api/photos/PhotosController.ts +++ b/backend/src/api/photos/PhotosController.ts @@ -11,9 +11,12 @@ import { SuccessResponse, } from 'tsoa'; import { getRepository, In } from 'typeorm'; +import Paginated from '../../business/pagination/Paginated'; +import mapPaginated from '../../business/utils/mapPaginated'; import Photo from '../../entities/Photo'; import Collection from '../../enum/Collection'; import getLngLatForIdentifier from '../../repositories/getLngLatForIdentifier'; +import { getPaginated } from '../../repositories/paginationUtils'; import { PhotoApiModel } from './PhotoApiModel'; import photoToApi from './photoToApi'; @@ -100,18 +103,36 @@ export class PhotosController extends Controller { @Get('/outtake-summaries') public async getOuttakeSummaries( - @Query() collection: Collection = Collection.FOURTIES - ): Promise<{ identifier: string }[]> { + @Query() collection: Collection = Collection.FOURTIES, + + // pagination + @Query('pageToken') pageToken?: string, + @Query('pageSize') pageSize = 100 + ): Promise> { const photoRepo = getRepository(Photo); - const photos = await photoRepo.find({ - where: { isOuttake: true, collection: collection }, - select: ['identifier'], - order: { - identifier: 'ASC', + const qb = photoRepo + .createQueryBuilder('photo') + .select(['photo.identifier']) + .where({ isOuttake: true, collection: collection }); + + const page = await getPaginated( + qb, + { + key: 'identifier', + sortDirection: 'ASC', + getSerializedToken: (photo) => photo.identifier, + deserializeToken: (token) => token, }, - }); - return photos.map(({ identifier }) => ({ identifier })); + { + pageToken, + pageSize, + } + ); + + return mapPaginated(page, ({ identifier }: Pick) => ({ + identifier, + })); } @Get('/{identifier}') diff --git a/backend/src/api/stories/StoriesController.ts b/backend/src/api/stories/StoriesController.ts index 16a9b893..fb5bdd5e 100644 --- a/backend/src/api/stories/StoriesController.ts +++ b/backend/src/api/stories/StoriesController.ts @@ -16,7 +16,10 @@ import { Security, } from 'tsoa'; +import { UserData as NetlifyUserData } from 'gotrue-js'; import { BadRequest, Forbidden, NotFound } from 'http-errors'; +import EmailCampaignService from '../../business/email/EmailCampaignService'; +import Paginated from '../../business/pagination/Paginated'; import { backfillUserStoryEmails, onStateTransition, @@ -26,6 +29,9 @@ import { verifyStoryToken, } from '../../business/stories/StoryTokenService'; import { validateRecaptchaToken } from '../../business/utils/grecaptcha'; +import mapPaginated from '../../business/utils/mapPaginated'; +import normalizeEmail from '../../business/utils/normalizeEmail'; +import required from '../../business/utils/required'; import Story from '../../entities/Story'; import StoryState from '../../enum/StoryState'; import StoryType from '../../enum/StoryType'; @@ -42,10 +48,6 @@ import { toDraftStoryResponse, toPublicStoryResponse, } from './storyToApi'; -import EmailCampaignService from '../../business/email/EmailCampaignService'; -import normalizeEmail from '../../business/utils/normalizeEmail'; -import { UserData as NetlifyUserData } from 'gotrue-js'; -import required from '../../business/utils/required'; function updateModelFromRequest( story: Story, @@ -213,19 +215,29 @@ export class StoriesController extends Controller { @Get('/') public async getStories( - @Query('forPhotoIdentifier') identifier?: string - ): Promise { - let stories: Story[]; + @Query('forPhotoIdentifier') identifier?: string, + + // pagination + @Query('pageToken') pageToken?: string, + @Query('pageSize') pageSize = 100 + ): Promise> { + let stories: Paginated; + + const paginationInput = { + pageToken, + pageSize, + }; if (identifier) { stories = await StoryRepository().findPublishedForPhotoIdentifier( - identifier + identifier, + paginationInput ); } else { - stories = await StoryRepository().findPublished(); + stories = await StoryRepository().findPublished(paginationInput); } - return map(stories, toPublicStoryResponse); + return mapPaginated(stories, toPublicStoryResponse); } @Security('netlify', ['moderator']) diff --git a/backend/src/business/pagination/Paginated.ts b/backend/src/business/pagination/Paginated.ts new file mode 100644 index 00000000..37509f11 --- /dev/null +++ b/backend/src/business/pagination/Paginated.ts @@ -0,0 +1,8 @@ +interface Paginated { + items: T[]; + total: number; + hasNextPage: boolean; + nextToken?: string; +} + +export default Paginated; diff --git a/backend/src/business/pagination/PaginationInput.ts b/backend/src/business/pagination/PaginationInput.ts new file mode 100644 index 00000000..5e08ce46 --- /dev/null +++ b/backend/src/business/pagination/PaginationInput.ts @@ -0,0 +1,6 @@ +interface PaginationInput { + pageToken?: string; + pageSize: number; +} + +export default PaginationInput; diff --git a/backend/src/business/pagination/PaginationOptions.ts b/backend/src/business/pagination/PaginationOptions.ts new file mode 100644 index 00000000..3b26bcc7 --- /dev/null +++ b/backend/src/business/pagination/PaginationOptions.ts @@ -0,0 +1,10 @@ +import { ObjectLiteral } from 'typeorm'; + +interface PaginationOptions { + key: string; + sortDirection?: 'ASC' | 'DESC'; + getSerializedToken: (entity: E) => string; + deserializeToken: (token: string) => unknown; +} + +export default PaginationOptions; diff --git a/backend/src/business/utils/mapPaginated.ts b/backend/src/business/utils/mapPaginated.ts new file mode 100644 index 00000000..45b215d5 --- /dev/null +++ b/backend/src/business/utils/mapPaginated.ts @@ -0,0 +1,12 @@ +import map from 'lodash/map'; +import Paginated from '../pagination/Paginated'; + +export default function mapPaginated( + paginated: Paginated, + fn: (I) => O +): Paginated { + return { + ...paginated, + items: map(paginated.items, fn), + }; +} diff --git a/backend/src/repositories/StoryRepository.ts b/backend/src/repositories/StoryRepository.ts index aa7096b2..bb198895 100644 --- a/backend/src/repositories/StoryRepository.ts +++ b/backend/src/repositories/StoryRepository.ts @@ -1,20 +1,24 @@ import { Brackets, getRepository, In, IsNull, Repository } from 'typeorm'; +import Paginated from '../business/pagination/Paginated'; +import PaginationInput from '../business/pagination/PaginationInput'; import Story from '../entities/Story'; import StoryState from '../enum/StoryState'; import getLngLatForIdentifier from './getLngLatForIdentifier'; +import { getPaginated } from './paginationUtils'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -- better type is inferred const StoryRepository = () => getRepository(Story).extend({ async findPublishedForPhotoIdentifier( this: Repository, - identifier: string + identifier: string, + pagination: PaginationInput ) { // Get the lng,lat for this photo, so we can return stories // for this photo and stories for other photos in the same location const maybeLngLat = await getLngLatForIdentifier(identifier); - return this.createQueryBuilder('story') + const qb = this.createQueryBuilder('story') .where({ state: StoryState.PUBLISHED }) .andWhere( new Brackets((qb) => { @@ -31,19 +35,39 @@ const StoryRepository = () => ) .leftJoinAndSelect('story.photo', 'photo') .leftJoinAndSelect('photo.effectiveAddress', 'effectiveAddress') - .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode') - .orderBy('story.created_at', 'DESC') - .getMany(); + .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode'); + return getPaginated( + qb, + { + key: 'createdAt', + sortDirection: 'DESC', + getSerializedToken: (story) => story.createdAt.toISOString(), + deserializeToken: (token) => Date.parse(token), + }, + pagination + ); }, - async findPublished(this: Repository) { - return this.createQueryBuilder('story') + async findPublished( + this: Repository, + pagination: PaginationInput + ): Promise> { + const qb = this.createQueryBuilder('story') .where({ state: StoryState.PUBLISHED }) - .orderBy('story.created_at', 'DESC') .leftJoinAndSelect('story.photo', 'photo') .leftJoinAndSelect('photo.effectiveAddress', 'effectiveAddress') - .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode') - .getMany(); + .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode'); + + return getPaginated( + qb, + { + key: 'createdAt', + sortDirection: 'DESC', + getSerializedToken: (story) => story.createdAt.toISOString(), + deserializeToken: (token) => new Date(token), + }, + pagination + ); }, /** diff --git a/backend/src/repositories/paginationUtils.ts b/backend/src/repositories/paginationUtils.ts new file mode 100644 index 00000000..a9ba18f8 --- /dev/null +++ b/backend/src/repositories/paginationUtils.ts @@ -0,0 +1,60 @@ +import last from 'lodash/last'; +import { + FindOperator, + IsNull, + LessThan, + MoreThan, + Not, + ObjectLiteral, + SelectQueryBuilder, +} from 'typeorm'; +import Paginated from '../business/pagination/Paginated'; +import PaginationInput from '../business/pagination/PaginationInput'; +import PaginationOptions from '../business/pagination/PaginationOptions'; +import required from '../business/utils/required'; + +export async function getPaginated( + qb: SelectQueryBuilder, + { + key, + sortDirection, + getSerializedToken, + deserializeToken, + }: PaginationOptions, + { pageToken: nextToken, pageSize }: PaginationInput +): Promise> { + let filterOp: FindOperator = Not(IsNull()); + if (nextToken) { + const tokenDeserialized = deserializeToken(nextToken); + filterOp = + sortDirection === 'ASC' + ? MoreThan(tokenDeserialized) + : LessThan(tokenDeserialized); + } + + const hasWhere = qb.expressionMap.wheres.length > 0; + + // Note clone doesn't actually seem to do anything, so it's important to do count before adding where clauses + const count = await qb.clone().getCount(); + + const results = await qb + .clone() + .orderBy(`${qb.alias}.${key}`, sortDirection) + [hasWhere ? 'andWhere' : 'where']({ + [key]: filterOp, + }) + .take(pageSize + 1) + .getMany(); + + const items = results.slice(0, pageSize); + const hasNextPage = results.length > pageSize; + const nextNextToken = hasNextPage + ? getSerializedToken(required(last(items), 'last')) + : undefined; + return { + items, + total: count, + hasNextPage, + nextToken: nextNextToken, + }; +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 35589287..75af41ad 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -33,6 +33,7 @@ "react-transition-group": "^4.4.1", "react-virtualized-auto-sizer": "^1.0.7", "react-window": "^1.8.7", + "react-window-infinite-loader": "^1.0.9", "zustand": "^4.1.5" }, "devDependencies": { @@ -48,11 +49,13 @@ "@types/copy-webpack-plugin": "^6.4.0", "@types/core-js": "^2.5.5", "@types/eslint": "^8.4.7", + "@types/eslint-config-prettier": "~6.11.3", "@types/eslint-plugin-prettier": "^3.1.0", "@types/geojson": "^7946.0.12", "@types/gtag.js": "0.0.12", "@types/html-webpack-plugin": "^3.2.6", "@types/less": "^3.0.2", + "@types/lint-staged": "~13.3.0", "@types/lodash": "^4.14.191", "@types/mapbox-gl": "^1.13.0", "@types/mini-css-extract-plugin": "^2.5.1", @@ -63,10 +66,12 @@ "@types/react-autosuggest": "^10.1.5", "@types/react-dom": "^18", "@types/react-modal": "^3.13.1", + "@types/react-refresh": "~0.14.5", "@types/react-router-dom": "^5.3.3", "@types/react-transition-group": "^4.4.7", "@types/react-virtualized-auto-sizer": "^1.0.1", "@types/react-window": "^1.8.5", + "@types/react-window-infinite-loader": "~1.0.9", "@types/webpack": "^5.28.0", "@types/webpack-bundle-analyzer": "^4.6.0", "@types/webpack-dev-server": "^3.11.2", @@ -2698,6 +2703,12 @@ "@types/json-schema": "*" } }, + "node_modules/@types/eslint-config-prettier": { + "version": "6.11.3", + "resolved": "https://registry.npmjs.org/@types/eslint-config-prettier/-/eslint-config-prettier-6.11.3.tgz", + "integrity": "sha512-3wXCiM8croUnhg9LdtZUJQwNcQYGWxxdOWDjPe1ykCqJFPVpzAKfs/2dgSoCtAvdPeaponcWPI7mPcGGp9dkKQ==", + "dev": true + }, "node_modules/@types/eslint-plugin-prettier": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@types/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.0.tgz", @@ -2838,6 +2849,12 @@ "integrity": "sha512-62vfe65cMSzYaWmpmhqCMMNl0khen89w57mByPi1OseGfcV/LV03fO8YVrNj7rFQsRWNJo650WWyh6m7p8vZmA==", "dev": true }, + "node_modules/@types/lint-staged": { + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/@types/lint-staged/-/lint-staged-13.3.0.tgz", + "integrity": "sha512-WxGjVP+rA4OJlEdbZdT9MS9PFKQ7kVPhLn26gC+2tnBWBEFEj/KW+IbFfz6sxdxY5U6V7BvyF+3BzCGsAMHhNg==", + "dev": true + }, "node_modules/@types/lodash": { "version": "4.14.191", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", @@ -2941,6 +2958,16 @@ "@types/react": "*" } }, + "node_modules/@types/react-refresh": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/@types/react-refresh/-/react-refresh-0.14.5.tgz", + "integrity": "sha512-ccDC4ikvxnRi7We6Lf3hWJrgF7PdPms9HEQPdKYAjxzh26LPD0j+8QQOjDQpFK6P7a3iaZjZDJAoaG7HvGAbxw==", + "dev": true, + "dependencies": { + "@types/babel__core": "*", + "csstype": "^3.0.2" + } + }, "node_modules/@types/react-router": { "version": "5.1.20", "resolved": "https://registry.npmjs.org/@types/react-router/-/react-router-5.1.20.tgz", @@ -2988,6 +3015,16 @@ "@types/react": "*" } }, + "node_modules/@types/react-window-infinite-loader": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/react-window-infinite-loader/-/react-window-infinite-loader-1.0.9.tgz", + "integrity": "sha512-gEInTjQwURCnDOFyIEK2+fWB5gTjqwx30O62QfxA9stE5aiB6EWkGj4UMhc0axq7/FV++Gs/TGW8FtgEx0S6Tw==", + "dev": true, + "dependencies": { + "@types/react": "*", + "@types/react-window": "*" + } + }, "node_modules/@types/relateurl": { "version": "0.2.29", "resolved": "https://registry.npmjs.org/@types/relateurl/-/relateurl-0.2.29.tgz", @@ -10483,6 +10520,18 @@ "react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/react-window-infinite-loader": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.9.tgz", + "integrity": "sha512-5Hg89IdU4Vrp0RT8kZYKeTIxWZYhNkVXeI1HbKo01Vm/Z7qztDvXljwx16sMzsa9yapRJQW3ODZfMUw38SOWHw==", + "engines": { + "node": ">8.0.0" + }, + "peerDependencies": { + "react": "^15.3.0 || ^16.0.0-alpha || ^17.0.0 || ^18.0.0", + "react-dom": "^15.3.0 || ^16.0.0-alpha || ^17.0.0 || ^18.0.0" + } + }, "node_modules/readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -15077,6 +15126,12 @@ "@types/json-schema": "*" } }, + "@types/eslint-config-prettier": { + "version": "6.11.3", + "resolved": "https://registry.npmjs.org/@types/eslint-config-prettier/-/eslint-config-prettier-6.11.3.tgz", + "integrity": "sha512-3wXCiM8croUnhg9LdtZUJQwNcQYGWxxdOWDjPe1ykCqJFPVpzAKfs/2dgSoCtAvdPeaponcWPI7mPcGGp9dkKQ==", + "dev": true + }, "@types/eslint-plugin-prettier": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@types/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.0.tgz", @@ -15219,6 +15274,12 @@ "integrity": "sha512-62vfe65cMSzYaWmpmhqCMMNl0khen89w57mByPi1OseGfcV/LV03fO8YVrNj7rFQsRWNJo650WWyh6m7p8vZmA==", "dev": true }, + "@types/lint-staged": { + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/@types/lint-staged/-/lint-staged-13.3.0.tgz", + "integrity": "sha512-WxGjVP+rA4OJlEdbZdT9MS9PFKQ7kVPhLn26gC+2tnBWBEFEj/KW+IbFfz6sxdxY5U6V7BvyF+3BzCGsAMHhNg==", + "dev": true + }, "@types/lodash": { "version": "4.14.191", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", @@ -15321,6 +15382,16 @@ "@types/react": "*" } }, + "@types/react-refresh": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/@types/react-refresh/-/react-refresh-0.14.5.tgz", + "integrity": "sha512-ccDC4ikvxnRi7We6Lf3hWJrgF7PdPms9HEQPdKYAjxzh26LPD0j+8QQOjDQpFK6P7a3iaZjZDJAoaG7HvGAbxw==", + "dev": true, + "requires": { + "@types/babel__core": "*", + "csstype": "^3.0.2" + } + }, "@types/react-router": { "version": "5.1.20", "resolved": "https://registry.npmjs.org/@types/react-router/-/react-router-5.1.20.tgz", @@ -15368,6 +15439,16 @@ "@types/react": "*" } }, + "@types/react-window-infinite-loader": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/react-window-infinite-loader/-/react-window-infinite-loader-1.0.9.tgz", + "integrity": "sha512-gEInTjQwURCnDOFyIEK2+fWB5gTjqwx30O62QfxA9stE5aiB6EWkGj4UMhc0axq7/FV++Gs/TGW8FtgEx0S6Tw==", + "dev": true, + "requires": { + "@types/react": "*", + "@types/react-window": "*" + } + }, "@types/relateurl": { "version": "0.2.29", "resolved": "https://registry.npmjs.org/@types/relateurl/-/relateurl-0.2.29.tgz", @@ -20927,6 +21008,12 @@ "memoize-one": ">=3.1.1 <6" } }, + "react-window-infinite-loader": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.9.tgz", + "integrity": "sha512-5Hg89IdU4Vrp0RT8kZYKeTIxWZYhNkVXeI1HbKo01Vm/Z7qztDvXljwx16sMzsa9yapRJQW3ODZfMUw38SOWHw==", + "requires": {} + }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 805ab009..e5719a0b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -41,6 +41,7 @@ "react-transition-group": "^4.4.1", "react-virtualized-auto-sizer": "^1.0.7", "react-window": "^1.8.7", + "react-window-infinite-loader": "^1.0.9", "zustand": "^4.1.5" }, "devDependencies": { @@ -56,11 +57,13 @@ "@types/copy-webpack-plugin": "^6.4.0", "@types/core-js": "^2.5.5", "@types/eslint": "^8.4.7", + "@types/eslint-config-prettier": "~6.11.3", "@types/eslint-plugin-prettier": "^3.1.0", "@types/geojson": "^7946.0.12", "@types/gtag.js": "0.0.12", "@types/html-webpack-plugin": "^3.2.6", "@types/less": "^3.0.2", + "@types/lint-staged": "~13.3.0", "@types/lodash": "^4.14.191", "@types/mapbox-gl": "^1.13.0", "@types/mini-css-extract-plugin": "^2.5.1", @@ -71,10 +74,12 @@ "@types/react-autosuggest": "^10.1.5", "@types/react-dom": "^18", "@types/react-modal": "^3.13.1", + "@types/react-refresh": "~0.14.5", "@types/react-router-dom": "^5.3.3", "@types/react-transition-group": "^4.4.7", "@types/react-virtualized-auto-sizer": "^1.0.1", "@types/react-window": "^1.8.5", + "@types/react-window-infinite-loader": "~1.0.9", "@types/webpack": "^5.28.0", "@types/webpack-bundle-analyzer": "^4.6.0", "@types/webpack-dev-server": "^3.11.2", diff --git a/frontend/src/screens/App/screens/AllStories/index.tsx b/frontend/src/screens/App/screens/AllStories/index.tsx index 4826fb57..dfc4d818 100644 --- a/frontend/src/screens/App/screens/AllStories/index.tsx +++ b/frontend/src/screens/App/screens/AllStories/index.tsx @@ -6,9 +6,10 @@ import { Link, useHistory, useParams } from 'react-router-dom'; import { Story } from 'screens/App/shared/types/Story'; import Grid from 'shared/components/Grid'; import StoryView from 'shared/components/Story'; +import Paginated from 'shared/types/Paginated'; +import { PHOTO_BASE } from 'shared/utils/apiConstants'; import { getAllStories } from 'shared/utils/StoryApi'; import stylesheet from './AllStories.less'; -import { PHOTO_BASE } from 'shared/utils/apiConstants'; const TARGET_WIDTH = 420; const ASPECT = 420 / 630; @@ -18,13 +19,61 @@ export default function Outtakes({ }: { className?: string; }): JSX.Element { + const nextToken = React.useRef(undefined); + const [storiesPage, setStoriesPage] = React.useState | null>( + null + ); const [stories, setStories] = React.useState([]); + const isLoading = React.useRef(false); - React.useEffect(() => { - void getAllStories().then((data) => { - setStories(data); - }); - }, []); + const handleNewPage = React.useCallback( + (newPage: Paginated) => { + nextToken.current = newPage.nextToken; + setStoriesPage(newPage); + setStories([...stories, ...newPage.items]); + }, + [stories] + ); + + React.useEffect( + () => { + isLoading.current = true; + void getAllStories() + .then(handleNewPage) + .finally(() => { + isLoading.current = false; + }); + }, + + // eslint-disable-next-line react-hooks/exhaustive-deps + [] + ); + + const loadNextPage = React.useCallback( + async (pageSize = 20) => { + if (!storiesPage) return; + if (!storiesPage.hasNextPage) return; + if (isLoading.current) { + return; + } + + isLoading.current = true; + + return getAllStories(Math.max(pageSize, 100), nextToken.current) + .then((data) => handleNewPage(data)) + .finally(() => { + isLoading.current = false; + }); + }, + [storiesPage, handleNewPage] + ); + + const handleLoadMoreItems = React.useCallback( + (untilIndex: number) => { + return loadNextPage(untilIndex - stories.length + 1); + }, + [stories, loadNextPage] + ); const history = useHistory(); const { identifier: selectedIdentifier } = useParams<{ @@ -49,10 +98,15 @@ export default function Outtakes({
{ + if (!story) { + return
; + } const identifier = story.photo; return (
diff --git a/frontend/src/screens/App/screens/Outtakes/index.tsx b/frontend/src/screens/App/screens/Outtakes/index.tsx index dd0c281e..aabf8e96 100644 --- a/frontend/src/screens/App/screens/Outtakes/index.tsx +++ b/frontend/src/screens/App/screens/Outtakes/index.tsx @@ -6,6 +6,7 @@ import { getOuttakeSummaries, PhotoSummary } from 'shared/utils/photosApi'; import { Link, useHistory, useParams } from 'react-router-dom'; import Grid from 'shared/components/Grid'; +import Paginated from 'shared/types/Paginated'; import { PHOTO_BASE } from 'shared/utils/apiConstants'; import stylesheet from './Outtakes.less'; @@ -17,15 +18,61 @@ export default function Outtakes({ }: { className?: string; }): JSX.Element { + const nextToken = React.useRef(undefined); + const [photoSummariesPage, setPhotoSummariesPage] = + React.useState | null>(null); const [photoSummaries, setPhotoSummaries] = React.useState( [] ); + const isLoading = React.useRef(false); - React.useEffect(() => { - void getOuttakeSummaries().then((data) => { - setPhotoSummaries(data); - }); - }, []); + const handleNewPage = React.useCallback( + (newPage: Paginated): void => { + nextToken.current = newPage.nextToken; + setPhotoSummariesPage(newPage); + setPhotoSummaries([...photoSummaries, ...newPage.items]); + }, + [photoSummaries] + ); + + React.useEffect( + () => { + isLoading.current = true; + void getOuttakeSummaries(500) + .then(handleNewPage) + .finally(() => { + isLoading.current = false; + }); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [] + ); + + const loadNextPage = React.useCallback( + async (pageSize: number) => { + if (!photoSummariesPage) return; + if (!photoSummariesPage.hasNextPage) return; + if (isLoading.current) { + return; + } + + isLoading.current = true; + + return getOuttakeSummaries(Math.max(pageSize, 500), nextToken.current) + .then((data) => handleNewPage(data)) + .finally(() => { + isLoading.current = false; + }); + }, + [photoSummariesPage, handleNewPage] + ); + + const handleLoadMoreItems = React.useCallback( + (untilIndex: number) => { + return loadNextPage(untilIndex - photoSummaries.length + 1); + }, + [photoSummaries, loadNextPage] + ); const history = useHistory(); const { identifier: selectedIdentifier } = useParams<{ @@ -46,7 +93,10 @@ export default function Outtakes({ targetWidth={TARGET_IMAGE_WIDTH} aspectRatio={ASPECT} className={stylesheet.grid} + totalItems={photoSummariesPage?.total ?? 0} + loadMoreItems={handleLoadMoreItems} renderItem={(photoSummary) => { + if (!photoSummary) return null; const { identifier } = photoSummary; return ( { getStoriesForPhoto(photoIdentifier) .then((stories) => { - setStories(stories); + // TODO handle pagination + setStories(stories.items); }) .catch((err) => { console.error(err); diff --git a/frontend/src/shared/components/Grid/index.tsx b/frontend/src/shared/components/Grid/index.tsx index 24cddfa6..b3d6c425 100644 --- a/frontend/src/shared/components/Grid/index.tsx +++ b/frontend/src/shared/components/Grid/index.tsx @@ -5,6 +5,7 @@ import throttle from 'lodash/throttle'; import { FixedSizeList as List } from 'react-window'; import AutoSizer, { Size } from 'react-virtualized-auto-sizer'; +import InfiniteLoader from 'react-window-infinite-loader'; const calculateItemsPerRow = ( targetWidth: number, @@ -25,7 +26,9 @@ type GridProps = { targetWidth: number; aspectRatio: number; items: T[]; + totalItems: number; renderItem: (item: T) => JSX.Element; + loadMoreItems: (upTo: number) => Promise; }; type PrivateProps = { @@ -35,6 +38,7 @@ type PrivateProps = { function Grid({ items, + totalItems, renderItem, className, targetWidth, @@ -45,6 +49,8 @@ function Grid({ visibleItemIRef: visibleImageIRef, listRef, + + loadMoreItems, }: GridProps & Size & PrivateProps): JSX.Element { const itemsPerRow = calculateItemsPerRow(targetWidth, containerWidth); const itemHeight = (1 / aspectRatio) * (containerWidth / itemsPerRow); @@ -67,52 +73,81 @@ function Grid({ { leading: false } ); + const rowCount = Math.ceil(totalItems / itemsPerRow); + const isRowLoaded = (index: number): boolean => { + const firstItemI = index * itemsPerRow; + const lastItemI = Math.min(firstItemI + itemsPerRow - 1, totalItems - 1); + return lastItemI < items.length; + }; + + const handleLoadMoreItems = React.useCallback( + async (_startIndex: number, stopIndex: number) => { + const firstItemI = stopIndex * itemsPerRow; + const lastItemI = Math.min(firstItemI + itemsPerRow - 1, totalItems - 1); + await loadMoreItems(lastItemI); + }, + [itemsPerRow, loadMoreItems, totalItems] + ); + return ( - { - const imageI = visibleStartIndex * itemsPerRow; - if (containerWidth === 0) { - return; - } - // We'll scroll back to here - // Record on a delay to allow time for reflow - updateVisibleImageI(imageI); - }} + - {({ index: rowIndex, style }) => { - const firstItemI = rowIndex * itemsPerRow; - return ( -
- {range( - firstItemI, - Math.min(firstItemI + itemsPerRow, items.length) - ).map((i) => { - const item = items[i]; - return ( -
- {renderItem(item)} -
- ); - })} -
- ); - }} -
+ {({ onItemsRendered, ref }) => ( + { + ref(el); + listRef.current = el; + }} + className={className} + width={containerWidth} + height={containerHeight} + itemSize={itemHeight} + itemCount={rowCount} + useIsScrolling + onItemsRendered={(props) => { + const { visibleStartIndex } = props; + const imageI = visibleStartIndex * itemsPerRow; + if (containerWidth === 0) { + return; + } + // We'll scroll back to here + // Record on a delay to allow time for reflow + updateVisibleImageI(imageI); + onItemsRendered(props); + }} + > + {({ index: rowIndex, style }) => { + const firstItemI = rowIndex * itemsPerRow; + return ( +
+ {range( + firstItemI, + Math.min(firstItemI + itemsPerRow, items.length) + ).map((i) => { + const item = items[i]; + return ( +
+ {renderItem(item)} +
+ ); + })} +
+ ); + }} +
+ )} + ); } diff --git a/frontend/src/shared/types/Paginated.ts b/frontend/src/shared/types/Paginated.ts new file mode 100644 index 00000000..37509f11 --- /dev/null +++ b/frontend/src/shared/types/Paginated.ts @@ -0,0 +1,8 @@ +interface Paginated { + items: T[]; + total: number; + hasNextPage: boolean; + nextToken?: string; +} + +export default Paginated; diff --git a/frontend/src/shared/utils/StoryApi.ts b/frontend/src/shared/utils/StoryApi.ts index cca5b644..91c4efeb 100644 --- a/frontend/src/shared/utils/StoryApi.ts +++ b/frontend/src/shared/utils/StoryApi.ts @@ -1,4 +1,5 @@ import memoize from 'lodash/memoize'; +import Paginated from 'shared/types/Paginated'; import api from 'shared/utils/api'; import { AdminStory, @@ -38,8 +39,8 @@ export async function updateStory( export const getStoriesForPhoto = memoize(async function getStoriesForPhoto( photoIdentifier: string -): Promise { - const resp = await api.get( +): Promise> { + const resp = await api.get>( `/stories?forPhotoIdentifier=${photoIdentifier}` ); return resp.data; @@ -59,12 +60,18 @@ export async function getStoryByToken(storyAuthToken: string): Promise { return resp.data; } -export const getAllStories = memoize(async function getAllStories(): Promise< - Story[] -> { - const resp = await api.get(`/stories`); +export const getAllStories = async function getAllStories( + pageSize = 100, + pageToken?: string +): Promise> { + const resp = await api.get>(`/stories`, { + params: { + pageToken, + pageSize, + }, + }); return resp.data; -}); +}; export async function getStoriesForReview(): Promise { const resp = await api.get('/stories/needs-review'); diff --git a/frontend/src/shared/utils/photosApi.ts b/frontend/src/shared/utils/photosApi.ts index 9831cf0b..f0500a63 100644 --- a/frontend/src/shared/utils/photosApi.ts +++ b/frontend/src/shared/utils/photosApi.ts @@ -1,3 +1,4 @@ +import Paginated from 'shared/types/Paginated'; import api from './api'; type Collection = '1940' | '1980'; @@ -37,8 +38,19 @@ export async function closest(latLng: { return resp.data.identifier; } -export async function getOuttakeSummaries(): Promise { - const resp = await api.get('/photos/outtake-summaries'); +export async function getOuttakeSummaries( + pageSize = 100, + pageToken?: string +): Promise> { + const resp = await api.get>( + '/photos/outtake-summaries', + { + params: { + pageToken, + pageSize, + }, + } + ); return resp.data; }