diff --git a/.circleci/config.yml b/.circleci/config.yml index 976dbdaf392..6dd78505f59 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -235,11 +235,11 @@ jobs: npm run start:migration -- $ENV fi - run: - name: Run Postgres Migration + name: Run Migration Postgres Expand command: | OUTPUT_ONLY=true npm run deploy:allColors $ENV POSTGRES_HOST=$(cat ./web-api/terraform/applyables/allColors/output.json | jq -r '.rds_host_name.value') - POSTGRES_HOST=$POSTGRES_HOST POSTGRES_USER=$POSTGRES_USER NODE_ENV=production npm run migration:postgres + POSTGRES_HOST=$POSTGRES_HOST POSTGRES_USER=$POSTGRES_USER NODE_ENV=production npm run migration:postgres:expand - run: name: Enable Check Migration Status Cron command: | @@ -555,6 +555,12 @@ jobs: if [ $MIGRATE_FLAG == true ]; then npm run migration:cleanup -- $ENV fi + - run: + name: Run Migration Postgres Contract + command: | + OUTPUT_ONLY=true npm run deploy:allColors $ENV + POSTGRES_HOST=$(cat ./web-api/terraform/applyables/allColors/output.json | jq -r '.rds_host_name.value') + POSTGRES_HOST=$POSTGRES_HOST POSTGRES_USER=$POSTGRES_USER NODE_ENV=production npm run migration:postgres:contract ######################## GLUE JOB: Jobs that run in a lower environment ######################## diff --git a/docs/postgres/Expand and Contract Migrations.md b/docs/postgres/Expand and Contract Migrations.md new file mode 100644 index 00000000000..f445d86b361 --- /dev/null +++ b/docs/postgres/Expand and Contract Migrations.md @@ -0,0 +1,59 @@ +# Expand and Contract Migrations + +https://www.tim-wellhausen.de/papers/ExpandAndContract/ExpandAndContract.html + +Hard rules: +- Always Expand & Contract +- Up, should do what you want aka changes +- Down, always revert back to previous working state + +## Circle Flow (need to implement) + +- Deploy +- Migrate + - Expand Migration +- Switch colors +- Cleanup + - Contract Migration + +## Renaming field that is partially in Postgres (case.caption) +- DynamoDB Migration +- Expand & Contract + - Add an expand migration that adds a new field called body + - Add a contract migration that removes message field + - In database-types.ts, change MessageTable.message to be MessageTable.body + - Update seed data (fixtures) so that message.message is now message.body in + - Update the mapper so that Message.message = message.body from DB + - (OPTIONAL) Update business entity Message.ts to have Message.body instead of Messge.message + - Deploy changes + +## Renaming field that is fully in Postgres (message.message -> message.body) +- Expand & Contract + - Add an expand migration that adds a new field called body + - Add a contract migration that removes message field + - In database-types.ts, change MessageTable.message to be MessageTable.body + - Update seed data (fixtures) so that message.message is now message.body in + - Update the mapper so that Message.message = message.body from DB + - (OPTIONAL) Update business entity Message.ts to have Message.body instead of Messge.message + - Deploy changes + +## Dropping a field that is fully in Postgres +- Contract migration + +## Dropping a field that is partially in Postgres +- DynamoDB Migration +- Contract migration + +## Adding a field that is fully in Postgres +- Expand migration + +## Adding a field that is partially in Postgres +- DynamoDB Migration +- Expand migration + +## Modifying Validation Rules (default values) is partially in Postgres +- TBD + + +## Modifying Validation Rules (default values) is fully in Postgres +- TBD diff --git a/docs/postgres/expand-contract.drawio b/docs/postgres/expand-contract.drawio new file mode 100644 index 00000000000..04ee45d7882 --- /dev/null +++ b/docs/postgres/expand-contract.drawio @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/package.json b/package.json index 8104ebddb82..9c96f47a913 100644 --- a/package.json +++ b/package.json @@ -183,8 +183,9 @@ "maintenance:engage:local": ". ./setup-local-env.sh && npx ts-node --transpile-only scripts/set-maintenance-mode-locally.ts true", "migrate:local": "AWS_ACCESS_KEY_ID=S3RVER AWS_SECRET_ACCESS_KEY=S3RVER SOURCE_TABLE=efcms-local DESTINATION_TABLE=efcms-local-1 ts-node --transpile-only ./web-api/run-local-migration.ts", "migration:cleanup": "./scripts/migration/update-deploy-table-after-migration.sh", - "migration:postgres": "npx ts-node --transpile-only ./web-api/src/persistence/postgres/utils/migrate/migrate.ts", - "migration:rollback:postgres": "npx ts-node --transpile-only ./web-api/src/persistence/postgres/utils//rollback.ts", + "migration:postgres:expand": "npx ts-node --transpile-only ./web-api/src/persistence/postgres/utils/migrate/migrate.ts expand", + "migration:postgres:contract": "npx ts-node --transpile-only ./web-api/src/persistence/postgres/utils/migrate/migrate.ts contract", + "migration:rollback:postgres": "npx ts-node --transpile-only ./web-api/src/persistence/postgres/utils/migrate/rollback.ts", "pending-color-switch": "./web-client/pending-color-switch.sh", "postinstall": "husky install", "print:success": "echo 'Build successful. You can access the site at http://localhost:1234'", diff --git a/run-local.sh b/run-local.sh index 27279c6f2d4..c27260cb87f 100755 --- a/run-local.sh +++ b/run-local.sh @@ -55,7 +55,8 @@ else fi fi -npm run migration:postgres +npm run migration:postgres:expand +npm run migration:postgres:contract npm run seed:postgres echo "Seeding cognito-local users" diff --git a/web-api/src/database-types.ts b/web-api/src/database-types.ts index c1bee19ec32..53f949a7aec 100644 --- a/web-api/src/database-types.ts +++ b/web-api/src/database-types.ts @@ -21,7 +21,7 @@ export interface MessageTable { isRead: boolean; isRepliedTo: boolean; leadDocketNumber?: string; - message: string; + body: string; messageId: string; parentMessageId: string; subject: string; diff --git a/web-api/src/persistence/postgres/messages/mapper.ts b/web-api/src/persistence/postgres/messages/mapper.ts index de02d27ec7a..0c5ad39e68d 100644 --- a/web-api/src/persistence/postgres/messages/mapper.ts +++ b/web-api/src/persistence/postgres/messages/mapper.ts @@ -4,9 +4,10 @@ import { NewMessageKysely, UpdateMessageKysely } from '@web-api/database-types'; import { RawMessage } from '@shared/business/entities/Message'; import { transformNullToUndefined } from '@web-api/persistence/postgres/utils/transformNullToUndefined'; -function pickFields(message) { +function pickFields(message): NewMessageKysely { return { attachments: JSON.stringify(message.attachments), + body: message.message, completedAt: message.completedAt, completedBy: message.completedBy, completedBySection: message.completedBySection, @@ -20,7 +21,6 @@ function pickFields(message) { isCompleted: message.isCompleted, isRead: message.isRead, isRepliedTo: message.isRepliedTo, - message: message.message, messageId: message.messageId, parentMessageId: message.parentMessageId, subject: message.subject, @@ -60,6 +60,7 @@ export function messageResultEntity(message) { caseTitle: Case.getCaseTitle(message.caption || ''), completedAt: message.completedAt?.toISOString(), createdAt: message.createdAt.toISOString(), + message: message.body, trialDate: message.trialDate?.toISOString(), }), ); diff --git a/web-api/src/persistence/postgres/utils/migrate/migrate.ts b/web-api/src/persistence/postgres/utils/migrate/migrate.ts index 823beb2a723..f0372a98e4d 100644 --- a/web-api/src/persistence/postgres/utils/migrate/migrate.ts +++ b/web-api/src/persistence/postgres/utils/migrate/migrate.ts @@ -3,7 +3,11 @@ import { FileMigrationProvider, Migrator } from 'kysely'; import { promises as fs } from 'fs'; import { getDbWriter } from '../../../../database'; -async function migrateToLatest() { +async function migrateToLatest(migrationType: string) { + if (migrationType !== 'expand' && migrationType !== 'contract') { + throw new Error(`Unable to run unknown migration type: ${migrationType}`); + } + await getDbWriter(async writer => { const migrator = new Migrator({ db: writer, @@ -14,26 +18,65 @@ async function migrateToLatest() { }), }); - const { error, results } = await migrator.migrateToLatest(); + const migrations = await migrator.getMigrations(); - results?.forEach(it => { - if (it.status === 'Success') { - console.log( - `migration "${it.migrationName}" was executed successfully`, - ); - } else if (it.status === 'Error') { - console.error(`failed to execute migration "${it.migrationName}"`); - } - }); + for (const migration of migrations) { + if ( + migration.name.includes(`.${migrationType}`) && + migration.executedAt === undefined + ) { + const { error, results } = await migrator.migrateTo(migration.name); + results?.forEach(it => { + if (it.status === 'Success') { + console.log( + `migration "${it.migrationName}" was executed successfully`, + ); + } else if (it.status === 'Error') { + console.error(`failed to execute migration "${it.migrationName}"`); + } + }); - if (error) { - console.error('failed to migrate'); - console.error(error); - process.exit(1); + if (error) { + console.error('failed to migrate'); + console.error(error); + process.exit(1); + } + } } await writer.destroy(); }); } -migrateToLatest().catch; +migrateToLatest(process.argv[2]).catch; + +// eslint-disable-next-line spellcheck/spell-checker +/* +DB Now +0001-init 1st +0002-init-indexes 2nd +293474924-init-indexes 3rd +nov5-init-indexes 4th +nov3-init-indexes 5th + +0003-add-message-body.expand.ts +0004-add-something.expand.ts +0003-add-message-body.contract.ts +0004-add-something.contract.ts + +0001-init.expand +0002-init-indexes.expand +0003-add-message-body.expand.ts +0004-add-something.expand.ts +0003-add-message-body.contract.ts +0004-add-something.contract.ts + +In Repo +0001-init.expand +0002-init-indexes.exapand +0003-add-message-body.expand.ts +0003-remove-message-message.contract.ts +0004-add-something.expand.ts +0004-remove-something.contract.ts + +*/ diff --git a/web-api/src/persistence/postgres/utils/migrate/migrations/0001-init.ts b/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_01-init.expand.ts similarity index 100% rename from web-api/src/persistence/postgres/utils/migrate/migrations/0001-init.ts rename to web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_01-init.expand.ts diff --git a/web-api/src/persistence/postgres/utils/migrate/migrations/0002-init-indexes.ts b/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_02-init-indexes.ts similarity index 100% rename from web-api/src/persistence/postgres/utils/migrate/migrations/0002-init-indexes.ts rename to web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_02-init-indexes.ts diff --git a/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_22-add-message-body.expand.ts b/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_22-add-message-body.expand.ts new file mode 100644 index 00000000000..95ca24b4674 --- /dev/null +++ b/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_22-add-message-body.expand.ts @@ -0,0 +1,15 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('dwMessage').addColumn('body', 'text').execute(); + await db + .updateTable('dwMessage') + .set({ + body: sql`message`, + }) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('dwMessage').dropColumn('body').execute(); +} diff --git a/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_22-remove-message-message.contract.ts b/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_22-remove-message-message.contract.ts new file mode 100644 index 00000000000..e7b0f64dfb9 --- /dev/null +++ b/web-api/src/persistence/postgres/utils/migrate/migrations/2024_10_22-remove-message-message.contract.ts @@ -0,0 +1,18 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('dwMessage').dropColumn('message').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable('dwMessage') + .addColumn('message', 'varchar') + .execute(); + await db + .updateTable('dwMessage') + .set({ + message: sql`body`, + }) + .execute(); +} diff --git a/web-api/src/persistence/postgres/utils/seed/fixtures/messages.ts b/web-api/src/persistence/postgres/utils/seed/fixtures/messages.ts index a8d86b14a18..2ace88f85ca 100644 --- a/web-api/src/persistence/postgres/utils/seed/fixtures/messages.ts +++ b/web-api/src/persistence/postgres/utils/seed/fixtures/messages.ts @@ -1,10 +1,11 @@ import { NewMessageKysely } from '@web-api/database-types'; +import { calculateDate } from '@shared/business/utilities/DateHandler'; -/* eslint-disable @miovision/disallow-date/no-new-date */ export const messages: NewMessageKysely[] = [ { attachments: JSON.stringify([]), - createdAt: new Date('2020-06-05T18:02:25.280Z'), + body: 'hello!', + createdAt: calculateDate({ dateString: '2020-06-05T18:02:25.280Z' }), docketNumber: '105-20', from: 'Test Petitionsclerk', fromSection: 'petitions', @@ -12,7 +13,6 @@ export const messages: NewMessageKysely[] = [ isCompleted: false, isRead: false, isRepliedTo: false, - message: 'hello!', messageId: 'eb0a139a-8951-4de1-8b83-f02a27504105', parentMessageId: 'eb0a139a-8951-4de1-8b83-f02a27504105', subject: 'message to myself', @@ -26,7 +26,8 @@ export const messages: NewMessageKysely[] = [ documentId: '4796a931-14fb-43e6-948f-d2b67ce4c1cb', }, ]), - createdAt: new Date('2023-06-02T21:15:50.105Z'), + body: 'Could you please review this?', + createdAt: calculateDate({ dateString: '2023-06-02T21:15:50.105Z' }), docketNumber: '103-20', from: 'Test Admissions Clerk', fromSection: 'admissions', @@ -34,7 +35,6 @@ export const messages: NewMessageKysely[] = [ isCompleted: false, isRead: false, isRepliedTo: false, - message: 'Could you please review this?', messageId: '1d4c1fd9-5265-4e46-894f-b8426d3a6836', parentMessageId: '1d4c1fd9-5265-4e46-894f-b8426d3a6836', subject: 'Administrative Record', @@ -48,7 +48,8 @@ export const messages: NewMessageKysely[] = [ documentId: '8ed9bad9-db58-43c8-b03f-c2e3ad92995f', }, ]), - createdAt: new Date('2020-08-18T18:07:36.333Z'), + body: 'Test message with deleted document.', + createdAt: calculateDate({ dateString: '2020-08-18T18:07:36.333Z' }), docketNumber: '104-19', from: 'Test Docketclerk', fromSection: 'docket', @@ -56,7 +57,6 @@ export const messages: NewMessageKysely[] = [ isCompleted: false, isRead: false, isRepliedTo: false, - message: 'Test message with deleted document.', messageId: '2d1191d3-4597-454a-a2b2-84e267ccf01e', parentMessageId: '2d1191d3-4597-454a-a2b2-84e267ccf01e', subject: 'Order',