From 40a1e7fc47b868f1dd39f78d13931b0ed0d53d4e Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Tue, 23 Jan 2024 10:57:42 -0700 Subject: [PATCH] CUMULUS-3432/CUMULUS-3506 - Upgrade to Aurora PG v13 (#3516) (#3553) * CUMULUS-3432/CUMULUS-3506 - Upgrade to Aurora PG v13 (#3516) * Add migration notes, update engine version defaults and parameter group family default * Update migration notes * Add changelog note for ticket * Update migration notes * Update IngestGranuleFailureSpec to match PG v13 failure case * Minor CL edits * Update test stack DB image * Update CL with postgres 13 test stack update * Update PG version to 3.12 * skip-integration-tests minor CL update --------- Co-authored-by: Jonathan Kovarik Co-authored-by: Jonathan Kovarik * Fix merge typo/spacing * Update CHANGELOG * Fix CHANGELOG wording --------- Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- CHANGELOG.md | 41 +++++++++++++++++++ bamboo/docker-compose.yml | 2 +- .../postgres-database-deployment.md | 4 +- example/rds-cluster-tf/variables.tf | 2 +- .../ingestGranule/IngestGranuleFailureSpec.js | 2 +- tf-modules/cumulus-rds-tf/variables.tf | 4 +- 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb4f67adda1..cf8688ce4b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,49 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## Unreleased +### Migration Notes + +For the v16.1 release series, Cumulus Core will be tested against PostgreSQL v13. Users +should migrate their datastores to Aurora PostgreSQL 13.12+ compatible data +stores as soon as possible after upgrading to this release. + +**IMPORTANT** -- When upgrading from v16.1.x users should update to a release +following 18.2.0/the first forward release supporting Postgres v13, as versions +between 16.1.x and 18.2.x+ are unsupported on Aurora Postgres v13. + +#### Engine Upgrade when using `cumulus-rds-tf` + +Users utilizing the `cumulus-rds-tf` module will have upgraded/had their +database clusters forcibly upgraded at the next maintenance window after February 29, 2024. + +To upgrade your engine version, we recommend a manual (outside of +terraform) upgrade. This will result in the cluster being upgraded with a +manually set parameter group not managed by terraform. + +There are several options that AWS provides for upgrading your cluster engine, +such as using the AWS console or CLI. For more information, visit their +[documentation](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless.modifying.html#aurora-serverless.modifying.upgrade). + +Once you have manually upgraded your database engine and the cluster is now on +version 13.12+, to continue using the `cumulus-rds-tf` module *once upgraded*, +update following module configuration values if set, or allow their defaults to +be utilized: + +```terraform +parameter_group_family = "aurora-postgresql13" +engine_version = 13.12 +``` + +**Please Note**: When you apply this update, the original PostgreSQL v11 +parameter group will be removed, and recreated using PG13 defaults/configured +terraform values and it will also update the database cluster to use the new configuration. + ## [v16.1.3] 2024-1-15 +Please note changes in 16.1.3 may not yet be released in future versions, as this +is a backport/patch release on the 16.x series of releases. Updates that are +included in the future will have a corresponding CHANGELOG entry in future releases. + ### Changed - **CUMULUS_3499 diff --git a/bamboo/docker-compose.yml b/bamboo/docker-compose.yml index 75eb1e0a31c..6b9a5aa2cc4 100644 --- a/bamboo/docker-compose.yml +++ b/bamboo/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: postgres: - image: postgres:11.13-alpine + image: postgres:13.9-alpine environment: - POSTGRES_PASSWORD=password network_mode: "service:build_env" diff --git a/docs/deployment/postgres-database-deployment.md b/docs/deployment/postgres-database-deployment.md index 98b5ecf51b5..6f59bdef429 100644 --- a/docs/deployment/postgres-database-deployment.md +++ b/docs/deployment/postgres-database-deployment.md @@ -6,7 +6,7 @@ hide_title: false ## Overview -Cumulus deployments require an Aurora [PostgreSQL 11](https://www.postgresql.org/) compatible database to be provided as the primary data store for Cumulus with Elasticsearch for non-authoritative querying/state data for the API and other applications that require more complex queries. Note that Cumulus is tested with an Aurora Postgres database. +Cumulus deployments require an Aurora [PostgreSQL 13](https://www.postgresql.org/) compatible database to be provided as the primary data store for Cumulus with Elasticsearch for non-authoritative querying/state data for the API and other applications that require more complex queries. Note that Cumulus is tested with an Aurora Postgres database. Users are *strongly* encouraged to plan for and implement a database solution that scales to their use requirements, meets their security posture and maintenance needs and/or allows for multi-tenant cluster usage. @@ -16,7 +16,7 @@ configured [Aurora Serverless](https://aws.amazon.com/rds/aurora/serverless/) cl To that end, Cumulus provides a terraform module [`cumulus-rds-tf`](https://github.com/nasa/cumulus/tree/master/tf-modules/cumulus-rds-tf) -that will deploy an AWS RDS Aurora Serverless PostgreSQL 11 compatible [database cluster](https://aws.amazon.com/rds/aurora/postgresql-features/), and optionally provision a single deployment database with credentialed secrets for use with Cumulus. +that will deploy an AWS RDS Aurora Serverless PostgreSQL 13 compatible [database cluster](https://aws.amazon.com/rds/aurora/postgresql-features/), and optionally provision a single deployment database with credentialed secrets for use with Cumulus. We have provided an example terraform deployment using this module in the [Cumulus template-deploy repository](https://github.com/nasa/cumulus-template-deploy/tree/master/rds-cluster-tf) on GitHub. diff --git a/example/rds-cluster-tf/variables.tf b/example/rds-cluster-tf/variables.tf index 2a6dcc92257..7ab210fa3b0 100644 --- a/example/rds-cluster-tf/variables.tf +++ b/example/rds-cluster-tf/variables.tf @@ -63,7 +63,7 @@ variable "tags" { variable "engine_version" { description = "Postgres engine version for Serverless cluster" type = string - default = "10.12" + default = "13.12" } variable "vpc_tag_name" { diff --git a/example/spec/parallel/ingestGranule/IngestGranuleFailureSpec.js b/example/spec/parallel/ingestGranule/IngestGranuleFailureSpec.js index b3b088636af..a52b4e47fae 100644 --- a/example/spec/parallel/ingestGranule/IngestGranuleFailureSpec.js +++ b/example/spec/parallel/ingestGranule/IngestGranuleFailureSpec.js @@ -252,7 +252,7 @@ describe('The Ingest Granule failure workflow', () => { const isSchemaValidationError = (error.Error === 'CumulusMessageAdapterExecutionError') && error.Cause.includes('jsonschema.exceptions.ValidationError'); const isPostgresWriteError = error.Error.includes('Failed writing files to PostgreSQL') && - error.Cause.includes('null value in column "bucket" violates not-null constraint'); + error.Cause.includes('null value in column "bucket" of relation "files" violates not-null constraint'); expect(isSchemaValidationError || isPostgresWriteError).toBeTrue(); }); }); diff --git a/tf-modules/cumulus-rds-tf/variables.tf b/tf-modules/cumulus-rds-tf/variables.tf index 483fa8ef54e..10d8f972e42 100644 --- a/tf-modules/cumulus-rds-tf/variables.tf +++ b/tf-modules/cumulus-rds-tf/variables.tf @@ -88,13 +88,13 @@ variable "vpc_id" { variable "engine_version" { description = "Postgres engine version for serverless cluster" type = string - default = "11.13" + default = "13.12" } variable "parameter_group_family" { description = "Database family to use for creating database parameter group" type = string - default = "aurora-postgresql11" + default = "aurora-postgresql13" } variable "max_capacity" {