From 1db13e503e19027fe0c2f9df05e960ab4ef18b0c Mon Sep 17 00:00:00 2001 From: Vinayak Hariharmath Date: Mon, 19 Aug 2024 16:44:27 +0530 Subject: [PATCH] db upgrage: upgrade postgres 15 to postgres 16 The Postgres upgrade from 15 to 16 or any later versions can be done by adding the env variable named POSTGRES_UPGRADE = copy in the pg db pod. More about upgrade process is here: https://www.postgresql.org/docs/current/pgupgrade.html In our case, we achieve the upgrade by setting this env var in noobaa-db-pg STS. Once after upgrade is done, we have to remove the POSTGRES_UPGRADE env from the STS. Otherwise pg db pod fails to come up because the db pod check PG directory version against upgrade image version and exits if there is no upgrade required and POSTGRES_UPGRADE is still provided. To remove this env, we store the upgrade status in noobaa-core CR after the PG upgrade and refer the same status during db reconcile. If the upgrade is already done, we remove this env from desired sts status. This will help to upgrade the db smoothly Signed-off-by: Vinayak Hariharmath --- deploy/internal/statefulset-postgres-db.yaml | 2 + pkg/bundle/deploy.go | 4 +- pkg/options/options.go | 2 +- pkg/system/phase2_creating.go | 85 +++++++++++++++++++- pkg/system/system.go | 25 ------ 5 files changed, 90 insertions(+), 28 deletions(-) diff --git a/deploy/internal/statefulset-postgres-db.yaml b/deploy/internal/statefulset-postgres-db.yaml index c2c2db69a4..f7e354608a 100644 --- a/deploy/internal/statefulset-postgres-db.yaml +++ b/deploy/internal/statefulset-postgres-db.yaml @@ -26,6 +26,8 @@ spec: - name: db image: NOOBAA_DB_IMAGE env: + - name: POSTGRESQL_UPGRADE + value: copy - name: POSTGRESQL_DATABASE value: nbcore - name: LC_COLLATE diff --git a/pkg/bundle/deploy.go b/pkg/bundle/deploy.go index 03b2b68030..77658fb0ad 100644 --- a/pkg/bundle/deploy.go +++ b/pkg/bundle/deploy.go @@ -5070,7 +5070,7 @@ spec: resource: limits.memory ` -const Sha256_deploy_internal_statefulset_postgres_db_yaml = "37a6c36928ba426ca04fd89e1eb2685e10d1a5f65c63ebb40c68a4f5c37645de" +const Sha256_deploy_internal_statefulset_postgres_db_yaml = "46460879b565c80900a09017d145e835ca0491a0c9e465ff0c8b9172535843fe" const File_deploy_internal_statefulset_postgres_db_yaml = `apiVersion: apps/v1 kind: StatefulSet @@ -5100,6 +5100,8 @@ spec: - name: db image: NOOBAA_DB_IMAGE env: + - name: POSTGRESQL_UPGRADE + value: copy - name: POSTGRESQL_DATABASE value: nbcore - name: LC_COLLATE diff --git a/pkg/options/options.go b/pkg/options/options.go index ebb9e46b12..64db8d198f 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -72,7 +72,7 @@ var NooBaaImage = ContainerImage // DBImage is the default db image url // it can be overridden for testing or different registry locations. -var DBImage = "quay.io/sclorg/postgresql-15-c9s" +var DBImage = "quay.io/sclorg/postgresql-16-c9s" // Psql12Image is the default postgres12 db image url // currently it can not be overridden. diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go index b94f0626c5..af9e910c3d 100644 --- a/pkg/system/phase2_creating.go +++ b/pkg/system/phase2_creating.go @@ -223,6 +223,80 @@ func (r *Reconciler) SetDesiredServiceDBForPostgres() error { return nil } +// Check db update status and add or remove the POSTGRES_UPDATE env from STS +func (r *Reconciler) addOrRemovePGUpgradeEnvInSTS() { + if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseFinished { + for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + envVars := container.Env + newEnvVar := corev1.EnvVar{ + Name: "POSTGRESQL_UPGRADE", + Value: "copy", + } + envVars = append(envVars, newEnvVar) + r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = envVars + + break + } + } + + return + } + + if r.NooBaa.Status.PostgresUpdatePhase == nbv1.UpgradePhaseFinished { + for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + newEnvVars := []corev1.EnvVar{} + for _, env := range container.Env { + if env.Name != "POSTGRESQL_UPGRADE" { + newEnvVars = append(newEnvVars, env) + } + } + r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = newEnvVars + + break + } + } + } +} + +func (r *Reconciler) isPostgresDBUpgradeReqd() bool { + if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseFinished { + for _, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers { + if container.Name == "db" { + for _, env := range container.Env { + if env.Name == "POSTGRESQL_UPGRADE" && env.Value == "copy" { + return true + } + } + } + } + } + return false +} + +// GetDesiredDBImage returns the desired DB image according to spec or env or default (in options) +func (r *Reconciler) GetDesiredDBImage(currentImage string) string { + // Postgres upgrade failure workaround + // if the current Postgres image is a postgresql-12 image, use NOOBAA_PSQL12_IMAGE. otherwise use GetdesiredDBImage + if IsPostgresql12Image(currentImage) { + psql12Image, ok := os.LookupEnv("NOOBAA_PSQL_12_IMAGE") + util.Logger().Warnf("The current Postgres image is a postgresql-12 image. using (%s)", psql12Image) + if !ok { + psql12Image = currentImage + util.Logger().Warnf("NOOBAA_PSQL_12_IMAGE is not set. using the current image %s", currentImage) + } + return psql12Image + } + + if r.isPostgresDBUpgradeReqd() { + r.NooBaa.Status.BeforeUpgradeDbImage = r.NooBaa.Spec.DBImage + r.NooBaa.Spec.DBImage = &options.DBImage + } + + return options.DBImage +} + // SetDesiredNooBaaDB updates the NooBaaDB as desired for reconciling func (r *Reconciler) SetDesiredNooBaaDB() error { var NooBaaDBTemplate *appsv1.StatefulSet = nil @@ -239,6 +313,9 @@ func (r *Reconciler) SetDesiredNooBaaDB() error { NooBaaDB.Spec.ServiceName = r.ServiceDbPg.Name NooBaaDBTemplate = util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet) + // Check db update status and add or remove the POSTGRES_UPDATE env to db STS + r.addOrRemovePGUpgradeEnvInSTS() + podSpec := &NooBaaDB.Spec.Template.Spec podSpec.ServiceAccountName = "noobaa-db" defaultUID := int64(10001) @@ -256,7 +333,7 @@ func (r *Reconciler) SetDesiredNooBaaDB() error { for i := range podSpec.Containers { c := &podSpec.Containers[i] if c.Name == "db" { - c.Image = GetDesiredDBImage(r.NooBaa, c.Image) + c.Image = r.GetDesiredDBImage(c.Image) if r.NooBaa.Spec.DBResources != nil { c.Resources = *r.NooBaa.Spec.DBResources } @@ -1133,6 +1210,7 @@ func (r *Reconciler) ReconcileDB() error { result, reconcilePostgresError := r.reconcileObjectAndGetResult(r.NooBaaPostgresDB, r.SetDesiredNooBaaDB, false) if reconcilePostgresError != nil { + return reconcilePostgresError } if !r.isObjectUpdated(result) && (isDBConfUpdated) { @@ -1143,6 +1221,11 @@ func (r *Reconciler) ReconcileDB() error { } } + + if r.isObjectUpdated(result) { + r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished + } + return nil } diff --git a/pkg/system/system.go b/pkg/system/system.go index 83785e2aa5..d7863f6adc 100644 --- a/pkg/system/system.go +++ b/pkg/system/system.go @@ -1115,31 +1115,6 @@ func Connect(isExternal bool) (*Client, error) { }, nil } -// GetDesiredDBImage returns the desired DB image according to spec or env or default (in options) -func GetDesiredDBImage(sys *nbv1.NooBaa, currentImage string) string { - // Postgres upgrade failure workaround - // if the current Postgres image is a postgresql-12 image, use NOOBAA_PSQL12_IMAGE. otherwise use GetdesiredDBImage - if IsPostgresql12Image(currentImage) { - psql12Image, ok := os.LookupEnv("NOOBAA_PSQL_12_IMAGE") - util.Logger().Warnf("The current Postgres image is a postgresql-12 image. using (%s)", psql12Image) - if !ok { - psql12Image = currentImage - util.Logger().Warnf("NOOBAA_PSQL_12_IMAGE is not set. using the current image %s", currentImage) - } - return psql12Image - } - - if sys.Spec.DBImage != nil { - return *sys.Spec.DBImage - } - - if os.Getenv("NOOBAA_DB_IMAGE") != "" { - return os.Getenv("NOOBAA_DB_IMAGE") - } - - return options.DBImage -} - // IsPostgresql12Image checks if the image is a postgresql-12 image func IsPostgresql12Image(image string) bool { return strings.Contains(image, "postgresql-12")