db upgrage: upgrade postgres 15 to postgres 16

The Postgres upgrade from 15 to 16 or any later versions can be done by adding the env variable named POSTGRES_UPGRADE = copy in the pg db pod.

More about upgrade process is here: https://www.postgresql.org/docs/current/pgupgrade.html

In our case, we achieve the upgrade by setting this env var in noobaa-db-pg STS.

Once after upgrade is done, we have to remove the POSTGRES_UPGRADE env from the STS. Otherwise pg db pod fails to come up because the db pod check PG directory version against upgrade image version and exits if there is no upgrade required and POSTGRES_UPGRADE is still provided.

To remove this env, we store the upgrade status in noobaa-core CR after the PG upgrade and refer the same status during db reconcile. If the upgrade is already done, we remove this env from desired sts status.

This will help to upgrade the db smoothly.

Signed-off-by: Vinayak Hariharmath <[email protected]>

deploy/crds/noobaa.io_noobaas.yaml

@@ -1875,6 +1875,10 @@ spec:
                 description: Phase is a simple, high-level summary of where the System
                   is in its lifecycle
                 type: string
+              postgresMajorVersion:
+                description: PostgresMajorVersion is the major version of postgress
+                  db image
+                type: string
               postgresUpdatePhase:
                 description: Upgrade reports the status of the ongoing postgres upgrade
                   process

pkg/apis/noobaa/v1alpha1/noobaa_types.go

@@ -365,6 +365,10 @@ type NooBaaStatus struct {
 	// BeforeUpgradeDbImage is the db image used before last db upgrade
 	// +optional
 	BeforeUpgradeDbImage *string `json:"beforeUpgradeDbImage,omitempty"`
+
+	// PostgresMajorVersion is the major version of postgress db image
+	// +optional
+	PostgresMajorVersion PostgresMajorVersion `json:"postgresMajorVersion,omitempty"`
 }
 
 // SystemPhase is a string enum type for system phases
@@ -494,6 +498,12 @@ type EndpointsStatus struct {
 	VirtualHosts []string `json:"virtualHosts"`
 }
 
+type PostgresMajorVersion string
+
+const (
+	PostgresMajorVersionV16 PostgresMajorVersion = "16"
+)
+
 // UpgradePhase is a string enum type for upgrade phases
 type UpgradePhase string
 
@@ -505,7 +515,7 @@ const (
 
 	UpgradePhaseMigrate UpgradePhase = "Migrating"
 
-	UpgradePhaseClean UpgradePhase = "Cleanning"
+	UpgradePhaseClean UpgradePhase = "Cleaning"
 
 	UpgradePhaseFinished UpgradePhase = "DoneUpgrade"
 

pkg/bundle/deploy.go

@@ -1415,7 +1415,7 @@ spec:
       status: {}
 `
 
-const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "3f88c800238f25e5dd26f3f1bf19028571cc646e3aea0f868bfd2ff600ee3ed1"
+const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "02e3ea37610f25352f7668db2f13041bbb97e73ed0f3b1be0e4249f674c57d9b"
 
 const File_deploy_crds_noobaa_io_noobaas_yaml = `---
 apiVersion: apiextensions.k8s.io/v1
@@ -3294,6 +3294,10 @@ spec:
                 description: Phase is a simple, high-level summary of where the System
                   is in its lifecycle
                 type: string
+              postgresMajorVersion:
+                description: PostgresMajorVersion is the major version of postgress
+                  db image
+                type: string
               postgresUpdatePhase:
                 description: Upgrade reports the status of the ongoing postgres upgrade
                   process

pkg/options/options.go

@@ -4,6 +4,7 @@ import (
 	"github.com/noobaa/noobaa-operator/v5/pkg/util"
 	"github.com/noobaa/noobaa-operator/v5/version"
 
+	nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -72,7 +73,11 @@ var NooBaaImage = ContainerImage
 
 // DBImage is the default db image url
 // it can be overridden for testing or different registry locations.
-var DBImage = "quay.io/sclorg/postgresql-15-c9s"
+var DBImage = "quay.io/sclorg/postgresql-16-c9s"
+
+// Postgress Major version is the major version of Postgress image
+// This is referred for postgres upgrade.
+var PostgresMajorVersion = nbv1.PostgresMajorVersionV16
 
 // Psql12Image is the default postgres12 db image url
 // currently it can not be overridden.

pkg/system/phase2_creating.go

@@ -122,6 +122,10 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error {
 
 	// create the db only if postgres secret is not given
 	if r.NooBaa.Spec.ExternalPgSecret == nil {
+		if err := r.UpgradeDbPostgres16(); err != nil {
+			return err
+		}
+
 		if err := r.ReconcileDB(); err != nil {
 			return err
 		}
@@ -130,6 +134,7 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error {
 			return err
 		}
 	}
+
 	// create bucket logging pvc if not provided by user for 'Guaranteed' logging in ODF env
 	if r.NooBaa.Spec.BucketLogging.LoggingType == nbv1.BucketLoggingTypeGuaranteed {
 		if err := r.ReconcileODFBucketLoggingPVC(); err != nil {
@@ -152,6 +157,91 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error {
 	return nil
 }
 
+func (r *Reconciler) UpgradeDbPostgres16() error {
+	var err error = nil
+
+	switch r.NooBaa.Status.PostgresUpdatePhase {
+	case "":
+		// Check whether it is fresh install case
+		sts := util.KubeObject(bundle.File_deploy_internal_statefulset_postgres_db_yaml).(*appsv1.StatefulSet)
+		sts.Name = "noobaa-db-pg"
+		sts.Namespace = options.Namespace
+		//postgres database doesn't exists, no need to upgrade.
+		if !util.KubeCheckQuiet(sts) {
+			r.Logger.Infof("UpgradePostgresDB: old STS doesn't exist - no need for upgrade")
+			r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone
+			r.NooBaa.Status.PostgresMajorVersion = options.PostgresMajorVersion
+		} else {
+			// Check wheter upgrade is reqd, set phase depending the requirement
+			if r.isDBUpToDate() {
+				r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone
+			} else {
+				r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhasePrepare
+			}
+		}
+	case nbv1.UpgradePhaseNone:
+		// No upgrade reqd
+		r.Logger.Infof("UpgradePostgresDB: upgrade is not required")
+	case nbv1.UpgradePhasePrepare:
+		if err := r.setEndpointsDeploymentReplicas(0); err != nil {
+			r.Logger.Errorf("UpgradePostgresDB::got error on endpoints deployment reconcile %v", err)
+			return err
+		}
+
+		if err = r.ReconcileObject(
+			r.NooBaaPostgresDB, func() error {
+				r.updateDBImageForUpgrade()
+				r.setPGUpgradeEnvInSTS()
+				return nil
+			},
+		); err != nil {
+			r.Logger.Errorf("got error on postgres STS reconcile %v", err)
+			break
+		}
+
+		// Update the DB pod
+		restartError := r.RestartDbPods()
+		if restartError != nil {
+			r.Logger.Warn("UpgradePostgresDB: Unable to restart db pods")
+		}
+
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseUpgrade
+	case nbv1.UpgradePhaseUpgrade:
+		dbPod := &corev1.Pod{}
+		dbPod.Name = "noobaa-db-pg-0"
+		dbPod.Namespace = r.NooBaaPostgresDB.Namespace
+		if !util.KubeCheckQuiet(dbPod) {
+			return nil
+		}
+		// make sure previous step has finished
+		if dbPod.ObjectMeta.DeletionTimestamp != nil {
+			r.Logger.Infof("UpgradePostgresDB: upgrade-db is not yet running, phase is: %s and deletion time stamp is %v",
+				dbPod.Status.Phase, dbPod.ObjectMeta.DeletionTimestamp)
+			return nil
+		}
+		r.NooBaa.Status.PostgresMajorVersion = options.PostgresMajorVersion
+		// Set the upgrade finished flag here and refer the same to remove the
+		// POSTGRES_UPGRADE flag in next reconcile.
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseClean
+	case nbv1.UpgradePhaseClean:
+		if err = r.ReconcileObject(r.NooBaaPostgresDB, func() error {
+			r.unSetPGUpgradeEnvInSTS()
+			return nil
+		}); err != nil {
+			r.Logger.Infof("UpgradePostgresDB: Cleaning of POSTGRES_UPGRADE env failed: err %v, retry", err)
+			return nil
+		}
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseFinished
+	case nbv1.UpgradePhaseFinished:
+		if err := r.setEndpointsDeploymentReplicas(1); err != nil {
+			r.Logger.Errorf("UpgradePostgresDB::got error on endpoints deployment reconcile %v", err)
+			return err
+		}
+		r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone
+	}
+	return err
+}
+
 // SetDesiredServiceAccount updates the ServiceAccount as desired for reconciling
 func (r *Reconciler) SetDesiredServiceAccount() error {
 	if r.ServiceAccount.Annotations == nil {
@@ -223,6 +313,95 @@ func (r *Reconciler) SetDesiredServiceDBForPostgres() error {
 	return nil
 }
 
+func (r *Reconciler) updateDBImageForUpgrade() {
+	var podSpec = &r.NooBaaPostgresDB.Spec.Template.Spec
+
+	for i := range podSpec.Containers {
+		c := &podSpec.Containers[i]
+		if c.Name == "db" {
+			// Fetch the DB image from options.go
+			c.Image = options.DBImage
+			if r.NooBaa.Spec.DBResources != nil {
+				c.Resources = *r.NooBaa.Spec.DBResources
+			}
+
+			c.Lifecycle = &corev1.Lifecycle{
+				PreStop: &corev1.LifecycleHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{"/bin/sh", "-c", "pg_ctl -D /var/lib/pgsql/data/userdata/ -w -t 60 -m fast stop"},
+					},
+				},
+			}
+
+		}
+	}
+}
+
+// SetEndpointsDeploymentReplicas updates the number of replicas on the endpoints deployment
+func (r *Reconciler) setEndpointsDeploymentReplicas(replicas int32) error {
+	r.Logger.Infof("UpgradeMigrateDB:: setting endpoints replica count to %d", replicas)
+	return r.ReconcileObject(r.DeploymentEndpoint, func() error {
+		r.DeploymentEndpoint.Spec.Replicas = &replicas
+		return nil
+	})
+}
+
+// Set env "POSTGRESQL_UPGRADE=copy" in PG sts to iniate the upgrade
+func (r *Reconciler) isDBUpToDate() bool {
+	var currentImage string
+	desiredImage := options.DBImage
+
+	for _, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+		if container.Name == "db" {
+			currentImage = container.Image
+		}
+	}
+	if currentImage != desiredImage &&
+		options.PostgresMajorVersion > r.NooBaa.Status.PostgresMajorVersion {
+		return false
+	}
+	return true
+}
+
+// Set env "POSTGRESQL_UPGRADE=copy" in PG sts to iniate the upgrade
+func (r *Reconciler) setPGUpgradeEnvInSTS() {
+	for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+		if container.Name == "db" {
+			for _, env := range container.Env {
+				if env.Name == "POSTGRESQL_UPGRADE" {
+					return
+				}
+			}
+			envVars := container.Env
+			newEnvVar := corev1.EnvVar{
+				Name:  "POSTGRESQL_UPGRADE",
+				Value: "copy",
+			}
+
+			envVars = append(envVars, newEnvVar)
+			r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = envVars
+			return
+		}
+	}
+}
+
+// unSet env "POSTGRESQL_UPGRADE=copy" in PG sts after upgrade
+func (r *Reconciler) unSetPGUpgradeEnvInSTS() {
+	for i, container := range r.NooBaaPostgresDB.Spec.Template.Spec.Containers {
+		if container.Name == "db" {
+			newEnvVars := []corev1.EnvVar{}
+			for _, env := range container.Env {
+				if env.Name != "POSTGRESQL_UPGRADE" {
+					newEnvVars = append(newEnvVars, env)
+				}
+			}
+			r.NooBaaPostgresDB.Spec.Template.Spec.Containers[i].Env = newEnvVars
+			return
+		}
+	}
+	r.NooBaa.Status.PostgresUpdatePhase = nbv1.UpgradePhaseNone
+}
+
 // SetDesiredNooBaaDB updates the NooBaaDB as desired for reconciling
 func (r *Reconciler) SetDesiredNooBaaDB() error {
 	var NooBaaDBTemplate *appsv1.StatefulSet = nil
@@ -1120,7 +1299,10 @@ func (r *Reconciler) reconcileDBRBAC() error {
 
 // ReconcileDB choose between different types of DB
 func (r *Reconciler) ReconcileDB() error {
-
+	if r.NooBaa.Status.PostgresUpdatePhase != nbv1.UpgradePhaseNone {
+		r.Logger.Infof("Postgres Update in progress")
+		return nil
+	}
 	if r.NooBaa.Spec.ExternalPgSecret != nil {
 		return nil
 	}

pkg/system/system.go

@@ -1129,14 +1129,6 @@ func GetDesiredDBImage(sys *nbv1.NooBaa, currentImage string) string {
 		return psql12Image
 	}
 
-	if sys.Spec.DBImage != nil {
-		return *sys.Spec.DBImage
-	}
-
-	if os.Getenv("NOOBAA_DB_IMAGE") != "" {
-		return os.Getenv("NOOBAA_DB_IMAGE")
-	}
-
 	return options.DBImage
 }