Implement major upgrade result annotations

e2e/tests/test_e2e.py

@@ -1185,13 +1185,19 @@ def get_docker_image():
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_major_version_upgrade(self):
         """
-        Test major version upgrade
+        Test major version upgrade: with full upgrade, maintenance window, and annotation
         """
         def check_version():
             p = k8s.patroni_rest("acid-upgrade-test-0", "")
             version = p.get("server_version", 0) // 10000
             return version
 
+        def get_annotations():
+            pg_manifest = k8s.api.custom_objects_api.get_namespaced_custom_object(
+                "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test")
+            annotations = pg_manifest["metadata"]["annotations"]
+            return annotations
+
         k8s = self.k8s
         cluster_label = 'application=spilo,cluster-name=acid-upgrade-test'
 
@@ -1209,52 +1215,54 @@ def check_version():
 
         master_nodes, _ = k8s.get_cluster_nodes(cluster_labels=cluster_label)
         # should upgrade immediately
-        pg_patch_version_14 = {
+        pg_patch_version_13 = {
             "spec": {
                 "postgresql": {
-                    "version": "14"
+                    "version": "13"
                 }
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
-            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_14)
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_13)
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
-        # should have finish failover
         k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
-        self.eventuallyEqual(check_version, 14, "Version should be upgraded from 12 to 14")
+        self.eventuallyEqual(check_version, 13, "Version should be upgraded from 12 to 13")
+
+        # check if annotation for last upgrade's success is set
+        previous_annotations = get_annotations()
+        self.assertIsNotNone(previous_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set")
 
         # should not upgrade because current time is not in maintenanceWindow
         current_time = datetime.now()
         maintenance_window_future = f"{(current_time+timedelta(minutes=60)).strftime('%H:%M')}-{(current_time+timedelta(minutes=120)).strftime('%H:%M')}"
-        pg_patch_version_15 = {
+        pg_patch_version_14 = {
             "spec": {
                 "postgresql": {
-                    "version": "15"
+                    "version": "14"
                 },
                 "maintenanceWindows": [
                     maintenance_window_future
                 ]
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
-            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15)
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_14)
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
-        # should have finish failover
         k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
-        self.eventuallyEqual(check_version, 14, "Version should not be upgraded")
+        self.eventuallyEqual(check_version, 13, "Version should not be upgraded")
 
         # change the version again to trigger operator sync
         maintenance_window_current = f"{(current_time-timedelta(minutes=30)).strftime('%H:%M')}-{(current_time+timedelta(minutes=30)).strftime('%H:%M')}"
-        pg_patch_version_16 = {
+        pg_patch_version_15 = {
             "spec": {
                 "postgresql": {
-                    "version": "16"
+                    "version": "15"
                 },
                 "maintenanceWindows": [
                     maintenance_window_current
@@ -1263,14 +1271,40 @@ def check_version():
         }
 
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
-            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_16)
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15)
         self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
 
-        # should have finish failover
         k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
         k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
-        self.eventuallyEqual(check_version, 16, "Version should be upgraded from 14 to 16")
+        self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15")
+
+        # check if annotation for last upgrade's success is updated after second upgrade
+        new_annotations = get_annotations()
+        self.assertIsNotNone(new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set")
+        self.assertNotEqual(previous_annotations.get("last-major-upgrade-success"), new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated")
+
+        # test annotation with failed upgrade annotation
+        pg_patch_version_16 = {
+            "metadata": {
+                "annotations": {
+                    "last-major-upgrade-failure": "2024-01-02T15:04:05Z"
+                },
+            },
+            "spec": {
+                "postgresql": {
+                    "version": "16"
+                },
+            },
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_16)
+        self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync")
+
+        k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label)
+        k8s.wait_for_pod_start('spilo-role=master,' + cluster_label)
+        k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label)
+        self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's failure is set")
 
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def test_persistent_volume_claim_retention_policy(self):

pkg/cluster/majorversionupgrade.go

@@ -1,12 +1,16 @@
 package cluster
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/zalando/postgres-operator/pkg/spec"
 	"github.com/zalando/postgres-operator/pkg/util"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // VersionMap Map of version numbers
@@ -54,6 +58,58 @@ func (c *Cluster) isUpgradeAllowedForTeam(owningTeam string) bool {
 	return util.SliceContains(allowedTeams, owningTeam)
 }
 
+func (c *Cluster) PatchAnnotations(patchBytesAnnotation []byte) error {
+	resourceName := c.Name
+	_, err := c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), resourceName, types.MergePatchType, patchBytesAnnotation, metav1.PatchOptions{})
+	if err != nil {
+		c.logger.Errorf("failed to patch annotations: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (c *Cluster) annotatePostgresResource(isSuccess bool) error {
+	annotations := make(map[string]string)
+	currentTime := metav1.Now().Format("2006-01-02T15:04:05Z")
+	if isSuccess {
+		annotations["last-major-upgrade-success"] = currentTime
+	} else {
+		annotations["last-major-upgrade-failure"] = currentTime
+	}
+
+	patchAnnotation := map[string]map[string]map[string]string{
+		"metadata": {
+			"annotations": annotations,
+		},
+	}
+	patchBytes, err := json.Marshal(patchAnnotation)
+	if err != nil {
+		c.logger.Errorf("failed to marshal annotations: %v", err)
+		return err
+	}
+
+	err = c.PatchAnnotations(patchBytes)
+	if err != nil {
+		c.logger.Errorf("failed to patch annotations to Postgres resource: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (c *Cluster) getLastMajorUpgradeAnnotations() map[string]string {
+	annotations := c.ObjectMeta.GetAnnotations()
+	upgradeAnnotationKeys := []string{"last-major-upgrade-failure", "last-major-upgrade-success"}
+	upgradeAnnotationValues := make(map[string]string)
+	for _, annotationKey := range upgradeAnnotationKeys {
+		if val, exists := annotations[annotationKey]; exists {
+			upgradeAnnotationValues[annotationKey] = val
+		} else {
+			upgradeAnnotationValues[annotationKey] = ""
+		}
+	}
+	return upgradeAnnotationValues
+}
+
 /*
 Execute upgrade when mode is set to manual or full or when the owning team is allowed for upgrade (and mode is "off").
 
@@ -67,12 +123,23 @@ func (c *Cluster) majorVersionUpgrade() error {
 	}
 
 	desiredVersion := c.GetDesiredMajorVersionAsInt()
+	isUpgradeSuccess := true
+	lastMajorUpgradeAnnotations := c.getLastMajorUpgradeAnnotations()
 
 	if c.currentMajorVersion >= desiredVersion {
+		if lastMajorUpgradeAnnotations["last-major-upgrade-success"] == "" {
+			c.annotatePostgresResource(isUpgradeSuccess)
+			c.logger.Info("update last major upgrade success annotation to current timestamp as it was not set")
+		}
 		c.logger.Infof("cluster version up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion)
 		return nil
 	}
 
+	if lastMajorUpgradeAnnotations["last-major-upgrade-failure"] != "" {
+		c.logger.Infof("last major upgrade failed, skipping upgrade")
+		return nil
+	}
+
 	if !isInMainternanceWindow(c.Spec.MaintenanceWindows) {
 		c.logger.Infof("skipping major version upgrade, not in maintenance window")
 		return nil
@@ -103,6 +170,10 @@ func (c *Cluster) majorVersionUpgrade() error {
 
 	// Recheck version with newest data from Patroni
 	if c.currentMajorVersion >= desiredVersion {
+		if lastMajorUpgradeAnnotations["last-major-upgrade-success"] == "" {
+			c.annotatePostgresResource(isUpgradeSuccess)
+			c.logger.Info("update last major upgrade success annotation to current timestamp as it was not set")
+		}
 		c.logger.Infof("recheck cluster version is already up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion)
 		return nil
 	}
@@ -132,11 +203,14 @@ func (c *Cluster) majorVersionUpgrade() error {
 				result, err = c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand)
 			}
 			if err != nil {
+				isUpgradeSuccess = false
+				c.annotatePostgresResource(isUpgradeSuccess)
 				c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err)
 				return err
 			}
-			c.logger.Infof("upgrade action triggered and command completed: %s", result[:100])
 
+			c.annotatePostgresResource(isUpgradeSuccess)
+			c.logger.Infof("upgrade action triggered and command completed: %s", result[:100])
 			c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion)
 		}
 	}