From 1b76a1a5318445cf83d8b7a55a0b430e343a53dd Mon Sep 17 00:00:00 2001 From: idanovinda Date: Tue, 13 Aug 2024 12:47:30 +0200 Subject: [PATCH 01/15] Implemenet major upgrade result annotations --- pkg/cluster/majorversionupgrade.go | 70 +++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index 86c95b6a4..f4a8b05f7 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -1,12 +1,16 @@ package cluster import ( + "context" + "encoding/json" "fmt" "strings" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // VersionMap Map of version numbers @@ -54,6 +58,52 @@ func (c *Cluster) isUpgradeAllowedForTeam(owningTeam string) bool { return util.SliceContains(allowedTeams, owningTeam) } +func (c *Cluster) PatchAnnotations(patchBytesAnnotation []byte) error { + resourceName := c.Name + _, err := c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), resourceName, types.MergePatchType, patchBytesAnnotation, metav1.PatchOptions{}) + if err != nil { + c.logger.Errorf("failed to patch annotations: %v", err) + return err + } + return nil +} + +func (c *Cluster) annotatePostgresResource(isSuccess bool) error { + annotations := make(map[string]string) + if isSuccess { + annotations["last-major-upgrade-succeeded"] = "true" + } else { + annotations["last-major-upgrade-succeeded"] = "false" + } + annotations["last-major-upgrade-timestamp"] = metav1.Now().Format("2006-01-02T15:04:05Z") + + patchAnnotation := map[string]map[string]map[string]string{ + "metadata": { + "annotations": annotations, + }, + } + patchBytes, err := json.Marshal(patchAnnotation) + if err != nil { + c.logger.Errorf("failed to marshal annotations: %v", err) + return err + } + + err = c.PatchAnnotations(patchBytes) + if err != nil { + c.logger.Errorf("failed to patch annotations to Postgres resource: %v", err) + return err + } + return nil +} + +func (c *Cluster) getLastMajorUpgradeSucceededAnnotation() string { + annotations := c.ObjectMeta.GetAnnotations() + if val, ok := annotations["last-major-upgrade-succeeded"]; ok { + return val + } + return "" +} + /* Execute upgrade when mode is set to manual or full or when the owning team is allowed for upgrade (and mode is "off"). @@ -67,12 +117,23 @@ func (c *Cluster) majorVersionUpgrade() error { } desiredVersion := c.GetDesiredMajorVersionAsInt() + isUpgradeSuccess := true + lastMajorUpgradeSucceeded := c.getLastMajorUpgradeSucceededAnnotation() if c.currentMajorVersion >= desiredVersion { + if lastMajorUpgradeSucceeded == "false" { + c.annotatePostgresResource(isUpgradeSuccess) + c.logger.Info("update last major upgrade succeeded annotation to true") + } c.logger.Infof("cluster version up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil } + if lastMajorUpgradeSucceeded == "false" { + c.logger.Infof("last major upgrade failed, skipping upgrade") + return nil + } + if !c.isInMainternanceWindow() { c.logger.Infof("skipping major version upgrade, not in maintenance window") return nil @@ -103,6 +164,10 @@ func (c *Cluster) majorVersionUpgrade() error { // Recheck version with newest data from Patroni if c.currentMajorVersion >= desiredVersion { + if lastMajorUpgradeSucceeded == "false" { + c.annotatePostgresResource(isUpgradeSuccess) + c.logger.Info("update last major upgrade succeeded annotation to true") + } c.logger.Infof("recheck cluster version is already up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil } @@ -132,11 +197,14 @@ func (c *Cluster) majorVersionUpgrade() error { result, err = c.ExecCommand(podName, "/bin/su", "postgres", "-c", upgradeCommand) } if err != nil { + isUpgradeSuccess = false + c.annotatePostgresResource(isUpgradeSuccess) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeWarning, "Major Version Upgrade", "upgrade from %d to %d FAILED: %v", c.currentMajorVersion, desiredVersion, err) return err } - c.logger.Infof("upgrade action triggered and command completed: %s", result[:100]) + c.annotatePostgresResource(isUpgradeSuccess) + c.logger.Infof("upgrade action triggered and command completed: %s", result[:100]) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "upgrade from %d to %d finished", c.currentMajorVersion, desiredVersion) } } From dddf18ecfeb9352b14f151b3f9d9e5c4605c7dc5 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Tue, 13 Aug 2024 14:31:40 +0200 Subject: [PATCH 02/15] e2e test: extend major upgrade test --- e2e/tests/test_e2e.py | 44 +++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index bd7dfef57..dabf138c3 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1209,30 +1209,30 @@ def check_version(): master_nodes, _ = k8s.get_cluster_nodes(cluster_labels=cluster_label) # should upgrade immediately - pg_patch_version_14 = { + pg_patch_version_13 = { "spec": { "postgresql": { - "version": "14" + "version": "13" } } } k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_14) + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_13) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # should have finish failover k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(check_version, 14, "Version should be upgraded from 12 to 14") + self.eventuallyEqual(check_version, 14, "Version should be upgraded from 12 to 13") # should not upgrade because current time is not in maintenanceWindow current_time = datetime.now() maintenance_window_future = f"{(current_time+timedelta(minutes=60)).strftime('%H:%M')}-{(current_time+timedelta(minutes=120)).strftime('%H:%M')}" - pg_patch_version_15 = { + pg_patch_version_14 = { "spec": { "postgresql": { - "version": "15" + "version": "14" }, "maintenanceWindows": [ maintenance_window_future @@ -1240,7 +1240,7 @@ def check_version(): } } k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_14) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # should have finish failover @@ -1251,10 +1251,10 @@ def check_version(): # change the version again to trigger operator sync maintenance_window_current = f"{(current_time-timedelta(minutes=30)).strftime('%H:%M')}-{(current_time+timedelta(minutes=30)).strftime('%H:%M')}" - pg_patch_version_16 = { + pg_patch_version_15 = { "spec": { "postgresql": { - "version": "16" + "version": "15" }, "maintenanceWindows": [ maintenance_window_current @@ -1263,7 +1263,7 @@ def check_version(): } k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_16) + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") # should have finish failover @@ -1272,6 +1272,30 @@ def check_version(): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 16, "Version should be upgraded from 14 to 16") + # test annotation with failed upgrade + pg_patch_version_16 = { + "metadata": { + "annotations": { + "last-major-upgrade-succeeded": "false" + }, + }, + "spec": { + "postgresql": { + "version": "16" + }, + }, + } + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_16) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + # should have finish failover + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + self.eventuallyEqual(check_version, 16, "Version should not be upgraded because annotation for last upgrade's success is set to false") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): ''' From 3f2617962589ecf67ce9090aadcb8d0f8c1f9098 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Tue, 13 Aug 2024 16:03:44 +0200 Subject: [PATCH 03/15] e2e test: fix checking equal values --- e2e/tests/test_e2e.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index dabf138c3..2c1910a4b 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1224,7 +1224,7 @@ def check_version(): k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(check_version, 14, "Version should be upgraded from 12 to 13") + self.eventuallyEqual(check_version, 13, "Version should be upgraded from 12 to 13") # should not upgrade because current time is not in maintenanceWindow current_time = datetime.now() @@ -1247,7 +1247,7 @@ def check_version(): k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(check_version, 14, "Version should not be upgraded") + self.eventuallyEqual(check_version, 13, "Version should not be upgraded") # change the version again to trigger operator sync maintenance_window_current = f"{(current_time-timedelta(minutes=30)).strftime('%H:%M')}-{(current_time+timedelta(minutes=30)).strftime('%H:%M')}" @@ -1270,7 +1270,7 @@ def check_version(): k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(check_version, 16, "Version should be upgraded from 14 to 16") + self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") # test annotation with failed upgrade pg_patch_version_16 = { @@ -1294,7 +1294,7 @@ def check_version(): k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(check_version, 16, "Version should not be upgraded because annotation for last upgrade's success is set to false") + self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's success is set to false") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): From c0eb0612b1537031cd3f40b248537936240a039d Mon Sep 17 00:00:00 2001 From: inovindasari Date: Wed, 14 Aug 2024 14:55:32 +0200 Subject: [PATCH 04/15] e2e test: extend check annotation --- e2e/tests/test_e2e.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2c1910a4b..ce539b089 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1185,13 +1185,19 @@ def get_docker_image(): @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_major_version_upgrade(self): """ - Test major version upgrade + Test major version upgrade: with full upgrade, maintenance window, and annotation """ def check_version(): p = k8s.patroni_rest("acid-upgrade-test-0", "") version = p.get("server_version", 0) // 10000 return version + def get_annotation(): + pg_manifest = k8s.api.custom_objects_api.get_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test") + annotation = pg_manifest["metadata"]["annotations"] + return annotation + k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-upgrade-test' @@ -1220,7 +1226,6 @@ def check_version(): "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_13) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - # should have finish failover k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) @@ -1243,7 +1248,6 @@ def check_version(): "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_14) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - # should have finish failover k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) @@ -1266,13 +1270,16 @@ def check_version(): "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - # should have finish failover k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") - # test annotation with failed upgrade + # check if annotation for last upgrade's success is set + annotation = get_annotation() + self.assertEqual(annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not set") + + # test annotation with failed upgrade annotation pg_patch_version_16 = { "metadata": { "annotations": { @@ -1285,17 +1292,26 @@ def check_version(): }, }, } - k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_16) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - # should have finish failover k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's success is set to false") + # if we change the version to the current one, last upgrade's success annotation is set to true + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + annotation = get_annotation() + self.assertEqual(annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not True") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): ''' From 4c033c4567d330b923852577a29945b2c038fd15 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Wed, 14 Aug 2024 15:01:19 +0200 Subject: [PATCH 05/15] resolve conflict --- pkg/cluster/majorversionupgrade.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index f4a8b05f7..3113b6d68 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -134,7 +134,7 @@ func (c *Cluster) majorVersionUpgrade() error { return nil } - if !c.isInMainternanceWindow() { + if !isInMainternanceWindow(c.Spec.MaintenanceWindows) { c.logger.Infof("skipping major version upgrade, not in maintenance window") return nil } From 41fb815494c6d4613f1f83c54150600db4ada821 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Wed, 14 Aug 2024 15:43:44 +0200 Subject: [PATCH 06/15] e2e test: upgrade timestamp annotation should not be empty --- e2e/tests/test_e2e.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index ce539b089..b913f57e1 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1276,8 +1276,9 @@ def get_annotation(): self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") # check if annotation for last upgrade's success is set - annotation = get_annotation() - self.assertEqual(annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not set") + previous_annotation = get_annotation() + self.assertEqual(previous_annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not set") + self.assertNotEqual(previous_annotation.get("last-major-upgrade-timestamp"), None, "Annotation for last upgrade is not set") # test annotation with failed upgrade annotation pg_patch_version_16 = { @@ -1309,8 +1310,9 @@ def get_annotation(): k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - annotation = get_annotation() - self.assertEqual(annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not True") + new_annotation = get_annotation() + self.assertEqual(new_annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not True") + self.assertNotEqual(new_annotation.get("last-major-upgrade-timestamp"), previous_annotation.get("last-major-upgrade-timestamp"), "Annotation for last upgrade is not set") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): From 620c60e07bce2dead60939305026e9cf381a85c8 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Wed, 14 Aug 2024 16:10:53 +0200 Subject: [PATCH 07/15] e2e test: apply feedback --- e2e/tests/test_e2e.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index b913f57e1..f0ec68236 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1192,11 +1192,11 @@ def check_version(): version = p.get("server_version", 0) // 10000 return version - def get_annotation(): + def get_annotations(): pg_manifest = k8s.api.custom_objects_api.get_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test") - annotation = pg_manifest["metadata"]["annotations"] - return annotation + annotations = pg_manifest["metadata"]["annotations"] + return annotations k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-upgrade-test' @@ -1276,9 +1276,9 @@ def get_annotation(): self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") # check if annotation for last upgrade's success is set - previous_annotation = get_annotation() - self.assertEqual(previous_annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not set") - self.assertNotEqual(previous_annotation.get("last-major-upgrade-timestamp"), None, "Annotation for last upgrade is not set") + previous_annotations = get_annotations() + self.assertEqual(previous_annotations.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not set") + self.assertIsNotNone(previous_annotations.get("last-major-upgrade-timestamp"), "Annotation for upgrade timestamp is not set") # test annotation with failed upgrade annotation pg_patch_version_16 = { @@ -1310,9 +1310,9 @@ def get_annotation(): k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - new_annotation = get_annotation() - self.assertEqual(new_annotation.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not True") - self.assertNotEqual(new_annotation.get("last-major-upgrade-timestamp"), previous_annotation.get("last-major-upgrade-timestamp"), "Annotation for last upgrade is not set") + new_annotations = get_annotations() + self.assertEqual(new_annotations.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not True") + self.assertNotEqual(new_annotations.get("last-major-upgrade-timestamp"), previous_annotations.get("last-major-upgrade-timestamp"), "Annotation for upgrade timestamp is not updated") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): From 1a26af2bb965646a257932c67d7dea55d88e22b8 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Tue, 20 Aug 2024 13:17:07 +0200 Subject: [PATCH 08/15] change annotation keys --- e2e/tests/test_e2e.py | 9 ++++----- pkg/cluster/majorversionupgrade.go | 32 ++++++++++++++++++------------ 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index f0ec68236..dcc94b219 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1284,7 +1284,7 @@ def get_annotations(): pg_patch_version_16 = { "metadata": { "annotations": { - "last-major-upgrade-succeeded": "false" + "last-major-upgrade-failure": "2024-01-02T15:04:05Z" }, }, "spec": { @@ -1300,9 +1300,9 @@ def get_annotations(): k8s.wait_for_pod_failover(master_nodes, 'spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's success is set to false") + self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's failure is set") - # if we change the version to the current one, last upgrade's success annotation is set to true + # if we change the version to the current one, last upgrade's success annotation is set to a timestamp k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") @@ -1311,8 +1311,7 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) new_annotations = get_annotations() - self.assertEqual(new_annotations.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not True") - self.assertNotEqual(new_annotations.get("last-major-upgrade-timestamp"), previous_annotations.get("last-major-upgrade-timestamp"), "Annotation for upgrade timestamp is not updated") + self.assertNotEqual(new_annotations.get("last-major-upgrade-success", ""), "", "Annotation for last upgrade's success is not set") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index 3113b6d68..43beb825a 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -70,12 +70,12 @@ func (c *Cluster) PatchAnnotations(patchBytesAnnotation []byte) error { func (c *Cluster) annotatePostgresResource(isSuccess bool) error { annotations := make(map[string]string) + currentTime := metav1.Now().Format("2006-01-02T15:04:05Z") if isSuccess { - annotations["last-major-upgrade-succeeded"] = "true" + annotations["last-major-upgrade-success"] = currentTime } else { - annotations["last-major-upgrade-succeeded"] = "false" + annotations["last-major-upgrade-failure"] = currentTime } - annotations["last-major-upgrade-timestamp"] = metav1.Now().Format("2006-01-02T15:04:05Z") patchAnnotation := map[string]map[string]map[string]string{ "metadata": { @@ -96,12 +96,18 @@ func (c *Cluster) annotatePostgresResource(isSuccess bool) error { return nil } -func (c *Cluster) getLastMajorUpgradeSucceededAnnotation() string { +func (c *Cluster) getLastMajorUpgradeAnnotations() map[string]string { annotations := c.ObjectMeta.GetAnnotations() - if val, ok := annotations["last-major-upgrade-succeeded"]; ok { - return val + upgradeAnnotationKeys := []string{"last-major-upgrade-failure", "last-major-upgrade-success"} + upgradeAnnotationValues := make(map[string]string) + for _, annotationKey := range upgradeAnnotationKeys { + if val, exists := annotations[annotationKey]; exists { + upgradeAnnotationValues[annotationKey] = val + } else { + upgradeAnnotationValues[annotationKey] = "" + } } - return "" + return upgradeAnnotationValues } /* @@ -118,18 +124,18 @@ func (c *Cluster) majorVersionUpgrade() error { desiredVersion := c.GetDesiredMajorVersionAsInt() isUpgradeSuccess := true - lastMajorUpgradeSucceeded := c.getLastMajorUpgradeSucceededAnnotation() + lastMajorUpgradeAnnotations := c.getLastMajorUpgradeAnnotations() if c.currentMajorVersion >= desiredVersion { - if lastMajorUpgradeSucceeded == "false" { + if lastMajorUpgradeAnnotations["last-major-upgrade-success"] == "" { c.annotatePostgresResource(isUpgradeSuccess) - c.logger.Info("update last major upgrade succeeded annotation to true") + c.logger.Info("update last major upgrade success annotation to current timestamp as it was not set") } c.logger.Infof("cluster version up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil } - if lastMajorUpgradeSucceeded == "false" { + if lastMajorUpgradeAnnotations["last-major-upgrade-failure"] != "" { c.logger.Infof("last major upgrade failed, skipping upgrade") return nil } @@ -164,9 +170,9 @@ func (c *Cluster) majorVersionUpgrade() error { // Recheck version with newest data from Patroni if c.currentMajorVersion >= desiredVersion { - if lastMajorUpgradeSucceeded == "false" { + if lastMajorUpgradeAnnotations["last-major-upgrade-success"] == "" { c.annotatePostgresResource(isUpgradeSuccess) - c.logger.Info("update last major upgrade succeeded annotation to true") + c.logger.Info("update last major upgrade success annotation to current timestamp as it was not set") } c.logger.Infof("recheck cluster version is already up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil From 33e9c1d0c7f1ffbd2bf2dfa776c622ae06b6b9aa Mon Sep 17 00:00:00 2001 From: inovindasari Date: Tue, 20 Aug 2024 13:44:41 +0200 Subject: [PATCH 09/15] change annotation value check --- e2e/tests/test_e2e.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index dcc94b219..2ab5ec762 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1277,8 +1277,7 @@ def get_annotations(): # check if annotation for last upgrade's success is set previous_annotations = get_annotations() - self.assertEqual(previous_annotations.get("last-major-upgrade-succeeded"), "true", "Annotation for last upgrade's success is not set") - self.assertIsNotNone(previous_annotations.get("last-major-upgrade-timestamp"), "Annotation for upgrade timestamp is not set") + self.assertIsNotNone(previous_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") # test annotation with failed upgrade annotation pg_patch_version_16 = { @@ -1311,7 +1310,8 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) new_annotations = get_annotations() - self.assertNotEqual(new_annotations.get("last-major-upgrade-success", ""), "", "Annotation for last upgrade's success is not set") + self.assertIsNotNone(new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + self.assertNotEqual(previous_annotations.get("last-major-upgrade-success"), new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): From 1a2b1a6099d2edacca5e6ae89e9bc672b21b9e97 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Tue, 20 Aug 2024 15:34:27 +0200 Subject: [PATCH 10/15] move the comparison of annotations --- e2e/tests/test_e2e.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2ab5ec762..487c02c98 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1231,6 +1231,10 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 13, "Version should be upgraded from 12 to 13") + # check if annotation for last upgrade's success is set + previous_annotations = get_annotations() + self.assertIsNotNone(previous_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + # should not upgrade because current time is not in maintenanceWindow current_time = datetime.now() maintenance_window_future = f"{(current_time+timedelta(minutes=60)).strftime('%H:%M')}-{(current_time+timedelta(minutes=120)).strftime('%H:%M')}" @@ -1275,9 +1279,10 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") - # check if annotation for last upgrade's success is set - previous_annotations = get_annotations() - self.assertIsNotNone(previous_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + # check if annotation for last upgrade's success is updated after second upgrade + new_annotations = get_annotations() + self.assertIsNotNone(new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + self.assertNotEqual(previous_annotations.get("last-major-upgrade-success"), new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") # test annotation with failed upgrade annotation pg_patch_version_16 = { @@ -1301,18 +1306,6 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's failure is set") - # if we change the version to the current one, last upgrade's success annotation is set to a timestamp - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") - k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) - - k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) - k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - new_annotations = get_annotations() - self.assertIsNotNone(new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") - self.assertNotEqual(previous_annotations.get("last-major-upgrade-success"), new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): ''' From 2fa1145272146b80ba9515c95593ec2cd052d756 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Tue, 27 Aug 2024 12:51:25 +0200 Subject: [PATCH 11/15] apply feedback and discussion --- e2e/tests/test_e2e.py | 22 +++++++-- pkg/cluster/majorversionupgrade.go | 73 +++++++++++++----------------- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 487c02c98..920c95a84 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1232,8 +1232,8 @@ def get_annotations(): self.eventuallyEqual(check_version, 13, "Version should be upgraded from 12 to 13") # check if annotation for last upgrade's success is set - previous_annotations = get_annotations() - self.assertIsNotNone(previous_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + annotations = get_annotations() + self.assertIsNotNone(annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") # should not upgrade because current time is not in maintenanceWindow current_time = datetime.now() @@ -1280,9 +1280,9 @@ def get_annotations(): self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") # check if annotation for last upgrade's success is updated after second upgrade - new_annotations = get_annotations() - self.assertIsNotNone(new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") - self.assertNotEqual(previous_annotations.get("last-major-upgrade-success"), new_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") + second_annotations = get_annotations() + self.assertIsNotNone(second_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + self.assertNotEqual(annotations.get("last-major-upgrade-success"), second_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") # test annotation with failed upgrade annotation pg_patch_version_16 = { @@ -1306,6 +1306,18 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 15, "Version should not be upgraded because annotation for last upgrade's failure is set") + # change the version back to 15 and should remove failure annotation + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_15) + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + + k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) + k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) + + third_annotations = get_annotations() + self.assertIsNone(third_annotations.get("last-major-upgrade-failure"), "Annotation for last upgrade's failure is not removed") + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self): ''' diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index 43beb825a..11fdb4356 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -22,6 +22,11 @@ var VersionMap = map[string]int{ "16": 160000, } +const ( + majorVersionUpgradeSuccessAnnotation = "last-major-upgrade-success" + majorVersionUpgradeFailureAnnotation = "last-major-upgrade-failure" +) + // IsBiggerPostgresVersion Compare two Postgres version numbers func IsBiggerPostgresVersion(old string, new string) bool { oldN := VersionMap[old] @@ -58,37 +63,20 @@ func (c *Cluster) isUpgradeAllowedForTeam(owningTeam string) bool { return util.SliceContains(allowedTeams, owningTeam) } -func (c *Cluster) PatchAnnotations(patchBytesAnnotation []byte) error { - resourceName := c.Name - _, err := c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), resourceName, types.MergePatchType, patchBytesAnnotation, metav1.PatchOptions{}) - if err != nil { - c.logger.Errorf("failed to patch annotations: %v", err) - return err - } - return nil -} - func (c *Cluster) annotatePostgresResource(isSuccess bool) error { annotations := make(map[string]string) currentTime := metav1.Now().Format("2006-01-02T15:04:05Z") if isSuccess { - annotations["last-major-upgrade-success"] = currentTime + annotations[majorVersionUpgradeSuccessAnnotation] = currentTime } else { - annotations["last-major-upgrade-failure"] = currentTime - } - - patchAnnotation := map[string]map[string]map[string]string{ - "metadata": { - "annotations": annotations, - }, + annotations[majorVersionUpgradeFailureAnnotation] = currentTime } - patchBytes, err := json.Marshal(patchAnnotation) + patchData, err := metaAnnotationsPatch(annotations) if err != nil { - c.logger.Errorf("failed to marshal annotations: %v", err) + c.logger.Errorf("could not form patch for %s postgresql resource: %v", c.Name, err) return err } - - err = c.PatchAnnotations(patchBytes) + _, err = c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), c.Name, types.MergePatchType, patchData, metav1.PatchOptions{}) if err != nil { c.logger.Errorf("failed to patch annotations to Postgres resource: %v", err) return err @@ -96,18 +84,24 @@ func (c *Cluster) annotatePostgresResource(isSuccess bool) error { return nil } -func (c *Cluster) getLastMajorUpgradeAnnotations() map[string]string { - annotations := c.ObjectMeta.GetAnnotations() - upgradeAnnotationKeys := []string{"last-major-upgrade-failure", "last-major-upgrade-success"} - upgradeAnnotationValues := make(map[string]string) - for _, annotationKey := range upgradeAnnotationKeys { - if val, exists := annotations[annotationKey]; exists { - upgradeAnnotationValues[annotationKey] = val - } else { - upgradeAnnotationValues[annotationKey] = "" - } +func (c *Cluster) removeFailuresAnnotation() error { + annotationToRemove := []map[string]string{ + { + "op": "remove", + "path": fmt.Sprintf("/metadata/annotations/%s", majorVersionUpgradeFailureAnnotation), + }, + } + removePatch, err := json.Marshal(annotationToRemove) + if err != nil { + c.logger.Errorf("could not form patch for %s postgresql resource: %v", c.Name, err) + return err + } + _, err = c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), c.Name, types.JSONPatchType, removePatch, metav1.PatchOptions{}) + if err != nil { + c.logger.Errorf("failed to patch annotations to Postgres resource: %v", err) + return err } - return upgradeAnnotationValues + return nil } /* @@ -124,18 +118,17 @@ func (c *Cluster) majorVersionUpgrade() error { desiredVersion := c.GetDesiredMajorVersionAsInt() isUpgradeSuccess := true - lastMajorUpgradeAnnotations := c.getLastMajorUpgradeAnnotations() if c.currentMajorVersion >= desiredVersion { - if lastMajorUpgradeAnnotations["last-major-upgrade-success"] == "" { - c.annotatePostgresResource(isUpgradeSuccess) - c.logger.Info("update last major upgrade success annotation to current timestamp as it was not set") + if _, exists := c.ObjectMeta.Annotations[majorVersionUpgradeFailureAnnotation]; exists { // if failure annotation exists, remove it + c.removeFailuresAnnotation() + c.logger.Infof("removing failure annotation as the cluster is already up to date") } c.logger.Infof("cluster version up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil } - if lastMajorUpgradeAnnotations["last-major-upgrade-failure"] != "" { + if _, exists := c.ObjectMeta.Annotations[majorVersionUpgradeFailureAnnotation]; exists { c.logger.Infof("last major upgrade failed, skipping upgrade") return nil } @@ -170,10 +163,6 @@ func (c *Cluster) majorVersionUpgrade() error { // Recheck version with newest data from Patroni if c.currentMajorVersion >= desiredVersion { - if lastMajorUpgradeAnnotations["last-major-upgrade-success"] == "" { - c.annotatePostgresResource(isUpgradeSuccess) - c.logger.Info("update last major upgrade success annotation to current timestamp as it was not set") - } c.logger.Infof("recheck cluster version is already up to date. current: %d, min desired: %d", c.currentMajorVersion, desiredVersion) return nil } From d645df2838e1401c1b4e43ff571456ec46503a43 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Tue, 27 Aug 2024 14:27:05 +0200 Subject: [PATCH 12/15] change logs --- pkg/cluster/majorversionupgrade.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index 11fdb4356..f07eee885 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -78,7 +78,7 @@ func (c *Cluster) annotatePostgresResource(isSuccess bool) error { } _, err = c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), c.Name, types.MergePatchType, patchData, metav1.PatchOptions{}) if err != nil { - c.logger.Errorf("failed to patch annotations to Postgres resource: %v", err) + c.logger.Errorf("failed to patch annotations to postgresql resource: %v", err) return err } return nil @@ -93,12 +93,12 @@ func (c *Cluster) removeFailuresAnnotation() error { } removePatch, err := json.Marshal(annotationToRemove) if err != nil { - c.logger.Errorf("could not form patch for %s postgresql resource: %v", c.Name, err) + c.logger.Errorf("could not form removal patch for %s postgresql resource: %v", c.Name, err) return err } _, err = c.KubeClient.Postgresqls(c.Namespace).Patch(context.Background(), c.Name, types.JSONPatchType, removePatch, metav1.PatchOptions{}) if err != nil { - c.logger.Errorf("failed to patch annotations to Postgres resource: %v", err) + c.logger.Errorf("failed to remove annotations from postgresql resource: %v", err) return err } return nil @@ -117,7 +117,6 @@ func (c *Cluster) majorVersionUpgrade() error { } desiredVersion := c.GetDesiredMajorVersionAsInt() - isUpgradeSuccess := true if c.currentMajorVersion >= desiredVersion { if _, exists := c.ObjectMeta.Annotations[majorVersionUpgradeFailureAnnotation]; exists { // if failure annotation exists, remove it @@ -167,6 +166,7 @@ func (c *Cluster) majorVersionUpgrade() error { return nil } + isUpgradeSuccess := true numberOfPods := len(pods) if allRunning && masterPod != nil { c.logger.Infof("healthy cluster ready to upgrade, current: %d desired: %d", c.currentMajorVersion, desiredVersion) From f1a388fdc9d8c7270159b0aa8f612c78753ef1cc Mon Sep 17 00:00:00 2001 From: inovindasari Date: Tue, 27 Aug 2024 17:04:51 +0200 Subject: [PATCH 13/15] add docs --- docs/administrator.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/administrator.md b/docs/administrator.md index d2b8e7039..ee97292eb 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -82,6 +82,12 @@ upgrade procedure, refer to the [corresponding PR in Spilo](https://github.com/z When `major_version_upgrade_mode` is set to `manual` the operator will run the upgrade script for you after the manifest is updated and pods are rotated. +### Upgrade annotations + +When an upgrade is executed, the operator sets an annotation in the PostgreSQL resource, either `last-major-upgrade-success` if the upgrade succeeds, or `last-major-upgrade-failure` if it fails. The value of the annotation is a timestamp indicating when the upgrade occurred. + +If a PostgreSQL resource contains a failure annotation, the operator will not attempt to retry the upgrade during a sync event. To remove the failure annotation, you can revert the PostgreSQL version back to the current version. This action will trigger the removal of the failure annotation. + ## Non-default cluster domain If your cluster uses a DNS domain other than the default `cluster.local`, this From ae97eb65f5a2195433eed1b5e559a45314240f4d Mon Sep 17 00:00:00 2001 From: Ida Novindasari Date: Wed, 28 Aug 2024 13:37:58 +0200 Subject: [PATCH 14/15] Update e2e/tests/test_e2e.py Co-authored-by: Polina Bungina <27892524+hughcapet@users.noreply.github.com> --- e2e/tests/test_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2e3f8e302..b8a1f1410 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1284,7 +1284,7 @@ def get_annotations(): self.assertIsNotNone(second_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") self.assertNotEqual(annotations.get("last-major-upgrade-success"), second_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") - # test annotation with failed upgrade annotation + # test upgrade with failed upgrade annotation pg_patch_version_16 = { "metadata": { "annotations": { From 4e6df31450e0dd51a300fb6aa280f2fb30eb20b4 Mon Sep 17 00:00:00 2001 From: inovindasari Date: Wed, 28 Aug 2024 13:42:50 +0200 Subject: [PATCH 15/15] add check failure annotation should not be set --- e2e/tests/test_e2e.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index b8a1f1410..f89e2fb86 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1257,6 +1257,9 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) self.eventuallyEqual(check_version, 13, "Version should not be upgraded") + second_annotations = get_annotations() + self.assertIsNone(second_annotations.get("last-major-upgrade-failure"), "Annotation for last upgrade's failure should not be set") + # change the version again to trigger operator sync maintenance_window_current = f"{(current_time-timedelta(minutes=30)).strftime('%H:%M')}-{(current_time+timedelta(minutes=30)).strftime('%H:%M')}" pg_patch_version_15 = { @@ -1280,9 +1283,9 @@ def get_annotations(): self.eventuallyEqual(check_version, 15, "Version should be upgraded from 13 to 15") # check if annotation for last upgrade's success is updated after second upgrade - second_annotations = get_annotations() - self.assertIsNotNone(second_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") - self.assertNotEqual(annotations.get("last-major-upgrade-success"), second_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") + third_annotations = get_annotations() + self.assertIsNotNone(third_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not set") + self.assertNotEqual(annotations.get("last-major-upgrade-success"), third_annotations.get("last-major-upgrade-success"), "Annotation for last upgrade's success is not updated") # test upgrade with failed upgrade annotation pg_patch_version_16 = { @@ -1315,8 +1318,8 @@ def get_annotations(): k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) k8s.wait_for_pod_start('spilo-role=replica,' + cluster_label) - third_annotations = get_annotations() - self.assertIsNone(third_annotations.get("last-major-upgrade-failure"), "Annotation for last upgrade's failure is not removed") + fourth_annotations = get_annotations() + self.assertIsNone(fourth_annotations.get("last-major-upgrade-failure"), "Annotation for last upgrade's failure is not removed") @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_persistent_volume_claim_retention_policy(self):