changeVersion return ready

mongodb · Oct 17, 2023 · 13703f1 · 13703f1
1 parent f4d5326
commit 13703f1
Show file tree

Hide file tree

Showing 5 changed files with 203 additions and 1 deletion.
diff --git a/cmd/readiness/main.go b/cmd/readiness/main.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config"
@@ -84,6 +85,13 @@ func isPodReady(conf config.Config) (bool, error) {
 
 // isOnWaitingStep returns true if the agent is stuck on waiting for the other Agents or something else to happen.
 func isOnWaitingStep(health health.Status) bool {
+	currentMove := findCurrentMove(health.MmsStatus)
+
+	if strings.ToLower(currentMove.Move) == "changeversion" {
+		logger.Info("Making ChangeVersion to return ready!")
+		return true
+	}
+
 	currentStep := findCurrentStep(health.MmsStatus)
 	if currentStep != nil {
 		return isWaitStep(currentStep)
@@ -136,6 +144,45 @@ func findCurrentStep(processStatuses map[string]health.MmsDirectorStatus) *healt
 	return lastStartedStep
 }
 
+// findCurrentStep returns the step which the Agent is working now.
+// The algorithm (described in https://github.com/10gen/ops-manager-kubernetes/pull/401#discussion_r333071555):
+//   - Obtain the latest plan (the last one in the plans array)
+//   - Find the last step, which has Started not nil and Completed nil. The Steps are processed as a tree in a BFS fashion.
+//     The last element is very likely to be the Step the Agent is performing at the moment. There are some chances that
+//     this is a waiting step, use isWaitStep to verify this.
+func findCurrentMove(processStatuses map[string]health.MmsDirectorStatus) *health.MoveStatus {
+	var currentPlan *health.PlanStatus
+	if len(processStatuses) == 0 {
+		// Seems shouldn't happen but let's check anyway - may be needs to be changed to Info if this happens
+		logger.Warnf("There is no information about Agent process plans")
+		return nil
+	}
+	if len(processStatuses) > 1 {
+		logger.Errorf("Only one process status is expected but got %d!", len(processStatuses))
+		return nil
+	}
+
+	// There is always only one process managed by the Agent - so there will be only one loop
+	for processName, processStatus := range processStatuses {
+		if len(processStatus.Plans) == 0 {
+			logger.Errorf("The process %s doesn't contain any plans!", processName)
+			return nil
+		}
+		currentPlan = processStatus.Plans[len(processStatus.Plans)-1]
+	}
+
+	if currentPlan.Completed != nil {
+		logger.Debugf("The Agent hasn't reported working on the new config yet, the last plan finished at %s",
+			currentPlan.Completed.Format(time.RFC3339))
+		return nil
+	}
+
+	lastStartedMove := currentPlan.Moves[len(currentPlan.Moves)-1]
+	logger.Debugf("Current Move: %s", lastStartedMove.Move)
+
+	return lastStartedMove
+}
+
 // isWaitStep returns true is the Agent is currently waiting for something to happen.
 //
 // Most of the time, the Agent waits for an initialization by other member of the cluster. In such case,

diff --git a/cmd/readiness/readiness_test.go b/cmd/readiness/readiness_test.go
@@ -266,6 +266,14 @@ func TestHeadlessAgentReachedGoal(t *testing.T) {
 	assert.Equal(t, map[string]string{"agent.mongodb.com/version": "5"}, thePod.Annotations)
 }
 
+// TestChangeVersion verifies that the probe reports "true" if we are in the changeVersion step.
+func TestChangeVersion(t *testing.T) {
+	c := testConfig("testdata/config-change-version.json")
+	ready, err := isPodReady(c)
+	assert.True(t, ready)
+	assert.NoError(t, err)
+}
+
 func testConfig(healthFilePath string) config.Config {
 	return testConfigWithMongoUp(healthFilePath, 15*time.Second)
 }

diff --git a/cmd/readiness/testdata/config-change-version.json b/cmd/readiness/testdata/config-change-version.json
@@ -0,0 +1,146 @@
+{
+    "statuses": {
+        "my-replica-set-2": {
+            "IsInGoalState": false,
+            "LastMongoUpTime": 1697534279,
+            "ExpectedToBeUp": true,
+            "ReplicationStatus": 2
+        }
+    },
+    "mmsStatus": {
+        "my-replica-set-2": {
+            "name": "my-replica-set-2",
+            "lastGoalVersionAchieved": -1,
+            "plans": [
+                {
+                    "automationConfigVersion": 2,
+                    "started": "2023-10-17T09:16:50.340423612Z",
+                    "completed": null,
+                    "moves": [
+                        {
+                            "move": "Start",
+                            "moveDoc": "Start the process",
+                            "steps": [
+                                {
+                                    "step": "StartFresh",
+                                    "stepDoc": "Start a mongo instance  (start fresh)",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:16:50.340448835Z",
+                                    "completed": null,
+                                    "result": "error"
+                                }
+                            ]
+                        },
+                        {
+                            "move": "WaitRsInit",
+                            "moveDoc": "Wait for the replica set to be initialized by another member",
+                            "steps": [
+                                {
+                                    "step": "WaitRsInit",
+                                    "stepDoc": "Wait for the replica set to be initialized by another member",
+                                    "isWaitStep": true,
+                                    "started": null,
+                                    "completed": null,
+                                    "result": ""
+                                }
+                            ]
+                        },
+                        {
+                            "move": "WaitFeatureCompatibilityVersionCorrect",
+                            "moveDoc": "Wait for featureCompatibilityVersion to be right",
+                            "steps": [
+                                {
+                                    "step": "WaitFeatureCompatibilityVersionCorrect",
+                                    "stepDoc": "Wait for featureCompatibilityVersion to be right",
+                                    "isWaitStep": true,
+                                    "started": null,
+                                    "completed": null,
+                                    "result": ""
+                                }
+                            ]
+                        }
+                    ]
+                },
+                {
+                    "automationConfigVersion": 2,
+                    "started": "2023-10-17T09:16:52.72943033Z",
+                    "completed": null,
+                    "moves": [
+                        {
+                            "move": "ChangeVersion",
+                            "moveDoc": "Change MongoDB Version",
+                            "steps": [
+                                {
+                                    "step": "CheckWrongVersion",
+                                    "stepDoc": "Check that MongoDB version is wrong",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:16:52.72945587Z",
+                                    "completed": "2023-10-17T09:16:52.72965925Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "CheckRsCorrect",
+                                    "stepDoc": "Check that replica set configuration is correct",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:16:52.729661497Z",
+                                    "completed": "2023-10-17T09:16:52.729932439Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "WaitAllRouterConfigsFlushedForUpgrade",
+                                    "stepDoc": "Wait until flushRouterConfig has been run on all mongoses",
+                                    "isWaitStep": true,
+                                    "started": "2023-10-17T09:16:52.729937288Z",
+                                    "completed": "2023-10-17T09:16:52.730140084Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "WaitCanUpdate",
+                                    "stepDoc": "Wait until the update can be made",
+                                    "isWaitStep": true,
+                                    "started": "2023-10-17T09:16:52.730142421Z",
+                                    "completed": "2023-10-17T09:16:52.730326363Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "DisableBalancerIfFirst",
+                                    "stepDoc": "Disable the balancer (may take a while)",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:16:52.730328642Z",
+                                    "completed": "2023-10-17T09:16:52.790512333Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "Stop",
+                                    "stepDoc": "Shutdown the process",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:16:52.790516813Z",
+                                    "completed": "2023-10-17T09:17:53.221577128Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "RemoveDbFilesIfArbiterDowngrade",
+                                    "stepDoc": "Delete db files if this is an arbiter downgrade.",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:17:53.221580106Z",
+                                    "completed": "2023-10-17T09:17:53.328152196Z",
+                                    "result": "success"
+                                },
+                                {
+                                    "step": "StartWithUpgrade",
+                                    "stepDoc": "Start a mongo instance  (upgrade)",
+                                    "isWaitStep": false,
+                                    "started": "2023-10-17T09:17:53.328156144Z",
+                                    "completed": null,
+                                    "result": "error"
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ],
+            "errorCode": 0,
+            "errorString": ""
+        }
+    }
+}
diff --git a/helm-charts b/helm-charts
diff --git a/pkg/readiness/health/health.go b/pkg/readiness/health/health.go
@@ -53,6 +53,7 @@ type PlanStatus struct {
 
 type MoveStatus struct {
 	Steps []*StepStatus `json:"steps"`
+	Move  string        `json:"move"`
 }
 type StepStatus struct {
 	Step       string     `json:"step"`
+2 −2		charts/atlas-operator-crds/Chart.yaml
+0 −78		charts/atlas-operator-crds/templates/atlas.mongodb.com_atlasdeployments.yaml
+0 −172		charts/atlas-operator-crds/templates/atlas.mongodb.com_atlasfederatedauths.yaml
+0 −53		charts/atlas-operator-crds/templates/atlas.mongodb.com_atlasprojects.yaml
+3 −3		charts/atlas-operator/Chart.yaml
+0 −18		charts/atlas-operator/templates/_helpers.tpl
+2 −2		charts/community-operator-crds/Chart.yaml
+2 −106		charts/community-operator-crds/templates/mongodbcommunity.mongodb.com_mongodbcommunity.yaml
+3 −3		charts/community-operator/Chart.yaml
+3 −3		charts/community-operator/values.yaml
+1 −1		charts/enterprise-operator/Chart.yaml
+0 −20		charts/enterprise-operator/crds/mongodb.com_mongodb.yaml
+0 −33		charts/enterprise-operator/crds/mongodb.com_mongodbmulticluster.yaml
+16 −285		charts/enterprise-operator/crds/mongodb.com_opsmanagers.yaml
+2 −2		charts/enterprise-operator/templates/operator.yaml
+6 −13		charts/enterprise-operator/values-openshift.yaml
+5 −5		charts/enterprise-operator/values.yaml