show intermediate cluster upgrade status (#4432)

pkg/api/handlers/types/types.go

@@ -37,6 +37,7 @@ type ResponseApp struct {
 	UpdateCheckerSpec string              `json:"updateCheckerSpec"`
 	AutoDeploy        apptypes.AutoDeploy `json:"autoDeploy"`
 	Namespace         string              `json:"namespace"`
+	AppState          string              `json:"appState"`
 
 	IsGitOpsSupported              bool   `json:"isGitOpsSupported"`
 	IsIdentityServiceSupported     bool   `json:"isIdentityServiceSupported"`

pkg/embeddedcluster/monitor.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
+	appstatetypes "github.com/replicatedhq/kots/pkg/appstate/types"
 	"github.com/replicatedhq/kots/pkg/logger"
 	"github.com/replicatedhq/kots/pkg/store"
 	"github.com/replicatedhq/kots/pkg/util"
@@ -21,7 +22,7 @@ var stateMut = sync.Mutex{}
 // - The app has an embedded cluster configuration.
 // - The app embedded cluster configuration differs from the current embedded cluster config.
 // - The current cluster config (as part of the Installation object) already exists in the cluster.
-func MaybeStartClusterUpgrade(ctx context.Context, client kubernetes.Interface, store store.Store, conf *v1beta1.Config) error {
+func MaybeStartClusterUpgrade(ctx context.Context, store store.Store, conf *v1beta1.Config, appID string) error {
 	if conf == nil {
 		return nil
 	}
@@ -43,13 +44,37 @@ func MaybeStartClusterUpgrade(ctx context.Context, client kubernetes.Interface,
 	} else if !upgrade {
 		return nil
 	}
-	if err := startClusterUpgrade(ctx, spec); err != nil {
-		return fmt.Errorf("failed to start cluster upgrade: %w", err)
-	}
 
-	go watchClusterState(ctx, store)
+	// we need to wait for the application to be ready before we can start the upgrade.
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
 
-	return nil
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+		}
+
+		appStatus, err := store.GetAppStatus(appID)
+		if err != nil {
+			return fmt.Errorf("failed to get app status: %w", err)
+		}
+
+		if appStatus.State != appstatetypes.StateReady {
+			logger.Infof("waiting for app to be ready before starting cluster upgrade. current state: %s", appStatus.State)
+			continue
+		}
+
+		if err := startClusterUpgrade(ctx, spec); err != nil {
+			return fmt.Errorf("failed to start cluster upgrade: %w", err)
+		}
+		logger.Info("started cluster upgrade")
+
+		go watchClusterState(ctx, store)
+
+		return nil
+	}
 }
 
 // InitClusterState initializes the cluster state in the database. This should be called when the

pkg/handlers/app.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
 	"github.com/replicatedhq/kots/pkg/airgap"
 	downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types"
 	"github.com/replicatedhq/kots/pkg/api/handlers/types"
@@ -228,6 +229,12 @@ func responseAppFromApp(a *apptypes.App) (*types.ResponseApp, error) {
 		return nil, errors.Wrap(err, "failed to get license")
 	}
 
+	appStatus, err := store.GetStore().GetAppStatus(a.ID)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get app status")
+	}
+	appState := string(appStatus.State)
+
 	downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to list downstreams for app")
@@ -320,9 +327,12 @@ func responseAppFromApp(a *apptypes.App) (*types.ResponseApp, error) {
 	}
 
 	if util.IsEmbeddedCluster() {
-		embeddedClusterConfig, err := store.GetStore().GetEmbeddedClusterConfigForVersion(a.ID, a.CurrentSequence)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to get embedded cluster config")
+		var embeddedClusterConfig *embeddedclusterv1beta1.Config
+		if appVersions.CurrentVersion != nil {
+			embeddedClusterConfig, err = store.GetStore().GetEmbeddedClusterConfigForVersion(a.ID, appVersions.CurrentVersion.Sequence)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to get embedded cluster config")
+			}
 		}
 
 		if embeddedClusterConfig != nil {
@@ -365,6 +375,7 @@ func responseAppFromApp(a *apptypes.App) (*types.ResponseApp, error) {
 		IsConfigurable:                 a.IsConfigurable,
 		UpdateCheckerSpec:              a.UpdateCheckerSpec,
 		AutoDeploy:                     a.AutoDeploy,
+		AppState:                       appState,
 		IsGitOpsSupported:              isGitopsSupported,
 		IsIdentityServiceSupported:     license.Spec.IsIdentityServiceSupported,
 		IsAppIdentityServiceSupported:  isAppIdentityServiceSupported,

pkg/operator/operator.go

@@ -407,23 +407,16 @@ func (o *Operator) DeployApp(appID string, sequence int64) (deployed bool, deplo
 	}
 	deployed, err = o.client.DeployApp(deployArgs)
 	if err != nil {
-		if util.IsEmbeddedCluster() {
-			go func() {
-				logger.Info("app deploy failed, starting cluster upgrade in the background")
-				err2 := embeddedcluster.MaybeStartClusterUpgrade(context.Background(), o.k8sClientset, o.store, kotsKinds.EmbeddedClusterConfig)
-				if err2 != nil {
-					logger.Error(errors.Wrap(err2, "failed to start cluster upgrade"))
-				}
-				logger.Info("cluster upgrade started")
-			}()
-		}
-
 		return false, errors.Wrap(err, "failed to deploy app")
 	}
 
-	err = embeddedcluster.MaybeStartClusterUpgrade(context.TODO(), o.k8sClientset, o.store, kotsKinds.EmbeddedClusterConfig)
-	if err != nil {
-		return false, errors.Wrap(err, "failed to start cluster upgrade")
+	if deployed {
+		go func() {
+			err = embeddedcluster.MaybeStartClusterUpgrade(context.TODO(), o.store, kotsKinds.EmbeddedClusterConfig, app.ID)
+			if err != nil {
+				logger.Error(errors.Wrap(err, "failed to start cluster upgrade"))
+			}
+		}()
 	}
 
 	return deployed, nil

web/src/features/AppVersionHistory/AppVersionHistoryRow.tsx

@@ -175,6 +175,12 @@ function AppVersionHistoryRow(props: Props) {
     if (isHelmManaged) {
       return false;
     }
+    if (
+      Utilities.isPendingClusterUpgrade(selectedApp) &&
+      version.status === "deployed"
+    ) {
+      return true;
+    }
     if (version.status === "deploying") {
       return true;
     }
@@ -465,6 +471,23 @@ function AppVersionHistoryRow(props: Props) {
     });
 
     if (!isPastVersion && !isPendingDeployedVersion) {
+      if (
+        Utilities.isPendingClusterUpgrade(selectedApp) &&
+        version.status === "deployed"
+      ) {
+        return (
+          <span className="flex alignItems--center u-fontSize--small u-lineHeight--normal u-textColor--bodyCopy u-fontWeight--medium">
+            <Loader
+              className="flex alignItems--center u-marginRight--5"
+              size="16"
+            />
+            {selectedApp?.appState !== "ready"
+              ? "Waiting for app to be ready"
+              : "Updating cluster"}
+          </span>
+        );
+      }
+
       if (version.status === "deployed") {
         return (
           <div>

web/src/features/Dashboard/components/Dashboard.tsx

@@ -228,6 +228,11 @@ const Dashboard = () => {
   };
 
   const onUpdateDownloadStatusError = (data: Error) => {
+    if (Utilities.isPendingClusterUpgrade(app)) {
+      // if the cluster is upgrading, we don't want to show an error
+      return;
+    }
+
     setState({
       checkingForUpdates: false,
       checkingForUpdateError: true,
@@ -586,6 +591,11 @@ const Dashboard = () => {
   };
 
   const onError = (err: Error) => {
+    if (Utilities.isPendingClusterUpgrade(app)) {
+      // if the cluster is upgrading, we don't want to show an error
+      return;
+    }
+
     setState({
       checkingForUpdateError: true,
       checkingForUpdates: false,

web/src/features/Dashboard/components/DashboardVersionCard.tsx

@@ -218,6 +218,11 @@ const DashboardVersionCard = (props: Props) => {
   }, [location.search]);
 
   useEffect(() => {
+    if (Utilities.isPendingClusterUpgrade(selectedApp)) {
+      // if the cluster is upgrading, we don't want to show an error
+      return;
+    }
+
     if (latestDeployableVersionErrMsg instanceof Error) {
       setState({
         latestDeployableVersionErrMsg: `Failed to get latest deployable version: ${latestDeployableVersionErrMsg.message}`,
@@ -348,6 +353,23 @@ const DashboardVersionCard = (props: Props) => {
   };
 
   const getCurrentVersionStatus = (version: Version | null) => {
+    if (
+      Utilities.isPendingClusterUpgrade(selectedApp) &&
+      version?.status === "deployed"
+    ) {
+      return (
+        <span className="flex alignItems--center u-fontSize--small u-lineHeight--normal u-textColor--bodyCopy u-fontWeight--medium">
+          <Loader
+            className="flex alignItems--center u-marginRight--5"
+            size="16"
+          />
+          {selectedApp?.appState !== "ready"
+            ? "Waiting for app to be ready"
+            : "Updating cluster"}
+        </span>
+      );
+    }
+
     if (version?.status === "deployed" || version?.status === "pending") {
       return (
         <span className="status-tag success flex-auto">
@@ -691,7 +713,8 @@ const DashboardVersionCard = (props: Props) => {
                 <ReactTooltip effect="solid" className="replicated-tooltip" />
               </div>
             ) : null}
-            {currentVersion?.status === "deploying" ? null : (
+            {currentVersion?.status === "deploying" ||
+            Utilities.isPendingClusterUpgrade(selectedApp) ? null : (
               <div className="flex-column justifyContent--center u-marginLeft--10">
                 <button
                   className="secondary blue btn"

web/src/types/index.ts

@@ -26,6 +26,7 @@ export type App = {
   needsRegistry?: boolean;
   slug: string;
   updateCheckerSpec: string;
+  appState: string;
 };
 
 export type AppLicense = {

web/src/utilities/utilities.js

@@ -645,14 +645,11 @@ export const Utilities = {
     );
   },
 
-  shouldShowClusterUpgradeModal(apps) {
-    if (!apps || apps.length === 0) {
+  isPendingClusterUpgrade(app) {
+    if (!app) {
       return false;
     }
 
-    // embedded cluster can only have one app
-    const app = apps[0];
-
     const triedToDeploy =
       app.downstream?.currentVersion?.status === "deploying" ||
       app.downstream?.currentVersion?.status === "deployed" ||
@@ -661,14 +658,24 @@ export const Utilities = {
       return false;
     }
 
-    // show the upgrade modal if the user has tried to deploy the current version
+    // return true if the user has tried to deploy the current version
     // and the cluster will upgrade or is already upgrading
     return (
       app.downstream?.cluster?.requiresUpgrade ||
       Utilities.isClusterUpgrading(app.downstream?.cluster?.state)
     );
   },
 
+  shouldShowClusterUpgradeModal(apps) {
+    if (!apps || apps.length === 0) {
+      return false;
+    }
+
+    // embedded cluster can only have one app
+    const app = apps[0];
+    return this.isPendingClusterUpgrade(app);
+  },
+
   // Converts string to titlecase i.e. 'hello' -> 'Hello'
   // @returns {String}
   toTitleCase(word) {