feat: use an upgrade service deployment mechanism for embedded clusters

.github/workflows/build-test.yaml

@@ -1940,13 +1940,6 @@ jobs:
             exit $EXIT_CODE
           fi
 
-          # validate that preflight checks ran
-          JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}"
-          if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" | grep -c pending_preflight)" != "1" ]; then
-              echo "Preflight checks did not run"
-              exit 1
-          fi
-
           COUNTER=1
           while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
             ((COUNTER += 1))

Makefile

@@ -51,6 +51,8 @@ kots: capture-start-time kots-real report-metric
 
 .PHONY: kots-real
 kots-real:
+	mkdir -p web/dist
+	touch web/dist/README.md
 	go build ${LDFLAGS} -o bin/kots $(BUILDFLAGS) github.com/replicatedhq/kots/cmd/kots
 
 .PHONY: fmt
@@ -80,7 +82,7 @@ build: capture-start-time build-real report-metric
 .PHONY: build-real
 build-real:
 	mkdir -p web/dist
-	touch web/dist/THIS_IS_OKTETO  # we need this for go:embed, but it's not actually used in dev
+	touch web/dist/README.md
 	go build ${LDFLAGS} ${GCFLAGS} -v -o bin/kotsadm $(BUILDFLAGS) ./cmd/kotsadm
 
 .PHONY: tidy
@@ -112,21 +114,31 @@ debug-build:
 debug: debug-build
 	LOG_LEVEL=$(LOG_LEVEL) dlv --listen=:2345 --headless=true --api-version=2 exec ./bin/kotsadm-debug api
 
-.PHONY: build-ttl.sh
-build-ttl.sh: kots build
+.PHONY: web
+web:
 	source .image.env && ${MAKE} -C web build-kotsadm
-	docker build -f deploy/Dockerfile -t ttl.sh/${CURRENT_USER}/kotsadm:24h .
+
+.PHONY: build-ttl.sh
+build-ttl.sh: export GOOS ?= linux
+build-ttl.sh: export GOARCH ?= amd64
+build-ttl.sh: web kots build
+	docker build --platform $(GOOS)/$(GOARCH) -f deploy/Dockerfile -t ttl.sh/${CURRENT_USER}/kotsadm:24h .
 	docker push ttl.sh/${CURRENT_USER}/kotsadm:24h
 
 .PHONY: all-ttl.sh
+all-ttl.sh: export GOOS ?= linux
+all-ttl.sh: export GOARCH ?= amd64
 all-ttl.sh: build-ttl.sh
-	source .image.env && IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h make -C migrations build_schema
+	source .image.env && \
+		IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h \
+		DOCKER_BUILD_ARGS="--platform $(GOOS)/$(GOARCH)" \
+		make -C migrations build_schema
 
-	docker pull kotsadm/minio:${MINIO_TAG}
+	docker pull --platform $(GOOS)/$(GOARCH)" kotsadm/minio:${MINIO_TAG}
 	docker tag kotsadm/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG}
 	docker push ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG}
 
-	docker pull kotsadm/rqlite:${RQLITE_TAG}
+	docker pull --platform $(GOOS)/$(GOARCH)" kotsadm/rqlite:${RQLITE_TAG}
 	docker tag kotsadm/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG}
 	docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG}
 

cmd/kots/cli/admin-console-push-images.go

@@ -115,12 +115,15 @@ func genAndCheckPushOptions(endpoint string, namespace string, log *logger.CLILo
 		log.FinishSpinner()
 	}
 
+	registryEndpoint, registryNamespace := splitEndpointAndNamespace(endpoint)
+
 	options := imagetypes.PushImagesOptions{
 		KotsadmTag: v.GetString("kotsadm-tag"),
 		Registry: registrytypes.RegistryOptions{
-			Endpoint: endpoint,
-			Username: username,
-			Password: password,
+			Endpoint:  registryEndpoint,
+			Namespace: registryNamespace,
+			Username:  username,
+			Password:  password,
 		},
 		ProgressWriter: os.Stdout,
 	}

cmd/kots/cli/airgap-update.go

@@ -0,0 +1,252 @@
+package cli
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/kots/pkg/archives"
+	"github.com/replicatedhq/kots/pkg/auth"
+	registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types"
+	"github.com/replicatedhq/kots/pkg/image"
+	imagetypes "github.com/replicatedhq/kots/pkg/image/types"
+	"github.com/replicatedhq/kots/pkg/k8sutil"
+	"github.com/replicatedhq/kots/pkg/kotsutil"
+	"github.com/replicatedhq/kots/pkg/logger"
+	"github.com/replicatedhq/kots/pkg/tasks"
+	"github.com/replicatedhq/kots/pkg/upload"
+	"github.com/replicatedhq/kots/pkg/util"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func AirgapUpdateCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:           "airgap-update [appSlug]",
+		Short:         "Process and upload an airgap update to the admin console",
+		Long:          "",
+		SilenceUsage:  true,
+		SilenceErrors: false,
+		Hidden:        true,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			viper.BindPFlags(cmd.Flags())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := viper.GetViper()
+
+			if len(args) == 0 {
+				cmd.Help()
+				os.Exit(1)
+			}
+
+			appSlug := args[0]
+			log := logger.NewCLILogger(cmd.OutOrStdout())
+
+			airgapBundle := v.GetString("airgap-bundle")
+			if airgapBundle == "" {
+				return fmt.Errorf("--airgap-bundle is required")
+			}
+
+			namespace, err := getNamespaceOrDefault(v.GetString("namespace"))
+			if err != nil {
+				return errors.Wrap(err, "failed to get namespace")
+			}
+
+			clientset, err := k8sutil.GetClientset()
+			if err != nil {
+				return errors.Wrap(err, "failed to get clientset")
+			}
+
+			registryConfig, err := getRegistryConfig(v, clientset, appSlug)
+			if err != nil {
+				return errors.Wrap(err, "failed to get registry config")
+			}
+
+			pushOpts := imagetypes.PushImagesOptions{
+				KotsadmTag: v.GetString("kotsadm-tag"),
+				Registry: registrytypes.RegistryOptions{
+					Endpoint:  registryConfig.OverrideRegistry,
+					Namespace: registryConfig.OverrideNamespace,
+					Username:  registryConfig.Username,
+					Password:  registryConfig.Password,
+				},
+				ProgressWriter: getProgressWriter(v, log),
+				LogForUI:       v.GetBool("from-api"),
+			}
+
+			if _, err := os.Stat(airgapBundle); err == nil {
+				err = image.TagAndPushImagesFromBundle(airgapBundle, pushOpts)
+				if err != nil {
+					return errors.Wrap(err, "failed to push images")
+				}
+			} else {
+				return errors.Wrap(err, "failed to stat airgap bundle")
+			}
+
+			updateFiles, err := getAirgapUpdateFiles(airgapBundle)
+			if err != nil {
+				return errors.Wrap(err, "failed to get airgap update files")
+			}
+			airgapUpdate, err := archives.FilterAirgapBundle(airgapBundle, updateFiles)
+			if err != nil {
+				return errors.Wrap(err, "failed to create filtered airgap bundle")
+			}
+			defer os.RemoveAll(airgapUpdate)
+
+			var localPort int
+			if v.GetBool("from-api") {
+				localPort = 3000
+			} else {
+				stopCh := make(chan struct{})
+				defer close(stopCh)
+
+				lp, errChan, err := upload.StartPortForward(namespace, stopCh, log)
+				if err != nil {
+					return err
+				}
+				localPort = lp
+
+				go func() {
+					select {
+					case err := <-errChan:
+						if err != nil {
+							log.Error(err)
+							os.Exit(1)
+						}
+					case <-stopCh:
+					}
+				}()
+			}
+
+			uploadEndpoint := fmt.Sprintf("http://localhost:%d/api/v1/app/%s/airgap/update", localPort, url.PathEscape(appSlug))
+
+			log.ActionWithSpinner("Uploading airgap update")
+			if err := uploadAirgapUpdate(airgapUpdate, uploadEndpoint, namespace); err != nil {
+				log.FinishSpinnerWithError()
+				return errors.Wrap(err, "failed to upload airgap update")
+			}
+			log.FinishSpinner()
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringP("namespace", "n", "", "the namespace in which kots/kotsadm is installed")
+	cmd.Flags().String("airgap-bundle", "", "path to the application airgap bundle to upload")
+
+	cmd.Flags().Bool("from-api", false, "whether the airgap update command was triggered by the API")
+	cmd.Flags().String("task-id", "", "the task ID to use for tracking progress")
+	cmd.Flags().MarkHidden("from-api")
+	cmd.Flags().MarkHidden("task-id")
+
+	registryFlags(cmd.Flags())
+
+	return cmd
+}
+
+func getProgressWriter(v *viper.Viper, log *logger.CLILogger) io.Writer {
+	if v.GetBool("from-api") {
+		pipeReader, pipeWriter := io.Pipe()
+		go func() {
+			scanner := bufio.NewScanner(pipeReader)
+			for scanner.Scan() {
+				if err := tasks.SetTaskStatus(v.GetString("task-id"), scanner.Text(), "running"); err != nil {
+					log.Error(err)
+				}
+			}
+			pipeReader.CloseWithError(scanner.Err())
+		}()
+		return pipeWriter
+	}
+	return os.Stdout
+}
+
+func getAirgapUpdateFiles(airgapBundle string) ([]string, error) {
+	airgap, err := kotsutil.FindAirgapMetaInBundle(airgapBundle)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find airgap meta in bundle")
+	}
+
+	if airgap.Spec.EmbeddedClusterArtifacts == nil {
+		return nil, errors.New("embedded cluster artifacts not found in airgap bundle")
+	}
+
+	if airgap.Spec.EmbeddedClusterArtifacts.Metadata == "" {
+		return nil, errors.New("embedded cluster metadata not found in airgap bundle")
+	}
+
+	if airgap.Spec.EmbeddedClusterArtifacts.AdditionalArtifacts == nil {
+		return nil, errors.New("embedded cluster additional artifacts not found in airgap bundle")
+	}
+
+	files := []string{
+		"airgap.yaml",
+		"app.tar.gz",
+		airgap.Spec.EmbeddedClusterArtifacts.Metadata,
+		airgap.Spec.EmbeddedClusterArtifacts.AdditionalArtifacts["kots"],
+	}
+
+	return files, nil
+}
+
+func uploadAirgapUpdate(airgapBundle string, uploadEndpoint string, namespace string) error {
+	buffer := bytes.NewBuffer(nil)
+	writer := multipart.NewWriter(buffer)
+
+	part, err := writer.CreateFormFile("application.airgap", "application.airgap")
+	if err != nil {
+		return errors.Wrap(err, "failed to create form file")
+	}
+
+	f, err := os.Open(airgapBundle)
+	if err != nil {
+		return errors.Wrap(err, "failed to open airgap bundle")
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(part, f); err != nil {
+		return errors.Wrap(err, "failed to copy airgap bundle to form file")
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return errors.Wrap(err, "failed to close writer")
+	}
+
+	clientset, err := k8sutil.GetClientset()
+	if err != nil {
+		return errors.Wrap(err, "failed to get k8s clientset")
+	}
+
+	authSlug, err := auth.GetOrCreateAuthSlug(clientset, namespace)
+	if err != nil {
+		return errors.Wrap(err, "failed to get auth slug")
+	}
+
+	newReq, err := util.NewRequest("PUT", uploadEndpoint, buffer)
+	if err != nil {
+		return errors.Wrap(err, "failed to create request")
+	}
+	newReq.Header.Add("Content-Type", writer.FormDataContentType())
+	newReq.Header.Add("Authorization", authSlug)
+
+	resp, err := http.DefaultClient.Do(newReq)
+	if err != nil {
+		return errors.Wrap(err, "failed to make request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 404 {
+		return errors.New("App not found")
+	} else if resp.StatusCode != 200 {
+		return errors.Errorf("Unexpected status code: %d", resp.StatusCode)
+	}
+
+	return nil
+}

cmd/kots/cli/install.go

@@ -37,8 +37,8 @@ import (
 	"github.com/replicatedhq/kots/pkg/print"
 	"github.com/replicatedhq/kots/pkg/pull"
 	"github.com/replicatedhq/kots/pkg/replicatedapp"
-	"github.com/replicatedhq/kots/pkg/store/kotsstore"
 	storetypes "github.com/replicatedhq/kots/pkg/store/types"
+	"github.com/replicatedhq/kots/pkg/tasks"
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	"github.com/replicatedhq/troubleshoot/pkg/preflight"
 	"github.com/spf13/cobra"
@@ -648,7 +648,7 @@ func uploadAirgapArchive(deployOptions kotsadmtypes.DeployOptions, authSlug stri
 		return false, errors.Wrap(err, "failed to create form from file")
 	}
 
-	contents, err := archives.GetFileFromAirgap(filename, deployOptions.AirgapBundle)
+	contents, err := archives.GetFileContentFromTGZArchive(filename, deployOptions.AirgapBundle)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to get file from airgap")
 	}
@@ -887,7 +887,7 @@ func ValidateAutomatedInstall(deployOptions kotsadmtypes.DeployOptions, authSlug
 	return "", errors.New("timeout waiting for automated install. Use the --wait-duration flag to increase timeout.")
 }
 
-func getAutomatedInstallStatus(url string, authSlug string) (*kotsstore.TaskStatus, error) {
+func getAutomatedInstallStatus(url string, authSlug string) (*tasks.TaskStatus, error) {
 	newReq, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create request")
@@ -910,7 +910,7 @@ func getAutomatedInstallStatus(url string, authSlug string) (*kotsstore.TaskStat
 		return nil, errors.Wrap(err, "failed to read response body")
 	}
 
-	taskStatus := kotsstore.TaskStatus{}
+	taskStatus := tasks.TaskStatus{}
 	if err := json.Unmarshal(b, &taskStatus); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal task status")
 	}

cmd/kots/cli/install_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/replicatedhq/kots/pkg/handlers"
 	kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
 	preflighttypes "github.com/replicatedhq/kots/pkg/preflight/types"
-	"github.com/replicatedhq/kots/pkg/store/kotsstore"
+	"github.com/replicatedhq/kots/pkg/tasks"
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	"github.com/replicatedhq/troubleshoot/pkg/preflight"
 )
@@ -588,7 +588,7 @@ func createPreflightResponse(isFail bool, isWarn bool, hasPassingStrict bool, pe
 }
 
 func createTaskStatus(status string, message string) ([]byte, error) {
-	return json.Marshal(kotsstore.TaskStatus{
+	return json.Marshal(tasks.TaskStatus{
 		Message: message,
 		Status:  status,
 	})