feat(ec): ability to upgrade the cluster

.github/actions/cmx-versions/index.js

@@ -36,7 +36,9 @@ async function getClusterVersions() {
             latest_minor_versions: true,
         },
         eks: {
-            latest_version: true,
+            // latest_version: true,
+            // TODO: re-enable latest_version once we have compatibility with 1.30
+            versions: new Set(["1.29"]),
             instance_type: "m7g.large" // arm64
         },
         openshift: {

.github/workflows/arm64-arc-runner-set.yaml

@@ -33,6 +33,7 @@ jobs:
             --token '${{ secrets.C11Y_MATRIX_TOKEN }}' \
             --distribution eks \
             --instance-type m7g.2xlarge \
+            --disk 100 \
             --name "$CLUSTER_NAME" \
             --ttl 168h \
             --wait 120m
@@ -49,6 +50,7 @@ jobs:
           helm install arc \
             --namespace arc-systems \
             --create-namespace \
+            --version 0.9.2 \
             oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller
 
           kubectl create namespace arc-runners
@@ -60,6 +62,7 @@ jobs:
             --namespace arc-runners \
             --set githubConfigUrl="https://github.com/replicatedhq/kots" \
             --set githubConfigSecret="github-config-secret" \
+            --version 0.9.2 \
             oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
 
           # bubblewrap (which is a sandbox tool used by melange) requires privileged docker containers.

.image.env

@@ -1,7 +1,7 @@
 # Generated file, do not modify.  This file is generated from a text file containing a list of images. The
 # most recent tag is interpolated from the source repository and used to generate a fully qualified image
 # name.
-MINIO_TAG='0.20240613.225353-r0'
+MINIO_TAG='0.20240626.010618-r0'
 RQLITE_TAG='8.26.2-r0'
 DEX_TAG='2.40.0-r1'
 SCHEMAHERO_TAG='0.17.9'

Makefile

@@ -1,6 +1,6 @@
 include Makefile.build.mk
 CURRENT_USER := $(if $(GITHUB_USER),$(GITHUB_USER),$(shell id -u -n))
-MINIO_TAG ?= 0.20240613.225353-r0
+MINIO_TAG ?= 0.20240626.010618-r0
 RQLITE_TAG ?= 8.26.2-r0
 DEX_TAG ?= 2.40.0-r1
 LVP_TAG ?= v0.6.6
@@ -51,6 +51,8 @@ kots: capture-start-time kots-real report-metric
 
 .PHONY: kots-real
 kots-real:
+	mkdir -p web/dist
+	touch web/dist/README.md
 	go build ${LDFLAGS} -o bin/kots $(BUILDFLAGS) github.com/replicatedhq/kots/cmd/kots
 
 .PHONY: fmt
@@ -80,7 +82,7 @@ build: capture-start-time build-real report-metric
 .PHONY: build-real
 build-real:
 	mkdir -p web/dist
-	touch web/dist/THIS_IS_OKTETO  # we need this for go:embed, but it's not actually used in dev
+	touch web/dist/README.md
 	go build ${LDFLAGS} ${GCFLAGS} -v -o bin/kotsadm $(BUILDFLAGS) ./cmd/kotsadm
 
 .PHONY: tidy
@@ -112,21 +114,31 @@ debug-build:
 debug: debug-build
 	LOG_LEVEL=$(LOG_LEVEL) dlv --listen=:2345 --headless=true --api-version=2 exec ./bin/kotsadm-debug api
 
-.PHONY: build-ttl.sh
-build-ttl.sh: kots build
+.PHONY: web
+web:
 	source .image.env && ${MAKE} -C web build-kotsadm
-	docker build -f deploy/Dockerfile -t ttl.sh/${CURRENT_USER}/kotsadm:24h .
+
+.PHONY: build-ttl.sh
+build-ttl.sh: export GOOS ?= linux
+build-ttl.sh: export GOARCH ?= amd64
+build-ttl.sh: web kots build
+	docker build --platform $(GOOS)/$(GOARCH) -f deploy/Dockerfile -t ttl.sh/${CURRENT_USER}/kotsadm:24h .
 	docker push ttl.sh/${CURRENT_USER}/kotsadm:24h
 
 .PHONY: all-ttl.sh
+all-ttl.sh: export GOOS ?= linux
+all-ttl.sh: export GOARCH ?= amd64
 all-ttl.sh: build-ttl.sh
-	source .image.env && IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h make -C migrations build_schema
+	source .image.env && \
+		IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h \
+		DOCKER_BUILD_ARGS="--platform $(GOOS)/$(GOARCH)" \
+		make -C migrations build_schema
 
-	docker pull kotsadm/minio:${MINIO_TAG}
+	docker pull --platform $(GOOS)/$(GOARCH)" kotsadm/minio:${MINIO_TAG}
 	docker tag kotsadm/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG}
 	docker push ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG}
 
-	docker pull kotsadm/rqlite:${RQLITE_TAG}
+	docker pull --platform $(GOOS)/$(GOARCH)" kotsadm/rqlite:${RQLITE_TAG}
 	docker tag kotsadm/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG}
 	docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG}
 

cmd/kots/cli/admin-console-push-images.go

@@ -115,12 +115,15 @@ func genAndCheckPushOptions(endpoint string, namespace string, log *logger.CLILo
 		log.FinishSpinner()
 	}
 
+	registryEndpoint, registryNamespace := splitEndpointAndNamespace(endpoint)
+
 	options := imagetypes.PushImagesOptions{
 		KotsadmTag: v.GetString("kotsadm-tag"),
 		Registry: registrytypes.RegistryOptions{
-			Endpoint: endpoint,
-			Username: username,
-			Password: password,
+			Endpoint:  registryEndpoint,
+			Namespace: registryNamespace,
+			Username:  username,
+			Password:  password,
 		},
 		ProgressWriter: os.Stdout,
 	}

cmd/kots/cli/airgap-update.go

@@ -0,0 +1,223 @@
+package cli
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/kots/pkg/archives"
+	"github.com/replicatedhq/kots/pkg/auth"
+	registrytypes "github.com/replicatedhq/kots/pkg/docker/registry/types"
+	"github.com/replicatedhq/kots/pkg/image"
+	imagetypes "github.com/replicatedhq/kots/pkg/image/types"
+	"github.com/replicatedhq/kots/pkg/k8sutil"
+	"github.com/replicatedhq/kots/pkg/logger"
+	"github.com/replicatedhq/kots/pkg/tasks"
+	"github.com/replicatedhq/kots/pkg/upload"
+	"github.com/replicatedhq/kots/pkg/util"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func AirgapUpdateCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:           "airgap-update [appSlug]",
+		Short:         "Process and upload an airgap update to the admin console",
+		Long:          "",
+		SilenceUsage:  true,
+		SilenceErrors: false,
+		Hidden:        true,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			viper.BindPFlags(cmd.Flags())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := viper.GetViper()
+
+			if len(args) == 0 {
+				cmd.Help()
+				os.Exit(1)
+			}
+
+			appSlug := args[0]
+			log := logger.NewCLILogger(cmd.OutOrStdout())
+
+			airgapBundle := v.GetString("airgap-bundle")
+			if airgapBundle == "" {
+				return fmt.Errorf("--airgap-bundle is required")
+			}
+
+			namespace, err := getNamespaceOrDefault(v.GetString("namespace"))
+			if err != nil {
+				return errors.Wrap(err, "failed to get namespace")
+			}
+
+			clientset, err := k8sutil.GetClientset()
+			if err != nil {
+				return errors.Wrap(err, "failed to get clientset")
+			}
+
+			registryConfig, err := getRegistryConfig(v, clientset, appSlug)
+			if err != nil {
+				return errors.Wrap(err, "failed to get registry config")
+			}
+
+			pushOpts := imagetypes.PushImagesOptions{
+				KotsadmTag: v.GetString("kotsadm-tag"),
+				Registry: registrytypes.RegistryOptions{
+					Endpoint:  registryConfig.OverrideRegistry,
+					Namespace: registryConfig.OverrideNamespace,
+					Username:  registryConfig.Username,
+					Password:  registryConfig.Password,
+				},
+				ProgressWriter: getProgressWriter(v, log),
+				LogForUI:       v.GetBool("from-api"),
+			}
+
+			if _, err := os.Stat(airgapBundle); err == nil {
+				err = image.TagAndPushImagesFromBundle(airgapBundle, pushOpts)
+				if err != nil {
+					return errors.Wrap(err, "failed to push images")
+				}
+			} else {
+				return errors.Wrap(err, "failed to stat airgap bundle")
+			}
+
+			updateFiles := []string{
+				"airgap.yaml",
+				"app.tar.gz",
+			}
+			if util.IsEmbeddedCluster() {
+				updateFiles = append(updateFiles, "embedded-cluster/artifacts/kots.tar.gz")
+			}
+
+			airgapUpdate, err := archives.FilterAirgapBundle(airgapBundle, updateFiles)
+			if err != nil {
+				return errors.Wrap(err, "failed to create filtered airgap bundle")
+			}
+			defer os.RemoveAll(airgapUpdate)
+
+			var localPort int
+			if v.GetBool("from-api") {
+				localPort = 3000
+			} else {
+				stopCh := make(chan struct{})
+				defer close(stopCh)
+
+				lp, errChan, err := upload.StartPortForward(namespace, stopCh, log)
+				if err != nil {
+					return err
+				}
+				localPort = lp
+
+				go func() {
+					select {
+					case err := <-errChan:
+						if err != nil {
+							log.Error(err)
+							os.Exit(1)
+						}
+					case <-stopCh:
+					}
+				}()
+			}
+
+			uploadEndpoint := fmt.Sprintf("http://localhost:%d/api/v1/app/%s/airgap/update", localPort, url.PathEscape(appSlug))
+
+			log.ActionWithSpinner("Uploading airgap update")
+			if err := uploadAirgapUpdate(airgapUpdate, uploadEndpoint, namespace); err != nil {
+				log.FinishSpinnerWithError()
+				return errors.Wrap(err, "failed to upload airgap update")
+			}
+			log.FinishSpinner()
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringP("namespace", "n", "", "the namespace in which kots/kotsadm is installed")
+	cmd.Flags().String("airgap-bundle", "", "path to the application airgap bundle to upload")
+	cmd.Flags().Bool("from-api", false, "whether the airgap update command was triggered by the API")
+
+	registryFlags(cmd.Flags())
+
+	return cmd
+}
+
+func getProgressWriter(v *viper.Viper, log *logger.CLILogger) io.Writer {
+	if v.GetBool("from-api") {
+		pipeReader, pipeWriter := io.Pipe()
+		go func() {
+			scanner := bufio.NewScanner(pipeReader)
+			for scanner.Scan() {
+				if err := tasks.SetTaskStatus("update-download", scanner.Text(), "running"); err != nil {
+					log.Error(err)
+				}
+			}
+			pipeReader.CloseWithError(scanner.Err())
+		}()
+		return pipeWriter
+	}
+	return os.Stdout
+}
+
+func uploadAirgapUpdate(airgapBundle string, uploadEndpoint string, namespace string) error {
+	buffer := bytes.NewBuffer(nil)
+	writer := multipart.NewWriter(buffer)
+
+	part, err := writer.CreateFormFile("application.airgap", "application.airgap")
+	if err != nil {
+		return errors.Wrap(err, "failed to create form file")
+	}
+
+	f, err := os.Open(airgapBundle)
+	if err != nil {
+		return errors.Wrap(err, "failed to open airgap bundle")
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(part, f); err != nil {
+		return errors.Wrap(err, "failed to copy airgap bundle to form file")
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return errors.Wrap(err, "failed to close writer")
+	}
+
+	clientset, err := k8sutil.GetClientset()
+	if err != nil {
+		return errors.Wrap(err, "failed to get k8s clientset")
+	}
+
+	authSlug, err := auth.GetOrCreateAuthSlug(clientset, namespace)
+	if err != nil {
+		return errors.Wrap(err, "failed to get auth slug")
+	}
+
+	newReq, err := util.NewRequest("PUT", uploadEndpoint, buffer)
+	if err != nil {
+		return errors.Wrap(err, "failed to create request")
+	}
+	newReq.Header.Add("Content-Type", writer.FormDataContentType())
+	newReq.Header.Add("Authorization", authSlug)
+
+	resp, err := http.DefaultClient.Do(newReq)
+	if err != nil {
+		return errors.Wrap(err, "failed to make request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 404 {
+		return errors.New("App not found")
+	} else if resp.StatusCode != 200 {
+		return errors.Errorf("Unexpected status code: %d", resp.StatusCode)
+	}
+
+	return nil
+}

cmd/kots/cli/install.go

@@ -37,8 +37,8 @@ import (
 	"github.com/replicatedhq/kots/pkg/print"
 	"github.com/replicatedhq/kots/pkg/pull"
 	"github.com/replicatedhq/kots/pkg/replicatedapp"
-	"github.com/replicatedhq/kots/pkg/store/kotsstore"
 	storetypes "github.com/replicatedhq/kots/pkg/store/types"
+	"github.com/replicatedhq/kots/pkg/tasks"
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	"github.com/replicatedhq/troubleshoot/pkg/preflight"
 	"github.com/spf13/cobra"
@@ -648,7 +648,7 @@ func uploadAirgapArchive(deployOptions kotsadmtypes.DeployOptions, authSlug stri
 		return false, errors.Wrap(err, "failed to create form from file")
 	}
 
-	contents, err := archives.GetFileFromAirgap(filename, deployOptions.AirgapBundle)
+	contents, err := archives.GetFileContentFromTGZArchive(filename, deployOptions.AirgapBundle)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to get file from airgap")
 	}
@@ -887,7 +887,7 @@ func ValidateAutomatedInstall(deployOptions kotsadmtypes.DeployOptions, authSlug
 	return "", errors.New("timeout waiting for automated install. Use the --wait-duration flag to increase timeout.")
 }
 
-func getAutomatedInstallStatus(url string, authSlug string) (*kotsstore.TaskStatus, error) {
+func getAutomatedInstallStatus(url string, authSlug string) (*tasks.TaskStatus, error) {
 	newReq, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create request")
@@ -910,7 +910,7 @@ func getAutomatedInstallStatus(url string, authSlug string) (*kotsstore.TaskStat
 		return nil, errors.Wrap(err, "failed to read response body")
 	}
 
-	taskStatus := kotsstore.TaskStatus{}
+	taskStatus := tasks.TaskStatus{}
 	if err := json.Unmarshal(b, &taskStatus); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal task status")
 	}