merge 'install' and 'upgrade' helm spec generation code (#1321)

* remove k0s upgrade code from installation reconcile loop

* set state to kubernetesInstalled in initial installation object creation

* f

* wait for the installation object to exist

* make operator not update helm charts (but still track chart statuses)

* race condition fix?

* replace envtest with fake client

* Revert "replace envtest with fake client"

This reverts commit 4ab9a66.

* f

* deduplicate 'update chart status' and 'update charts' logic

* addons rework

* unit testing for chart generation

* add velero test

* return to having operator reconcile helm charts (for HA)

* use operator and ec-utils images from metadata

* debugging

* DEBUGGING

* better version mismatch logging

* compile operator version

* wait for a node to be ready in the cluster

* reinclude clusterResources collector in ci support bundles

* wait for node install in a better fashion

* include cluster resources for EC namespaces

* Revert "reinclude clusterResources collector in ci support bundles"

This reverts commit 0984c48.

* add 'version' command for operator

* print compiled versions at operator startup

* do not print full versions list on startup

* ?>

* add more logging to upgrade job

* logging to figure out why the wrong chart name is being set

* debug location

* allow upgrades in CI while working on the build process

* fix generateHelmConfigs unit test

* dedupe more string constants

* make adminconsole chart HA when relevant

* remove updateInfraChartsFromInstall

* first passing TestInstallationReconciler test

* all TestInstallationReconciler passing

* deduplicate test code

* test a few more error cases

* remove checking for '123m' on upgrades temporarily

* remove metadata mangling

* edit the embeddedclusteroperator/static/metadata.yaml at build time in CI

* f

* pull the operator location from the metadata file too

* fix script

* network and seaweedfs endpoint

* deduplicate seaweedfsLowerBandIPIndex

* proxy fix?

* offset commands in 5nodeairgapupgrade test

* set installation state to 'kubernetesInstalled' after restore

* cleanup logs

* handle TODOs

cmd/embedded-cluster/install.go

@@ -563,6 +563,7 @@ func installAndWaitForK0s(c *cli.Context, provider *defaults.Provider, applier *
 		metrics.ReportApplyFinished(c, err)
 		return nil, err
 	}
+
 	loading.Infof("Node installation finished!")
 	return cfg, nil
 }
@@ -799,13 +800,19 @@ func getAddonsApplier(c *cli.Context, runtimeConfig *ecv1beta1.RuntimeConfigSpec
 		opts = append(opts, addons.WithoutPrompt())
 	}
 	if l := c.String("license"); l != "" {
-		opts = append(opts, addons.WithLicense(l))
+		license, err := helpers.ParseLicense(l)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse license: %w", err)
+		}
+
+		opts = append(opts, addons.WithLicense(license))
+		opts = append(opts, addons.WithLicenseFile(l))
 	}
 	if ab := c.String("airgap-bundle"); ab != "" {
 		opts = append(opts, addons.WithAirgapBundle(ab))
 	}
 	if proxy != nil {
-		opts = append(opts, addons.WithProxy(proxy.HTTPProxy, proxy.HTTPSProxy, proxy.NoProxy))
+		opts = append(opts, addons.WithProxy(proxy))
 	}
 	if c.String("overrides") != "" {
 		eucfg, err := helpers.ParseEndUserConfig(c.String("overrides"))

cmd/embedded-cluster/restore.go

@@ -1258,6 +1258,11 @@ func restoreReconcileInstallationFromRuntimeConfig(ctx context.Context, runtimeC
 	if err := kcli.Update(ctx, in); err != nil {
 		return fmt.Errorf("update installation: %w", err)
 	}
+	in.Status.State = ecv1beta1.InstallationStateKubernetesInstalled
+	if err := kcli.Status().Update(ctx, in); err != nil {
+		return fmt.Errorf("update installation status: %w", err)
+	}
+
 	return nil
 }
 

dev/dockerfiles/operator/Dockerfile.ttlsh

@@ -13,6 +13,11 @@ COPY pkg/ pkg/
 COPY kinds/ kinds/
 COPY utils/ utils/
 
+ARG VERSION
+ENV VERSION=${VERSION}
+ARG K0S_VERSION
+ENV K0S_VERSION=${K0S_VERSION}
+
 RUN --mount=type=cache,target="/root/.cache/go-build" make -C operator build
 
 FROM debian:bookworm-slim

e2e/cluster/lxd/cluster.go

@@ -895,7 +895,7 @@ func PullImage(in *ClusterInput, image string) {
 func (c *Cluster) RunCommandsOnNode(node int, cmds [][]string, envs ...map[string]string) error {
 	for _, cmd := range cmds {
 		cmdstr := strings.Join(cmd, " ")
-		c.T.Logf("running `%s` node %d", cmdstr, node)
+		c.T.Logf("%s running `%s` node %d", time.Now().Format(time.RFC3339), cmdstr, node)
 		_, _, err := c.RunCommandOnNode(node, cmd, envs...)
 		if err != nil {
 			return err

e2e/install_test.go

@@ -2094,7 +2094,7 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) {
 		{"rm", "/assets/release.airgap"},
 		{"rm", "/usr/local/bin/embedded-cluster"},
 	}
-	runInParallel(t,
+	runInParallelOffset(t, time.Second*30,
 		func(t *testing.T) error {
 			err := tc.RunCommandsOnNode(1, joinCommandsSequence)
 			if err != nil {

e2e/scripts/check-postupgrade-state.sh

@@ -66,11 +66,11 @@ main() {
 
     # ensure that the embedded-cluster-operator has been updated
     kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator
-    kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator | grep -q "123m" # ensure new values are present
+    kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator | grep "embeddedClusterVersion:" | grep -q -e "-upgrade"
     kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator
     # ensure the new value made it into the pod
-    if ! kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator | grep -q "123m" ; then
-        echo "CPU request of 123m not found in embedded-cluster pod"
+    if ! kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator | grep "EMBEDDEDCLUSTER_VERSION" | grep -q -e "-upgrade" ; then
+        echo "Upgrade version not present in embedded-cluster-operator environment variable"
         kubectl logs -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator --tail=100
         exit 1
     fi

e2e/scripts/common.sh

@@ -359,6 +359,7 @@ validate_data_dirs() {
         echo "found openebs chart"
 
         openebsdatadir=$(kubectl -n kube-system get charts k0s-addon-chart-openebs -oyaml | grep -v apiVersion | grep "basePath:" | awk '{print $2}') 
+        echo "found openebsdatadir $openebsdatadir, want $expected_openebsdatadir"
         if [ "$openebsdatadir" != "$expected_openebsdatadir" ]; then
             echo "got unexpected openebsdatadir $openebsdatadir, want $expected_openebsdatadir"
             kubectl -n kube-system get charts k0s-addon-chart-openebs -oyaml | grep -v apiVersion | grep "basePath:" -A5 -B5
@@ -374,6 +375,7 @@ validate_data_dirs() {
         echo "found seaweedfs chart"
 
         seaweefdatadir=$(kubectl -n kube-system get charts k0s-addon-chart-seaweedfs -oyaml| grep -v apiVersion | grep -m 1 "hostPathPrefix:" | awk '{print $2}') 
+        echo "found seaweefdatadir $seaweefdatadir, want $expected_datadir/seaweedfs/(ssd|storage)"
         if ! echo "$seaweefdatadir" | grep -qE "^$expected_datadir/seaweedfs/(ssd|storage)$" ; then
             echo "got unexpected seaweefdatadir $seaweefdatadir, want $expected_datadir/seaweedfs/(ssd|storage)"
             kubectl -n kube-system get charts k0s-addon-chart-seaweedfs -oyaml| grep -v apiVersion | grep -m 1 "hostPathPrefix:" -A5 -B5
@@ -389,6 +391,7 @@ validate_data_dirs() {
         echo "found velero chart"
 
         velerodatadir=$(kubectl -n kube-system get charts k0s-addon-chart-velero -oyaml | grep -v apiVersion | grep "podVolumePath:" | awk '{print $2}') 
+        echo "found velerodatadir $velerodatadir, want $expected_k0sdatadir/kubelet/pods"
         if [ "$velerodatadir" != "$expected_k0sdatadir/kubelet/pods" ]; then
             echo "got unexpected velerodatadir $velerodatadir, want $expected_openebsdatadir/kubelet/pods"
             kubectl -n kube-system get charts k0s-addon-chart-velero -oyaml | grep -v apiVersion | grep "podVolumePath:" -A5 -B5

e2e/utils.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 )
 
 func RequireEnvVars(t *testing.T, envVars []string) {
@@ -71,10 +72,15 @@ func k8sVersionPrevious() string {
 }
 
 func runInParallel(t *testing.T, fns ...func(t *testing.T) error) {
+	runInParallelOffset(t, time.Duration(0), fns...)
+}
+
+func runInParallelOffset(t *testing.T, offset time.Duration, fns ...func(t *testing.T) error) {
 	t.Helper()
 	errCh := make(chan error, len(fns))
-	for _, fn := range fns {
+	for idx, fn := range fns {
 		go func(fn func(t *testing.T) error) {
+			time.Sleep(offset * time.Duration(idx))
 			errCh <- fn(t)
 		}(fn)
 	}

operator/Makefile

@@ -11,6 +11,7 @@ ARCH ?= $(shell go env GOARCH)
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 0.0.1
+K0S_VERSION ?= v1.29.9+k0s.0-ec.0
 
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
@@ -26,6 +27,10 @@ endif
 
 export PATH := $(shell pwd)/bin:$(PATH)
 
+LD_FLAGS = \
+	-X github.com/replicatedhq/embedded-cluster/pkg/versions.K0sVersion=$(K0S_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/versions.Version=$(VERSION)
+
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
@@ -72,7 +77,7 @@ test: manifests envtest ## Run tests.
 build: ## Build manager binary.
 	CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build \
 		-tags osusergo,netgo \
-		-ldflags="-s -w -extldflags=-static" \
+		-ldflags="-s -w $(LD_FLAGS) -extldflags=-static" \
 		-o bin/manager main.go
 
 .PHONY: run
@@ -159,8 +164,13 @@ build-operator-image-chainguard:
 
 .PHONY: build-operator-image-dockerfile
 build-operator-image-dockerfile:
-	docker build --platform="$(OS)/$(ARCH)" -t "$(IMAGE_NAME):$(call image-tag,$(PACKAGE_VERSION))" -f ../dev/dockerfiles/operator/Dockerfile.ttlsh ../
+	docker build --platform="$(OS)/$(ARCH)" \
+		-t "$(IMAGE_NAME):$(call image-tag,$(PACKAGE_VERSION))" \
+		--build-arg VERSION=$(VERSION) \
+		--build-arg K0S_VERSION=$(K0S_VERSION) \
+		-f ../dev/dockerfiles/operator/Dockerfile.ttlsh ../
 	mkdir -p build
+	echo "Version $(VERSION) k0s $(K0S_VERSION)"
 	echo "$(IMAGE_NAME):$(call image-tag,$(PACKAGE_VERSION))" > build/image
 
 .PHONY: build-and-push-operator-image

operator/charts/embedded-cluster-operator/troubleshoot/cluster-support-bundle.yaml

@@ -7,6 +7,16 @@ metadata:
 spec:
   uri: https://raw.githubusercontent.com/replicatedhq/embedded-cluster/main/operator/charts/embedded-cluster-operator/troubleshoot/cluster-support-bundle.yaml
   collectors:
+  - clusterInfo: {}
+  - clusterResources:
+      namespaces:
+      - kube-system
+      - openebs
+      - registry
+      - embedded-cluster
+      - seaweedfs
+      - default
+      - velero
   - logs:
       name: podlogs/embedded-cluster-operator
       namespace: embedded-cluster
@@ -70,6 +80,41 @@ spec:
       - app=registry
       limits:
         maxAge: 720h
+  - logs:
+      name: podlogs/seaweedfs
+      namespace: seaweedfs
+      selector:
+      - app.kubernetes.io/component=objectstorage-provisioner
+      limits:
+        maxAge: 720h
+  - logs:
+      name: podlogs/seaweedfs
+      namespace: seaweedfs
+      selector:
+      - app.kubernetes.io/component=filer
+      limits:
+        maxAge: 720h
+  - logs:
+      name: podlogs/seaweedfs
+      namespace: seaweedfs
+      selector:
+      - app.kubernetes.io/component=master
+      limits:
+        maxAge: 720h
+  - logs:
+      name: podlogs/seaweedfs
+      namespace: seaweedfs
+      selector:
+      - app.kubernetes.io/component=s3
+      limits:
+        maxAge: 720h
+  - logs:
+      name: podlogs/seaweedfs
+      namespace: seaweedfs
+      selector:
+      - app.kubernetes.io/component=volume
+      limits:
+        maxAge: 720h
   analyzers:
   - textAnalyze:
       checkName: Cluster installation status