Merge branch 'main' into em/log-preflight-errors

.github/workflows/build-test.yaml

@@ -2824,7 +2824,7 @@ jobs:
       fail-fast: false
       matrix:
         cluster: [
-          {distribution: kind, version: v1.27}
+          {distribution: kind, version: v1.27, instance-type: r1.medium}
         ]
     env:
       APP_SLUG: remove-app
@@ -2842,6 +2842,7 @@ jobs:
           cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
           timeout-minutes: '120'
           ttl: 2h
+          instance-type: ${{ matrix.cluster.instance-type }}
           export-kubeconfig: true
 
       - name: download kots binary
@@ -3261,6 +3262,33 @@ jobs:
             sleep 1
           done
 
+          # validate that the conditional chart is installed
+          COUNTER=1
+          while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" == "" ]; do
+            ((COUNTER += 1))
+            if [ $COUNTER -gt 120 ]; then
+              echo "Timed out waiting for my-conditional-chart-release to be installed"
+              helm ls -n "$APP_SLUG"
+              exit 1
+            fi
+            sleep 1
+          done
+
+          # toggle the config option to exclude the conditional chart
+          ./bin/kots set config "$APP_SLUG" install_conditional_chart=0 --deploy --namespace "$APP_SLUG"
+
+          # wait for my-conditional-chart-release to be uninstalled
+          COUNTER=1
+          while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" != "" ]; do
+            ((COUNTER += 1))
+            if [ $COUNTER -gt 120 ]; then
+              echo "Timed out waiting for my-conditional-chart-release to be uninstalled"
+              helm ls -n "$APP_SLUG"
+              exit 1
+            fi
+            sleep 1
+          done
+
 
       - name: Generate support bundle on failure
         if: failure()

.image.env

@@ -1,8 +1,8 @@
 # Generated file, do not modify.  This file is generated from a text file containing a list of images. The
 # most recent tag is interpolated from the source repository and used to generate a fully qualified image
 # name.
-MINIO_TAG='RELEASE.2023-08-23T10-07-06Z'
-MC_TAG='RELEASE.2023-08-18T21-57-55Z'
+MINIO_TAG='RELEASE.2023-09-23T03-47-50Z'
+MC_TAG='RELEASE.2023-09-22T05-07-46Z'
 RQLITE_TAG='7.21.4'
 DEX_TAG='v2.37.0'
 SCHEMAHERO_TAG='0.14.0'

Makefile

@@ -1,7 +1,7 @@
 include Makefile.build.mk
 CURRENT_USER := $(shell id -u -n)
-MINIO_TAG ?= RELEASE.2023-08-23T10-07-06Z
-MC_TAG ?= RELEASE.2023-08-18T21-57-55Z
+MINIO_TAG ?= RELEASE.2023-09-23T03-47-50Z
+MC_TAG ?= RELEASE.2023-09-22T05-07-46Z
 RQLITE_TAG ?= 7.21.4
 DEX_TAG ?= v2.37.0
 LVP_TAG ?= v0.5.4

deploy/Dockerfile

@@ -136,9 +136,9 @@ RUN curl -fsSL -o kustomize.tar.gz "${KUSTOMIZE5_URL}" \
 ENV KOTS_HELM_BIN_DIR=/usr/local/bin
 
 # Install helm v3
-ENV HELM3_VERSION=3.12.2
+ENV HELM3_VERSION=3.13.0
 ENV HELM3_URL=https://get.helm.sh/helm-v${HELM3_VERSION}-linux-amd64.tar.gz
-ENV HELM3_SHA256SUM=2b6efaa009891d3703869f4be80ab86faa33fa83d9d5ff2f6492a8aebe97b219
+ENV HELM3_SHA256SUM=138676351483e61d12dfade70da6c03d471bbdcac84eaadeb5e1d06fa114a24f
 RUN cd /tmp && curl -fsSL -o helm.tar.gz "${HELM3_URL}" \
   && echo "${HELM3_SHA256SUM} helm.tar.gz" | sha256sum -c - \
   && tar -xzvf helm.tar.gz \

deploy/okteto/okteto-v2.Dockerfile

@@ -136,9 +136,9 @@ RUN curl -fsSL -o kustomize.tar.gz "${KUSTOMIZE5_URL}" \
 ENV KOTS_HELM_BIN_DIR=/usr/local/bin
 
 # Install helm v3
-ENV HELM3_VERSION=3.12.2
+ENV HELM3_VERSION=3.13.0
 ENV HELM3_URL=https://get.helm.sh/helm-v${HELM3_VERSION}-linux-amd64.tar.gz
-ENV HELM3_SHA256SUM=2b6efaa009891d3703869f4be80ab86faa33fa83d9d5ff2f6492a8aebe97b219
+ENV HELM3_SHA256SUM=138676351483e61d12dfade70da6c03d471bbdcac84eaadeb5e1d06fa114a24f
 RUN cd /tmp && curl -fsSL -o helm.tar.gz "${HELM3_URL}" \
   && echo "${HELM3_SHA256SUM} helm.tar.gz" | sha256sum -c - \
   && tar -xzvf helm.tar.gz \

deploy/okteto/okteto.Dockerfile

@@ -137,9 +137,9 @@ RUN curl -fsSL -o kustomize.tar.gz "${KUSTOMIZE5_URL}" \
 ENV KOTS_HELM_BIN_DIR=/usr/local/bin
 
 # Install helm v3
-ENV HELM3_VERSION=3.12.2
+ENV HELM3_VERSION=3.13.0
 ENV HELM3_URL=https://get.helm.sh/helm-v${HELM3_VERSION}-linux-amd64.tar.gz
-ENV HELM3_SHA256SUM=2b6efaa009891d3703869f4be80ab86faa33fa83d9d5ff2f6492a8aebe97b219
+ENV HELM3_SHA256SUM=138676351483e61d12dfade70da6c03d471bbdcac84eaadeb5e1d06fa114a24f
 RUN cd /tmp && curl -fsSL -o helm.tar.gz "${HELM3_URL}" \
   && echo "${HELM3_SHA256SUM} helm.tar.gz" | sha256sum -c - \
   && tar -xzvf helm.tar.gz \

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe
+	github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82
 	github.com/replicatedhq/kurlkinds v1.3.6
 	github.com/replicatedhq/troubleshoot v0.72.1
 	github.com/replicatedhq/yaml/v3 v3.0.0-beta5-replicatedhq

go.sum

@@ -1525,8 +1525,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
-github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe h1:3AJInd06UxzqHmgy8+24CPsT2tYSE0zToJZyuX9q+MA=
-github.com/replicatedhq/kotskinds v0.0.0-20230724164735-f83482cc9cfe/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
+github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82 h1:QniKgIpcXu4wBMM4xIXGz+lkAU+hSIXFuVM+vxkNk0Y=
+github.com/replicatedhq/kotskinds v0.0.0-20231004174055-e6676d808a82/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
 github.com/replicatedhq/kurlkinds v1.3.6 h1:/dhS32cSSZR4yS4vA8EquBvz+VgJCyTqBO9Xw+6eI4M=
 github.com/replicatedhq/kurlkinds v1.3.6/go.mod h1:c5+hoAkuARgftB2Ft3RCyWRZZPhL0clHEaw7XoGDAg4=
 github.com/replicatedhq/termui/v3 v3.1.1-0.20200811145416-f40076d26851 h1:eRlNDHxGfVkPCRXbA4BfQJvt5DHjFiTtWy3R/t4djyY=

hack/dev/skaffold.Dockerfile

@@ -153,9 +153,9 @@ RUN curl -fsSL -o kustomize.tar.gz "${KUSTOMIZE5_URL}" \
 ENV KOTS_HELM_BIN_DIR=/usr/local/bin
 
 # Install helm v3
-ENV HELM3_VERSION=3.12.2
+ENV HELM3_VERSION=3.13.0
 ENV HELM3_URL=https://get.helm.sh/helm-v${HELM3_VERSION}-linux-amd64.tar.gz
-ENV HELM3_SHA256SUM=2b6efaa009891d3703869f4be80ab86faa33fa83d9d5ff2f6492a8aebe97b219
+ENV HELM3_SHA256SUM=138676351483e61d12dfade70da6c03d471bbdcac84eaadeb5e1d06fa114a24f
 RUN cd /tmp && curl -fsSL -o helm.tar.gz "${HELM3_URL}" \
   && echo "${HELM3_SHA256SUM} helm.tar.gz" | sha256sum -c - \
   && tar -xzvf helm.tar.gz \

pkg/airgap/airgap.go

@@ -227,6 +227,7 @@ func CreateAppFromAirgap(opts CreateAirgapAppOpts) (finalError error) {
 			Password:   opts.RegistryPassword,
 			IsReadOnly: opts.RegistryIsReadOnly,
 		},
+		AppID:                  opts.PendingApp.ID,
 		AppSlug:                opts.PendingApp.Slug,
 		AppSequence:            0,
 		AppVersionLabel:        instParams.AppVersionLabel,

pkg/airgap/update.go

@@ -173,6 +173,7 @@ func UpdateAppFromPath(a *apptypes.App, airgapRoot string, airgapBundlePath stri
 		Silent:                 true,
 		RewriteImages:          true,
 		RewriteImageOptions:    registrySettings,
+		AppID:                  a.ID,
 		AppSlug:                a.Slug,
 		AppSequence:            appSequence,
 		SkipCompatibilityCheck: skipCompatibilityCheck,

pkg/apiserver/server.go

@@ -89,9 +89,9 @@ func Start(params *APIServerParams) {
 
 	if !util.IsHelmManaged() {
 		client := &client.Client{
-			TargetNamespace:   util.AppNamespace(),
-			ExistingInformers: map[string]bool{},
-			HookStopChans:     []chan struct{}{},
+			TargetNamespace:       util.AppNamespace(),
+			ExistingHookInformers: map[string]bool{},
+			HookStopChans:         []chan struct{}{},
 		}
 		store := store.GetStore()
 		k8sClientset, err := k8sutil.GetClientset()

pkg/apparchive/helm-v1beta2.go

@@ -73,6 +73,10 @@ type WriteV1Beta2HelmChartsOptions struct {
 
 // WriteV1Beta2HelmCharts copies the upstream helm chart archive and rendered values to the helm directory and processes online images (if necessary)
 func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error {
+	// clear the previous helm dir before writing
+	helmDir := opts.Upstream.GetHelmDir(opts.WriteUpstreamOptions)
+	os.RemoveAll(helmDir)
+
 	if opts.KotsKinds == nil || opts.KotsKinds.V1Beta2HelmCharts == nil {
 		return nil
 	}
@@ -91,7 +95,7 @@ func WriteV1Beta2HelmCharts(opts WriteV1Beta2HelmChartsOptions) error {
 			}
 		}
 
-		chartDir := path.Join(opts.Upstream.GetHelmDir(opts.WriteUpstreamOptions), helmChart.GetDirName())
+		chartDir := path.Join(helmDir, helmChart.GetDirName())
 		if err := os.MkdirAll(chartDir, 0744); err != nil {
 			return errors.Wrap(err, "failed to create chart dir")
 		}

pkg/base/replicated.go

@@ -23,6 +23,7 @@ import (
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	kotsv1beta2 "github.com/replicatedhq/kotskinds/apis/kots/v1beta2"
 	kotsscheme "github.com/replicatedhq/kotskinds/client/kotsclientset/scheme"
+	"github.com/replicatedhq/kotskinds/pkg/helmchart"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	troubleshootscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -502,7 +503,7 @@ func getKotsKinds(u *upstreamtypes.Upstream) (*kotsutil.KotsKinds, error) {
 
 // FindHelmChartArchiveInRelease iterates through all files in the release (upstreamFiles), looking for a helm chart archive
 // that matches the chart name and version specified in the kotsHelmChart parameter
-func FindHelmChartArchiveInRelease(upstreamFiles []upstreamtypes.UpstreamFile, kotsHelmChart kotsutil.HelmChartInterface) ([]byte, error) {
+func FindHelmChartArchiveInRelease(upstreamFiles []upstreamtypes.UpstreamFile, kotsHelmChart helmchart.HelmChartInterface) ([]byte, error) {
 	for _, upstreamFile := range upstreamFiles {
 		if !isHelmChart(upstreamFile.Content) {
 			continue

pkg/handlers/custom_metrics.go

@@ -79,6 +79,10 @@ func validateCustomMetricsData(data ApplicationMetricsData) error {
 
 	for key, val := range data {
 		valType := reflect.TypeOf(val)
+		if valType == nil {
+			return errors.Errorf("%s value is nil, only scalar values are allowed", key)
+		}
+
 		switch valType.Kind() {
 		case reflect.Slice:
 			return errors.Errorf("%s value is an array, only scalar values are allowed", key)

pkg/handlers/custom_metrics_test.go

@@ -53,6 +53,14 @@ func Test_validateCustomMetricsData(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "nil value",
+			data: ApplicationMetricsData{
+				"key1": nil,
+				"key2": 4,
+			},
+			wantErr: true,
+		},
 	}
 
 	for _, test := range tests {

pkg/handlers/handlers.go

@@ -259,22 +259,35 @@ func RegisterSessionAuthRoutes(r *mux.Router, kotsStore store.Store, handler KOT
 		HandlerFunc(middleware.EnforceAccess(policy.BackupRead, handler.GetVeleroStatus))
 
 	// KURL
-	r.Name("Kurl").Path("/api/v1/kurl").HandlerFunc(NotImplemented) // I'm not sure why this is here
-	r.Name("GenerateNodeJoinCommandWorker").Path("/api/v1/kurl/generate-node-join-command-worker").Methods("POST").
-		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateNodeJoinCommandWorker))
-	r.Name("GenerateNodeJoinCommandMaster").Path("/api/v1/kurl/generate-node-join-command-master").Methods("POST").
-		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateNodeJoinCommandMaster))
-	r.Name("GenerateNodeJoinCommandSecondary").Path("/api/v1/kurl/generate-node-join-command-secondary").Methods("POST").
-		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateNodeJoinCommandSecondary))
-	r.Name("GenerateNodeJoinCommandPrimary").Path("/api/v1/kurl/generate-node-join-command-primary").Methods("POST").
-		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateNodeJoinCommandPrimary))
-	r.Name("DrainNode").Path("/api/v1/kurl/nodes/{nodeName}/drain").Methods("POST").
-		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DrainNode))
-	r.Name("DeleteNode").Path("/api/v1/kurl/nodes/{nodeName}").Methods("DELETE").
-		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DeleteNode))
+	r.Name("Kurl").Path("/api/v1/kurl").HandlerFunc(NotImplemented)
+	r.Name("GenerateKurlNodeJoinCommandWorker").Path("/api/v1/kurl/generate-node-join-command-worker").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateKurlNodeJoinCommandWorker))
+	r.Name("GenerateKurlNodeJoinCommandMaster").Path("/api/v1/kurl/generate-node-join-command-master").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateKurlNodeJoinCommandMaster))
+	r.Name("GenerateKurlNodeJoinCommandSecondary").Path("/api/v1/kurl/generate-node-join-command-secondary").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateKurlNodeJoinCommandSecondary))
+	r.Name("GenerateKurlNodeJoinCommandPrimary").Path("/api/v1/kurl/generate-node-join-command-primary").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateKurlNodeJoinCommandPrimary))
+	r.Name("DrainKurlNode").Path("/api/v1/kurl/nodes/{nodeName}/drain").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DrainKurlNode))
+	r.Name("DeleteKurlNode").Path("/api/v1/kurl/nodes/{nodeName}").Methods("DELETE").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DeleteKurlNode))
 	r.Name("GetKurlNodes").Path("/api/v1/kurl/nodes").Methods("GET").
 		HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.GetKurlNodes))
 
+	// HelmVM
+	r.Name("HelmVM").Path("/api/v1/helmvm").HandlerFunc(NotImplemented)
+	r.Name("GenerateHelmVMNodeJoinCommandSecondary").Path("/api/v1/helmvm/generate-node-join-command-secondary").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateHelmVMNodeJoinCommandSecondary))
+	r.Name("GenerateHelmVMNodeJoinCommandPrimary").Path("/api/v1/helmvm/generate-node-join-command-primary").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.GenerateHelmVMNodeJoinCommandPrimary))
+	r.Name("DrainHelmVMNode").Path("/api/v1/helmvm/nodes/{nodeName}/drain").Methods("POST").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DrainHelmVMNode))
+	r.Name("DeleteHelmVMNode").Path("/api/v1/helmvm/nodes/{nodeName}").Methods("DELETE").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterWrite, handler.DeleteHelmVMNode))
+	r.Name("GetHelmVMNodes").Path("/api/v1/helmvm/nodes").Methods("GET").
+		HandlerFunc(middleware.EnforceAccess(policy.ClusterRead, handler.GetHelmVMNodes))
+
 	// Prometheus
 	r.Name("SetPrometheusAddress").Path("/api/v1/prometheus").Methods("POST").
 		HandlerFunc(middleware.EnforceAccess(policy.PrometheussettingsWrite, handler.SetPrometheusAddress))