feat: fixes unsupported overrides for extensions (#254)

fixes the unsupported overrides being overwritten. adds a more complete end to end test case.
replicatedhq · Jan 9, 2024 · 386984a · 386984a
1 parent 78417c1
commit 386984a
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 40 deletions.
diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go
@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	k0sv1beta1 "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/k0sproject/rig"
 	"github.com/k0sproject/rig/log"
@@ -24,6 +25,7 @@ import (
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
+	"github.com/replicatedhq/embedded-cluster/pkg/customization"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
 	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
@@ -153,18 +155,6 @@ func updateConfig(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("unable to read cluster config: %w", err)
 	}
-	cfg.Spec.K0s.Version = k0sversion.MustParse(defaults.K0sVersion)
-	if c.String("overrides") != "" {
-		eucfg, err := parseEndUserConfig(c.String("overrides"))
-		if err != nil {
-			return fmt.Errorf("unable to process overrides file: %w", err)
-		}
-		if err := config.ApplyEmbeddedUnsupportedOverrides(
-			cfg, eucfg.Spec.UnsupportedOverrides.K0s,
-		); err != nil {
-			return fmt.Errorf("unable to apply overrides: %w", err)
-		}
-	}
 	opts := []addons.Option{}
 	if c.Bool("no-prompt") {
 		opts = append(opts, addons.WithoutPrompt())
@@ -175,6 +165,10 @@ func updateConfig(c *cli.Context) error {
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
+	cfg.Spec.K0s.Version = k0sversion.MustParse(defaults.K0sVersion)
+	if err := applyUnsupportedOverrides(c, cfg); err != nil {
+		return fmt.Errorf("unable to apply unsupported overrides: %w", err)
+	}
 	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("unable to create config file: %w", err)
@@ -186,6 +180,31 @@ func updateConfig(c *cli.Context) error {
 	return nil
 }
 
+// applyUnsupportedOverrides applies overrides to the k0s configuration. Applies first the
+// overrides embedded into the binary and after the ones provided by the user (--overrides).
+func applyUnsupportedOverrides(c *cli.Context, cfg *k0sv1beta1.Cluster) error {
+	if embcfg, err := customization.GetEmbeddedClusterConfig(); err != nil {
+		return fmt.Errorf("unable to get embedded cluster config: %w", err)
+	} else if embcfg != nil {
+		overrides := embcfg.Spec.UnsupportedOverrides.K0s
+		if err := config.ApplyEmbeddedUnsupportedOverrides(cfg, overrides); err != nil {
+			return fmt.Errorf("unable to apply embedded overrides: %w", err)
+		}
+	}
+	if c.String("overrides") == "" {
+		return nil
+	}
+	eucfg, err := parseEndUserConfig(c.String("overrides"))
+	if err != nil {
+		return fmt.Errorf("unable to process overrides file: %w", err)
+	}
+	overrides := eucfg.Spec.UnsupportedOverrides.K0s
+	if err := config.ApplyEmbeddedUnsupportedOverrides(cfg, overrides); err != nil {
+		return fmt.Errorf("unable to apply overrides: %w", err)
+	}
+	return nil
+}
+
 // copyUserProvidedConfig copies the user provided configuration to the config dir.
 func copyUserProvidedConfig(c *cli.Context) error {
 	usercfg := c.String("config")
@@ -249,17 +268,6 @@ func ensureK0sctlConfig(c *cli.Context, useprompt bool) error {
 	if err != nil {
 		return fmt.Errorf("unable to render config: %w", err)
 	}
-	if c.String("overrides") != "" {
-		eucfg, err := parseEndUserConfig(c.String("overrides"))
-		if err != nil {
-			return fmt.Errorf("unable to process overrides file: %w", err)
-		}
-		if err := config.ApplyEmbeddedUnsupportedOverrides(
-			cfg, eucfg.Spec.UnsupportedOverrides.K0s,
-		); err != nil {
-			return fmt.Errorf("unable to apply overrides: %w", err)
-		}
-	}
 	opts := []addons.Option{}
 	if c.Bool("no-prompt") {
 		opts = append(opts, addons.WithoutPrompt())
@@ -270,6 +278,9 @@ func ensureK0sctlConfig(c *cli.Context, useprompt bool) error {
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
+	if err := applyUnsupportedOverrides(c, cfg); err != nil {
+		return fmt.Errorf("unable to apply unsupported overrides: %w", err)
+	}
 	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("unable to create config file: %w", err)

diff --git a/e2e/scripts/unsupported-overrides.sh b/e2e/scripts/unsupported-overrides.sh
@@ -18,6 +18,47 @@ spec:
             values:
               allowedUnsafeSysctls:
               - net.ipv4.ip_forward
+          extensions:
+            helm:
+              charts:
+              - chartname: openebs/openebs
+                name: openebs
+                namespace: openebs
+                order: 1
+                values: |
+                  localprovisioner:
+                    hostpathClass:
+                      isDefaultClass: true
+                  ndm:
+                    enabled: false
+                  ndmOperator:
+                    enabled: false
+                version: 3.10.0
+              - chartname: oci://registry.replicated.com/library/embedded-cluster-operator
+                name: embedded-cluster-operator
+                namespace: embedded-cluster
+                order: 2
+                version: 0.13.0
+              - chartname: oci://registry.replicated.com/library/admin-console
+                name: admin-console
+                namespace: embedded-cluster
+                order: 3
+                version: 1.105.1
+                values: |
+                  isHelmManaged: false
+                  kotsApplication: default value
+                  minimalRBAC: false
+                  service:
+                    nodePort: 30000
+                    type: NodePort
+              - chartname: oci://registry-1.docker.io/bitnamicharts/memcached
+                name: memcached
+                namespace: embedded-cluster
+                order: 4
+                version: 6.6.2
+              repositories:
+              - name: openebs
+                url: https://openebs.github.io/charts
 "
 
 embed_cluster_config() {
@@ -61,6 +102,27 @@ override_applied() {
       cat "$K0SCTLCONFIG"
       return 1
     fi
+    if ! grep "memcached" "$K0SCTLCONFIG"; then
+      echo "override not applied, expected memcached helmchart not found, actual config:"
+      cat "$K0SCTLCONFIG"
+      return 1
+    fi
+}
+
+wait_for_memcached_pods() {
+    ready=$(kubectl get pods -n embedded-cluster | grep -c memcached || true)
+    counter=0
+    while [ "$ready" -lt "1" ]; do
+        if [ "$counter" -gt 36 ]; then
+            return 1
+        fi
+        sleep 5
+        counter=$((counter+1))
+        echo "Waiting for memcached pods"
+        ready=$(kubectl get pods -n embedded-cluster | grep -c memcached || true)
+        kubectl get pods -n embedded-cluster 2>&1 || true
+        echo "$ready"
+    done
 }
 
 main() {
@@ -79,6 +141,10 @@ main() {
         echo "Expected override to be applied"
         exit 1
     fi
+    if ! wait_for_memcached_pods; then
+        echo "Failed waiting for memcached pods"
+        exit 1
+    fi
 }
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -19,7 +19,6 @@ import (
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	k0sversion "github.com/k0sproject/version"
-	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 	k8syaml "sigs.k8s.io/yaml"
@@ -52,33 +51,17 @@ func ReadConfigFile(cfgPath string) (*v1beta1.Cluster, error) {
 
 // RenderClusterConfig renders a cluster configuration interactively.
 func RenderClusterConfig(ctx context.Context, multi bool) (*v1beta1.Cluster, error) {
-	clusterConfig, err := customization.GetEmbeddedClusterConfig()
-	if err != nil {
-		return nil, fmt.Errorf("unable to get embedded cluster config: %w", err)
-	} else if clusterConfig == nil {
-		clusterConfig = &embeddedclusterv1beta1.Config{}
-	}
 	if multi {
 		cfg, err := renderMultiNodeConfig(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("unable to render multi-node config: %w", err)
 		}
-		if err := ApplyEmbeddedUnsupportedOverrides(
-			cfg, clusterConfig.Spec.UnsupportedOverrides.K0s,
-		); err != nil {
-			return nil, fmt.Errorf("unable to apply unsupported overrides: %w", err)
-		}
 		return cfg, nil
 	}
 	cfg, err := renderSingleNodeConfig(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("unable to render single-node config: %w", err)
 	}
-	if err := ApplyEmbeddedUnsupportedOverrides(
-		cfg, clusterConfig.Spec.UnsupportedOverrides.K0s,
-	); err != nil {
-		return nil, fmt.Errorf("unable to apply unsupported overrides: %w", err)
-	}
 	return cfg, nil
 }