From 78b60a4afdae858b3bf0f644e3872ac443698a79 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 20 Nov 2023 11:17:07 -0500 Subject: [PATCH] use custom 'controller' role name if available (#184) * use custom 'controller' role name if available * reconcile overrides formats * handle nil * actually parse config from the binary * ensure node has controller override role * intentionally break test * update code * undo intentional breakage * test both that the role was used and custom labels were applied * remove unused struct --- cmd/embedded-cluster/install.go | 16 +++- e2e/kots-release/config.yaml | 24 ++++++ e2e/scripts/single-node-install.sh | 17 +++++ .../embeddedclusteroperator.go | 19 +---- pkg/config/config.go | 73 +++++++++++-------- pkg/config/config_test.go | 18 +++-- pkg/config/testdata/override-change-name.yaml | 7 +- .../testdata/override-enable-telemetry.yaml | 9 +-- .../testdata/override-setting-ip-forward.yaml | 15 ++-- .../testdata/override-zero-out-sans-list.yaml | 9 +-- pkg/customization/customization.go | 18 ++++- 11 files changed, 144 insertions(+), 81 deletions(-) create mode 100644 e2e/kots-release/config.yaml diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index c60746265..25399ef6e 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -16,9 +16,11 @@ import ( "github.com/k0sproject/rig" "github.com/k0sproject/rig/log" k0sversion "github.com/k0sproject/version" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "gopkg.in/yaml.v2" + kyaml "sigs.k8s.io/yaml" "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/config" @@ -160,7 +162,12 @@ func updateConfig(c *cli.Context) error { if err != nil { return fmt.Errorf("unable to read overrides file: %w", err) } - config.ApplyEmbeddedUnsupportedOverrides(cfg, data) + var overrideCfg embeddedclusterv1beta1.Config + if err := kyaml.Unmarshal(data, &overrideCfg); err != nil { + return fmt.Errorf("unable to unmarshal overrides file: %w", err) + } + + config.ApplyEmbeddedUnsupportedOverrides(cfg, overrideCfg.Spec.UnsupportedOverrides.K0s) } opts := []addons.Option{} @@ -257,7 +264,12 @@ func ensureK0sctlConfig(c *cli.Context, useprompt bool) error { if err != nil { return fmt.Errorf("unable to read overrides file: %w", err) } - config.ApplyEmbeddedUnsupportedOverrides(cfg, data) + var overrideCfg embeddedclusterv1beta1.Config + if err := kyaml.Unmarshal(data, &overrideCfg); err != nil { + return fmt.Errorf("unable to unmarshal overrides file: %w", err) + } + + config.ApplyEmbeddedUnsupportedOverrides(cfg, overrideCfg.Spec.UnsupportedOverrides.K0s) } opts := []addons.Option{} diff --git a/e2e/kots-release/config.yaml b/e2e/kots-release/config.yaml new file mode 100644 index 000000000..1751d11e7 --- /dev/null +++ b/e2e/kots-release/config.yaml @@ -0,0 +1,24 @@ +apiVersion: embeddedcluster.replicated.com/v1beta1 +kind: Config +metadata: + name: "testconfig" +spec: + controller: + labels: + controller-label: controller-label-value + name: controller-test + custom: + - labels: + abc-test-label: abc-test-label-value + abc-test-label-two: abc-test-label-value-2 + name: abc + - labels: + xyz-test-label: xyz-value + name: xyz + unsupportedOverrides: + k0s: | + metadata: + name: foo + spec: + telemetry: + enabled: false diff --git a/e2e/scripts/single-node-install.sh b/e2e/scripts/single-node-install.sh index 2e1ebc034..1b7fc1b84 100644 --- a/e2e/scripts/single-node-install.sh +++ b/e2e/scripts/single-node-install.sh @@ -14,6 +14,7 @@ wait_for_healthy_node() { ready=$(kubectl get nodes | grep -v NotReady | grep -c Ready || true) kubectl get nodes || true done + return 0 } @@ -45,6 +46,18 @@ wait_for_pods_running() { done } +ensure_node_config() { + if ! kubectl describe node | grep "controller-label" ; then + echo "Failed to find controller-label" + return 1 + fi + + if ! kubectl describe node | grep "controller-test" ; then + echo "Failed to find controller-test" + return 1 + fi +} + main() { if ! embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then cat /etc/os-release @@ -59,6 +72,10 @@ main() { echo "Failed to install embedded-cluster" exit 1 fi + if ! ensure_node_config; then + echo "Cluster did not respect node config" + exit 1 + fi if ! wait_for_pods_running 900; then echo "Failed to install embedded-cluster" exit 1 diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go index 9d7dfdb73..c93b25d24 100644 --- a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go +++ b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go @@ -13,7 +13,6 @@ import ( "gopkg.in/yaml.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kyaml "sigs.k8s.io/yaml" "github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole" "github.com/replicatedhq/embedded-cluster/pkg/customization" @@ -76,22 +75,6 @@ func (e *EmbeddedClusterOperator) GenerateHelmConfig() ([]v1beta1.Chart, []v1bet return []v1beta1.Chart{chartConfig}, nil, nil } -// readEmbeddedClusterConfig reads and unmarshal the Config object from the embedded cluster -// that has been embedded into this binary through a release. -func (e *EmbeddedClusterOperator) readEmbeddedClusterConfig() (*embeddedclusterv1beta1.Config, error) { - rawcfg, err := customization.AdminConsole{}.EmbeddedClusterConfig() - if err != nil { - return nil, fmt.Errorf("unable to read embeddec cluster config: %w", err) - } else if rawcfg == nil { - return nil, nil - } - var cfg embeddedclusterv1beta1.Config - if err := kyaml.Unmarshal(rawcfg, &cfg); err != nil { - return nil, fmt.Errorf("unable to unmarshal embedded cluster config: %w", err) - } - return &cfg, nil -} - // Outro is executed after the cluster deployment. func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client) error { loading := pb.Start() @@ -101,7 +84,7 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client) return err } loading.Closef("Embedded Cluster Operator is ready!") - cfg, err := e.readEmbeddedClusterConfig() + cfg, err := customization.AdminConsole{}.EmbeddedClusterConfig() if err != nil { return err } diff --git a/pkg/config/config.go b/pkg/config/config.go index f7ea3feb3..3a07bf107 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,6 +19,7 @@ import ( "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1" "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster" k0sversion "github.com/k0sproject/version" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1" "github.com/sirupsen/logrus" yamlv2 "gopkg.in/yaml.v2" "sigs.k8s.io/yaml" @@ -51,23 +52,25 @@ func ReadConfigFile(cfgPath string) (*v1beta1.Cluster, error) { // RenderClusterConfig renders a cluster configuration interactively. func RenderClusterConfig(ctx context.Context, multi bool) (*v1beta1.Cluster, error) { - embconfig, err := customization.AdminConsole{}.EmbeddedClusterConfig() + clusterConfig, err := customization.AdminConsole{}.EmbeddedClusterConfig() if err != nil { return nil, fmt.Errorf("unable to get embedded cluster config: %w", err) + } else if clusterConfig == nil { + clusterConfig = &embeddedclusterv1beta1.Config{} } if multi { cfg, err := renderMultiNodeConfig(ctx) if err != nil { return nil, fmt.Errorf("unable to render multi-node config: %w", err) } - ApplyEmbeddedUnsupportedOverrides(cfg, embconfig) + ApplyEmbeddedUnsupportedOverrides(cfg, clusterConfig.Spec.UnsupportedOverrides.K0s) return cfg, nil } cfg, err := renderSingleNodeConfig(ctx) if err != nil { return nil, fmt.Errorf("unable to render single-node config: %w", err) } - ApplyEmbeddedUnsupportedOverrides(cfg, embconfig) + ApplyEmbeddedUnsupportedOverrides(cfg, clusterConfig.Spec.UnsupportedOverrides.K0s) return renderSingleNodeConfig(ctx) } @@ -251,16 +254,18 @@ func renderSingleNodeConfig(ctx context.Context) (*v1beta1.Cluster, error) { if err != nil { return nil, fmt.Errorf("unable to get preferred node IP address: %w", err) } + + labels := additionalControllerLabels() + labels["kots.io/embedded-cluster-role-0"] = getControllerRoleName() + labels["kots.io/embedded-cluster-role"] = "total-1" + host := &hostcfg{ Address: ipaddr, Role: "controller+worker", Port: 22, User: usr.Username, KeyPath: defaults.SSHKeyPath(), - Labels: map[string]string{ - "kots.io/embedded-cluster-role-0": "controller", - "kots.io/embedded-cluster-role": "total-1", - }, + Labels: labels, } if err := host.testConnection(); err != nil { return nil, fmt.Errorf("unable to connect to %s: %w", ipaddr, err) @@ -309,32 +314,11 @@ func UpdateHelmConfigs(cfg *v1beta1.Cluster, opts ...addons.Option) error { return nil } -// UnsupportedConfigOverrides is a auxiliary struct for parsing the unsupported overrides -// as provided in the Kots release. XXX This should eventually become a CRD. -type UnsupportedConfigOverrides struct { - Spec struct { - UnsupportedOverrides struct { - K0s *cluster.K0s `yaml:"k0s"` - } `yaml:"unsupportedOverrides"` - } `yaml:"spec"` -} - // ApplyEmbeddedUnsupportedOverrides applies the custom configuration to the cluster config. -func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig []byte) error { - if embconfig == nil { +func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig string) error { + if embconfig == "" { return nil } - var overrides UnsupportedConfigOverrides - if err := yaml.Unmarshal(embconfig, &overrides); err != nil { - return fmt.Errorf("unable to parse cluster config overrides: %w", err) - } - if overrides.Spec.UnsupportedOverrides.K0s == nil { - return nil - } - origConfigBytes, err := yaml.Marshal(overrides.Spec.UnsupportedOverrides.K0s.Config) - if err != nil { - return fmt.Errorf("unable to marshal cluster config overrides: %w", err) - } newConfigBytes, err := yaml.Marshal(config.Spec.K0s.Config) if err != nil { return fmt.Errorf("unable to marshal original cluster config: %w", err) @@ -343,7 +327,7 @@ func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig []byte if err != nil { return fmt.Errorf("unable to convert cluster config overrides to json: %w", err) } - target, err := fmtconvert.YAMLToJSON(origConfigBytes) + target, err := fmtconvert.YAMLToJSON([]byte(embconfig)) if err != nil { return fmt.Errorf("unable to convert original cluster config to json: %w", err) } @@ -362,3 +346,30 @@ func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig []byte config.Spec.K0s.Config = newK0sConfig return nil } + +func getControllerRoleName() string { + clusterConfig, err := customization.AdminConsole{}.EmbeddedClusterConfig() + + controllerRoleName := "controller" + if err == nil { + if clusterConfig != nil { + if clusterConfig.Spec.Controller.Name != "" { + controllerRoleName = clusterConfig.Spec.Controller.Name + } + } + } + return controllerRoleName +} + +func additionalControllerLabels() map[string]string { + clusterConfig, err := customization.AdminConsole{}.EmbeddedClusterConfig() + + if err == nil { + if clusterConfig != nil { + if clusterConfig.Spec.Controller.Labels != nil { + return clusterConfig.Spec.Controller.Labels + } + } + } + return map[string]string{} +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a27d85ebf..b5766312d 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -7,8 +7,11 @@ import ( "testing" "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + kyaml "sigs.k8s.io/yaml" ) //go:embed testdata/*yaml @@ -39,16 +42,21 @@ func TestApplyUnsupportedOverrides(t *testing.T) { } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { + req := require.New(t) + var config v1beta1.Cluster err := yaml.Unmarshal([]byte(tt.Config), &config) - assert.NoError(t, err) - err = ApplyEmbeddedUnsupportedOverrides(&config, []byte(tt.Override)) - assert.NoError(t, err) + req.NoError(err) + var cfg embeddedclusterv1beta1.Config + err = kyaml.Unmarshal([]byte(tt.Override), &cfg) + req.NoError(err) + err = ApplyEmbeddedUnsupportedOverrides(&config, cfg.Spec.UnsupportedOverrides.K0s) + req.NoError(err) result, err := yaml.Marshal(config) - assert.NoError(t, err) + req.NoError(err) resultString := strings.TrimSpace(string(result)) expectedString := strings.TrimSpace(string(tt.Expected)) - assert.Equal(t, resultString, expectedString) + req.Equal(expectedString, resultString) }) } } diff --git a/pkg/config/testdata/override-change-name.yaml b/pkg/config/testdata/override-change-name.yaml index 191a1b92c..6d1c7df70 100644 --- a/pkg/config/testdata/override-change-name.yaml +++ b/pkg/config/testdata/override-change-name.yaml @@ -32,10 +32,9 @@ override: |- kind: Config spec: unsupportedOverrides: - k0s: - config: - metadata: - name: foo + k0s: | + metadata: + name: foo expected: |- apiVersion: k0sctl.k0sproject.io/v1beta1 kind: Cluster diff --git a/pkg/config/testdata/override-enable-telemetry.yaml b/pkg/config/testdata/override-enable-telemetry.yaml index 7d9d95a15..853496273 100644 --- a/pkg/config/testdata/override-enable-telemetry.yaml +++ b/pkg/config/testdata/override-enable-telemetry.yaml @@ -32,11 +32,10 @@ override: |- kind: Config spec: unsupportedOverrides: - k0s: - config: - spec: - telemetry: - enabled: true + k0s: | + spec: + telemetry: + enabled: true expected: |- apiVersion: k0sctl.k0sproject.io/v1beta1 kind: Cluster diff --git a/pkg/config/testdata/override-setting-ip-forward.yaml b/pkg/config/testdata/override-setting-ip-forward.yaml index b5e4aade3..94da52599 100644 --- a/pkg/config/testdata/override-setting-ip-forward.yaml +++ b/pkg/config/testdata/override-setting-ip-forward.yaml @@ -32,14 +32,13 @@ override: |- kind: Config spec: unsupportedOverrides: - k0s: - config: - spec: - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward + k0s: | + spec: + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward expected: |- apiVersion: k0sctl.k0sproject.io/v1beta1 kind: Cluster diff --git a/pkg/config/testdata/override-zero-out-sans-list.yaml b/pkg/config/testdata/override-zero-out-sans-list.yaml index c4add6066..742f0f581 100644 --- a/pkg/config/testdata/override-zero-out-sans-list.yaml +++ b/pkg/config/testdata/override-zero-out-sans-list.yaml @@ -36,11 +36,10 @@ override: |- kind: Config spec: unsupportedOverrides: - k0s: - config: - spec: - api: - sans: null + k0s: | + spec: + api: + sans: null expected: |- apiVersion: k0sctl.k0sproject.io/v1beta1 kind: Cluster diff --git a/pkg/customization/customization.go b/pkg/customization/customization.go index 9a51aace0..e2bf62b15 100644 --- a/pkg/customization/customization.go +++ b/pkg/customization/customization.go @@ -10,6 +10,7 @@ import ( "os" "runtime" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1" "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "sigs.k8s.io/yaml" @@ -95,12 +96,14 @@ func (a AdminConsole) processSection(section *elf.Section) (*ParsedSection, erro continue } result.HostPreflights = append(result.HostPreflights, content.Bytes()) + continue } if bytes.Contains(content.Bytes(), []byte("apiVersion: embeddedcluster.replicated.com/v1beta1")) { - if bytes.Contains(content.Bytes(), []byte("kind: Config")) { + if !bytes.Contains(content.Bytes(), []byte("kind: Config")) { continue } result.EmbeddedClusterConfig = content.Bytes() + continue } } } @@ -165,7 +168,7 @@ func (a AdminConsole) Application() ([]byte, error) { } // EmbeddedClusterConfig reads the embedded cluster config from the embedded Kots Application Release. -func (a AdminConsole) EmbeddedClusterConfig() ([]byte, error) { +func (a AdminConsole) EmbeddedClusterConfig() (*embeddedclusterv1beta1.Config, error) { if runtime.GOOS != "linux" { return nil, nil } @@ -175,5 +178,14 @@ func (a AdminConsole) EmbeddedClusterConfig() ([]byte, error) { } else if section == nil { return nil, nil } - return section.EmbeddedClusterConfig, nil + + rawcfg := section.EmbeddedClusterConfig + if rawcfg == nil { + return nil, nil + } + var cfg embeddedclusterv1beta1.Config + if err := yaml.Unmarshal(rawcfg, &cfg); err != nil { + return nil, fmt.Errorf("unable to unmarshal embedded cluster config: %w", err) + } + return &cfg, nil }