From 46bd74d6f266bc37d09e65da9af1144be4b1d505 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 24 Sep 2024 00:38:37 +0200 Subject: [PATCH] use the entire installation spec object in node join response (#1211) * use unified join command response also check/enforce EC version mismatch * add kots image override * unit tests and preflights * update kotsadm image * f * Spec -> InstallationSpec * updated adminconsole version * remove override image --------- Co-authored-by: laverya <2318911+laverya@users.noreply.github.com> --- cmd/embedded-cluster/join.go | 95 ++++++++++---------- cmd/embedded-cluster/join_test.go | 11 ++- cmd/embedded-cluster/preflights.go | 18 ++-- pkg/addons/adminconsole/static/metadata.yaml | 18 ++-- 4 files changed, 78 insertions(+), 64 deletions(-) diff --git a/cmd/embedded-cluster/join.go b/cmd/embedded-cluster/join.go index b3bb3dfad..232965d03 100644 --- a/cmd/embedded-cluster/join.go +++ b/cmd/embedded-cluster/join.go @@ -31,21 +31,17 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/netutils" "github.com/replicatedhq/embedded-cluster/pkg/prompts" "github.com/replicatedhq/embedded-cluster/pkg/spinner" + "github.com/replicatedhq/embedded-cluster/pkg/versions" ) // JoinCommandResponse is the response from the kots api we use to fetch the k0s join token. type JoinCommandResponse struct { - K0sJoinCommand string `json:"k0sJoinCommand"` - K0sToken string `json:"k0sToken"` - ClusterID uuid.UUID `json:"clusterID"` - K0sUnsupportedOverrides string `json:"k0sUnsupportedOverrides"` - EndUserK0sConfigOverrides string `json:"endUserK0sConfigOverrides"` - // MetricsBaseURL is the https://replicated.app endpoint url - MetricsBaseURL string `json:"metricsBaseURL"` - AirgapRegistryAddress string `json:"airgapRegistryAddress"` - Proxy *ecv1beta1.ProxySpec `json:"proxy"` - Network *ecv1beta1.NetworkSpec `json:"network"` - LocalArtifactMirrorPort int `json:"localArtifactMirrorPort,omitempty"` + K0sJoinCommand string `json:"k0sJoinCommand"` + K0sToken string `json:"k0sToken"` + ClusterID uuid.UUID `json:"clusterID"` + EmbeddedClusterVersion string `json:"embeddedClusterVersion"` + AirgapRegistryAddress string `json:"airgapRegistryAddress"` + InstallationSpec ecv1beta1.InstallationSpec `json:"installationSpec,omitempty"` } // extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that @@ -69,13 +65,13 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map // EndUserOverrides returns a dig.Mapping that can be applied on top of a k0s configuration. // This patch is assembled based on the EndUserK0sConfigOverrides field. func (j JoinCommandResponse) EndUserOverrides() (dig.Mapping, error) { - return j.extractK0sConfigOverridePatch([]byte(j.EndUserK0sConfigOverrides)) + return j.extractK0sConfigOverridePatch([]byte(j.InstallationSpec.EndUserK0sConfigOverrides)) } // EmbeddedOverrides returns a dig.Mapping that can be applied on top of a k0s configuration. // This patch is assembled based on the K0sUnsupportedOverrides field. func (j JoinCommandResponse) EmbeddedOverrides() (dig.Mapping, error) { - return j.extractK0sConfigOverridePatch([]byte(j.K0sUnsupportedOverrides)) + return j.extractK0sConfigOverridePatch([]byte(j.InstallationSpec.Config.UnsupportedOverrides.K0s)) } // getJoinToken issues a request to the kots api to get the actual join command @@ -114,7 +110,7 @@ func startAndWaitForK0s(c *cli.Context, jcmd *JoinCommandResponse) error { logrus.Debugf("starting %s service", binName) if err := startK0sService(); err != nil { err := fmt.Errorf("unable to start service: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } @@ -122,7 +118,7 @@ func startAndWaitForK0s(c *cli.Context, jcmd *JoinCommandResponse) error { logrus.Debugf("waiting for k0s to be ready") if err := waitForK0s(); err != nil { err := fmt.Errorf("unable to wait for node: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } @@ -198,13 +194,18 @@ var joinCommand = &cli.Command{ return fmt.Errorf("unable to get join token: %w", err) } - setProxyEnv(jcmd.Proxy) - proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy, networkInterface) + // check to make sure the version returned by the join token is the same as the one we are running + if jcmd.EmbeddedClusterVersion != versions.Version { + return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion) + } + + setProxyEnv(jcmd.InstallationSpec.Proxy) + proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface) if err != nil { return fmt.Errorf("failed to check proxy config for local IP: %w", err) } if !proxyOK { - return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP) + return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.InstallationSpec.Proxy.NoProxy, localIP) } isAirgap := c.String("airgap-bundle") != "" @@ -216,25 +217,25 @@ var joinCommand = &cli.Command{ } } - metrics.ReportJoinStarted(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID) + metrics.ReportJoinStarted(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID) logrus.Debugf("materializing %s binaries", binName) if err := materializeFiles(c); err != nil { - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } - applier, err := getAddonsApplier(c, "", jcmd.Proxy) + applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy) if err != nil { - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } - // jcmd.MetricsBaseURL is the replicated.app endpoint url - replicatedAPIURL := jcmd.MetricsBaseURL + // jcmd.InstallationSpec.MetricsBaseURL is the replicated.app endpoint url + replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) - if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.Proxy); err != nil { + if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil { err := fmt.Errorf("unable to run host preflights locally: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } @@ -246,54 +247,54 @@ var joinCommand = &cli.Command{ logrus.Debugf("saving token to disk") if err := saveTokenToDisk(jcmd.K0sToken); err != nil { err := fmt.Errorf("unable to save token to disk: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } logrus.Debugf("installing %s binaries", binName) if err := installK0sBinary(); err != nil { err := fmt.Errorf("unable to install k0s binary: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } if jcmd.AirgapRegistryAddress != "" { if err := airgap.AddInsecureRegistry(jcmd.AirgapRegistryAddress); err != nil { err := fmt.Errorf("unable to add insecure registry: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } } logrus.Debugf("creating systemd unit files") localArtifactMirrorPort := defaults.LocalArtifactMirrorPort - if jcmd.LocalArtifactMirrorPort > 0 { - localArtifactMirrorPort = jcmd.LocalArtifactMirrorPort + if jcmd.InstallationSpec.LocalArtifactMirror != nil && jcmd.InstallationSpec.LocalArtifactMirror.Port > 0 { + localArtifactMirrorPort = jcmd.InstallationSpec.LocalArtifactMirror.Port } // both controller and worker nodes will have 'worker' in the join command - if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy, localArtifactMirrorPort); err != nil { + if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.InstallationSpec.Proxy, localArtifactMirrorPort); err != nil { err := fmt.Errorf("unable to create systemd unit files: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } logrus.Debugf("overriding network configuration") if err := applyNetworkConfiguration(jcmd, networkInterface); err != nil { err := fmt.Errorf("unable to apply network configuration: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) } logrus.Debugf("applying configuration overrides") if err := applyJoinConfigurationOverrides(jcmd); err != nil { err := fmt.Errorf("unable to apply configuration overrides: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } logrus.Debugf("joining node to cluster") if err := runK0sInstallCommand(jcmd.K0sJoinCommand, networkInterface); err != nil { err := fmt.Errorf("unable to join node to cluster: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } @@ -302,7 +303,7 @@ var joinCommand = &cli.Command{ } if !strings.Contains(jcmd.K0sJoinCommand, "controller") { - metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID) + metrics.ReportJoinSucceeded(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID) logrus.Debugf("worker node join finished") return nil } @@ -310,37 +311,37 @@ var joinCommand = &cli.Command{ kcli, err := kubeutils.KubeClient() if err != nil { err := fmt.Errorf("unable to get kube client: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } hostname, err := os.Hostname() if err != nil { err := fmt.Errorf("unable to get hostname: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } if err := waitForNode(c.Context, kcli, hostname); err != nil { err := fmt.Errorf("unable to wait for node: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } if c.Bool("enable-ha") { if err := maybeEnableHA(c.Context, kcli); err != nil { err := fmt.Errorf("unable to enable high availability: %w", err) - metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err) + metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err) return err } } - metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID) + metrics.ReportJoinSucceeded(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID) logrus.Debugf("controller node join finished") return nil }, } func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface string) error { - if jcmd.Network != nil { + if jcmd.InstallationSpec.Network != nil { clusterSpec := config.RenderK0sConfig() address, err := netutils.FirstValidAddress(networkInterface) if err != nil { @@ -350,13 +351,13 @@ func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface strin clusterSpec.Spec.Storage.Etcd.PeerAddress = address // NOTE: we should be copying everything from the in cluster config spec and overriding // the node specific config from clusterSpec.GetClusterWideConfig() - clusterSpec.Spec.Network.PodCIDR = jcmd.Network.PodCIDR - clusterSpec.Spec.Network.ServiceCIDR = jcmd.Network.ServiceCIDR - if jcmd.Network.NodePortRange != "" { + clusterSpec.Spec.Network.PodCIDR = jcmd.InstallationSpec.Network.PodCIDR + clusterSpec.Spec.Network.ServiceCIDR = jcmd.InstallationSpec.Network.ServiceCIDR + if jcmd.InstallationSpec.Network.NodePortRange != "" { if clusterSpec.Spec.API.ExtraArgs == nil { clusterSpec.Spec.API.ExtraArgs = map[string]string{} } - clusterSpec.Spec.API.ExtraArgs["service-node-port-range"] = jcmd.Network.NodePortRange + clusterSpec.Spec.API.ExtraArgs["service-node-port-range"] = jcmd.InstallationSpec.Network.NodePortRange } clusterSpecYaml, err := k8syaml.Marshal(clusterSpec) diff --git a/cmd/embedded-cluster/join_test.go b/cmd/embedded-cluster/join_test.go index cd3118dc8..e2fb45324 100644 --- a/cmd/embedded-cluster/join_test.go +++ b/cmd/embedded-cluster/join_test.go @@ -10,6 +10,7 @@ import ( "github.com/k0sproject/dig" k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -101,8 +102,14 @@ func TestJoinCommandResponseOverrides(t *testing.T) { t.Run(tname, func(t *testing.T) { req := require.New(t) join := JoinCommandResponse{ - K0sUnsupportedOverrides: tt.EmbeddedOverrides, - EndUserK0sConfigOverrides: tt.EndUserOverrides, + InstallationSpec: ecv1beta1.InstallationSpec{ + Config: &ecv1beta1.ConfigSpec{ + UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{ + K0s: tt.EmbeddedOverrides, + }, + }, + EndUserK0sConfigOverrides: tt.EndUserOverrides, + }, } embedded, err := join.EmbeddedOverrides() diff --git a/cmd/embedded-cluster/preflights.go b/cmd/embedded-cluster/preflights.go index 94b7268dd..7833e82bc 100644 --- a/cmd/embedded-cluster/preflights.go +++ b/cmd/embedded-cluster/preflights.go @@ -7,6 +7,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/defaults" "github.com/replicatedhq/embedded-cluster/pkg/netutils" + "github.com/replicatedhq/embedded-cluster/pkg/versions" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -126,13 +127,18 @@ var joinRunPreflightsCommand = &cli.Command{ return fmt.Errorf("unable to get join token: %w", err) } - setProxyEnv(jcmd.Proxy) - proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy, networkInterface) + // check to make sure the version returned by the join token is the same as the one we are running + if jcmd.EmbeddedClusterVersion != versions.Version { + return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion) + } + + setProxyEnv(jcmd.InstallationSpec.Proxy) + proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface) if err != nil { return fmt.Errorf("failed to check proxy config for local IP: %w", err) } if !proxyOK { - return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP) + return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.InstallationSpec.Proxy.NoProxy, localIP) } isAirgap := c.String("airgap-bundle") != "" @@ -142,15 +148,15 @@ var joinRunPreflightsCommand = &cli.Command{ return err } - applier, err := getAddonsApplier(c, "", jcmd.Proxy) + applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy) if err != nil { return err } logrus.Debugf("running host preflights") - replicatedAPIURL := jcmd.MetricsBaseURL + replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress) - if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.Proxy); err != nil { + if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil { err := fmt.Errorf("unable to run host preflights locally: %w", err) return err } diff --git a/pkg/addons/adminconsole/static/metadata.yaml b/pkg/addons/adminconsole/static/metadata.yaml index 6ba05ab8e..b6e0a4edb 100644 --- a/pkg/addons/adminconsole/static/metadata.yaml +++ b/pkg/addons/adminconsole/static/metadata.yaml @@ -5,26 +5,26 @@ # $ make buildtools # $ output/bin/buildtools update addon # -version: 1.117.2 +version: 1.117.3 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console images: kotsadm: repo: proxy.replicated.com/anonymous/kotsadm/kotsadm tag: - amd64: v1.117.2-amd64@sha256:eb36a5a9af101164598f1e2f7f0745aad59bdb8d8870e09e2702d8c75c753380 - arm64: v1.117.2-arm64@sha256:a60cf01f935a785b1a5e693d65a18bd6f4e8d7aac7be30a3311c4c17f3b8222f + amd64: v1.117.3-amd64@sha256:d47ac4df627ac357452efffb717776adb452c3ab9b466ef3ccaf808df722b7a6 + arm64: v1.117.3-arm64@sha256:6b1b7bfc42fb3c7f21fe077b4645194fbdd1d497f92f4b12c00ceadb40969b8e kotsadm-migrations: repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations tag: - amd64: v1.117.2-amd64@sha256:afb070977dfc16d3b85d2ae9e3cf68fc317e17e3ab05e6ad08d353201f1682ef - arm64: v1.117.2-arm64@sha256:cff8c43da1094d7969ef689eca3fd11c17aae936f45d1584fd00e25085a6112b + amd64: v1.117.3-amd64@sha256:56d2765497a57c06ef6e9f7705cf5218d21a978d197575a3c22fe7d84db07f0a + arm64: v1.117.3-arm64@sha256:25ff2b4697425be55d576f5f068480a3ab8fe6216341f2ec3b3c1962d3ac08ba kurl-proxy: repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy tag: - amd64: v1.117.2-amd64@sha256:b2e4a7ee3ca4cedd1028fbac0678fcab091b1e0a81fd39d6b8d5d20f509b2304 - arm64: v1.117.2-arm64@sha256:056b640c03ff824553adb2fc95fb4930e4cdea18c89b656ed81c1937d8ab3c73 + amd64: v1.117.3-amd64@sha256:816bcbc273ec51255d7b459e49831ce09fd361db4a295d31f61d7af02177860f + arm64: v1.117.3-arm64@sha256:c4f5632ca9ea3fae2208b4ca7923900c6e958a0906b4a9dd4f6b5ebdab4b9d89 rqlite: repo: proxy.replicated.com/anonymous/kotsadm/rqlite tag: - amd64: 8.30.3-r0-amd64@sha256:2f0c6a90c462b8be8f08c5a1a83f71932be96b846beb3dfbd9fa0e3d13c66d63 - arm64: 8.30.3-r0-arm64@sha256:4e96628236b7a1e9549bfdb16d44ff71ee7607b2450548b7e0f86f1f48b36c4e + amd64: 8.30.4-r0-amd64@sha256:884ac56b236e059e420858c94d90a083fe48b666c8b3433da612b9380906ce41 + arm64: 8.30.4-r0-arm64@sha256:cfb55508de1fb59cfc5d14586f52d8a09b776e766ea612351df77d19ae2e6d9a