Skip to content

Commit

Permalink
use the entire installation spec object in node join response (#1211)
Browse files Browse the repository at this point in the history 
* use unified join command response

also check/enforce EC version mismatch

* add kots image override

* unit tests and preflights

* update kotsadm image

* f

* Spec -> InstallationSpec

* updated adminconsole version

* remove override image

---------

Co-authored-by: laverya <[email protected]>
  • Loading branch information
@laverya
laverya and laverya authored Sep 23, 2024
1 parent 9570e97 commit 46bd74d
Show file tree
Hide file tree
Showing 4 changed files with 78 additions and 64 deletions.
95 changes: 48 additions & 47 deletions cmd/embedded-cluster/join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,17 @@ import (
"github.com/replicatedhq/embedded-cluster/pkg/netutils"
"github.com/replicatedhq/embedded-cluster/pkg/prompts"
"github.com/replicatedhq/embedded-cluster/pkg/spinner"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
)

// JoinCommandResponse is the response from the kots api we use to fetch the k0s join token.
type JoinCommandResponse struct {
K0sJoinCommand string `json:"k0sJoinCommand"`
K0sToken string `json:"k0sToken"`
ClusterID uuid.UUID `json:"clusterID"`
K0sUnsupportedOverrides string `json:"k0sUnsupportedOverrides"`
EndUserK0sConfigOverrides string `json:"endUserK0sConfigOverrides"`
// MetricsBaseURL is the https://replicated.app endpoint url
MetricsBaseURL string `json:"metricsBaseURL"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
Proxy *ecv1beta1.ProxySpec `json:"proxy"`
Network *ecv1beta1.NetworkSpec `json:"network"`
LocalArtifactMirrorPort int `json:"localArtifactMirrorPort,omitempty"`
K0sJoinCommand string `json:"k0sJoinCommand"`
K0sToken string `json:"k0sToken"`
ClusterID uuid.UUID `json:"clusterID"`
EmbeddedClusterVersion string `json:"embeddedClusterVersion"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
InstallationSpec ecv1beta1.InstallationSpec `json:"installationSpec,omitempty"`
}

// extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
Expand All @@ -69,13 +65,13 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map
// EndUserOverrides returns a dig.Mapping that can be applied on top of a k0s configuration.
// This patch is assembled based on the EndUserK0sConfigOverrides field.
func (j JoinCommandResponse) EndUserOverrides() (dig.Mapping, error) {
return j.extractK0sConfigOverridePatch([]byte(j.EndUserK0sConfigOverrides))
return j.extractK0sConfigOverridePatch([]byte(j.InstallationSpec.EndUserK0sConfigOverrides))
}

// EmbeddedOverrides returns a dig.Mapping that can be applied on top of a k0s configuration.
// This patch is assembled based on the K0sUnsupportedOverrides field.
func (j JoinCommandResponse) EmbeddedOverrides() (dig.Mapping, error) {
return j.extractK0sConfigOverridePatch([]byte(j.K0sUnsupportedOverrides))
return j.extractK0sConfigOverridePatch([]byte(j.InstallationSpec.Config.UnsupportedOverrides.K0s))
}

// getJoinToken issues a request to the kots api to get the actual join command
Expand Down Expand Up @@ -114,15 +110,15 @@ func startAndWaitForK0s(c *cli.Context, jcmd *JoinCommandResponse) error {
logrus.Debugf("starting %s service", binName)
if err := startK0sService(); err != nil {
err := fmt.Errorf("unable to start service: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

loading.Infof("Waiting for %s node to be ready", binName)
logrus.Debugf("waiting for k0s to be ready")
if err := waitForK0s(); err != nil {
err := fmt.Errorf("unable to wait for node: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

Expand Down Expand Up @@ -198,13 +194,18 @@ var joinCommand = &cli.Command{
return fmt.Errorf("unable to get join token: %w", err)
}

setProxyEnv(jcmd.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy, networkInterface)
// check to make sure the version returned by the join token is the same as the one we are running
if jcmd.EmbeddedClusterVersion != versions.Version {
return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion)
}

setProxyEnv(jcmd.InstallationSpec.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface)
if err != nil {
return fmt.Errorf("failed to check proxy config for local IP: %w", err)
}
if !proxyOK {
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP)
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.InstallationSpec.Proxy.NoProxy, localIP)
}

isAirgap := c.String("airgap-bundle") != ""
Expand All @@ -216,25 +217,25 @@ var joinCommand = &cli.Command{
}
}

metrics.ReportJoinStarted(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
metrics.ReportJoinStarted(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID)
logrus.Debugf("materializing %s binaries", binName)
if err := materializeFiles(c); err != nil {
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

applier, err := getAddonsApplier(c, "", jcmd.Proxy)
applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy)
if err != nil {
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

// jcmd.MetricsBaseURL is the replicated.app endpoint url
replicatedAPIURL := jcmd.MetricsBaseURL
// jcmd.InstallationSpec.MetricsBaseURL is the replicated.app endpoint url
replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress)
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.Proxy); err != nil {
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil {
err := fmt.Errorf("unable to run host preflights locally: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

Expand All @@ -246,54 +247,54 @@ var joinCommand = &cli.Command{
logrus.Debugf("saving token to disk")
if err := saveTokenToDisk(jcmd.K0sToken); err != nil {
err := fmt.Errorf("unable to save token to disk: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("installing %s binaries", binName)
if err := installK0sBinary(); err != nil {
err := fmt.Errorf("unable to install k0s binary: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

if jcmd.AirgapRegistryAddress != "" {
if err := airgap.AddInsecureRegistry(jcmd.AirgapRegistryAddress); err != nil {
err := fmt.Errorf("unable to add insecure registry: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
}

logrus.Debugf("creating systemd unit files")
localArtifactMirrorPort := defaults.LocalArtifactMirrorPort
if jcmd.LocalArtifactMirrorPort > 0 {
localArtifactMirrorPort = jcmd.LocalArtifactMirrorPort
if jcmd.InstallationSpec.LocalArtifactMirror != nil && jcmd.InstallationSpec.LocalArtifactMirror.Port > 0 {
localArtifactMirrorPort = jcmd.InstallationSpec.LocalArtifactMirror.Port
}
// both controller and worker nodes will have 'worker' in the join command
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy, localArtifactMirrorPort); err != nil {
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.InstallationSpec.Proxy, localArtifactMirrorPort); err != nil {
err := fmt.Errorf("unable to create systemd unit files: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("overriding network configuration")
if err := applyNetworkConfiguration(jcmd, networkInterface); err != nil {
err := fmt.Errorf("unable to apply network configuration: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
}

logrus.Debugf("applying configuration overrides")
if err := applyJoinConfigurationOverrides(jcmd); err != nil {
err := fmt.Errorf("unable to apply configuration overrides: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("joining node to cluster")
if err := runK0sInstallCommand(jcmd.K0sJoinCommand, networkInterface); err != nil {
err := fmt.Errorf("unable to join node to cluster: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

Expand All @@ -302,45 +303,45 @@ var joinCommand = &cli.Command{
}

if !strings.Contains(jcmd.K0sJoinCommand, "controller") {
metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
metrics.ReportJoinSucceeded(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID)
logrus.Debugf("worker node join finished")
return nil
}

kcli, err := kubeutils.KubeClient()
if err != nil {
err := fmt.Errorf("unable to get kube client: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
hostname, err := os.Hostname()
if err != nil {
err := fmt.Errorf("unable to get hostname: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
if err := waitForNode(c.Context, kcli, hostname); err != nil {
err := fmt.Errorf("unable to wait for node: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

if c.Bool("enable-ha") {
if err := maybeEnableHA(c.Context, kcli); err != nil {
err := fmt.Errorf("unable to enable high availability: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
}

metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
metrics.ReportJoinSucceeded(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID)
logrus.Debugf("controller node join finished")
return nil
},
}

func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface string) error {
if jcmd.Network != nil {
if jcmd.InstallationSpec.Network != nil {
clusterSpec := config.RenderK0sConfig()
address, err := netutils.FirstValidAddress(networkInterface)
if err != nil {
Expand All @@ -350,13 +351,13 @@ func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface strin
clusterSpec.Spec.Storage.Etcd.PeerAddress = address
// NOTE: we should be copying everything from the in cluster config spec and overriding
// the node specific config from clusterSpec.GetClusterWideConfig()
clusterSpec.Spec.Network.PodCIDR = jcmd.Network.PodCIDR
clusterSpec.Spec.Network.ServiceCIDR = jcmd.Network.ServiceCIDR
if jcmd.Network.NodePortRange != "" {
clusterSpec.Spec.Network.PodCIDR = jcmd.InstallationSpec.Network.PodCIDR
clusterSpec.Spec.Network.ServiceCIDR = jcmd.InstallationSpec.Network.ServiceCIDR
if jcmd.InstallationSpec.Network.NodePortRange != "" {
if clusterSpec.Spec.API.ExtraArgs == nil {
clusterSpec.Spec.API.ExtraArgs = map[string]string{}
}
clusterSpec.Spec.API.ExtraArgs["service-node-port-range"] = jcmd.Network.NodePortRange
clusterSpec.Spec.API.ExtraArgs["service-node-port-range"] = jcmd.InstallationSpec.Network.NodePortRange
}
clusterSpecYaml, err := k8syaml.Marshal(clusterSpec)

Expand Down
11 changes: 9 additions & 2 deletions cmd/embedded-cluster/join_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/k0sproject/dig"
k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -101,8 +102,14 @@ func TestJoinCommandResponseOverrides(t *testing.T) {
t.Run(tname, func(t *testing.T) {
req := require.New(t)
join := JoinCommandResponse{
K0sUnsupportedOverrides: tt.EmbeddedOverrides,
EndUserK0sConfigOverrides: tt.EndUserOverrides,
InstallationSpec: ecv1beta1.InstallationSpec{
Config: &ecv1beta1.ConfigSpec{
UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{
K0s: tt.EmbeddedOverrides,
},
},
EndUserK0sConfigOverrides: tt.EndUserOverrides,
},
}

embedded, err := join.EmbeddedOverrides()
Expand Down
18 changes: 12 additions & 6 deletions cmd/embedded-cluster/preflights.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/replicatedhq/embedded-cluster/pkg/defaults"
"github.com/replicatedhq/embedded-cluster/pkg/netutils"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
Expand Down Expand Up @@ -126,13 +127,18 @@ var joinRunPreflightsCommand = &cli.Command{
return fmt.Errorf("unable to get join token: %w", err)
}

setProxyEnv(jcmd.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy, networkInterface)
// check to make sure the version returned by the join token is the same as the one we are running
if jcmd.EmbeddedClusterVersion != versions.Version {
return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion)
}

setProxyEnv(jcmd.InstallationSpec.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface)
if err != nil {
return fmt.Errorf("failed to check proxy config for local IP: %w", err)
}
if !proxyOK {
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP)
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.InstallationSpec.Proxy.NoProxy, localIP)
}

isAirgap := c.String("airgap-bundle") != ""
Expand All @@ -142,15 +148,15 @@ var joinRunPreflightsCommand = &cli.Command{
return err
}

applier, err := getAddonsApplier(c, "", jcmd.Proxy)
applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy)
if err != nil {
return err
}

logrus.Debugf("running host preflights")
replicatedAPIURL := jcmd.MetricsBaseURL
replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress)
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.Proxy); err != nil {
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil {
err := fmt.Errorf("unable to run host preflights locally: %w", err)
return err
}
Expand Down
18 changes: 9 additions & 9 deletions pkg/addons/adminconsole/static/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,26 @@
# $ make buildtools
# $ output/bin/buildtools update addon <addon name>
#
version: 1.117.2
version: 1.117.3
location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console
images:
kotsadm:
repo: proxy.replicated.com/anonymous/kotsadm/kotsadm
tag:
amd64: v1.117.2-amd64@sha256:eb36a5a9af101164598f1e2f7f0745aad59bdb8d8870e09e2702d8c75c753380
arm64: v1.117.2-arm64@sha256:a60cf01f935a785b1a5e693d65a18bd6f4e8d7aac7be30a3311c4c17f3b8222f
amd64: v1.117.3-amd64@sha256:d47ac4df627ac357452efffb717776adb452c3ab9b466ef3ccaf808df722b7a6
arm64: v1.117.3-arm64@sha256:6b1b7bfc42fb3c7f21fe077b4645194fbdd1d497f92f4b12c00ceadb40969b8e
kotsadm-migrations:
repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations
tag:
amd64: v1.117.2-amd64@sha256:afb070977dfc16d3b85d2ae9e3cf68fc317e17e3ab05e6ad08d353201f1682ef
arm64: v1.117.2-arm64@sha256:cff8c43da1094d7969ef689eca3fd11c17aae936f45d1584fd00e25085a6112b
amd64: v1.117.3-amd64@sha256:56d2765497a57c06ef6e9f7705cf5218d21a978d197575a3c22fe7d84db07f0a
arm64: v1.117.3-arm64@sha256:25ff2b4697425be55d576f5f068480a3ab8fe6216341f2ec3b3c1962d3ac08ba
kurl-proxy:
repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy
tag:
amd64: v1.117.2-amd64@sha256:b2e4a7ee3ca4cedd1028fbac0678fcab091b1e0a81fd39d6b8d5d20f509b2304
arm64: v1.117.2-arm64@sha256:056b640c03ff824553adb2fc95fb4930e4cdea18c89b656ed81c1937d8ab3c73
amd64: v1.117.3-amd64@sha256:816bcbc273ec51255d7b459e49831ce09fd361db4a295d31f61d7af02177860f
arm64: v1.117.3-arm64@sha256:c4f5632ca9ea3fae2208b4ca7923900c6e958a0906b4a9dd4f6b5ebdab4b9d89
rqlite:
repo: proxy.replicated.com/anonymous/kotsadm/rqlite
tag:
amd64: 8.30.3-r0-amd64@sha256:2f0c6a90c462b8be8f08c5a1a83f71932be96b846beb3dfbd9fa0e3d13c66d63
arm64: 8.30.3-r0-arm64@sha256:4e96628236b7a1e9549bfdb16d44ff71ee7607b2450548b7e0f86f1f48b36c4e
amd64: 8.30.4-r0-amd64@sha256:884ac56b236e059e420858c94d90a083fe48b666c8b3433da612b9380906ce41
arm64: 8.30.4-r0-arm64@sha256:cfb55508de1fb59cfc5d14586f52d8a09b776e766ea612351df77d19ae2e6d9a

0 comments on commit 46bd74d

Please sign in to comment.