node join commands should be generated properly

pkg/embeddedcluster/node_join.go

@@ -2,15 +2,15 @@ package embeddedcluster
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/replicatedhq/kots/pkg/embeddedcluster/types"
+	"github.com/replicatedhq/kots/pkg/k8sutil"
 	"github.com/replicatedhq/kots/pkg/util"
-	corev1 "k8s.io/api/core/v1"
-	kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -24,6 +24,25 @@ type joinTokenEntry struct {
 var joinTokenMapMut = sync.Mutex{}
 var joinTokenMap = map[string]*joinTokenEntry{}
 
+const k0sTokenTemplate = `apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: %s
+    server: https://%s:%d
+  name: k0s
+contexts:
+- context:
+    cluster: k0s
+    user: %s
+  name: k0s
+current-context: k0s
+kind: Config
+users:
+- name: %s
+  user:
+    token: %s
+`
+
 // GenerateAddNodeToken will generate the embedded cluster node add command for a node with the specified roles
 // join commands will last for 24 hours, and will be cached for 1 hour after first generation
 func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) {
@@ -44,9 +63,9 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node
 		return joinToken.Token, nil
 	}
 
-	newToken, err := runAddNodeCommandPod(ctx, client, nodeRole)
+	newToken, err := makeK0sToken(ctx, client, nodeRole)
 	if err != nil {
-		return "", fmt.Errorf("failed to run add node command pod: %w", err)
+		return "", fmt.Errorf("failed to generate k0s token: %w", err)
 	}
 
 	now := time.Now()
@@ -56,161 +75,60 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node
 	return newToken, nil
 }
 
-// run a pod that will generate the add node token
-func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) {
-	podName := "k0s-token-generator-"
-	suffix := strings.Replace(nodeRole, "+", "-", -1)
-	podName += suffix
-
-	// cleanup the pod if it already exists
-	err := client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{})
+func makeK0sToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) {
+	rawToken, err := k8sutil.GenerateK0sBootstrapToken(client, time.Hour, nodeRole)
 	if err != nil {
-		if !kuberneteserrors.IsNotFound(err) {
-			return "", fmt.Errorf("failed to delete pod: %w", err)
-		}
+		return "", fmt.Errorf("failed to generate bootstrap token: %w", err)
 	}
 
-	hostPathFile := corev1.HostPathFile
-	hostPathDir := corev1.HostPathDirectory
-	_, err = client.CoreV1().Pods("kube-system").Create(ctx, &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      podName,
-			Namespace: "kube-system",
-			Labels: map[string]string{
-				"replicated.app/embedded-cluster": "true",
-			},
-		},
-		Spec: corev1.PodSpec{
-			RestartPolicy: corev1.RestartPolicyOnFailure,
-			HostNetwork:   true,
-			Volumes: []corev1.Volume{
-				{
-					Name: "bin",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/usr/local/bin/k0s",
-							Type: &hostPathFile,
-						},
-					},
-				},
-				{
-					Name: "lib",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/var/lib/k0s",
-							Type: &hostPathDir,
-						},
-					},
-				},
-				{
-					Name: "etc",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/etc/k0s",
-							Type: &hostPathDir,
-						},
-					},
-				},
-				{
-					Name: "run",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/run/k0s",
-							Type: &hostPathDir,
-						},
-					},
-				},
-			},
-			Affinity: &corev1.Affinity{
-				NodeAffinity: &corev1.NodeAffinity{
-					RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
-						NodeSelectorTerms: []corev1.NodeSelectorTerm{
-							{
-								MatchExpressions: []corev1.NodeSelectorRequirement{
-									{
-										Key:      "node.k0sproject.io/role",
-										Operator: corev1.NodeSelectorOpIn,
-										Values: []string{
-											"control-plane",
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			Containers: []corev1.Container{
-				{
-					Name:    "k0s-token-generator",
-					Image:   "ubuntu:jammy", // this will not work on airgap, but it needs to be debian based at the moment
-					Command: []string{"/mnt/k0s"},
-					Args: []string{
-						"token",
-						"create",
-						"--expiry",
-						"12h",
-						"--role",
-						nodeRole,
-					},
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							Name:      "bin",
-							MountPath: "/mnt/k0s",
-						},
-						{
-							Name:      "lib",
-							MountPath: "/var/lib/k0s",
-						},
-						{
-							Name:      "etc",
-							MountPath: "/etc/k0s",
-						},
-						{
-							Name:      "run",
-							MountPath: "/run/k0s",
-						},
-					},
-				},
-			},
-		},
-	}, metav1.CreateOptions{})
+	cert, err := k8sutil.GetClusterCaCert(ctx, client)
 	if err != nil {
-		return "", fmt.Errorf("failed to create pod: %w", err)
+		return "", fmt.Errorf("failed to get cluster ca cert: %w", err)
 	}
+	cert = base64.StdEncoding.EncodeToString([]byte(cert))
 
-	// wait for the pod to complete
-	for {
-		pod, err := client.CoreV1().Pods("kube-system").Get(ctx, podName, metav1.GetOptions{})
-		if err != nil {
-			return "", fmt.Errorf("failed to get pod: %w", err)
-		}
-
-		if pod.Status.Phase == corev1.PodSucceeded {
-			break
-		}
-
-		if pod.Status.Phase == corev1.PodFailed {
-			return "", fmt.Errorf("pod failed")
-		}
+	firstPrimary, err := firstPrimaryIpAddress(ctx, client)
+	if err != nil {
+		return "", fmt.Errorf("failed to get first primary ip address: %w", err)
+	}
 
-		time.Sleep(time.Second)
+	userName := "kubelet-bootstrap"
+	port := 6443
+	if nodeRole == "controller" {
+		userName = "controller-bootstrap"
+		port = 9443
 	}
 
-	// get the logs from the completed pod
-	podLogs, err := client.CoreV1().Pods("kube-system").GetLogs(podName, &corev1.PodLogOptions{}).DoRaw(ctx)
+	fullToken := fmt.Sprintf(k0sTokenTemplate, cert, firstPrimary, port, userName, userName, rawToken)
+	gzipToken, err := util.GzipData([]byte(fullToken))
 	if err != nil {
-		return "", fmt.Errorf("failed to get pod logs: %w", err)
+		return "", fmt.Errorf("failed to gzip token: %w", err)
 	}
+	b64Token := base64.StdEncoding.EncodeToString(gzipToken)
+
+	return b64Token, nil
+}
 
-	// delete the completed pod
-	err = client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{})
+func firstPrimaryIpAddress(ctx context.Context, client kubernetes.Interface) (string, error) {
+	nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
 	if err != nil {
-		return "", fmt.Errorf("failed to delete pod: %w", err)
+		return "", fmt.Errorf("failed to list nodes: %w", err)
+	}
+
+	for _, node := range nodes.Items {
+		if cp, ok := node.Labels["node-role.kubernetes.io/control-plane"]; !ok || cp != "true" {
+			continue
+		}
+
+		for _, address := range node.Status.Addresses {
+			if address.Type == "InternalIP" {
+				return address.Address, nil
+			}
+		}
+
 	}
 
-	// the logs are just a join token, which needs to be added to other things to get a join command
-	return string(podLogs), nil
+	return "", fmt.Errorf("failed to find controller node")
 }
 
 // GenerateAddNodeCommand returns the command a user should run to add a node with the provided token

pkg/embeddedcluster/util.go

@@ -86,7 +86,7 @@ func GetCurrentInstallation(ctx context.Context) (*embeddedclusterv1beta1.Instal
 	}
 	scheme := runtime.NewScheme()
 	embeddedclusterv1beta1.AddToScheme(scheme)
-	kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme})
+	kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme, WarningHandler: kbclient.WarningHandlerOptions{SuppressWarnings: true}})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get kubebuilder client: %w", err)
 	}

pkg/k8sutil/cluster.go

@@ -16,6 +16,35 @@ import (
 // GenerateBootstrapToken will generate a node join token for kubeadm.
 // ttl defines the time to live for this token.
 func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (string, error) {
+	data := map[string][]byte{}
+	data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte("Token auto generated by Kotsadm.")
+	for _, usage := range []string{"authentication", "signing"} {
+		data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
+	}
+	data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token")
+
+	return generateJoinTokenInternal(client, ttl, data)
+}
+
+func GenerateK0sBootstrapToken(client kubernetes.Interface, ttl time.Duration, role string) (string, error) {
+	data := make(map[string][]byte)
+
+	// these 'data' entries are taken from k0s: https://github.com/replicatedhq/k0s/blob/7bc57553ea8ccb6847fdd8249701554ee8be1ab0/pkg/token/manager.go#L69
+	data["usage-bootstrap-api-auth"] = []byte("true")
+	if role == "worker" {
+		data["description"] = []byte("Worker bootstrap token generated by Kotsadm for k0s")
+		data["usage-bootstrap-authentication"] = []byte("true")
+		data["usage-bootstrap-api-worker-calls"] = []byte("true")
+	} else {
+		data["description"] = []byte("Controller bootstrap token generated by Kotsadm for k0s")
+		data["usage-bootstrap-authentication"] = []byte("false")
+		data["usage-bootstrap-signing"] = []byte("false")
+		data["usage-controller-join"] = []byte("true")
+	}
+	return generateJoinTokenInternal(client, ttl, data)
+}
+
+func generateJoinTokenInternal(client kubernetes.Interface, ttl time.Duration, data map[string][]byte) (string, error) {
 	token, err := bootstraputil.GenerateBootstrapToken()
 	if err != nil {
 		return "", errors.Wrap(err, "generate kubeadm token")
@@ -24,21 +53,12 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str
 	tokenID := substrs[1]
 	tokenSecret := substrs[2]
 
-	data := map[string][]byte{
-		bootstrapapi.BootstrapTokenIDKey:     []byte(tokenID),
-		bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret),
-	}
-	data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte("Token auto generated by Kotsadm.")
+	data[bootstrapapi.BootstrapTokenIDKey] = []byte(tokenID)
+	data[bootstrapapi.BootstrapTokenSecretKey] = []byte(tokenSecret)
 
 	expirationString := time.Now().Add(ttl).UTC().Format(time.RFC3339)
 	data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString)
 
-	for _, usage := range []string{"authentication", "signing"} {
-		data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
-	}
-
-	data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token")
-
 	secretName := fmt.Sprintf("%s%s", bootstrapapi.BootstrapTokenSecretPrefix, tokenID)
 	bootstrapToken := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -55,3 +75,17 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str
 
 	return token, nil
 }
+
+func GetClusterCaCert(ctx context.Context, client kubernetes.Interface) (string, error) {
+	cert, err := client.CoreV1().ConfigMaps("kube-system").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{})
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get kube-root-ca.crt")
+	}
+
+	caCert, ok := cert.Data["ca.crt"]
+	if !ok {
+		return "", fmt.Errorf("ca.crt not found in kube-root-ca.crt, actual data was %v", cert.Data)
+	}
+
+	return caCert, nil
+}