diff --git a/pkg/embeddedcluster/node_join.go b/pkg/embeddedcluster/node_join.go index 62ef585b40..725d457567 100644 --- a/pkg/embeddedcluster/node_join.go +++ b/pkg/embeddedcluster/node_join.go @@ -2,15 +2,15 @@ package embeddedcluster import ( "context" + "encoding/base64" "fmt" "strings" "sync" "time" "github.com/replicatedhq/kots/pkg/embeddedcluster/types" + "github.com/replicatedhq/kots/pkg/k8sutil" "github.com/replicatedhq/kots/pkg/util" - corev1 "k8s.io/api/core/v1" - kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -24,6 +24,25 @@ type joinTokenEntry struct { var joinTokenMapMut = sync.Mutex{} var joinTokenMap = map[string]*joinTokenEntry{} +const k0sTokenTemplate = `apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: %s + server: https://%s:%d + name: k0s +contexts: +- context: + cluster: k0s + user: %s + name: k0s +current-context: k0s +kind: Config +users: +- name: %s + user: + token: %s +` + // GenerateAddNodeToken will generate the embedded cluster node add command for a node with the specified roles // join commands will last for 24 hours, and will be cached for 1 hour after first generation func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) { @@ -44,9 +63,9 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node return joinToken.Token, nil } - newToken, err := runAddNodeCommandPod(ctx, client, nodeRole) + newToken, err := makeK0sToken(ctx, client, nodeRole) if err != nil { - return "", fmt.Errorf("failed to run add node command pod: %w", err) + return "", fmt.Errorf("failed to generate k0s token: %w", err) } now := time.Now() @@ -56,161 +75,60 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node return newToken, nil } -// run a pod that will generate the add node token -func runAddNodeCommandPod(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) { - podName := "k0s-token-generator-" - suffix := strings.Replace(nodeRole, "+", "-", -1) - podName += suffix - - // cleanup the pod if it already exists - err := client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{}) +func makeK0sToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) { + rawToken, err := k8sutil.GenerateK0sBootstrapToken(client, time.Hour, nodeRole) if err != nil { - if !kuberneteserrors.IsNotFound(err) { - return "", fmt.Errorf("failed to delete pod: %w", err) - } + return "", fmt.Errorf("failed to generate bootstrap token: %w", err) } - hostPathFile := corev1.HostPathFile - hostPathDir := corev1.HostPathDirectory - _, err = client.CoreV1().Pods("kube-system").Create(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: "kube-system", - Labels: map[string]string{ - "replicated.app/embedded-cluster": "true", - }, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - HostNetwork: true, - Volumes: []corev1.Volume{ - { - Name: "bin", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/usr/local/bin/k0s", - Type: &hostPathFile, - }, - }, - }, - { - Name: "lib", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/k0s", - Type: &hostPathDir, - }, - }, - }, - { - Name: "etc", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/k0s", - Type: &hostPathDir, - }, - }, - }, - { - Name: "run", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/k0s", - Type: &hostPathDir, - }, - }, - }, - }, - Affinity: &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "node.k0sproject.io/role", - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - "control-plane", - }, - }, - }, - }, - }, - }, - }, - }, - Containers: []corev1.Container{ - { - Name: "k0s-token-generator", - Image: "ubuntu:jammy", // this will not work on airgap, but it needs to be debian based at the moment - Command: []string{"/mnt/k0s"}, - Args: []string{ - "token", - "create", - "--expiry", - "12h", - "--role", - nodeRole, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "bin", - MountPath: "/mnt/k0s", - }, - { - Name: "lib", - MountPath: "/var/lib/k0s", - }, - { - Name: "etc", - MountPath: "/etc/k0s", - }, - { - Name: "run", - MountPath: "/run/k0s", - }, - }, - }, - }, - }, - }, metav1.CreateOptions{}) + cert, err := k8sutil.GetClusterCaCert(ctx, client) if err != nil { - return "", fmt.Errorf("failed to create pod: %w", err) + return "", fmt.Errorf("failed to get cluster ca cert: %w", err) } + cert = base64.StdEncoding.EncodeToString([]byte(cert)) - // wait for the pod to complete - for { - pod, err := client.CoreV1().Pods("kube-system").Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - return "", fmt.Errorf("failed to get pod: %w", err) - } - - if pod.Status.Phase == corev1.PodSucceeded { - break - } - - if pod.Status.Phase == corev1.PodFailed { - return "", fmt.Errorf("pod failed") - } + firstPrimary, err := firstPrimaryIpAddress(ctx, client) + if err != nil { + return "", fmt.Errorf("failed to get first primary ip address: %w", err) + } - time.Sleep(time.Second) + userName := "kubelet-bootstrap" + port := 6443 + if nodeRole == "controller" { + userName = "controller-bootstrap" + port = 9443 } - // get the logs from the completed pod - podLogs, err := client.CoreV1().Pods("kube-system").GetLogs(podName, &corev1.PodLogOptions{}).DoRaw(ctx) + fullToken := fmt.Sprintf(k0sTokenTemplate, cert, firstPrimary, port, userName, userName, rawToken) + gzipToken, err := util.GzipData([]byte(fullToken)) if err != nil { - return "", fmt.Errorf("failed to get pod logs: %w", err) + return "", fmt.Errorf("failed to gzip token: %w", err) } + b64Token := base64.StdEncoding.EncodeToString(gzipToken) + + return b64Token, nil +} - // delete the completed pod - err = client.CoreV1().Pods("kube-system").Delete(ctx, podName, metav1.DeleteOptions{}) +func firstPrimaryIpAddress(ctx context.Context, client kubernetes.Interface) (string, error) { + nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - return "", fmt.Errorf("failed to delete pod: %w", err) + return "", fmt.Errorf("failed to list nodes: %w", err) + } + + for _, node := range nodes.Items { + if cp, ok := node.Labels["node-role.kubernetes.io/control-plane"]; !ok || cp != "true" { + continue + } + + for _, address := range node.Status.Addresses { + if address.Type == "InternalIP" { + return address.Address, nil + } + } + } - // the logs are just a join token, which needs to be added to other things to get a join command - return string(podLogs), nil + return "", fmt.Errorf("failed to find controller node") } // GenerateAddNodeCommand returns the command a user should run to add a node with the provided token diff --git a/pkg/embeddedcluster/util.go b/pkg/embeddedcluster/util.go index 7119541e5c..ee02903682 100644 --- a/pkg/embeddedcluster/util.go +++ b/pkg/embeddedcluster/util.go @@ -86,7 +86,7 @@ func GetCurrentInstallation(ctx context.Context) (*embeddedclusterv1beta1.Instal } scheme := runtime.NewScheme() embeddedclusterv1beta1.AddToScheme(scheme) - kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme}) + kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme, WarningHandler: kbclient.WarningHandlerOptions{SuppressWarnings: true}}) if err != nil { return nil, fmt.Errorf("failed to get kubebuilder client: %w", err) } diff --git a/pkg/k8sutil/cluster.go b/pkg/k8sutil/cluster.go index c8b2c54460..b46af29f3d 100644 --- a/pkg/k8sutil/cluster.go +++ b/pkg/k8sutil/cluster.go @@ -16,6 +16,35 @@ import ( // GenerateBootstrapToken will generate a node join token for kubeadm. // ttl defines the time to live for this token. func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (string, error) { + data := map[string][]byte{} + data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte("Token auto generated by Kotsadm.") + for _, usage := range []string{"authentication", "signing"} { + data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") + } + data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token") + + return generateJoinTokenInternal(client, ttl, data) +} + +func GenerateK0sBootstrapToken(client kubernetes.Interface, ttl time.Duration, role string) (string, error) { + data := make(map[string][]byte) + + // these 'data' entries are taken from k0s: https://github.com/replicatedhq/k0s/blob/7bc57553ea8ccb6847fdd8249701554ee8be1ab0/pkg/token/manager.go#L69 + data["usage-bootstrap-api-auth"] = []byte("true") + if role == "worker" { + data["description"] = []byte("Worker bootstrap token generated by Kotsadm for k0s") + data["usage-bootstrap-authentication"] = []byte("true") + data["usage-bootstrap-api-worker-calls"] = []byte("true") + } else { + data["description"] = []byte("Controller bootstrap token generated by Kotsadm for k0s") + data["usage-bootstrap-authentication"] = []byte("false") + data["usage-bootstrap-signing"] = []byte("false") + data["usage-controller-join"] = []byte("true") + } + return generateJoinTokenInternal(client, ttl, data) +} + +func generateJoinTokenInternal(client kubernetes.Interface, ttl time.Duration, data map[string][]byte) (string, error) { token, err := bootstraputil.GenerateBootstrapToken() if err != nil { return "", errors.Wrap(err, "generate kubeadm token") @@ -24,21 +53,12 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str tokenID := substrs[1] tokenSecret := substrs[2] - data := map[string][]byte{ - bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), - bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), - } - data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte("Token auto generated by Kotsadm.") + data[bootstrapapi.BootstrapTokenIDKey] = []byte(tokenID) + data[bootstrapapi.BootstrapTokenSecretKey] = []byte(tokenSecret) expirationString := time.Now().Add(ttl).UTC().Format(time.RFC3339) data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) - for _, usage := range []string{"authentication", "signing"} { - data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") - } - - data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token") - secretName := fmt.Sprintf("%s%s", bootstrapapi.BootstrapTokenSecretPrefix, tokenID) bootstrapToken := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -55,3 +75,17 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str return token, nil } + +func GetClusterCaCert(ctx context.Context, client kubernetes.Interface) (string, error) { + cert, err := client.CoreV1().ConfigMaps("kube-system").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to get kube-root-ca.crt") + } + + caCert, ok := cert.Data["ca.crt"] + if !ok { + return "", fmt.Errorf("ca.crt not found in kube-root-ca.crt, actual data was %v", cert.Data) + } + + return caCert, nil +}