diff --git a/go.mod b/go.mod index 39349bfa5f..bf2ef57a09 100644 --- a/go.mod +++ b/go.mod @@ -264,7 +264,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect - github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect + github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/microsoft/go-mssqldb v1.7.1 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect @@ -401,6 +401,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/c9s/goprocinfo v0.0.0-20190309065803-0b2ad9ac246b // indirect github.com/containers/storage v1.53.0 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/kopia/kopia v0.10.7 // indirect diff --git a/pkg/embeddedcluster/monitor.go b/pkg/embeddedcluster/monitor.go index 9291b26380..b5f71030f2 100644 --- a/pkg/embeddedcluster/monitor.go +++ b/pkg/embeddedcluster/monitor.go @@ -9,6 +9,7 @@ import ( embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" appstatetypes "github.com/replicatedhq/kots/pkg/appstate/types" + "github.com/replicatedhq/kots/pkg/k8sutil" "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/logger" "github.com/replicatedhq/kots/pkg/store" @@ -32,8 +33,13 @@ func MaybeStartClusterUpgrade(ctx context.Context, store store.Store, kotsKinds return nil } + kbClient, err := k8sutil.GetKubeClient(ctx) + if err != nil { + return fmt.Errorf("failed to get kubeclient: %w", err) + } + spec := kotsKinds.EmbeddedClusterConfig.Spec - if upgrade, err := RequiresUpgrade(ctx, spec); err != nil { + if upgrade, err := RequiresUpgrade(ctx, kbClient, spec); err != nil { // if there is no installation object we can't start an upgrade. this is a valid // scenario specially during cluster bootstrap. as we do not need to upgrade the // cluster just after its installation we can return nil here. @@ -123,7 +129,11 @@ func watchClusterState(ctx context.Context, store store.Store) { // by reading the latest embedded cluster installation CRD. // If the lastState is the same as the current state, it will not update the database. func updateClusterState(ctx context.Context, store store.Store, lastState string) (string, error) { - installation, err := GetCurrentInstallation(ctx) + kbClient, err := k8sutil.GetKubeClient(ctx) + if err != nil { + return "", fmt.Errorf("failed to get kubeclient: %w", err) + } + installation, err := GetCurrentInstallation(ctx, kbClient) if err != nil { return "", fmt.Errorf("failed to get current installation: %w", err) } diff --git a/pkg/embeddedcluster/node_join.go b/pkg/embeddedcluster/node_join.go index 8d5a5062ae..e3e93b3dd3 100644 --- a/pkg/embeddedcluster/node_join.go +++ b/pkg/embeddedcluster/node_join.go @@ -11,9 +11,10 @@ import ( "github.com/replicatedhq/kots/pkg/embeddedcluster/types" "github.com/replicatedhq/kots/pkg/k8sutil" "github.com/replicatedhq/kots/pkg/util" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" + k8stypes "k8s.io/apimachinery/pkg/types" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) type joinTokenEntry struct { @@ -46,7 +47,7 @@ users: // GenerateAddNodeToken will generate the embedded cluster node add command for a node with the specified roles // join commands will last for 24 hours, and will be cached for 1 hour after first generation -func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) { +func GenerateAddNodeToken(ctx context.Context, client kbclient.Client, nodeRole string) (string, error) { // get the joinToken struct entry for this node role joinTokenMapMut.Lock() if _, ok := joinTokenMap[nodeRole]; !ok { @@ -76,7 +77,7 @@ func GenerateAddNodeToken(ctx context.Context, client kubernetes.Interface, node return newToken, nil } -func makeK0sToken(ctx context.Context, client kubernetes.Interface, nodeRole string) (string, error) { +func makeK0sToken(ctx context.Context, client kbclient.Client, nodeRole string) (string, error) { rawToken, err := k8sutil.GenerateK0sBootstrapToken(client, time.Hour, nodeRole) if err != nil { return "", fmt.Errorf("failed to generate bootstrap token: %w", err) @@ -110,9 +111,9 @@ func makeK0sToken(ctx context.Context, client kubernetes.Interface, nodeRole str return b64Token, nil } -func firstPrimaryIpAddress(ctx context.Context, client kubernetes.Interface) (string, error) { - nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { +func firstPrimaryIpAddress(ctx context.Context, client kbclient.Client) (string, error) { + var nodes corev1.NodeList + if err := client.List(ctx, &nodes); err != nil { return "", fmt.Errorf("failed to list nodes: %w", err) } @@ -126,7 +127,6 @@ func firstPrimaryIpAddress(ctx context.Context, client kubernetes.Interface) (st return address.Address, nil } } - } return "", fmt.Errorf("failed to find controller node") @@ -134,22 +134,22 @@ func firstPrimaryIpAddress(ctx context.Context, client kubernetes.Interface) (st // GenerateAddNodeCommand returns the command a user should run to add a node with the provided token // the command will be of the form 'embeddedcluster node join ip:port UUID' -func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, token string, isAirgap bool) (string, error) { - cm, err := ReadConfigMap(client) +func GenerateAddNodeCommand(ctx context.Context, kbClient kbclient.Client, token string, isAirgap bool) (string, error) { + installation, err := GetCurrentInstallation(ctx, kbClient) if err != nil { - return "", fmt.Errorf("failed to read configmap: %w", err) + return "", fmt.Errorf("failed to get current installation: %w", err) } - binaryName := cm.Data["embedded-binary-name"] + binaryName := installation.Spec.BinaryName // get the IP of a controller node - nodeIP, err := getControllerNodeIP(ctx, client) + nodeIP, err := getControllerNodeIP(ctx, kbClient) if err != nil { return "", fmt.Errorf("failed to get controller node IP: %w", err) } // get the port of the 'admin-console' service - port, err := getAdminConsolePort(ctx, client) + port, err := getAdminConsolePort(ctx, kbClient) if err != nil { return "", fmt.Errorf("failed to get admin console port: %w", err) } @@ -165,8 +165,8 @@ func GenerateAddNodeCommand(ctx context.Context, client kubernetes.Interface, to // GenerateK0sJoinCommand returns the k0s node join command, without the token but with all other required flags // (including node labels generated from the roles etc) -func GenerateK0sJoinCommand(ctx context.Context, client kubernetes.Interface, roles []string) (string, error) { - controllerRoleName, err := ControllerRoleName(ctx) +func GenerateK0sJoinCommand(ctx context.Context, kbClient kbclient.Client, roles []string) (string, error) { + controllerRoleName, err := ControllerRoleName(ctx, kbClient) if err != nil { return "", fmt.Errorf("failed to get controller role name: %w", err) } @@ -183,7 +183,7 @@ func GenerateK0sJoinCommand(ctx context.Context, client kubernetes.Interface, ro cmd = append(cmd, "--enable-worker", "--no-taints") } - labels, err := getRolesNodeLabels(ctx, roles) + labels, err := getRolesNodeLabels(ctx, kbClient, roles) if err != nil { return "", fmt.Errorf("failed to get role labels: %w", err) } @@ -193,11 +193,11 @@ func GenerateK0sJoinCommand(ctx context.Context, client kubernetes.Interface, ro } // gets the port of the 'admin-console' or 'kurl-proxy-kotsadm' service -func getAdminConsolePort(ctx context.Context, client kubernetes.Interface) (int32, error) { - kurlProxyPort, err := getAdminConsolePortImpl(ctx, client, "kurl-proxy-kotsadm") +func getAdminConsolePort(ctx context.Context, kbClient kbclient.Client) (int32, error) { + kurlProxyPort, err := getAdminConsolePortImpl(ctx, kbClient, "kurl-proxy-kotsadm") if err != nil { if errors.IsNotFound(err) { - adminConsolePort, err := getAdminConsolePortImpl(ctx, client, "admin-console") + adminConsolePort, err := getAdminConsolePortImpl(ctx, kbClient, "admin-console") if err != nil { return -1, fmt.Errorf("failed to get admin-console port: %w", err) } @@ -208,9 +208,9 @@ func getAdminConsolePort(ctx context.Context, client kubernetes.Interface) (int3 return kurlProxyPort, nil } -func getAdminConsolePortImpl(ctx context.Context, client kubernetes.Interface, svcName string) (int32, error) { - svc, err := client.CoreV1().Services(util.PodNamespace).Get(ctx, svcName, metav1.GetOptions{}) - if err != nil { +func getAdminConsolePortImpl(ctx context.Context, kbClient kbclient.Client, svcName string) (int32, error) { + var svc corev1.Service + if err := kbClient.Get(ctx, k8stypes.NamespacedName{Name: svcName, Namespace: util.PodNamespace}, &svc); err != nil { return -1, fmt.Errorf("failed to get %s service: %w", svcName, err) } @@ -227,9 +227,9 @@ func getAdminConsolePortImpl(ctx context.Context, client kubernetes.Interface, s } // getControllerNodeIP gets the IP of a healthy controller node -func getControllerNodeIP(ctx context.Context, client kubernetes.Interface) (string, error) { - nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { +func getControllerNodeIP(ctx context.Context, kbClient kbclient.Client) (string, error) { + var nodes corev1.NodeList + if err := kbClient.List(ctx, &nodes); err != nil { return "", fmt.Errorf("failed to list nodes: %w", err) } @@ -247,16 +247,15 @@ func getControllerNodeIP(ctx context.Context, client kubernetes.Interface) (stri } } } - } return "", fmt.Errorf("failed to find healthy controller node") } -func getRolesNodeLabels(ctx context.Context, roles []string) (string, error) { +func getRolesNodeLabels(ctx context.Context, kbClient kbclient.Client, roles []string) (string, error) { roleListLabels := getRoleListLabels(roles) - labels, err := getRoleNodeLabels(ctx, roles) + labels, err := getRoleNodeLabels(ctx, kbClient, roles) if err != nil { return "", fmt.Errorf("failed to get node labels for roles %v: %w", roles, err) } diff --git a/pkg/embeddedcluster/node_join_test.go b/pkg/embeddedcluster/node_join_test.go index 85350722eb..33090bf193 100644 --- a/pkg/embeddedcluster/node_join_test.go +++ b/pkg/embeddedcluster/node_join_test.go @@ -3,12 +3,15 @@ package embeddedcluster import ( "context" "testing" + "time" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" "github.com/replicatedhq/kots/pkg/util" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestGenerateAddNodeCommand(t *testing.T) { @@ -17,15 +20,21 @@ func TestGenerateAddNodeCommand(t *testing.T) { util.PodNamespace = "" }() + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + embeddedclusterv1beta1.AddToScheme(scheme) + // Create a fake clientset - clientset := fake.NewSimpleClientset( - &corev1.ConfigMap{ + kbClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &embeddedclusterv1beta1.Installation{ ObjectMeta: metav1.ObjectMeta{ - Name: "embedded-cluster-config", - Namespace: "embedded-cluster", + Name: time.Now().Format("20060102150405"), + Labels: map[string]string{ + "replicated.com/disaster-recovery": "ec-install", + }, }, - Data: map[string]string{ - "embedded-binary-name": "my-app", + Spec: embeddedclusterv1beta1.InstallationSpec{ + BinaryName: "my-app", }, }, &corev1.Node{ @@ -66,12 +75,12 @@ func TestGenerateAddNodeCommand(t *testing.T) { }, }, }, - ) + ).Build() req := require.New(t) // Generate the add node command for online - gotCommand, err := GenerateAddNodeCommand(context.Background(), clientset, "token", false) + gotCommand, err := GenerateAddNodeCommand(context.Background(), kbClient, "token", false) if err != nil { t.Fatalf("Failed to generate add node command: %v", err) } @@ -81,7 +90,7 @@ func TestGenerateAddNodeCommand(t *testing.T) { req.Equal(wantCommand, gotCommand) // Generate the add node command for airgap - gotCommand, err = GenerateAddNodeCommand(context.Background(), clientset, "token", true) + gotCommand, err = GenerateAddNodeCommand(context.Background(), kbClient, "token", true) if err != nil { t.Fatalf("Failed to generate add node command: %v", err) } diff --git a/pkg/embeddedcluster/roles.go b/pkg/embeddedcluster/roles.go index 6dc0d80e86..83407cecc4 100644 --- a/pkg/embeddedcluster/roles.go +++ b/pkg/embeddedcluster/roles.go @@ -8,6 +8,7 @@ import ( "strings" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) const DEFAULT_CONTROLLER_ROLE_NAME = "controller" @@ -15,8 +16,8 @@ const DEFAULT_CONTROLLER_ROLE_NAME = "controller" var labelValueRegex = regexp.MustCompile(`[^a-zA-Z0-9-_.]+`) // GetRoles will get a list of role names -func GetRoles(ctx context.Context) ([]string, error) { - config, err := ClusterConfig(ctx) +func GetRoles(ctx context.Context, kbClient kbclient.Client) ([]string, error) { + config, err := ClusterConfig(ctx, kbClient) if err != nil { return nil, fmt.Errorf("failed to get cluster config: %w", err) } @@ -45,8 +46,8 @@ func GetRoles(ctx context.Context) ([]string, error) { // ControllerRoleName determines the name for the 'controller' role // this might be part of the config, or it might be the default -func ControllerRoleName(ctx context.Context) (string, error) { - conf, err := ClusterConfig(ctx) +func ControllerRoleName(ctx context.Context, kbClient kbclient.Client) (string, error) { + conf, err := ClusterConfig(ctx, kbClient) if err != nil { return "", fmt.Errorf("failed to get cluster config: %w", err) } @@ -89,8 +90,8 @@ func SortRoles(controllerRole string, inputRoles []string) []string { } // getRoleNodeLabels looks up roles in the cluster config and determines the additional labels to be applied from that -func getRoleNodeLabels(ctx context.Context, roles []string) ([]string, error) { - config, err := ClusterConfig(ctx) +func getRoleNodeLabels(ctx context.Context, kbClient kbclient.Client, roles []string) ([]string, error) { + config, err := ClusterConfig(ctx, kbClient) if err != nil { return nil, fmt.Errorf("failed to get cluster config: %w", err) } diff --git a/pkg/embeddedcluster/util.go b/pkg/embeddedcluster/util.go index 5a253304dd..068cf41951 100644 --- a/pkg/embeddedcluster/util.go +++ b/pkg/embeddedcluster/util.go @@ -11,40 +11,21 @@ import ( embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" "github.com/replicatedhq/kots/pkg/k8sutil" kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) -const configMapName = "embedded-cluster-config" -const configMapNamespace = "embedded-cluster" - // ErrNoInstallations is returned when no installation object is found in the cluster. var ErrNoInstallations = fmt.Errorf("no installations found") -// ReadConfigMap will read the Kurl config from a configmap -func ReadConfigMap(client kubernetes.Interface) (*corev1.ConfigMap, error) { - return client.CoreV1().ConfigMaps(configMapNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) -} - func IsHA(clientset kubernetes.Interface) (bool, error) { return true, nil } -func ClusterID(client kubernetes.Interface) (string, error) { - configMap, err := ReadConfigMap(client) - if err != nil { - return "", fmt.Errorf("failed to read configmap: %w", err) - } - - return configMap.Data["embedded-cluster-id"], nil -} - // RequiresUpgrade returns true if the provided configuration differs from the latest active configuration. -func RequiresUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSpec) (bool, error) { - curcfg, err := ClusterConfig(ctx) +func RequiresUpgrade(ctx context.Context, kbClient kbclient.Client, newcfg embeddedclusterv1beta1.ConfigSpec) (bool, error) { + curcfg, err := ClusterConfig(ctx, kbClient) if err != nil { return false, fmt.Errorf("failed to get current cluster config: %w", err) } @@ -60,8 +41,8 @@ func RequiresUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSp } // GetCurrentInstallation returns the most recent installation object from the cluster. -func GetCurrentInstallation(ctx context.Context) (*embeddedclusterv1beta1.Installation, error) { - installations, err := ListInstallations(ctx) +func GetCurrentInstallation(ctx context.Context, kbClient kbclient.Client) (*embeddedclusterv1beta1.Installation, error) { + installations, err := ListInstallations(ctx, kbClient) if err != nil { return nil, fmt.Errorf("failed to list installations: %w", err) } @@ -74,20 +55,9 @@ func GetCurrentInstallation(ctx context.Context) (*embeddedclusterv1beta1.Instal return &installations[0], nil } -func ListInstallations(ctx context.Context) ([]embeddedclusterv1beta1.Installation, error) { - clientConfig, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, fmt.Errorf("failed to get cluster config: %w", err) - } - scheme := runtime.NewScheme() - embeddedclusterv1beta1.AddToScheme(scheme) - kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme, WarningHandler: kbclient.WarningHandlerOptions{SuppressWarnings: true}}) - if err != nil { - return nil, fmt.Errorf("failed to get kubebuilder client: %w", err) - } +func ListInstallations(ctx context.Context, kbClient kbclient.Client) ([]embeddedclusterv1beta1.Installation, error) { var installationList embeddedclusterv1beta1.InstallationList - err = kbClient.List(ctx, &installationList, &kbclient.ListOptions{}) - if err != nil { + if err := kbClient.List(ctx, &installationList, &kbclient.ListOptions{}); err != nil { return nil, fmt.Errorf("failed to list installations: %w", err) } return installationList.Items, nil @@ -95,8 +65,8 @@ func ListInstallations(ctx context.Context) ([]embeddedclusterv1beta1.Installati // ClusterConfig will extract the current cluster configuration from the latest installation // object found in the cluster. -func ClusterConfig(ctx context.Context) (*embeddedclusterv1beta1.ConfigSpec, error) { - latest, err := GetCurrentInstallation(ctx) +func ClusterConfig(ctx context.Context, kbClient kbclient.Client) (*embeddedclusterv1beta1.ConfigSpec, error) { + latest, err := GetCurrentInstallation(ctx, kbClient) if err != nil { return nil, fmt.Errorf("failed to get current installation: %w", err) } @@ -118,17 +88,11 @@ func getArtifactsFromInstallation(installation kotsv1beta1.Installation, appSlug // startClusterUpgrade will create a new installation with the provided config. func startClusterUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.ConfigSpec, artifacts *embeddedclusterv1beta1.ArtifactsLocation, license kotsv1beta1.License) error { - clientConfig, err := k8sutil.GetClusterConfig() - if err != nil { - return fmt.Errorf("failed to get cluster config: %w", err) - } - scheme := runtime.NewScheme() - embeddedclusterv1beta1.AddToScheme(scheme) - kbClient, err := kbclient.New(clientConfig, kbclient.Options{Scheme: scheme}) + kbClient, err := k8sutil.GetKubeClient(ctx) if err != nil { - return fmt.Errorf("failed to get kubebuilder client: %w", err) + return fmt.Errorf("failed to get kubeclient: %w", err) } - current, err := GetCurrentInstallation(ctx) + current, err := GetCurrentInstallation(ctx, kbClient) if err != nil { return fmt.Errorf("failed to get current installation: %w", err) } diff --git a/pkg/handlers/app.go b/pkg/handlers/app.go index bf06e7d3eb..891a4a8490 100644 --- a/pkg/handlers/app.go +++ b/pkg/handlers/app.go @@ -290,23 +290,26 @@ func responseAppFromApp(a *apptypes.App) (*types.ResponseApp, error) { } if embeddedClusterConfig != nil { - cluster.RequiresUpgrade, err = embeddedcluster.RequiresUpgrade(context.TODO(), embeddedClusterConfig.Spec) + kbClient, err := k8sutil.GetKubeClient(context.TODO()) + if err != nil { + return nil, fmt.Errorf("failed to get kubeclient: %w", err) + } + + cluster.RequiresUpgrade, err = embeddedcluster.RequiresUpgrade(context.TODO(), kbClient, embeddedClusterConfig.Spec) if err != nil { return nil, errors.Wrap(err, "failed to check if cluster requires upgrade") } - embeddedClusterInstallations, err := embeddedcluster.ListInstallations(context.TODO()) + embeddedClusterInstallations, err := embeddedcluster.ListInstallations(context.TODO(), kbClient) if err != nil { return nil, errors.Wrap(err, "failed to list installations") } - cluster.NumInstallations = len(embeddedClusterInstallations) - currentInstallation, err := embeddedcluster.GetCurrentInstallation(context.TODO()) + currentInstallation, err := embeddedcluster.GetCurrentInstallation(context.TODO(), kbClient) if err != nil { return nil, errors.Wrap(err, "failed to get latest installation") } - if currentInstallation != nil { cluster.State = string(currentInstallation.Status.State) } diff --git a/pkg/handlers/embedded_cluster_get.go b/pkg/handlers/embedded_cluster_get.go index f8725702e1..2cfc000fa2 100644 --- a/pkg/handlers/embedded_cluster_get.go +++ b/pkg/handlers/embedded_cluster_get.go @@ -68,7 +68,14 @@ func (h *Handler) GetEmbeddedClusterRoles(w http.ResponseWriter, r *http.Request return } - roles, err := embeddedcluster.GetRoles(r.Context()) + kbClient, err := k8sutil.GetKubeClient(r.Context()) + if err != nil { + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + roles, err := embeddedcluster.GetRoles(r.Context(), kbClient) if err != nil { logger.Error(err) w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/handlers/embedded_cluster_node_join_command.go b/pkg/handlers/embedded_cluster_node_join_command.go index 4355784d42..d27bcabb94 100644 --- a/pkg/handlers/embedded_cluster_node_join_command.go +++ b/pkg/handlers/embedded_cluster_node_join_command.go @@ -54,13 +54,6 @@ func (h *Handler) GenerateEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, return } - client, err := k8sutil.GetClientset() - if err != nil { - logger.Error(fmt.Errorf("failed to get clientset: %w", err)) - w.WriteHeader(http.StatusInternalServerError) - return - } - apps, err := store.GetStore().ListInstalledApps() if err != nil { logger.Error(fmt.Errorf("failed to list installed apps: %w", err)) @@ -74,7 +67,14 @@ func (h *Handler) GenerateEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, } app := apps[0] - nodeJoinCommand, err := embeddedcluster.GenerateAddNodeCommand(r.Context(), client, token, app.IsAirgap) + kbClient, err := k8sutil.GetKubeClient(r.Context()) + if err != nil { + logger.Error(fmt.Errorf("failed to get kubeclient: %w", err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + + nodeJoinCommand, err := embeddedcluster.GenerateAddNodeCommand(r.Context(), kbClient, token, app.IsAirgap) if err != nil { logger.Error(fmt.Errorf("failed to generate add node command: %w", err)) w.WriteHeader(http.StatusInternalServerError) @@ -104,15 +104,15 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht } // use roles to generate join token etc - client, err := k8sutil.GetClientset() + kbClient, err := k8sutil.GetKubeClient(r.Context()) if err != nil { - logger.Error(fmt.Errorf("failed to get clientset: %w", err)) + logger.Error(fmt.Errorf("failed to get kubeclient: %w", err)) w.WriteHeader(http.StatusInternalServerError) return } k0sRole := "worker" - controllerRoleName, err := embeddedcluster.ControllerRoleName(r.Context()) + controllerRoleName, err := embeddedcluster.ControllerRoleName(r.Context(), kbClient) if err != nil { logger.Error(fmt.Errorf("failed to get controller role name: %w", err)) w.WriteHeader(http.StatusInternalServerError) @@ -129,14 +129,14 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht // sort roles by name, but put controller first roles = embeddedcluster.SortRoles(controllerRoleName, roles) - k0sToken, err := embeddedcluster.GenerateAddNodeToken(r.Context(), client, k0sRole) + k0sToken, err := embeddedcluster.GenerateAddNodeToken(r.Context(), kbClient, k0sRole) if err != nil { logger.Error(fmt.Errorf("failed to generate add node token: %w", err)) w.WriteHeader(http.StatusInternalServerError) return } - k0sJoinCommand, err := embeddedcluster.GenerateK0sJoinCommand(r.Context(), client, roles) + k0sJoinCommand, err := embeddedcluster.GenerateK0sJoinCommand(r.Context(), kbClient, roles) if err != nil { logger.Error(fmt.Errorf("failed to generate k0s join command: %w", err)) w.WriteHeader(http.StatusInternalServerError) @@ -145,20 +145,15 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht logger.Infof("k0s join command: %q", k0sJoinCommand) - clusterID, err := embeddedcluster.ClusterID(client) - if err != nil { - logger.Error(fmt.Errorf("failed to get cluster id: %w", err)) - w.WriteHeader(http.StatusInternalServerError) - return - } - - // extracts the configuration overrides from the current active installation object. - install, err := embeddedcluster.GetCurrentInstallation(r.Context()) + // get the current active installation object + install, err := embeddedcluster.GetCurrentInstallation(r.Context(), kbClient) if err != nil { logger.Error(fmt.Errorf("failed to get current install: %w", err)) w.WriteHeader(http.StatusInternalServerError) return } + + // extract the configuration overrides from the installation object endUserK0sConfigOverrides := install.Spec.EndUserK0sConfigOverrides var k0sUnsupportedOverrides, ecVersion string if install.Spec.Config != nil { @@ -168,11 +163,17 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht airgapRegistryAddress := "" if install.Spec.AirGap { - airgapRegistryAddress, _, _ = kotsutil.GetEmbeddedRegistryCreds(client) + clientset, err := k8sutil.GetClientset() + if err != nil { + logger.Error(fmt.Errorf("failed to get clientset: %w", err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + airgapRegistryAddress, _, _ = kotsutil.GetEmbeddedRegistryCreds(clientset) } JSON(w, http.StatusOK, GetEmbeddedClusterNodeJoinCommandResponse{ - ClusterID: clusterID, + ClusterID: install.Spec.ClusterID, K0sJoinCommand: k0sJoinCommand, K0sToken: k0sToken, K0sUnsupportedOverrides: k0sUnsupportedOverrides, diff --git a/pkg/k8sutil/clientset.go b/pkg/k8sutil/clientset.go index 30f96bb72b..3948ca3ace 100644 --- a/pkg/k8sutil/clientset.go +++ b/pkg/k8sutil/clientset.go @@ -1,9 +1,12 @@ package k8sutil import ( + "context" + "io" "strconv" "github.com/pkg/errors" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" flag "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" @@ -14,7 +17,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" ) const ( @@ -134,3 +140,20 @@ func GetDynamicResourceInterface(gvk *schema.GroupVersionKind, namespace string) return dr, nil } + +func GetKubeClient(ctx context.Context) (kbclient.Client, error) { + k8slogger := zap.New(func(o *zap.Options) { + o.DestWriter = io.Discard + }) + log.SetLogger(k8slogger) + cfg, err := GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + kcli, err := kbclient.New(cfg, kbclient.Options{}) + if err != nil { + return nil, errors.Wrap(err, "failed to create kubebuilder client") + } + embeddedclusterv1beta1.AddToScheme(kcli.Scheme()) + return kcli, nil +} diff --git a/pkg/k8sutil/cluster.go b/pkg/k8sutil/cluster.go index b46af29f3d..38a9ceb48e 100644 --- a/pkg/k8sutil/cluster.go +++ b/pkg/k8sutil/cluster.go @@ -3,14 +3,17 @@ package k8sutil import ( "context" "fmt" - "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" "time" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/cluster-bootstrap/token/api" bootstraputil "k8s.io/cluster-bootstrap/token/util" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) // GenerateBootstrapToken will generate a node join token for kubeadm. @@ -23,10 +26,17 @@ func GenerateBootstrapToken(client kubernetes.Interface, ttl time.Duration) (str } data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:kubeadm:default-node-token") - return generateJoinTokenInternal(client, ttl, data) + token, secret, err := generateJoinToken(ttl, data) + if err != nil { + return "", errors.Wrap(err, "failed to generate join token") + } + if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { + return "", errors.Wrapf(err, "failed to create bootstrap token secret with name %s", secret.ObjectMeta.Name) + } + return token, nil } -func GenerateK0sBootstrapToken(client kubernetes.Interface, ttl time.Duration, role string) (string, error) { +func GenerateK0sBootstrapToken(client kbclient.Client, ttl time.Duration, role string) (string, error) { data := make(map[string][]byte) // these 'data' entries are taken from k0s: https://github.com/replicatedhq/k0s/blob/7bc57553ea8ccb6847fdd8249701554ee8be1ab0/pkg/token/manager.go#L69 @@ -41,13 +51,21 @@ func GenerateK0sBootstrapToken(client kubernetes.Interface, ttl time.Duration, r data["usage-bootstrap-signing"] = []byte("false") data["usage-controller-join"] = []byte("true") } - return generateJoinTokenInternal(client, ttl, data) + + token, secret, err := generateJoinToken(ttl, data) + if err != nil { + return "", errors.Wrap(err, "failed to generate join token") + } + if err := client.Create(context.TODO(), secret); err != nil { + return "", errors.Wrapf(err, "failed to create bootstrap token secret with name %s", secret.ObjectMeta.Name) + } + return token, nil } -func generateJoinTokenInternal(client kubernetes.Interface, ttl time.Duration, data map[string][]byte) (string, error) { +func generateJoinToken(ttl time.Duration, data map[string][]byte) (string, *corev1.Secret, error) { token, err := bootstraputil.GenerateBootstrapToken() if err != nil { - return "", errors.Wrap(err, "generate kubeadm token") + return "", nil, errors.Wrap(err, "generate bootstrap token") } substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) tokenID := substrs[1] @@ -60,7 +78,7 @@ func generateJoinTokenInternal(client kubernetes.Interface, ttl time.Duration, d data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) secretName := fmt.Sprintf("%s%s", bootstrapapi.BootstrapTokenSecretPrefix, tokenID) - bootstrapToken := &corev1.Secret{ + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, Namespace: metav1.NamespaceSystem, @@ -69,16 +87,12 @@ func generateJoinTokenInternal(client kubernetes.Interface, ttl time.Duration, d Data: data, } - if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(context.TODO(), bootstrapToken, metav1.CreateOptions{}); err != nil { - return "", errors.Wrapf(err, "failed to create bootstrap token with name %s", secretName) - } - - return token, nil + return token, secret, nil } -func GetClusterCaCert(ctx context.Context, client kubernetes.Interface) (string, error) { - cert, err := client.CoreV1().ConfigMaps("kube-system").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{}) - if err != nil { +func GetClusterCaCert(ctx context.Context, client kbclient.Client) (string, error) { + var cert corev1.ConfigMap + if err := client.Get(ctx, k8stypes.NamespacedName{Name: "kube-root-ca.crt", Namespace: "kube-system"}, &cert); err != nil { return "", errors.Wrap(err, "failed to get kube-root-ca.crt") } diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index abd77860bd..f9cb071385 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -14,7 +14,6 @@ import ( "github.com/pkg/errors" downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" apptypes "github.com/replicatedhq/kots/pkg/app/types" - "github.com/replicatedhq/kots/pkg/embeddedcluster" "github.com/replicatedhq/kots/pkg/k8sutil" "github.com/replicatedhq/kots/pkg/kotsadm" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" @@ -375,14 +374,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if util.IsEmbeddedCluster() { backupAnnotations["kots.io/embedded-cluster"] = "true" backupAnnotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID() - clusterConfig, err := embeddedcluster.ClusterConfig(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to get embedded cluster config") - } else if clusterConfig == nil { - return nil, errors.New("embedded cluster config is nil") - } - - backupAnnotations["kots.io/embedded-cluster-version"] = clusterConfig.Version + backupAnnotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion() } includeClusterResources := true diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index f70cbd6441..b2178abd2b 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -114,8 +114,8 @@ func GetReportingInfo(appID string) *types.ReportingInfo { InstanceID: appID, KOTSInstallID: os.Getenv("KOTS_INSTALL_ID"), KURLInstallID: os.Getenv("KURL_INSTALL_ID"), - EmbeddedClusterID: os.Getenv("EMBEDDED_CLUSTER_ID"), - EmbeddedClusterVersion: os.Getenv("EMBEDDED_CLUSTER_VERSION"), + EmbeddedClusterID: util.EmbeddedClusterID(), + EmbeddedClusterVersion: util.EmbeddedClusterVersion(), UserAgent: buildversion.GetUserAgent(), } diff --git a/pkg/util/util.go b/pkg/util/util.go index b8962882f8..e0e2048f0d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -173,6 +173,10 @@ func EmbeddedClusterID() string { return os.Getenv("EMBEDDED_CLUSTER_ID") } +func EmbeddedClusterVersion() string { + return os.Getenv("EMBEDDED_CLUSTER_VERSION") +} + func GetValueFromMapPath(m interface{}, path []string) interface{} { if len(path) == 0 { return nil