From 715977adde89fe0e58bf7aada25a0d4c47067989 Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Mon, 25 Nov 2024 15:10:02 +0000 Subject: [PATCH] chore: refactor endpoint to return full endpoint list vs node ips --- pkg/embeddedcluster/node_join.go | 42 ++++- pkg/embeddedcluster/node_join_test.go | 146 ++++++++++++++---- .../embedded_cluster_node_join_command.go | 10 +- ...embedded_cluster_node_join_command_test.go | 14 +- 4 files changed, 167 insertions(+), 45 deletions(-) diff --git a/pkg/embeddedcluster/node_join.go b/pkg/embeddedcluster/node_join.go index 849221f72d..b6f06ccc3d 100644 --- a/pkg/embeddedcluster/node_join.go +++ b/pkg/embeddedcluster/node_join.go @@ -77,9 +77,47 @@ func GenerateAddNodeToken(ctx context.Context, client kbclient.Client, nodeRole return newToken, nil } -// GetAllNodeIPAddresses returns the internal IP addresses of all the ready nodes in the cluster grouped by +// GetendpointsToCheck returns the list of endpoints that should be checked by a node joining the cluster +// based on the array of roles the node will have +func GetEndpointsToCheck(ctx context.Context, client kbclient.Client, roles []string) ([]string, error) { + controllerRoleName, err := ControllerRoleName(ctx, client) + if err != nil { + return nil, fmt.Errorf("failed to get controller role name: %w", err) + } + + isController := false + for _, role := range roles { + if role == controllerRoleName { + isController = true + break + } + } + controllerAddr, workerAddr, err := getAllNodeIPAddresses(ctx, client) + if err != nil { + return nil, fmt.Errorf("failed to get all node IP addresses: %w", err) + } + + endpoints := []string{} + for _, addr := range controllerAddr { + // any joining node should be able to reach the kube-api port and k0s-api port on all the controllers + endpoints = append(endpoints, fmt.Sprintf("%s:6443", addr), fmt.Sprintf("%s:9443", addr)) + if isController { + // controllers should be able to reach the etcd and kubelet ports on the controllers + endpoints = append(endpoints, fmt.Sprintf("%s:2380", addr), fmt.Sprintf("%s:10250", addr)) + } + } + if isController { + for _, addr := range workerAddr { + // controllers should be able to reach the kubelet port on the workers + endpoints = append(endpoints, fmt.Sprintf("%s:10250", addr)) + } + } + return endpoints, nil +} + +// getAllNodeIPAddresses returns the internal IP addresses of all the ready nodes in the cluster grouped by // controller and worker nodes respectively -func GetAllNodeIPAddresses(ctx context.Context, client kbclient.Client) ([]string, []string, error) { +func getAllNodeIPAddresses(ctx context.Context, client kbclient.Client) ([]string, []string, error) { var nodes corev1.NodeList if err := client.List(ctx, &nodes); err != nil { return nil, nil, fmt.Errorf("failed to list nodes: %w", err) diff --git a/pkg/embeddedcluster/node_join_test.go b/pkg/embeddedcluster/node_join_test.go index 4e8213817c..fc4ecc9ce0 100644 --- a/pkg/embeddedcluster/node_join_test.go +++ b/pkg/embeddedcluster/node_join_test.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -101,23 +102,60 @@ func TestGenerateAddNodeCommand(t *testing.T) { } func TestGetAllNodeIPAddresses(t *testing.T) { + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + embeddedclusterv1beta1.AddToScheme(scheme) tests := []struct { - name string - nodes []corev1.Node - expectedControllerIPs []string - expectedWorkerIPs []string + name string + roles []string + kbClient kbclient.Client + expectedEndpoints []string }{ { - name: "no nodes", - nodes: []corev1.Node{}, - expectedControllerIPs: []string{}, - expectedWorkerIPs: []string{}, + name: "no nodes", + roles: []string{"some-role"}, + kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: time.Now().Format("20060102150405"), + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + BinaryName: "my-app", + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "v1.100.0", + Roles: embeddedclusterv1beta1.Roles{ + Controller: embeddedclusterv1beta1.NodeRole{ + Name: "controller-role", + }, + }, + }, + }, + }, + ).Build(), + expectedEndpoints: []string{}, }, { - name: "1 controller, 1 worker", - nodes: []corev1.Node{ - { + name: "worker node joining cluster with 1 controller and 1 worker", + roles: []string{"some-role"}, + kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: time.Now().Format("20060102150405"), + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + BinaryName: "my-app", + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "v1.100.0", + Roles: embeddedclusterv1beta1.Roles{ + Controller: embeddedclusterv1beta1.NodeRole{ + Name: "controller-role", + }, + }, + }, + }, + }, + &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "controller", Labels: map[string]string{ @@ -139,7 +177,7 @@ func TestGetAllNodeIPAddresses(t *testing.T) { }, }, }, - { + &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "worker", Labels: map[string]string{ @@ -161,14 +199,30 @@ func TestGetAllNodeIPAddresses(t *testing.T) { }, }, }, - }, - expectedControllerIPs: []string{"192.168.0.100"}, - expectedWorkerIPs: []string{"192.168.0.101"}, + ).Build(), + expectedEndpoints: []string{"192.168.0.100:6443", "192.168.0.100:9443"}, }, { - name: "1 controller ready, 1 controller not ready, 1 worker ready, 1 worker not ready", - nodes: []corev1.Node{ - { + name: "controller node joining cluster with 2 controller ready, 1 controller not ready, 1 worker ready, 1 worker not ready", + roles: []string{"controller-role"}, + kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: time.Now().Format("20060102150405"), + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + BinaryName: "my-app", + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "v1.100.0", + Roles: embeddedclusterv1beta1.Roles{ + Controller: embeddedclusterv1beta1.NodeRole{ + Name: "controller-role", + }, + }, + }, + }, + }, + &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "controller 1", Labels: map[string]string{ @@ -190,7 +244,7 @@ func TestGetAllNodeIPAddresses(t *testing.T) { }, }, }, - { + &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "controller 2", Labels: map[string]string{ @@ -212,7 +266,29 @@ func TestGetAllNodeIPAddresses(t *testing.T) { }, }, }, - { + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "controller 3", + Labels: map[string]string{ + "node-role.kubernetes.io/control-plane": "true", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: "192.168.0.102", + }, + }, + }, + }, + &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "worker 1", Labels: map[string]string{}, @@ -227,12 +303,12 @@ func TestGetAllNodeIPAddresses(t *testing.T) { Addresses: []corev1.NodeAddress{ { Type: corev1.NodeInternalIP, - Address: "192.168.0.102", + Address: "192.168.0.103", }, }, }, }, - { + &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "worker 2", Labels: map[string]string{ @@ -249,30 +325,32 @@ func TestGetAllNodeIPAddresses(t *testing.T) { Addresses: []corev1.NodeAddress{ { Type: corev1.NodeInternalIP, - Address: "192.168.0.103", + Address: "192.168.0.104", }, }, }, }, + ).Build(), + expectedEndpoints: []string{ + "192.168.0.100:6443", + "192.168.0.100:9443", + "192.168.0.100:2380", + "192.168.0.100:10250", + "192.168.0.102:6443", + "192.168.0.102:9443", + "192.168.0.102:2380", + "192.168.0.102:10250", + "192.168.0.103:10250", }, - expectedControllerIPs: []string{"192.168.0.100"}, - expectedWorkerIPs: []string{"192.168.0.102"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { req := require.New(t) - - // Create a fake clientset - scheme := runtime.NewScheme() - corev1.AddToScheme(scheme) - kbClient := fake.NewClientBuilder().WithScheme(scheme).WithLists(&corev1.NodeList{Items: test.nodes}).Build() - - controllerAddr, workerAddr, err := GetAllNodeIPAddresses(context.Background(), kbClient) + endpoints, err := GetEndpointsToCheck(context.Background(), test.kbClient, test.roles) req.NoError(err) - req.Equal(test.expectedControllerIPs, controllerAddr) - req.Equal(test.expectedWorkerIPs, workerAddr) + req.Equal(test.expectedEndpoints, endpoints) }) } } diff --git a/pkg/handlers/embedded_cluster_node_join_command.go b/pkg/handlers/embedded_cluster_node_join_command.go index 2316cd4b88..5efc2f88c4 100644 --- a/pkg/handlers/embedded_cluster_node_join_command.go +++ b/pkg/handlers/embedded_cluster_node_join_command.go @@ -25,8 +25,7 @@ type GetEmbeddedClusterNodeJoinCommandResponse struct { K0sToken string `json:"k0sToken"` EmbeddedClusterVersion string `json:"embeddedClusterVersion"` AirgapRegistryAddress string `json:"airgapRegistryAddress"` - WorkerNodeIPs []string `json:"workerNodeIPs"` - ControllerNodeIPs []string `json:"controllerNodeIPs"` + TCPConnectionsRequired []string `json:"tcpConnectionsRequired"` InstallationSpec ecv1beta1.InstallationSpec `json:"installationSpec,omitempty"` } @@ -171,8 +170,8 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht airgapRegistryAddress, _, _ = kotsutil.GetEmbeddedRegistryCreds(clientset) } - // get all the healthy node ip addresses, to be used for preflight checks in the upcoming join - controllerNodeIPs, workerNodeIPs, err := embeddedcluster.GetAllNodeIPAddresses(r.Context(), kbClient) + // get all the endpoints a joining node needs to ensure connectivity to + endpoints, err := embeddedcluster.GetEndpointsToCheck(r.Context(), kbClient, roles) if err != nil { logger.Error(fmt.Errorf("failed to get the node ip addresses: %w", err)) w.WriteHeader(http.StatusInternalServerError) @@ -185,8 +184,7 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht K0sToken: k0sToken, EmbeddedClusterVersion: ecVersion, AirgapRegistryAddress: airgapRegistryAddress, - WorkerNodeIPs: workerNodeIPs, - ControllerNodeIPs: controllerNodeIPs, + TCPConnectionsRequired: endpoints, InstallationSpec: install.Spec, }) } diff --git a/pkg/handlers/embedded_cluster_node_join_command_test.go b/pkg/handlers/embedded_cluster_node_join_command_test.go index aacccc9184..9c3cce131c 100644 --- a/pkg/handlers/embedded_cluster_node_join_command_test.go +++ b/pkg/handlers/embedded_cluster_node_join_command_test.go @@ -143,14 +143,22 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) { ).Build(), }, { - name: "controller and worker node IPs are returned", + name: "tcp connections required are returned based on the controller role provided", httpStatus: http.StatusOK, embeddedClusterID: "cluster-id", validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *GetEmbeddedClusterNodeJoinCommandResponse) { req := require.New(t) - req.Equal([]string{"192.168.0.101"}, r.WorkerNodeIPs) - req.Equal([]string{"192.168.0.100"}, r.ControllerNodeIPs) + req.Equal([]string{ + "192.168.0.100:6443", + "192.168.0.100:9443", + "192.168.0.100:2380", + "192.168.0.100:10250", + "192.168.0.101:10250", + }, r.TCPConnectionsRequired) + }, + getRoles: func(t *testing.T, token string) ([]string, error) { + return []string{"controller-role"}, nil }, kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects( &embeddedclusterv1beta1.Installation{