Skip to content

Commit

Permalink
chore: refactor endpoint to return full endpoint list vs node ips
Browse files Browse the repository at this point in the history
  • Loading branch information
@JGAntunes
JGAntunes committed Nov 25, 2024
1 parent 80ae205 commit 715977a
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 45 deletions.
42 changes: 40 additions & 2 deletions pkg/embeddedcluster/node_join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,47 @@ func GenerateAddNodeToken(ctx context.Context, client kbclient.Client, nodeRole
return newToken, nil
}

// GetAllNodeIPAddresses returns the internal IP addresses of all the ready nodes in the cluster grouped by
// GetendpointsToCheck returns the list of endpoints that should be checked by a node joining the cluster
// based on the array of roles the node will have
func GetEndpointsToCheck(ctx context.Context, client kbclient.Client, roles []string) ([]string, error) {
controllerRoleName, err := ControllerRoleName(ctx, client)
if err != nil {
return nil, fmt.Errorf("failed to get controller role name: %w", err)
}

isController := false
for _, role := range roles {
if role == controllerRoleName {
isController = true
break
}
}
controllerAddr, workerAddr, err := getAllNodeIPAddresses(ctx, client)
if err != nil {
return nil, fmt.Errorf("failed to get all node IP addresses: %w", err)
}

endpoints := []string{}
for _, addr := range controllerAddr {
// any joining node should be able to reach the kube-api port and k0s-api port on all the controllers
endpoints = append(endpoints, fmt.Sprintf("%s:6443", addr), fmt.Sprintf("%s:9443", addr))
if isController {
// controllers should be able to reach the etcd and kubelet ports on the controllers
endpoints = append(endpoints, fmt.Sprintf("%s:2380", addr), fmt.Sprintf("%s:10250", addr))
}
}
if isController {
for _, addr := range workerAddr {
// controllers should be able to reach the kubelet port on the workers
endpoints = append(endpoints, fmt.Sprintf("%s:10250", addr))
}
}
return endpoints, nil
}

// getAllNodeIPAddresses returns the internal IP addresses of all the ready nodes in the cluster grouped by
// controller and worker nodes respectively
func GetAllNodeIPAddresses(ctx context.Context, client kbclient.Client) ([]string, []string, error) {
func getAllNodeIPAddresses(ctx context.Context, client kbclient.Client) ([]string, []string, error) {
var nodes corev1.NodeList
if err := client.List(ctx, &nodes); err != nil {
return nil, nil, fmt.Errorf("failed to list nodes: %w", err)
Expand Down
146 changes: 112 additions & 34 deletions pkg/embeddedcluster/node_join_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

Expand Down Expand Up @@ -101,23 +102,60 @@ func TestGenerateAddNodeCommand(t *testing.T) {
}

func TestGetAllNodeIPAddresses(t *testing.T) {
scheme := runtime.NewScheme()
corev1.AddToScheme(scheme)
embeddedclusterv1beta1.AddToScheme(scheme)

tests := []struct {
name string
nodes []corev1.Node
expectedControllerIPs []string
expectedWorkerIPs []string
name string
roles []string
kbClient kbclient.Client
expectedEndpoints []string
}{
{
name: "no nodes",
nodes: []corev1.Node{},
expectedControllerIPs: []string{},
expectedWorkerIPs: []string{},
name: "no nodes",
roles: []string{"some-role"},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
).Build(),
expectedEndpoints: []string{},
},
{
name: "1 controller, 1 worker",
nodes: []corev1.Node{
{
name: "worker node joining cluster with 1 controller and 1 worker",
roles: []string{"some-role"},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller",
Labels: map[string]string{
Expand All @@ -139,7 +177,7 @@ func TestGetAllNodeIPAddresses(t *testing.T) {
},
},
},
{
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "worker",
Labels: map[string]string{
Expand All @@ -161,14 +199,30 @@ func TestGetAllNodeIPAddresses(t *testing.T) {
},
},
},
},
expectedControllerIPs: []string{"192.168.0.100"},
expectedWorkerIPs: []string{"192.168.0.101"},
).Build(),
expectedEndpoints: []string{"192.168.0.100:6443", "192.168.0.100:9443"},
},
{
name: "1 controller ready, 1 controller not ready, 1 worker ready, 1 worker not ready",
nodes: []corev1.Node{
{
name: "controller node joining cluster with 2 controller ready, 1 controller not ready, 1 worker ready, 1 worker not ready",
roles: []string{"controller-role"},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 1",
Labels: map[string]string{
Expand All @@ -190,7 +244,7 @@ func TestGetAllNodeIPAddresses(t *testing.T) {
},
},
},
{
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 2",
Labels: map[string]string{
Expand All @@ -212,7 +266,29 @@ func TestGetAllNodeIPAddresses(t *testing.T) {
},
},
},
{
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 3",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "true",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.102",
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "worker 1",
Labels: map[string]string{},
Expand All @@ -227,12 +303,12 @@ func TestGetAllNodeIPAddresses(t *testing.T) {
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.102",
Address: "192.168.0.103",
},
},
},
},
{
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "worker 2",
Labels: map[string]string{
Expand All @@ -249,30 +325,32 @@ func TestGetAllNodeIPAddresses(t *testing.T) {
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.103",
Address: "192.168.0.104",
},
},
},
},
).Build(),
expectedEndpoints: []string{
"192.168.0.100:6443",
"192.168.0.100:9443",
"192.168.0.100:2380",
"192.168.0.100:10250",
"192.168.0.102:6443",
"192.168.0.102:9443",
"192.168.0.102:2380",
"192.168.0.102:10250",
"192.168.0.103:10250",
},
expectedControllerIPs: []string{"192.168.0.100"},
expectedWorkerIPs: []string{"192.168.0.102"},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := require.New(t)

// Create a fake clientset
scheme := runtime.NewScheme()
corev1.AddToScheme(scheme)
kbClient := fake.NewClientBuilder().WithScheme(scheme).WithLists(&corev1.NodeList{Items: test.nodes}).Build()

controllerAddr, workerAddr, err := GetAllNodeIPAddresses(context.Background(), kbClient)
endpoints, err := GetEndpointsToCheck(context.Background(), test.kbClient, test.roles)
req.NoError(err)
req.Equal(test.expectedControllerIPs, controllerAddr)
req.Equal(test.expectedWorkerIPs, workerAddr)
req.Equal(test.expectedEndpoints, endpoints)
})
}
}
10 changes: 4 additions & 6 deletions pkg/handlers/embedded_cluster_node_join_command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ type GetEmbeddedClusterNodeJoinCommandResponse struct {
K0sToken string `json:"k0sToken"`
EmbeddedClusterVersion string `json:"embeddedClusterVersion"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
WorkerNodeIPs []string `json:"workerNodeIPs"`
ControllerNodeIPs []string `json:"controllerNodeIPs"`
TCPConnectionsRequired []string `json:"tcpConnectionsRequired"`
InstallationSpec ecv1beta1.InstallationSpec `json:"installationSpec,omitempty"`
}

Expand Down Expand Up @@ -171,8 +170,8 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht
airgapRegistryAddress, _, _ = kotsutil.GetEmbeddedRegistryCreds(clientset)
}

// get all the healthy node ip addresses, to be used for preflight checks in the upcoming join
controllerNodeIPs, workerNodeIPs, err := embeddedcluster.GetAllNodeIPAddresses(r.Context(), kbClient)
// get all the endpoints a joining node needs to ensure connectivity to
endpoints, err := embeddedcluster.GetEndpointsToCheck(r.Context(), kbClient, roles)
if err != nil {
logger.Error(fmt.Errorf("failed to get the node ip addresses: %w", err))
w.WriteHeader(http.StatusInternalServerError)
Expand All @@ -185,8 +184,7 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht
K0sToken: k0sToken,
EmbeddedClusterVersion: ecVersion,
AirgapRegistryAddress: airgapRegistryAddress,
WorkerNodeIPs: workerNodeIPs,
ControllerNodeIPs: controllerNodeIPs,
TCPConnectionsRequired: endpoints,
InstallationSpec: install.Spec,
})
}
14 changes: 11 additions & 3 deletions pkg/handlers/embedded_cluster_node_join_command_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,22 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
).Build(),
},
{
name: "controller and worker node IPs are returned",
name: "tcp connections required are returned based on the controller role provided",
httpStatus: http.StatusOK,
embeddedClusterID: "cluster-id",
validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *GetEmbeddedClusterNodeJoinCommandResponse) {
req := require.New(t)

req.Equal([]string{"192.168.0.101"}, r.WorkerNodeIPs)
req.Equal([]string{"192.168.0.100"}, r.ControllerNodeIPs)
req.Equal([]string{
"192.168.0.100:6443",
"192.168.0.100:9443",
"192.168.0.100:2380",
"192.168.0.100:10250",
"192.168.0.101:10250",
}, r.TCPConnectionsRequired)
},
getRoles: func(t *testing.T, token string) ([]string, error) {
return []string{"controller-role"}, nil
},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
Expand Down

0 comments on commit 715977a

Please sign in to comment.