Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ec_join): update handler to return tcp connections required #5004

Merged
merged 9 commits into from
Nov 26, 2024
Merged

feat(ec_join): update handler to return tcp connections required #5004

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
99bac82
feat(ec_join): add method to return all ready node ip addresses
JGAntunes Nov 13, 2024
a30fc5b
feat(ec_join): update handler to return node ips
JGAntunes Nov 13, 2024
d67dcb1
chore(test): create a struct and interface to allow kube client mocks…
JGAntunes Nov 13, 2024
a88e61e
chore: use newly created struct
JGAntunes Nov 13, 2024
2ac19a8
chore: tests for join handler
JGAntunes Nov 14, 2024
e932a81
chore: moaaar tests
JGAntunes Nov 15, 2024
80ae205
chore: tests for the worker and controller node IPs
JGAntunes Nov 15, 2024
715977a
chore: refactor endpoint to return full endpoint list vs node ips
JGAntunes Nov 25, 2024
dcd6f84
Merge branch 'main' into jgantunes/sc-113015/preflight-remote-access-…
JGAntunes Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/apiserver/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ func Start(params *APIServerParams) {
loggingRouter := r.NewRoute().Subrouter()
loggingRouter.Use(handlers.LoggingMiddleware)

handler := &handlers.Handler{}
handler := handlers.NewHandler()

/**********************************************************************
* Unauthenticated routes
Expand Down
88 changes: 82 additions & 6 deletions pkg/embeddedcluster/node_join.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,75 @@ func GenerateAddNodeToken(ctx context.Context, client kbclient.Client, nodeRole
return newToken, nil
}

// GetendpointsToCheck returns the list of endpoints that should be checked by a node joining the cluster
// based on the array of roles the node will have
func GetEndpointsToCheck(ctx context.Context, client kbclient.Client, roles []string) ([]string, error) {
controllerRoleName, err := ControllerRoleName(ctx, client)
if err != nil {
return nil, fmt.Errorf("failed to get controller role name: %w", err)
}

isController := false
for _, role := range roles {
if role == controllerRoleName {
isController = true
break
}
}
controllerAddr, workerAddr, err := getAllNodeIPAddresses(ctx, client)
if err != nil {
return nil, fmt.Errorf("failed to get all node IP addresses: %w", err)
}

endpoints := []string{}
for _, addr := range controllerAddr {
// any joining node should be able to reach the kube-api port and k0s-api port on all the controllers
endpoints = append(endpoints, fmt.Sprintf("%s:6443", addr), fmt.Sprintf("%s:9443", addr))
if isController {
// controllers should be able to reach the etcd and kubelet ports on the controllers
endpoints = append(endpoints, fmt.Sprintf("%s:2380", addr), fmt.Sprintf("%s:10250", addr))
}
}
if isController {
for _, addr := range workerAddr {
// controllers should be able to reach the kubelet port on the workers
endpoints = append(endpoints, fmt.Sprintf("%s:10250", addr))
}
}
return endpoints, nil
}

// getAllNodeIPAddresses returns the internal IP addresses of all the ready nodes in the cluster grouped by
// controller and worker nodes respectively
func getAllNodeIPAddresses(ctx context.Context, client kbclient.Client) ([]string, []string, error) {
var nodes corev1.NodeList
if err := client.List(ctx, &nodes); err != nil {
return nil, nil, fmt.Errorf("failed to list nodes: %w", err)
}

controllerAddr := []string{}
workerAddr := []string{}
for _, node := range nodes.Items {
// Only consider nodes that are ready
if !isReady(node) {
continue
}

// Filter nodes by control-plane and worker roles
if cp, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok && cp == "true" {
if addr := findInternalIPAddress(node.Status.Addresses); addr != nil {
controllerAddr = append(controllerAddr, addr.Address)
}
} else {
if addr := findInternalIPAddress(node.Status.Addresses); addr != nil {
workerAddr = append(workerAddr, addr.Address)
}
}
}

return controllerAddr, workerAddr, nil
}

func makeK0sToken(ctx context.Context, client kbclient.Client, nodeRole string) (string, error) {
rawToken, err := k8sutil.GenerateK0sBootstrapToken(client, time.Hour, nodeRole)
if err != nil {
Expand All @@ -89,7 +158,7 @@ func makeK0sToken(ctx context.Context, client kbclient.Client, nodeRole string)
}
cert = base64.StdEncoding.EncodeToString([]byte(cert))

firstPrimary, err := firstPrimaryIpAddress(ctx, client)
firstPrimary, err := firstPrimaryIPAddress(ctx, client)
if err != nil {
return "", fmt.Errorf("failed to get first primary ip address: %w", err)
}
Expand All @@ -111,7 +180,7 @@ func makeK0sToken(ctx context.Context, client kbclient.Client, nodeRole string)
return b64Token, nil
}

func firstPrimaryIpAddress(ctx context.Context, client kbclient.Client) (string, error) {
func firstPrimaryIPAddress(ctx context.Context, client kbclient.Client) (string, error) {
var nodes corev1.NodeList
if err := client.List(ctx, &nodes); err != nil {
return "", fmt.Errorf("failed to list nodes: %w", err)
Expand All @@ -122,16 +191,23 @@ func firstPrimaryIpAddress(ctx context.Context, client kbclient.Client) (string,
continue
}

for _, address := range node.Status.Addresses {
if address.Type == "InternalIP" {
return address.Address, nil
}
if addr := findInternalIPAddress(node.Status.Addresses); addr != nil {
return addr.Address, nil
}
}

return "", fmt.Errorf("failed to find controller node")
}

func findInternalIPAddress(addresses []corev1.NodeAddress) *corev1.NodeAddress {
for _, address := range addresses {
if address.Type == "InternalIP" {
return &address
}
}
return nil
}

// GenerateAddNodeCommand returns the command a user should run to add a node with the provided token
// the command will be of the form 'embeddedcluster node join ip:port UUID'
func GenerateAddNodeCommand(ctx context.Context, kbClient kbclient.Client, token string, isAirgap bool) (string, error) {
Expand Down
255 changes: 255 additions & 0 deletions pkg/embeddedcluster/node_join_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

Expand Down Expand Up @@ -99,3 +100,257 @@ func TestGenerateAddNodeCommand(t *testing.T) {
wantCommand = "sudo ./my-app join --airgap-bundle my-app.airgap 192.168.0.100:30000 token"
req.Equal(wantCommand, gotCommand)
}

func TestGetAllNodeIPAddresses(t *testing.T) {
scheme := runtime.NewScheme()
corev1.AddToScheme(scheme)
embeddedclusterv1beta1.AddToScheme(scheme)

tests := []struct {
name string
roles []string
kbClient kbclient.Client
expectedEndpoints []string
}{
{
name: "no nodes",
roles: []string{"some-role"},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
).Build(),
expectedEndpoints: []string{},
},
{
name: "worker node joining cluster with 1 controller and 1 worker",
roles: []string{"some-role"},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "true",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.100",
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "worker",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "false",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.101",
},
},
},
},
).Build(),
expectedEndpoints: []string{"192.168.0.100:6443", "192.168.0.100:9443"},
},
{
name: "controller node joining cluster with 2 controller ready, 1 controller not ready, 1 worker ready, 1 worker not ready",
roles: []string{"controller-role"},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 1",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "true",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.100",
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 2",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "true",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionFalse,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.101",
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 3",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "true",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.102",
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "worker 1",
Labels: map[string]string{},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.103",
},
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "worker 2",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "false",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionFalse,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.104",
},
},
},
},
).Build(),
expectedEndpoints: []string{
"192.168.0.100:6443",
"192.168.0.100:9443",
"192.168.0.100:2380",
"192.168.0.100:10250",
"192.168.0.102:6443",
"192.168.0.102:9443",
"192.168.0.102:2380",
"192.168.0.102:10250",
"192.168.0.103:10250",
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := require.New(t)
endpoints, err := GetEndpointsToCheck(context.Background(), test.kbClient, test.roles)
req.NoError(err)
req.Equal(test.expectedEndpoints, endpoints)
})
}
}
Loading
Loading