adding multiple Nodes to e2e cluster

cmd/e2e/utilities_test.go

@@ -97,20 +97,30 @@ func validateThatAllocatedServersHaveReadyForPlayersUnblocked(ctx context.Contex
 		return err
 	}
 
-	if len(nodeAgentPodList.Items) != 1 {
-		return fmt.Errorf("expected 1 node agent pod, got %d", len(nodeAgentPodList.Items))
-	}
-	nodeAgentPod := nodeAgentPodList.Items[0]
-
-	nodeAgentLogs, err := getContainerLogs(ctx, coreClient, nodeAgentPod.Name, nodeAgentName, thundernetesSystemNamespace)
-	if err != nil {
-		return err
+	if len(nodeAgentPodList.Items) != 3 {
+		return fmt.Errorf("expected 3 NodeAgent Pods, got %d", len(nodeAgentPodList.Items))
 	}
 
 	for _, gameServer := range activeGameServers {
+		// find the NodeAgent Pod for this GameServer (they have been scheduled in the same Node)
+		var nodeAgentLogs string
+		var err error
+		for _, nodeAgentPod := range nodeAgentPodList.Items {
+			// when running on kind, the GameServer.Status.PublicIP is equal to the private IP of the Node
+			if nodeAgentPod.Status.HostIP == gameServer.Status.PublicIP {
+				nodeAgentLogs, err = getContainerLogs(ctx, coreClient, nodeAgentPod.Name, nodeAgentName, thundernetesSystemNamespace)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		if nodeAgentLogs == "" {
+			return fmt.Errorf("could not find NodeAgent Pod for GameServer %s", gameServer.Name)
+		}
+
 		Eventually(func() error {
 			if !strings.Contains(nodeAgentLogs, "sessionCookie:randomCookie") {
-				return fmt.Errorf("expected to find 'sessionCookie:randomCookie' in nodeAgent logs, got %s", nodeAgentLogs)
+				return fmt.Errorf("expected to find 'sessionCookie:randomCookie' in NodeAgent logs, got %s", nodeAgentLogs)
 			}
 
 			containerLogs, err := getContainerLogs(ctx, coreClient, gameServer.Name, containerName, gameServer.Namespace)

cmd/gameserverapi/deploy.yaml → cmd/gameserverapi/deploy/default/deploy.yaml

@@ -16,7 +16,7 @@ spec:
         app: thundernetes-gameserverapi
     spec:
       containers:
-        - image: thundernetes-gameserverapi:${IMAGE_TAG}
+        - image: thundernetes-gameserverapi
           name: gameserverapi
           imagePullPolicy: IfNotPresent
           resources:

cmd/gameserverapi/deploy/default/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- deploy.yaml

cmd/gameserverapi/deploy/e2e/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+  - ../default
+patchesStrategicMerge:
+  - nodeselector.yaml

cmd/gameserverapi/deploy/e2e/nodeselector.yaml

@@ -0,0 +1,12 @@
+# this YAML file, when used with kustomize, it adds the nodeSelector field to the controller deployment
+# it's used in e2e tests, so that the controller is scheduled on a different Node than the GameServer Pods
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: thundernetes-gameserverapi
+  namespace: thundernetes-system
+spec:
+  template:
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: kind-worker

cmd/nodeagent/nodeagentmanager.go

@@ -247,6 +247,7 @@ func (n *NodeAgentManager) gameServerCreatedOrUpdated(obj *unstructured.Unstruct
 
 	// server is Active, so get session details as well initial players details
 	sessionID, sessionCookie, initialPlayers := parseSessionDetails(obj, gameServerName, gameServerNamespace)
+	// sessionCookie:<valueOfCookie> string is looked for in the e2e tests, be careful not to modify it!
 	logger.Infof("getting values from allocation - GameServer CR, sessionID:%s, sessionCookie:%s, initialPlayers: %v", sessionID, sessionCookie, initialPlayers)
 
 	// create the GameServerDetails CR

e2e/kind-config.yaml

@@ -2,6 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
+- role: worker
   extraPortMappings:
   - containerPort: 5000
     hostPort: 5000
@@ -11,11 +12,5 @@ nodes:
     hostPort: 5001
     listenAddress: "0.0.0.0"
     protocol: tcp
-  - containerPort: 10000
-    hostPort: 10000
-    listenAddress: "0.0.0.0"
-    protocol: tcp
-  - containerPort: 10001
-    hostPort: 10001
-    listenAddress: "0.0.0.0"
-    protocol: tcp
+- role: worker
+- role: worker

e2e/run.sh

@@ -44,10 +44,12 @@ export FAKE_TLS_PUBLIC=/tmp/${RANDOM}.pem
 openssl req -x509 -newkey rsa:4096 -nodes -keyout ${FAKE_TLS_PRIVATE} -out ${FAKE_TLS_PUBLIC} -days 365 -subj '/CN=localhost'
 
 echo "-----Compiling, building and deploying the operator to local Kubernetes cluster-----"
-IMG=${IMAGE_NAME_OPERATOR}:${IMAGE_TAG} API_SERVICE_SECURITY=usetls make -C "${DIR}"/../pkg/operator deploy
+IMG=${IMAGE_NAME_OPERATOR}:${IMAGE_TAG} API_SERVICE_SECURITY=usetls make -C "${DIR}"/../pkg/operator deploye2e
 
 echo "-----Deploying GameServer API-----"
-IMAGE_TAG=${IMAGE_TAG} envsubst < cmd/gameserverapi/deploy.yaml | kubectl apply -f -
+cd cmd/gameserverapi/deploy/default
+"${DIR}"/../pkg/operator/bin/kustomize edit set image thundernetes-gameserverapi=thundernetes-gameserverapi:${IMAGE_TAG}
+"${DIR}"/../pkg/operator/bin/kustomize build ../e2e | kubectl apply -f -
 
 echo "-----Waiting for Controller deployment-----"
 kubectl wait --for=condition=available --timeout=300s deployment/thundernetes-controller-manager -n thundernetes-system
@@ -56,7 +58,7 @@ echo "-----Waiting for GameServer API deployment-----"
 kubectl wait --for=condition=ready --timeout=300s pod -n thundernetes-system -l app=thundernetes-gameserverapi
 
 echo "-----Running end to end tests-----"
-cd cmd/e2e
+cd "${DIR}"/../cmd/e2e
 # create the test namespaces
 kubectl create namespace gameserverapi
 kubectl create namespace e2e

pkg/operator/Makefile

@@ -114,6 +114,11 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | envsubst | kubectl apply -f -
 
+.PHONY: deploye2e
+deploye2e: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/e2e | envsubst | kubectl apply -f -
+
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -

pkg/operator/config/e2e/kustomization.yaml

@@ -0,0 +1,5 @@
+bases:
+- ../default
+
+patchesStrategicMerge:
+- nodeselector.yaml

pkg/operator/config/e2e/nodeselector.yaml

@@ -0,0 +1,14 @@
+# this YAML file, when used with kustomize, it adds the nodeSelector field to the controller deployment
+# it's used in e2e tests, so that the controller is scheduled on a different Node than the GameServer Pods
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+  labels:
+    control-plane: controller-manager
+spec:
+  template:
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: kind-worker