diff --git a/cmd/e2e/build_crashing_test.go b/cmd/e2e/build_crashing_test.go index 78fb4fac..38cd8f68 100644 --- a/cmd/e2e/build_crashing_test.go +++ b/cmd/e2e/build_crashing_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "net" "time" . "github.com/onsi/ginkgo/v2" @@ -40,13 +39,12 @@ var _ = Describe("Crashing Build", func() { g.Expect(verifyGameServerBuildOverall(ctx, kubeClient, state)).To(Succeed()) }, 45*time.Second, interval).Should(Succeed()) // bigger timeout because of the time crashes take to occur and captured by the controller - // we are updating the GameServerBuild to be able to have more crashes for it to become Unhealthy + // we are updating the GameServerBuild with a big CrashesToMarkUnhealthy to give time to the GameServerBuild to become Healthy gsb := &mpsv1alpha1.GameServerBuild{} err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildCrashingName, Namespace: testNamespace}, gsb) Expect(err).ToNot(HaveOccurred()) patch := client.MergeFrom(gsb.DeepCopy()) - gsb.Spec.CrashesToMarkUnhealthy = 10 - + gsb.Spec.CrashesToMarkUnhealthy = 1000 err = kubeClient.Patch(ctx, gsb, patch) Expect(err).ToNot(HaveOccurred()) @@ -66,7 +64,16 @@ var _ = Describe("Crashing Build", func() { g.Expect(verifyGameServerBuildOverall(ctx, kubeClient, state)).To(Succeed()) }, 10*time.Second, interval).Should(Succeed()) - // but only temporarily, since the game servers will continue to crash + // we're decreasing the CrashesToMarkUnhealthy to 10 so that the + // GameServerBuild will eventually become Unhealthy + err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildCrashingName, Namespace: testNamespace}, gsb) + Expect(err).ToNot(HaveOccurred()) + patch = client.MergeFrom(gsb.DeepCopy()) + gsb.Spec.CrashesToMarkUnhealthy = 10 + err = kubeClient.Patch(ctx, gsb, patch) + Expect(err).ToNot(HaveOccurred()) + + // now, let's make sure that GameServerBuild is Unhealthy Eventually(func(g Gomega) { gsb := &mpsv1alpha1.GameServerBuild{} err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildCrashingName, Namespace: testNamespace}, gsb) @@ -83,16 +90,6 @@ var _ = Describe("Crashing Build", func() { } g.Expect(verifyGameServerBuildOverall(ctx, kubeClient, state)).To(Succeed()) }, 40*time.Second, interval).Should(Succeed()) - - Eventually(func(g Gomega) { - var gsList mpsv1alpha1.GameServerList - err := kubeClient.List(ctx, &gsList, client.MatchingLabels{LabelBuildName: testBuildCrashingName}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(gsList.Items)).To(Equal(2)) - gs := gsList.Items[0] - g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) - g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) - }, 10*time.Second, interval).Should(Succeed()) }) }) diff --git a/cmd/e2e/build_host_network_test.go b/cmd/e2e/build_host_network_test.go index 4a929c12..05e46415 100644 --- a/cmd/e2e/build_host_network_test.go +++ b/cmd/e2e/build_host_network_test.go @@ -105,16 +105,17 @@ var _ = Describe("Build with hostnetwork", func() { g.Expect(verifyPodsInHostNetwork(ctx, kubeClient, gsb, state)).To(Succeed()) }, timeout, interval).Should(Succeed()) + // make sure all GameServers have a Public IP and NodeName Eventually(func(g Gomega) { var gsList mpsv1alpha1.GameServerList err := kubeClient.List(ctx, &gsList, client.MatchingLabels{LabelBuildName: testBuildWithHostNetworkName}) Expect(err).ToNot(HaveOccurred()) Expect(len(gsList.Items)).To(Equal(3)) - gs := gsList.Items[0] - g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) - g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) + for _, gs := range gsList.Items { + g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) + g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) + } }, timeout, interval).Should(Succeed()) - }) }) diff --git a/cmd/e2e/build_sleep_before_readyforplayers_test.go b/cmd/e2e/build_sleep_before_readyforplayers_test.go index d18c8231..7fa70d30 100644 --- a/cmd/e2e/build_sleep_before_readyforplayers_test.go +++ b/cmd/e2e/build_sleep_before_readyforplayers_test.go @@ -99,16 +99,17 @@ var _ = Describe("Build which sleeps before calling GSDK ReadyForPlayers", func( g.Expect(verifyGameServerBuildOverall(ctx, kubeClient, state)).To(Succeed()) }, timeout, interval).Should(Succeed()) + // make sure all GameServers have a Public IP and NodeName Eventually(func(g Gomega) { var gsList mpsv1alpha1.GameServerList err := kubeClient.List(ctx, &gsList, client.MatchingLabels{LabelBuildName: testBuildSleepBeforeReadyForPlayersName}) Expect(err).ToNot(HaveOccurred()) Expect(len(gsList.Items)).To(Equal(3)) - gs := gsList.Items[0] - g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) - g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) + for _, gs := range gsList.Items { + g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) + g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) + } }, timeout, interval).Should(Succeed()) - }) }) diff --git a/cmd/e2e/build_without_readyforplayers_test.go b/cmd/e2e/build_without_readyforplayers_test.go index 2eb7ba78..dcf91545 100644 --- a/cmd/e2e/build_without_readyforplayers_test.go +++ b/cmd/e2e/build_without_readyforplayers_test.go @@ -13,22 +13,22 @@ import ( ) var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { - testBuildWithoutReadyForPlayers := "withoutreadyforplayers" + testBuildWithoutReadyForPlayersName := "withoutreadyforplayers" testWithoutReadyForPlayersBuildID := "85ffe8da-c82f-4035-86c5-9d2b5f42d6f8" It("should have GameServers stuck in Initializing", func() { ctx := context.Background() kubeConfig := ctrl.GetConfigOrDie() kubeClient, err := createKubeClient(kubeConfig) Expect(err).ToNot(HaveOccurred()) - err = kubeClient.Create(ctx, createBuildWithoutReadyForPlayers(testBuildWithoutReadyForPlayers, testWithoutReadyForPlayersBuildID, img)) + err = kubeClient.Create(ctx, createBuildWithoutReadyForPlayers(testBuildWithoutReadyForPlayersName, testWithoutReadyForPlayersBuildID, img)) Expect(err).ToNot(HaveOccurred()) Eventually(func(g Gomega) { gsb := &mpsv1alpha1.GameServerBuild{} - err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) g.Expect(err).ToNot(HaveOccurred()) state := buildState{ - buildName: testBuildWithoutReadyForPlayers, + buildName: testBuildWithoutReadyForPlayersName, buildID: testWithoutReadyForPlayersBuildID, initializingCount: 2, standingByCount: 0, @@ -40,7 +40,7 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { // update the GameServerBuild to 4 standingBy gsb := &mpsv1alpha1.GameServerBuild{} - err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) Expect(err).ToNot(HaveOccurred()) patch := client.MergeFrom(gsb.DeepCopy()) gsb.Spec.StandingBy = 4 @@ -49,10 +49,10 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { Eventually(func(g Gomega) { gsb := &mpsv1alpha1.GameServerBuild{} - err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) g.Expect(err).ToNot(HaveOccurred()) state := buildState{ - buildName: testBuildWithoutReadyForPlayers, + buildName: testBuildWithoutReadyForPlayersName, buildID: testWithoutReadyForPlayersBuildID, initializingCount: 4, standingByCount: 0, @@ -64,7 +64,7 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { // update the GameServerBuild to 0 standingBy gsb = &mpsv1alpha1.GameServerBuild{} - err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) Expect(err).ToNot(HaveOccurred()) patch = client.MergeFrom(gsb.DeepCopy()) gsb.Spec.StandingBy = 0 @@ -73,10 +73,10 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { Eventually(func(g Gomega) { gsb := &mpsv1alpha1.GameServerBuild{} - err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) g.Expect(err).ToNot(HaveOccurred()) state := buildState{ - buildName: testBuildWithoutReadyForPlayers, + buildName: testBuildWithoutReadyForPlayersName, buildID: testWithoutReadyForPlayersBuildID, initializingCount: 0, standingByCount: 0, @@ -88,7 +88,7 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { // update the GameServerBuild to 2 standingBy again gsb = &mpsv1alpha1.GameServerBuild{} - err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err = kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) Expect(err).ToNot(HaveOccurred()) patch = client.MergeFrom(gsb.DeepCopy()) gsb.Spec.StandingBy = 2 @@ -97,10 +97,10 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { Eventually(func(g Gomega) { gsb := &mpsv1alpha1.GameServerBuild{} - err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayers, Namespace: testNamespace}, gsb) + err := kubeClient.Get(ctx, client.ObjectKey{Name: testBuildWithoutReadyForPlayersName, Namespace: testNamespace}, gsb) g.Expect(err).ToNot(HaveOccurred()) state := buildState{ - buildName: testBuildWithoutReadyForPlayers, + buildName: testBuildWithoutReadyForPlayersName, buildID: testWithoutReadyForPlayersBuildID, initializingCount: 2, standingByCount: 0, @@ -110,14 +110,16 @@ var _ = Describe("GameServerBuild without ReadyForPlayers GSDK call", func() { g.Expect(verifyGameServerBuildOverall(ctx, kubeClient, state)).To(Succeed()) }, timeout, interval).Should(Succeed()) + // make sure all GameServers have a Public IP and NodeName Eventually(func(g Gomega) { var gsList mpsv1alpha1.GameServerList - err := kubeClient.List(ctx, &gsList, client.MatchingLabels{LabelBuildName: testBuildWithoutReadyForPlayers}) + err := kubeClient.List(ctx, &gsList, client.MatchingLabels{LabelBuildName: testBuildWithoutReadyForPlayersName}) Expect(err).ToNot(HaveOccurred()) Expect(len(gsList.Items)).To(Equal(2)) - gs := gsList.Items[0] - g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) - g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) + for _, gs := range gsList.Items { + g.Expect(gs.Status.NodeName).ToNot(BeEmpty()) + g.Expect(net.ParseIP(gs.Status.PublicIP)).ToNot(BeNil()) + } }, timeout, interval).Should(Succeed()) }) diff --git a/cmd/e2e/utilities_test.go b/cmd/e2e/utilities_test.go index a6cf5649..767f786a 100644 --- a/cmd/e2e/utilities_test.go +++ b/cmd/e2e/utilities_test.go @@ -40,8 +40,9 @@ const ( portKey string = "gameport" safeToEvictPodAttribute string = "cluster-autoscaler.kubernetes.io/safe-to-evict" timeout = time.Second * 30 - interval = time.Millisecond * 250 + interval = time.Second * 1 thundernetesSystemNamespace = "thundernetes-system" + testNodeCount = 3 ) type AllocationResult struct { @@ -97,8 +98,8 @@ func validateThatAllocatedServersHaveReadyForPlayersUnblocked(ctx context.Contex return err } - if len(nodeAgentPodList.Items) != 3 { - return fmt.Errorf("expected 3 NodeAgent Pods, got %d", len(nodeAgentPodList.Items)) + if len(nodeAgentPodList.Items) != testNodeCount { + return fmt.Errorf("expected %d NodeAgent Pods, got %d", testNodeCount, len(nodeAgentPodList.Items)) } for _, gameServer := range activeGameServers { diff --git a/cmd/gameserverapi/deploy/e2e/nodeselector.yaml b/cmd/gameserverapi/deploy/e2e/nodeselector.yaml deleted file mode 100644 index da95e008..00000000 --- a/cmd/gameserverapi/deploy/e2e/nodeselector.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# this YAML file, when used with kustomize, it adds the nodeSelector field to the controller deployment -# it's used in e2e tests, so that the controller is scheduled on a different Node than the GameServer Pods -apiVersion: apps/v1 -kind: Deployment -metadata: - name: thundernetes-gameserverapi - namespace: thundernetes-system -spec: - template: - spec: - nodeSelector: - kubernetes.io/hostname: kind-worker \ No newline at end of file diff --git a/cmd/gameserverapi/deploy/default/deploy.yaml b/cmd/gameserverapi/deployment/default/deploy.yaml similarity index 95% rename from cmd/gameserverapi/deploy/default/deploy.yaml rename to cmd/gameserverapi/deployment/default/deploy.yaml index 1a989f35..4fc1c11d 100644 --- a/cmd/gameserverapi/deploy/default/deploy.yaml +++ b/cmd/gameserverapi/deployment/default/deploy.yaml @@ -16,7 +16,7 @@ spec: app: thundernetes-gameserverapi spec: containers: - - image: thundernetes-gameserverapi + - image: thundernetes-gameserverapi:${IMAGE_TAG} name: gameserverapi imagePullPolicy: IfNotPresent resources: diff --git a/cmd/gameserverapi/deploy/default/kustomization.yaml b/cmd/gameserverapi/deployment/default/kustomization.yaml similarity index 100% rename from cmd/gameserverapi/deploy/default/kustomization.yaml rename to cmd/gameserverapi/deployment/default/kustomization.yaml diff --git a/cmd/gameserverapi/deploy/e2e/kustomization.yaml b/cmd/gameserverapi/deployment/e2e/kustomization.yaml similarity index 100% rename from cmd/gameserverapi/deploy/e2e/kustomization.yaml rename to cmd/gameserverapi/deployment/e2e/kustomization.yaml diff --git a/cmd/gameserverapi/deployment/e2e/nodeselector.yaml b/cmd/gameserverapi/deployment/e2e/nodeselector.yaml new file mode 100644 index 00000000..8f39ac84 --- /dev/null +++ b/cmd/gameserverapi/deployment/e2e/nodeselector.yaml @@ -0,0 +1,13 @@ +# When this YAML file is used with kustomize, it adds the nodeSelector field to the controller deployment +# It's used in e2e tests, so that the gameserverapi is scheduled on a specific Node, so that we can do the +# listening service port forwarding (5001) correctly +apiVersion: apps/v1 +kind: Deployment +metadata: + name: thundernetes-gameserverapi + namespace: thundernetes-system +spec: + template: + spec: + nodeSelector: + kubernetes.io/hostname: kind-worker \ No newline at end of file diff --git a/e2e/run.sh b/e2e/run.sh index f1582d7c..c595d65c 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -47,9 +47,8 @@ echo "-----Compiling, building and deploying the operator to local Kubernetes cl IMG=${IMAGE_NAME_OPERATOR}:${IMAGE_TAG} API_SERVICE_SECURITY=usetls make -C "${DIR}"/../pkg/operator deploye2e echo "-----Deploying GameServer API-----" -cd cmd/gameserverapi/deploy/default -"${DIR}"/../pkg/operator/bin/kustomize edit set image thundernetes-gameserverapi=thundernetes-gameserverapi:${IMAGE_TAG} -"${DIR}"/../pkg/operator/bin/kustomize build ../e2e | kubectl apply -f - +cd cmd/gameserverapi/deployment/default +"${DIR}"/../pkg/operator/bin/kustomize build ../e2e | IMAGE_TAG=${IMAGE_TAG} envsubst | kubectl apply -f - echo "-----Waiting for Controller deployment-----" kubectl wait --for=condition=available --timeout=300s deployment/thundernetes-controller-manager -n thundernetes-system diff --git a/pkg/operator/Makefile b/pkg/operator/Makefile index f8119f29..72dd7405 100644 --- a/pkg/operator/Makefile +++ b/pkg/operator/Makefile @@ -84,7 +84,6 @@ docker-push: ## Push docker image with the manager. .PHONY: create-install-files create-install-files: - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} ## Create install files for thundernetes installed without API security $(KUSTOMIZE) build config/default | API_SERVICE_SECURITY=none envsubst > ${INSTALL_FILES_FOLDER}/operator.yaml ## Create install files for thundernetes installed with API security @@ -111,12 +110,10 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | envsubst | kubectl apply -f - .PHONY: deploye2e deploye2e: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/e2e | envsubst | kubectl apply -f - .PHONY: undeploy diff --git a/pkg/operator/config/e2e/nodeselector.yaml b/pkg/operator/config/e2e/nodeselector.yaml index 5a78d9df..b33e9db3 100644 --- a/pkg/operator/config/e2e/nodeselector.yaml +++ b/pkg/operator/config/e2e/nodeselector.yaml @@ -1,5 +1,6 @@ -# this YAML file, when used with kustomize, it adds the nodeSelector field to the controller deployment -# it's used in e2e tests, so that the controller is scheduled on a different Node than the GameServer Pods +# When this YAML file is used with kustomize, it adds the nodeSelector field to the controller deployment +# It's used in e2e tests, so that the controller is scheduled on a specific Node, so that we can do the +# allocation API service port forwarding (5000) correctly apiVersion: apps/v1 kind: Deployment metadata: diff --git a/pkg/operator/config/manager/kustomization.yaml b/pkg/operator/config/manager/kustomization.yaml index ba65f4ce..c427e413 100755 --- a/pkg/operator/config/manager/kustomization.yaml +++ b/pkg/operator/config/manager/kustomization.yaml @@ -6,9 +6,3 @@ configMapGenerator: - files: - controller_manager_config.yaml name: manager-config -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: ghcr.io/playfab/thundernetes-operator - newTag: 0.4.1 diff --git a/pkg/operator/config/manager/manager.yaml b/pkg/operator/config/manager/manager.yaml index 2bdf8f38..31f9080e 100644 --- a/pkg/operator/config/manager/manager.yaml +++ b/pkg/operator/config/manager/manager.yaml @@ -33,7 +33,7 @@ spec: - /manager args: - --leader-elect - image: controller:latest + image: ${IMG} imagePullPolicy: IfNotPresent env: - name: MIN_PORT