From 37c3940492678ac486148732e61f00b33a69de80 Mon Sep 17 00:00:00 2001 From: Yurii Prokulevych Date: Wed, 30 Oct 2024 17:05:59 +0100 Subject: [PATCH] system-test: check MetalLB graceful restart --- .../internal/rdscorecommon/egress-service.go | 1 + .../rdscorecommon/metallb-graceful-restart.go | 682 ++++++++++++++++++ .../rdscore/internal/rdscoreconfig/config.go | 8 + .../internal/rdscoreconfig/default.yaml | 5 + .../rdscore/tests/00_validate_top_level.go | 4 + 5 files changed, 700 insertions(+) create mode 100644 tests/system-tests/rdscore/internal/rdscorecommon/metallb-graceful-restart.go diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go b/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go index 9c9940be1..66dd3768b 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go @@ -6,6 +6,7 @@ import ( "net/netip" "os/exec" "strings" + "time" "github.com/golang/glog" diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/metallb-graceful-restart.go b/tests/system-tests/rdscore/internal/rdscorecommon/metallb-graceful-restart.go new file mode 100644 index 000000000..066004023 --- /dev/null +++ b/tests/system-tests/rdscore/internal/rdscorecommon/metallb-graceful-restart.go @@ -0,0 +1,682 @@ +package rdscorecommon + +import ( + "fmt" + "net" + "net/netip" + "strconv" + "sync" + "time" + + "github.com/golang/glog" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift-kni/eco-goinfra/pkg/pod" + "github.com/openshift-kni/eco-goinfra/pkg/service" + . "github.com/openshift-kni/eco-gotests/tests/system-tests/rdscore/internal/rdscoreinittools" + "github.com/openshift-kni/eco-gotests/tests/system-tests/rdscore/internal/rdscoreparams" +) + +// Event structure stores basic data about connection attempts. +type Event struct { + Msg string + Timestamp time.Time +} + +// Counter structre is used to measure successful and failed connection attempts. +type Counter struct { + Count int64 + CounterLock sync.Mutex + Start time.Time + End time.Time + Success []Event + Failures []Event +} + +type myData struct { + TCPConn *net.TCPConn + TCPErr error +} + +// mTCPConnect dials to the TCP endpoint and terminates within a specied timeout. +func mTCPConnect(addr *net.TCPAddr, timeOut int) (*net.TCPConn, error) { + result := make(chan myData) + + go func(network string, laddr, raddr *net.TCPAddr, result chan myData) { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Trying to connect to %q at %v", raddr.String(), + time.Now().UnixMilli()) + + tConn, err := net.DialTCP(network, laddr, addr) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to connect to %q due to: %v", + raddr.String(), err) + + result <- myData{TCPConn: nil, TCPErr: err} + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Connected to %q", raddr.String()) + + result <- myData{TCPConn: tConn, TCPErr: nil} + }("tcp", nil, addr, result) + + select { + case data := <-result: + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Read from connection: %v\n", data) + + return data.TCPConn, data.TCPErr + case <-time.After(time.Duration(timeOut) * time.Millisecond): + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("timeout waiting for connection to be establised") + + return nil, fmt.Errorf("timeout waiting for connection to be establised") + } +} + +// verifySingleTCPConnection validates graceful restart feature by opening a TCP connection +// to the IP address of LoadBalancer service and using the connection to +// send/receive data. During the run frr-k8s pod is destroyed but this +// shouldn't result in the connection being dropped. +// +//nolint:funlen +func verifySingleTCPConnection(loadBalancerIP string, servicePort int32, + finished, ready chan bool, stats *Counter) { + var ( + err error + endPoint string + ) + + By(fmt.Sprintf("Accessing backend pods via %s IP", loadBalancerIP)) + + myIP, err := netip.ParseAddr(loadBalancerIP) + + Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address") + + if myIP.Is4() { + endPoint = fmt.Sprintf("%s:%d", loadBalancerIP, servicePort) + } + + if myIP.Is6() { + endPoint = fmt.Sprintf("[%s]:%d", loadBalancerIP, servicePort) + } + + getHTTPMsg := []byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", endPoint)) + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Resolving TCP endpoint %q", endPoint) + + addr, err := net.ResolveTCPAddr("tcp", endPoint) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed resolve TCP address %q : %v", endPoint, err) + + ready <- true + + return + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Dialing to the TCP endpoint %q", endPoint) + + lbConnection, err := mTCPConnect(addr, int(1000)) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed dailing to %q : %v", endPoint, err) + + ready <- true + + return + } + + defer lbConnection.Close() + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Enabling KeepAlive for TCP connection") + + err = lbConnection.SetKeepAlive(true) + + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("failed to enable KeepAlive for the connection: %v", err)) + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Setting KeepAlivePeriod to 1s") + + err = lbConnection.SetKeepAlivePeriod(time.Duration(1)) + + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("failed to enable KeepAlivePeriod for the connection: %v", err)) + + var stop bool + + for !stop { + select { + case <-finished: + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Received data on 'finishd' channel") + + stop = true + default: + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Writing to the TCP connection at %v", time.Now().UnixMilli()) + + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Dialing to the %s at %v", + // endPoint, time.Now().UnixMilli()) + + // endPoint := fmt.Sprintf("%s:%d", noramlizedIP, servicePort) + + // lbConnection, err := net.DialTimeout("tcp", endPoint, 300*time.Millisecond) + + // if err != nil { + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed dailing to %s : %v", endPoint, err) + // stats.CounterLock.Lock() + // stats.Count++ + // stats.Failures = append(stats.Failures, + // Event{Msg: fmt.Sprintf("Failed dailing to %s : %v", endPoint, err), Timestamp: time.Now()}) + // stats.CounterLock.Unlock() + + // continue + // } + + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Successfully connected to %s", endPoint) + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Local Address %s", lbConnection.LocalAddr()) + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Remote Address %s", lbConnection.RemoteAddr()) + + nSent, err := lbConnection.Write(getHTTPMsg) + + if err != nil || nSent == 0 { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to write to connection: %v", err) + + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed to write to connection: %v", err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + stop = true + ready <- true + } + + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Sent %d bytes", nSent) + + // err = lbConnection.SetReadDeadline(time.Now().Add(50 * time.Millisecond)) + + // if err != nil { + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to set ReadDeadline: %v", err) + // stats.CounterLock.Lock() + // stats.Count++ + // stats.Failures = append(stats.Failures, + // Event{Msg: fmt.Sprintf("Failed to set ReadDeadline: %v", err), Timestamp: time.Now()}) + // stats.CounterLock.Unlock() + + // // lbConnection.Close() + + // continue + // } + + // glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Reading reply") + + err = lbConnection.SetReadDeadline(time.Now().Add(300 * time.Millisecond)) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed set ReadDeadline: %v", err) + + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed to set ReadDeadline %v", err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + stop = true + } + + msgReply := make([]byte, 1024) + + bRead, err := lbConnection.Read(msgReply) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to read reply(%v): %v", + time.Now().UnixMilli(), err) + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed to read reply: %v", err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + stop = true + } else { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Read %d bytes", bRead) + + stats.CounterLock.Lock() + stats.Count++ + stats.Success = append(stats.Success, + Event{Msg: string(msgReply[0:bRead]), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + } + } + } + ready <- true +} + +// verifyMultipleTCPConnections simulates new clients connecting to the specified endpoint, +// while Metallb-FRR pod is killed. +// It opens new TCP connection to the specified endpoint during every iteration. +// 'finished' channel is used receive signal that metallb-frr pod was restarted +// 'ready' channel is used to communicate to the parent function that it's finished. +// +//nolint:funlen +func verifyMultipleTCPConnections(loadBalancerIP string, servicePort int32, + finished, ready chan bool, stats *Counter) { + var ( + err error + endPoint string + ) + + By(fmt.Sprintf("Accessing backend pods via %s IP", loadBalancerIP)) + + myIP, err := netip.ParseAddr(loadBalancerIP) + + Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address") + + if myIP.Is4() { + endPoint = fmt.Sprintf("%s:%d", loadBalancerIP, servicePort) + } + + if myIP.Is6() { + endPoint = fmt.Sprintf("[%s]:%d", loadBalancerIP, servicePort) + } + + getHTTPMsg := []byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", endPoint)) + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Resolving TCP endpoint %q", endPoint) + + addr, err := net.ResolveTCPAddr("tcp", endPoint) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed resolve TCP address %q : %v", endPoint, err) + + ready <- true + + return + } + + var stop bool + + for !stop { + select { + case <-finished: + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Received data on 'finished' channel") + + stop = true + default: + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Dialing to the TCP endpoint %q", endPoint) + + lbConnection, err := mTCPConnect(addr, int(300)) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed dailing to %q : %v", endPoint, err) + + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed dailing to %s : %v", endPoint, err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + continue + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t -> Successfully connected to %s", endPoint) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t -> Local Address %s", lbConnection.LocalAddr()) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t -> Remote Address %s", lbConnection.RemoteAddr()) + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Writing to the TCP connection at %v", time.Now().UnixMilli()) + + nSent, err := lbConnection.Write(getHTTPMsg) + + if err != nil || nSent == 0 { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to write to connection: %v", err) + + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed to write to connection: %v", err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + lbConnection.Close() + + continue + } + + err = lbConnection.SetReadDeadline(time.Now().Add(300 * time.Millisecond)) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed set ReadDeadline: %v", err) + + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed to set ReadDeadline %v", err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + lbConnection.Close() + + continue + } + + msgReply := make([]byte, 1024) + + bRead, err := lbConnection.Read(msgReply) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to read reply(%v): %v", + time.Now().UnixMilli(), err) + stats.CounterLock.Lock() + stats.Count++ + stats.Failures = append(stats.Failures, + Event{Msg: fmt.Sprintf("Failed to read reply: %v", err), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + + lbConnection.Close() + + continue + } else { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Read %d bytes", bRead) + + stats.CounterLock.Lock() + stats.Count++ + stats.Success = append(stats.Success, + Event{Msg: string(msgReply[0:bRead]), Timestamp: time.Now()}) + stats.CounterLock.Unlock() + } + + lbConnection.Close() + } + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("go routine 'verifyMultipleTCPConnections' finished") + + stats.CounterLock.Lock() + stats.End = time.Now() + stats.CounterLock.Unlock() + + ready <- true +} + +// restartMetallbFRRPod finds metallb-frr pod running on a node, +// deletes it, and waits for a new pod to start and reach ready state. +func restartMetallbFRRPod(node string, rebootFailed *bool, finished chan bool) { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Looking for a metallb-frr pod on %q node", node) + + mPodList, err := pod.List(APIClient, rdscoreparams.MetalLBOperatorNamespace, + metav1.ListOptions{LabelSelector: rdscoreparams.MetalLBFRRPodSelector}) + + if len(mPodList) == 0 { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to found pod: %v", err) + + *rebootFailed = true + + finished <- true + + return + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Filter pod running on %q", node) + + var prevPod *pod.Builder + + for _, _pod := range mPodList { + if _pod.Definition.Spec.NodeName == node { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Found pod %q running on %q", + _pod.Definition.Name, node) + + prevPod = _pod + + break + } + } + + if prevPod == nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to found frr-k8s pod running on %q", node) + + *rebootFailed = true + + finished <- true + + return + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Deleting pod %q", prevPod.Definition.Name) + + prevPod, err = prevPod.DeleteAndWait(15 * time.Second) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to delete pod %q: %v", + prevPod.Definition.Name, err) + + *rebootFailed = true + + finished <- true + + return + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Looking for a new metallb-frr pod") + + var newPod *pod.Builder + + for i := 0; i < 60; i++ { + mPodList, err = pod.List(APIClient, "metallb-system", + metav1.ListOptions{LabelSelector: "app=frr-k8s"}) + + if len(mPodList) == 0 { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to found pod: %v", err) + + *rebootFailed = true + + time.Sleep(1 * time.Second) + + continue + } + + for _, _pod := range mPodList { + if _pod.Definition.Spec.NodeName == node { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Found pod running on %q", node) + + newPod = _pod + + break + } + } + + if newPod.Definition.Name == prevPod.Definition.Name { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("No new frr-k8s pod found") + + *rebootFailed = true + newPod = nil + + time.Sleep(1 * time.Second) + + continue + } else { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t---> New frr-k8s pod found: %q", + newPod.Definition.Name) + + *rebootFailed = false + + break + } + } + + if newPod == nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("No new frr-k8s pod found on %q node", node) + + *rebootFailed = true + + finished <- true + + return + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Waiting for pod %q to be Ready", + newPod.Definition.Name) + + err = newPod.WaitUntilReady(3 * time.Minute) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Pod %q hasn't reached Ready state", + newPod.Definition.Name) + + *rebootFailed = true + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Goroutine restartMetallbFRRPod finished") + + finished <- true +} + +//nolint:funlen +func verifyGracefulFlow(svcName string) { + By(fmt.Sprintf("Pulling %q service configuration", svcName)) + + var ( + svcBuilder *service.Builder + err error + ctx SpecContext + ) + + Eventually(func() bool { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Pulling %q service from %q namespace", + svcName, RDSCoreConfig.GracefulRestartServiceNS) + + svcBuilder, err = service.Pull(APIClient, svcName, RDSCoreConfig.GracefulRestartServiceNS) + + if err != nil { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Error pulling %q service from %q namespace: %v", + svcName, RDSCoreConfig.GracefulRestartServiceNS, err) + + return false + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Successfully pulled %q service from %q namespace", + svcBuilder.Definition.Name, svcBuilder.Definition.Namespace) + + return true + }).WithContext(ctx).WithPolling(5*time.Second).WithTimeout(1*time.Minute).Should(BeTrue(), + fmt.Sprintf("Error obtaining service %q configuration", svcName)) + + By(fmt.Sprintf("Asserting service %q has LoadBalancer IP address", svcBuilder.Definition.Name)) + + Eventually(func() bool { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Check service %q in %q namespace has LoadBalancer IP", + svcBuilder.Definition.Name, svcBuilder.Definition.Namespace) + + refreshSVC := svcBuilder.Exists() + + if !refreshSVC { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to refresh service status") + + return false + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Service has %d IP addresses", + len(svcBuilder.Object.Status.LoadBalancer.Ingress)) + + return len(svcBuilder.Object.Status.LoadBalancer.Ingress) != 0 + }).WithContext(ctx).WithPolling(15*time.Second).WithTimeout(3*time.Minute).Should(BeTrue(), + "Service does not have LoadBalancer IP address") + + mPodList, err := pod.List(APIClient, RDSCoreConfig.GracefulRestartServiceNS, + metav1.ListOptions{LabelSelector: RDSCoreConfig.GracefulRestartAppLabel}) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to find pods due to: %v", err)) + + Expect(len(mPodList)).ToNot(BeZero(), "0 pods found") + Expect(len(mPodList)).To(Equal(1), "More then 1 pod found") + + nodeName := mPodList[0].Object.Spec.NodeName + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Will crash metallb-frr pod on %q node during a test", nodeName) + + finished := make(chan bool) + ready := make(chan bool) + + statistics := Counter{Start: time.Now()} + + for _, vip := range svcBuilder.Object.Status.LoadBalancer.Ingress { + loadBalancerIP := vip.IP + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Accessing workload via LoadBalancer's IP %s", loadBalancerIP) + + myIP, err := netip.ParseAddr(loadBalancerIP) + + Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address") + + if myIP.Is6() { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv6 not supported at the moment") + + // loadBalancerIP = fmt.Sprintf("[%s]", loadBalancerIP) + + continue + } + + normalizedPort, err := strconv.Atoi(RDSCoreConfig.GracefulRestartAppServicePort) + + Expect(err).ToNot(HaveOccurred(), "Failed to part service port") + + go verifySingleTCPConnection(loadBalancerIP, int32(normalizedPort), finished, ready, &statistics) + } + + // + // Remove metallb-frr pod + // + + var rebootFailed = false + + // go func(node string, rebootFailed *bool, finished chan bool) { + // ... + // }(nodeName, &rebootFailed, finished) + + go restartMetallbFRRPod(nodeName, &rebootFailed, finished) + + // Wait for metallb-frr pod to be restarted. + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t *** Waiting for go routines to finish ***") + + select { + case <-ready: + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("*** Received signal on 'ready' channel ***") + close(ready) + + if rebootFailed { + Fail("Error during metallb-frr pod restart") + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t>>> There were %d requests", statistics.Count) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t>>> Failed requests: %d", len(statistics.Failures)) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t>>> Successful requests: %d", len(statistics.Success)) + + if len(statistics.Failures) != 0 { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Failed request 0: %s", statistics.Failures[0].Msg) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Failed request 0: %v", statistics.Failures[0].Timestamp) + lIdx := len(statistics.Failures) - 1 + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Failed request last: %s", statistics.Failures[lIdx].Msg) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Failed request last: %v", statistics.Failures[lIdx].Timestamp) + } + + if len(statistics.Success) != 0 { + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Success request 0: %s", statistics.Success[0].Msg) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Success request 0: %v", statistics.Success[0].Timestamp) + lIdx := len(statistics.Success) - 1 + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Success request last: %s", statistics.Success[lIdx].Msg) + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t\t>>> Success request last: %v", statistics.Success[lIdx].Timestamp) + } + + // TODO(yprokule): update timeout after testing + case <-time.After(5 * time.Minute): + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Go routines canceled after 5 minutes") + + Fail("Error checking status of MetalLB Graceful Restart due to timeout") + } + + glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Finished waiting for go routines") +} + +// VerifyGracefulRestartETPCluster check MetalLB graceful restart. +func VerifyGracefulRestartETPCluster() { + verifyGracefulFlow(RDSCoreConfig.GracefulRestartServiceName) +} diff --git a/tests/system-tests/rdscore/internal/rdscoreconfig/config.go b/tests/system-tests/rdscore/internal/rdscoreconfig/config.go index c66755d62..ae95a8ff5 100644 --- a/tests/system-tests/rdscore/internal/rdscoreconfig/config.go +++ b/tests/system-tests/rdscore/internal/rdscoreconfig/config.go @@ -162,6 +162,8 @@ type CoreConfig struct { IPVlanNADThreeName string `yaml:"rdscore_ipvlan_nad_three_name" envconfig:"ECO_SYSTEM_RDSCORE_IPVLAN_NAD_THREE_NAME"` IPVlanNADFourName string `yaml:"rdscore_ipvlan_nad_four_name" envconfig:"ECO_SYSTEM_RDSCORE_IPVLAN_NAD_FOUR_NAME"` //nolint:lll + GracefulRestartAppLabel string `yaml:"rdscore_graceful_restart_app_label" envconfig:"ECO_RDSCORE_GRACEFUL_RESTART_APP_LABEL"` + //nolint:lll PerformanceProfileHTName string `yaml:"rdscore_performance_profile_ht_name" envconfig:"ECO_RDS_CORE_PERFORMANCE_PROFILE_HT_NAME"` //nolint:lll KDumpWorkerMCPNodeLabel string `yaml:"rdscore_kdump_worker_node_label" envconfig:"ECO_RDSCORE_KDUMP_WORKER_NODE_LABEL"` @@ -353,6 +355,12 @@ type CoreConfig struct { StorageCephRBDSCName string `yaml:"rdscore_sc_cephrbd_name" envconfig:"ECO_RDSCORE_SC_CEPHRBD_NAME"` EgressServiceNS string `yaml:"rdscore_egress_service_ns" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_NS"` //nolint:lll,nolintlint + GracefulRestartServiceNS string `yaml:"rdscore_graceful_restart_service_ns" envconfig:"ECO_RDSCORE_GRACEFUL_RESTART_SERVICE_NS"` + //nolint:lll,nolintlint + GracefulRestartServiceName string `yaml:"rdscore_graceful_restart_service_name" envconfig:"ECO_RDSCORE_GRACEFUL_RESTART_SERVICE_NAME"` + //nolint:lll,nolintlint + GracefulRestartAppServicePort string `yaml:"rdscore_graceful_restart_service_port" envconfig:"ECO_RDSCORE_GRACEFUL_RESTART_SERVICE_PORT"` + //nolint:lll,nolintlint EgressServiceRemoteIP string `yaml:"rdscore_egress_service_remote_ip" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_REMOTE_IP"` //nolint:lll,nolintlint EgressServiceRemoteIPv6 string `yaml:"rdscore_egress_service_remote_ipv6" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_REMOTE_IPV6"` diff --git a/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml b/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml index 1b8abcf64..42e2433ec 100644 --- a/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml +++ b/tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml @@ -148,3 +148,8 @@ rdscore_egress_service_2_node_selector: node-role.kubernetes.io/worker: '' rdscore_egress_service_deploy_2_img: '' rdscore_egress_service_deploy_2_ipaddr_pool: 'rds-local-pool' +# MetalLB graceful restart +rdscore_graceful_restart_service_ns: rds-graceful-app +rdscore_graceful_restart_service_name: rds-graceful-app +rdscore_graceful_restart_app_label: 'rds-app=rds-core-metallb-integration' +rdscore_graceful_restart_service_port: '8080' diff --git a/tests/system-tests/rdscore/tests/00_validate_top_level.go b/tests/system-tests/rdscore/tests/00_validate_top_level.go index 067503148..bd6025ca5 100644 --- a/tests/system-tests/rdscore/tests/00_validate_top_level.go +++ b/tests/system-tests/rdscore/tests/00_validate_top_level.go @@ -25,6 +25,10 @@ var _ = Describe( ContinueOnFailure, Label("rds-core-workflow"), func() { Context("Configured Cluster", Label("clean-cluster"), func() { + It("Verify MetalLB Graceful Restart", + Label("metallb-graceful"), + rdscorecommon.VerifyGracefulRestartETPCluster) + It("Verify EgressService with Cluster ExternalTrafficPolicy", Label("egress", "egress-etp-cluster"), reportxml.ID("76485"), rdscorecommon.VerifyEgressServiceWithClusterETP)