From 2cdf37f703830f916fae2c3a3ca7387881485ef3 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 13 Sep 2023 17:57:45 +0530 Subject: [PATCH] e2e: added test to verify read affinity functionality e2e test case is added to test if read affinity is enabled by verifying read_from_replica=localize option is passed Signed-off-by: Praveen M --- e2e/deployment.go | 18 ++++++++++++--- e2e/pod.go | 57 +++++++++++++++++++++++++++++++++++++++++++++++ e2e/rbd.go | 30 ++++++++++++++++++++++--- e2e/utils.go | 9 ++++++++ 4 files changed, 108 insertions(+), 6 deletions(-) diff --git a/e2e/deployment.go b/e2e/deployment.go index f4f63642af0c..27a831791e96 100644 --- a/e2e/deployment.go +++ b/e2e/deployment.go @@ -231,15 +231,19 @@ func (yr *yamlResource) Do(action kubectlAction) error { // replaceNamespaceInTemplate() on it. There are several options for adjusting // templates, each has their own comment. type yamlResourceNamespaced struct { - filename string - namespace string + filename string + namespace string + domainLabel string + crushLocationLabels string // set the number of replicas in a Deployment to 1. oneReplica bool // enable topology support (for RBD) enableTopology bool - domainLabel string + + // enable read affinity support (for RBD) + enableReadAffinity bool } func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error { @@ -260,6 +264,14 @@ func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error { data = addTopologyDomainsToDSYaml(data, yrn.domainLabel) } + if yrn.enableReadAffinity { + data = enableReadAffinityInTemplate(data) + } + + if yrn.crushLocationLabels != "" { + data = addCrsuhLocationLabels(data, yrn.crushLocationLabels) + } + err = retryKubectlInput(yrn.namespace, action, data, deployTimeout) if err != nil { return fmt.Errorf("failed to %s resource %q in namespace %q: %w", action, yrn.filename, yrn.namespace, err) diff --git a/e2e/pod.go b/e2e/pod.go index 427189b58bc4..1980206bc329 100644 --- a/e2e/pod.go +++ b/e2e/pod.go @@ -623,3 +623,60 @@ func verifySeLinuxMountOption( return nil } + +func verifyReadAffinity( + f *framework.Framework, + pvcPath, appPath, daemonSetName, cn, ns string, +) error { + readFromReplicaOption := fmt.Sprintf("read_from_replica=localize,crush_location=%s:%s|%s:%s", + strings.Split(crushLocationRegionLabel, "/")[1], crushLocationRegionValue, + strings.Split(crushLocationZoneLabel, "/")[1], crushLocationZoneValue, + ) + + // create PVC + pvc, err := loadPVC(pvcPath) + if err != nil { + return fmt.Errorf("failed to load pvc: %w", err) + } + pvc.Namespace = f.UniqueName + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + return fmt.Errorf("failed to create PVC: %w", err) + } + app, err := loadApp(appPath) + if err != nil { + return fmt.Errorf("failed to load application: %w", err) + } + app.Namespace = f.UniqueName + err = createApp(f.ClientSet, app, deployTimeout) + if err != nil { + return fmt.Errorf("failed to create application: %w", err) + } + + pod, err := f.ClientSet.CoreV1().Pods(f.UniqueName).Get(context.TODO(), app.Name, metav1.GetOptions{}) + if err != nil { + framework.Logf("Error occurred getting pod %s in namespace %s", app.Name, f.UniqueName) + + return fmt.Errorf("failed to get pod: %w", err) + } + + nodepluginPodName, err := getDaemonsetPodOnNode(f, daemonSetName, pod.Spec.NodeName, ns) + if err != nil { + return fmt.Errorf("failed to get daemonset pod on node: %w", err) + } + logs, err := frameworkPod.GetPodLogs(context.TODO(), f.ClientSet, ns, nodepluginPodName, cn) + if err != nil { + return fmt.Errorf("failed to get pod logs from container %s/%s/%s : %w", ns, nodepluginPodName, cn, err) + } + + if !strings.Contains(logs, readFromReplicaOption) { + return fmt.Errorf("read from relica option %s not found in logs: %s", readFromReplicaOption, logs) + } + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + return fmt.Errorf("failed to delete PVC and application: %w", err) + } + + return nil +} diff --git a/e2e/rbd.go b/e2e/rbd.go index 3361a39e5f22..3a6c58329dbd 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -65,6 +65,12 @@ var ( rbdTopologyPool = "newrbdpool" rbdTopologyDataPool = "replicapool" // NOTE: should be different than rbdTopologyPool for test to be effective + // CRUSH location node labels & values. + crushLocationRegionLabel = "topology.kubernetes.io/region" + crushLocationRegionValue = "east" + crushLocationZoneLabel = "topology.kubernetes.io/zone" + crushLocationZoneValue = "east-zone1" + // yaml files required for deployment. pvcPath = rbdExamplePath + "pvc.yaml" appPath = rbdExamplePath + "pod.yaml" @@ -161,9 +167,11 @@ func createORDeleteRbdResources(action kubectlAction) { }, // the node-plugin itself &yamlResourceNamespaced{ - filename: rbdDirPath + rbdNodePlugin, - namespace: cephCSINamespace, - domainLabel: nodeRegionLabel + "," + nodeZoneLabel, + filename: rbdDirPath + rbdNodePlugin, + namespace: cephCSINamespace, + domainLabel: nodeRegionLabel + "," + nodeZoneLabel, + enableReadAffinity: true, + crushLocationLabels: crushLocationRegionLabel + "," + crushLocationZoneLabel, }, } @@ -275,6 +283,14 @@ var _ = Describe("RBD", func() { if err != nil { framework.Failf("failed to create node label: %v", err) } + err = createNodeLabel(f, crushLocationRegionLabel, crushLocationRegionValue) + if err != nil { + framework.Failf("failed to create node label: %v", err) + } + err = createNodeLabel(f, crushLocationZoneLabel, crushLocationZoneValue) + if err != nil { + framework.Failf("failed to create node label: %v", err) + } if cephCSINamespace != defaultNs { err = createNamespace(c, cephCSINamespace) if err != nil { @@ -444,6 +460,14 @@ var _ = Describe("RBD", func() { }) } + By("verify readAffinity support", func() { + err := verifyReadAffinity(f, pvcPath, appPath, + rbdDaemonsetName, rbdContainerName, cephCSINamespace) + if err != nil { + framework.Failf("failed to verify readAffinity: %v", err) + } + }) + By("verify mountOptions support", func() { err := verifySeLinuxMountOption(f, pvcPath, appPath, rbdDaemonsetName, rbdContainerName, cephCSINamespace) diff --git a/e2e/utils.go b/e2e/utils.go index 48ee28c92adb..67e413f5e384 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -827,6 +827,15 @@ func enableTopologyInTemplate(data string) string { return strings.ReplaceAll(data, "--feature-gates=Topology=false", "--feature-gates=Topology=true") } +func enableReadAffinityInTemplate(template string) string { + return strings.ReplaceAll(template, "# - \"--enable-read-affinity=true\"", "- \"--enable-read-affinity=true\"") +} + +func addCrsuhLocationLabels(template, labels string) string { + return strings.ReplaceAll(template, "# - \"--crush-location-labels=topology.io/zone,topology.io/rack\"", + "- \"--crush-location-labels="+labels+"\"") +} + func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) (string, error) { filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" // write data in PVC