e2e: test vgs beta api

build.env

@@ -15,7 +15,8 @@ CSI_IMAGE_VERSION=canary
 CSI_UPGRADE_VERSION=v3.12.1
 
 # Ceph version to use
-BASE_IMAGE=quay.io/ceph/ceph:v19
+BASE_IMAGE=quay.ceph.io/ceph-ci/ceph:main
+#BASE_IMAGE=quay.io/ceph/ceph:v19
 CEPH_VERSION=squid
 
 # standard Golang options

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -145,14 +145,14 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: quay.io/madhupr001/csi-snapshotter:latest
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
             - "--timeout=150s"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots=true"
+            - "--feature-gates=CSIVolumeGroupSnapshot=true"
             - "--http-endpoint=$(POD_IP):8092"
           env:
             - name: ADDRESS
@@ -195,9 +195,7 @@ spec:
           imagePullPolicy: "IfNotPresent"
       volumes:
         - name: socket-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: host-sys
           hostPath:
             path: /sys
@@ -214,9 +212,7 @@ spec:
           configMap:
             name: ceph-csi-config
         - name: keys-tmp-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: ceph-csi-encryption-kms-config
           configMap:
             name: ceph-csi-encryption-kms-config

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -134,14 +134,14 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: quay.io/madhupr001/csi-snapshotter:latest
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
             - "--timeout=150s"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots=true"
+            - "--feature-gates=CSIVolumeGroupSnapshot=true"
             - "--http-endpoint=$(POD_IP):8092"
           env:
             - name: ADDRESS
@@ -265,9 +265,7 @@ spec:
           hostPath:
             path: /lib/modules
         - name: socket-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: ceph-config
           configMap:
             name: ceph-config
@@ -278,9 +276,7 @@ spec:
           configMap:
             name: ceph-csi-encryption-kms-config
         - name: keys-tmp-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: oidc-token
           projected:
             sources:

e2e/cephfs.go

@@ -349,6 +349,31 @@ var _ = Describe(cephfsType, func() {
 				}
 			})
 
+			By("test volumeGroupSnapshot", func() {
+				err := createCephfsStorageClass(f.ClientSet, f, true, nil)
+				if err != nil {
+					framework.Failf("failed to create CephFS storageclass: %v", err)
+				}
+				scName := "csi-cephfs-sc"
+				snapshotter, err := newCephFSVolumeGroupSnapshot(f, f.UniqueName, scName, false, deployTimeout, 20)
+				if err != nil {
+					framework.Failf("failed to create volumeGroupSnapshot Base: %v", err)
+				}
+
+				snapTestErr := snapshotter.TestVolumeGroupSnapshot()
+
+				if snapTestErr != nil {
+					framework.Failf("failed to test volumeGroupSnapshot: %v", snapTestErr)
+				}
+				err = deleteResource(cephFSExamplePath + "storageclass.yaml")
+				if err != nil {
+					framework.Failf("failed to delete CephFS storageclass: %v", err)
+				}
+
+				framework.Logf("[Debug] volumeGroupSnapshot test passed")
+				return
+			})
+
 			By("validate fuseMountOptions", func() {
 				params := map[string]string{
 					"mounter":          "fuse",
@@ -2478,19 +2503,6 @@ var _ = Describe(cephfsType, func() {
 				}
 			})
 
-			By("test volumeGroupSnapshot", func() {
-				scName := "csi-cephfs-sc"
-				snapshotter, err := newCephFSVolumeGroupSnapshot(f, f.UniqueName, scName, false, deployTimeout, 3)
-				if err != nil {
-					framework.Failf("failed to create volumeGroupSnapshot Base: %v", err)
-				}
-
-				err = snapshotter.TestVolumeGroupSnapshot()
-				if err != nil {
-					framework.Failf("failed to test volumeGroupSnapshot: %v", err)
-				}
-			})
-
 			By("verify rados objects are within a namespace", func() {
 				updateRadosNamespace := func(radosNamespaceName string) {
 					framework.Logf("updating configmap with rados namespace %s", radosNamespace)

e2e/e2e_test.go

@@ -39,7 +39,7 @@ func init() {
 	flag.BoolVar(&deployNFS, "deploy-nfs", false, "deploy nfs csi driver")
 	flag.BoolVar(&testCephFS, "test-cephfs", true, "test cephFS csi driver")
 	flag.BoolVar(&testCephFSFscrypt, "test-cephfs-fscrypt", false, "test CephFS csi driver fscrypt support")
-	flag.BoolVar(&testRBD, "test-rbd", true, "test rbd csi driver")
+	flag.BoolVar(&testRBD, "test-rbd", false, "test rbd csi driver")
 	flag.BoolVar(&testRBDFSCrypt, "test-rbd-fscrypt", false, "test rbd csi driver fscrypt support")
 	flag.BoolVar(&testNBD, "test-nbd", false, "test rbd csi driver with rbd-nbd mounter")
 	flag.BoolVar(&testNFS, "test-nfs", false, "test nfs csi driver")
@@ -59,6 +59,8 @@ func init() {
 	framework.AfterReadingAllFlags(&framework.TestContext)
 
 	fmt.Println("timeout for deploytimeout ", deployTimeout)
+	testRBD = true
+	testNFS = false
 }
 
 func setDefaultKubeconfig() {
@@ -88,7 +90,7 @@ func handleFlags() {
 	// variable, it is still possible to only run the NFS tests, when both
 	// CephFS and RBD are disabled.
 	if testCephFS {
-		testNFS = testCephFS
-		deployNFS = deployCephFS
+		//testNFS = testCephFS
+		//deployNFS = deployCephFS
 	}
 }

e2e/rbd.go

@@ -550,6 +550,29 @@ var _ = Describe("RBD", func() {
 				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
+			By("test volumeGroupSnapshot", func() {
+				supported, err := librbdSupportsVolumeGroupSnapshot(f)
+				if err != nil {
+					framework.Failf("failed to check for VolumeGroupSnapshot support: %v", err)
+				}
+				if !supported {
+					Skip("librbd does not support required VolumeGroupSnapshot function(s)")
+				}
+
+				scName := "csi-rbd-sc"
+				snapshotter, err := newRBDVolumeGroupSnapshot(f, f.UniqueName, scName, false, deployTimeout, 20)
+				if err != nil {
+					framework.Failf("failed to create RBDVolumeGroupSnapshot: %v", err)
+				}
+
+				snapTestErr := snapshotter.TestVolumeGroupSnapshot()
+				if snapTestErr != nil {
+					framework.Failf("failed to test volumeGroupSnapshot: %v", snapTestErr)
+				}
+
+				framework.Logf("[Debug] volumeGroupSnapshot test passed")
+				return
+			})
 			By("reattach the old PV to a new PVC and check if PVC metadata is updated on RBD image", func() {
 				reattachPVCNamespace := f.Namespace.Name + "-2"
 				pvc, err := loadPVC(pvcPath)
@@ -4871,27 +4894,6 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
-			By("test volumeGroupSnapshot", func() {
-				supported, err := librbdSupportsVolumeGroupSnapshot(f)
-				if err != nil {
-					framework.Failf("failed to check for VolumeGroupSnapshot support: %v", err)
-				}
-				if !supported {
-					Skip("librbd does not support required VolumeGroupSnapshot function(s)")
-				}
-
-				scName := "csi-rbd-sc"
-				snapshotter, err := newRBDVolumeGroupSnapshot(f, f.UniqueName, scName, false, deployTimeout, 3)
-				if err != nil {
-					framework.Failf("failed to create RBDVolumeGroupSnapshot: %v", err)
-				}
-
-				err = snapshotter.TestVolumeGroupSnapshot()
-				if err != nil {
-					framework.Failf("failed to test volumeGroupSnapshot: %v", err)
-				}
-			})
-
 			// delete RBD provisioner secret
 			err := deleteCephUser(f, keyringRBDProvisionerUsername)
 			if err != nil {

e2e/utils.go

@@ -1614,6 +1614,9 @@ const (
 	kubectlCreate = kubectlAction("create")
 	// kubectlDelete tells retryKubectlInput() to run "delete".
 	kubectlDelete = kubectlAction("delete")
+
+	//kubectlGet tells retryKubectlInput() to run "get".
+	kubectlGet = kubectlAction("get")
 )
 
 // String returns the string format of the kubectlAction, this is automatically
@@ -1637,7 +1640,7 @@ func retryKubectlInput(namespace string, action kubectlAction, data string, t in
 		}
 		cmd = append(cmd, []string{string(action), "-f", "-"}...)
 
-		_, err := e2ekubectl.RunKubectlInput(namespace, data, cmd...)
+		output, err := e2ekubectl.RunKubectlInput(namespace, data, cmd...)
 		if err != nil {
 			if isRetryableAPIError(err) {
 				return false, nil
@@ -1656,7 +1659,7 @@ func retryKubectlInput(namespace string, action kubectlAction, data string, t in
 
 			return false, fmt.Errorf("failed to run kubectl: %w", err)
 		}
-
+		framework.Logf("kubectl output: %s\n", output)
 		return true, nil
 	})
 }

e2e/volumegroupsnapshot.go

@@ -18,9 +18,10 @@ package e2e
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 
-	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
@@ -78,17 +79,29 @@ func (c *cephFSVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi
 		return fmt.Errorf("failed getting cephFS metadata pool name: %w", err)
 	}
 
-	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	vgsc, err := c.groupclient.VolumeGroupSnapshotContents().Get(ctx, *vgs.Status.BoundVolumeGroupSnapshotContentName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
+	sourcePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	// we are creating clones for each source PVC
-	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	clonePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	totalPVCCount := sourcePVCCount + clonePVCCount
 	validateSubvolumeCount(c.framework, totalPVCCount, fileSystemName, subvolumegroup)
 
 	// we are creating 1 snapshot for each source PVC, validate the snapshot count
-	for _, pvcSnap := range vgs.Status.PVCVolumeSnapshotRefList {
+	for _, snapshot := range vgsc.Status.VolumeSnapshotHandlePairList {
+		volumeHandle := snapshot.VolumeHandle
+		volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
+			string(vgsc.UID)+volumeHandle)))
+		volumeSnapshot, err := c.snapClient.VolumeSnapshots(vgs.Namespace).Get(ctx, volumeSnapshotName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to get VolumeSnapshot: %w", err)
+		}
+		pvcName := *volumeSnapshot.Spec.Source.PersistentVolumeClaimName
 		pvc, err := c.framework.ClientSet.CoreV1().PersistentVolumeClaims(vgs.Namespace).Get(ctx,
-			pvcSnap.PersistentVolumeClaimRef.Name,
-			metav1.GetOptions{})
+			pvcName, metav1.GetOptions{})
 		if err != nil {
 			return fmt.Errorf("failed to get PVC: %w", err)
 		}
@@ -165,8 +178,14 @@ func (rvgs *rbdVolumeGroupSnapshot) GetVolumeGroupSnapshotClass() (*groupsnapapi
 }
 
 func (rvgs *rbdVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error {
-	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
-	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+
+	vgsc, err := rvgs.groupclient.VolumeGroupSnapshotContents().Get(context.TODO(), *vgs.Status.BoundVolumeGroupSnapshotContentName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
+	sourcePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
+	clonePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	totalPVCCount := sourcePVCCount + clonePVCCount
 
 	validateOmapCount(rvgs.framework, totalPVCCount, rbdType, defaultRBDPool, volumesType)