From 575ea8197671bf0812e6fb5351bdbc098a23fe5b Mon Sep 17 00:00:00 2001 From: chihyuwu Date: Tue, 27 Aug 2024 10:00:29 +0800 Subject: [PATCH] Update to version 1.2.0 --- Dockerfile | 2 +- Makefile | 2 +- README.md | 29 +++- deploy/example/storageclass-nfs.yaml | 15 ++ deploy/helm/Chart.yaml | 2 +- deploy/kubernetes/v1.19/controller.yml | 2 +- deploy/kubernetes/v1.19/node.yml | 2 +- .../v1.19/snapshotter/snapshotter.yaml | 2 +- deploy/kubernetes/v1.20/controller.yml | 2 +- deploy/kubernetes/v1.20/node.yml | 2 +- .../v1.20/snapshotter/snapshotter.yaml | 2 +- pkg/driver/controllerserver.go | 46 +++++- pkg/driver/driver.go | 9 +- pkg/driver/nfs_utils.go | 40 +++++ pkg/driver/nodeserver.go | 140 +++++++++++++++++- pkg/dsm/service/dsm.go | 89 +++++++++-- pkg/dsm/service/share_volume.go | 21 ++- pkg/dsm/webapi/share.go | 89 +++++++++++ pkg/models/dsm_req_spec.go | 2 + pkg/utils/utils.go | 1 + scripts/deploy.sh | 29 +--- scripts/functions.sh | 29 ++++ scripts/uninstall.sh | 27 +--- 23 files changed, 481 insertions(+), 103 deletions(-) create mode 100644 deploy/example/storageclass-nfs.yaml create mode 100644 pkg/driver/nfs_utils.go create mode 100644 scripts/functions.sh diff --git a/Dockerfile b/Dockerfile index 3fe00fc..b1851bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ FROM alpine:latest LABEL maintainers="Synology Authors" \ description="Synology CSI Plugin" -RUN apk add --no-cache e2fsprogs e2fsprogs-extra xfsprogs xfsprogs-extra blkid util-linux iproute2 bash btrfs-progs ca-certificates cifs-utils +RUN apk add --no-cache e2fsprogs e2fsprogs-extra xfsprogs xfsprogs-extra blkid util-linux iproute2 bash btrfs-progs ca-certificates cifs-utils nfs-utils # Create symbolic link for chroot.sh WORKDIR / diff --git a/Makefile b/Makefile index 3bc209e..0717774 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REGISTRY_NAME=synology IMAGE_NAME=synology-csi -IMAGE_VERSION=v1.1.3 +IMAGE_VERSION=v1.2.0 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION) # For now, only build linux/amd64 platform diff --git a/README.md b/README.md index 57470ec..eaff2ac 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The official [Container Storage Interface](https://github.com/container-storage- Driver Name: csi.san.synology.com | Driver Version | Image | Supported K8s Version | | -------------------------------------------------------------------------------- | --------------------------------------------------------------------- | --------------------- | -| [v1.1.3](https://github.com/SynologyOpenSource/synology-csi/tree/release-v1.1.3) | [synology-csi:v1.1.3](https://hub.docker.com/r/synology/synology-csi) | 1.20+ | +| [v1.2.0](https://github.com/SynologyOpenSource/synology-csi/tree/release-v1.2.0) | [synology-csi:v1.2.0](https://hub.docker.com/r/synology/synology-csi) | 1.20+ | @@ -158,17 +158,36 @@ Create and apply StorageClasses with the properties you want. allowVolumeExpansion: true ``` + **NFS Protocol** + ``` + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: synostorage-nfs + provisioner: csi.san.synology.com + parameters: + protocol: "nfs" + dsm: "192.168.1.1" + location: '/volume1' + mountPermissions: '0755' + mountOptions: + - nfsvers=4.1 + reclaimPolicy: Delete + allowVolumeExpansion: true + ``` + 2. Configure the StorageClass properties by assigning the parameters in the table. You can also leave blank if you don’t have a preference: | Name | Type | Description | Default | Supported protocols | | ------------------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | ------------------- | - | *dsm* | string | The IPv4 address of your DSM, which must be included in the `client-info.yml` for the CSI driver to log in to DSM | - | iSCSI, SMB | - | *location* | string | The location (/volume1, /volume2, ...) on DSM where the LUN for *PersistentVolume* will be created | - | iSCSI, SMB | + | *dsm* | string | The IPv4 address of your DSM, which must be included in the `client-info.yml` for the CSI driver to log in to DSM | - | iSCSI, SMB, NFS | + | *location* | string | The location (/volume1, /volume2, ...) on DSM where the LUN for *PersistentVolume* will be created | - | iSCSI, SMB, NFS | | *fsType* | string | The formatting file system of the *PersistentVolumes* when you mount them on the pods. This parameter only works with iSCSI. For SMB, the fsType is always ‘cifs‘. | 'ext4' | iSCSI | - | *protocol* | string | The storage backend protocol. Enter ‘iscsi’ to create LUNs or ‘smb‘ to create shared folders on DSM. | 'iscsi' | iSCSI, SMB | - | *formatOptions* | string | Additional options/arguments passed to `mkfs.*` command. See a linux manual that corresponds with your FS of choice. | - | iSCSI | + | *protocol* | string | The storage backend protocol. Enter ‘iscsi’ to create LUNs, or ‘smb‘ or 'nfs' to create shared folders on DSM. | 'iscsi' | iSCSI, SMB, NFS | + | *formatOptions* | string | Additional options/arguments passed to `mkfs.*` command. See a linux manual that corresponds with your FS of choice. | - | iSCSI | | *csi.storage.k8s.io/node-stage-secret-name* | string | The name of node-stage-secret. Required if DSM shared folder is accessed via SMB. | - | SMB | | *csi.storage.k8s.io/node-stage-secret-namespace* | string | The namespace of node-stage-secret. Required if DSM shared folder is accessed via SMB. | - | SMB | + | *mountPermissions* | string | Mounted folder permissions. If set as non-zero, driver will perform `chmod` after mount | '0750' | NFS | **Notice** diff --git a/deploy/example/storageclass-nfs.yaml b/deploy/example/storageclass-nfs.yaml new file mode 100644 index 0000000..e26ae47 --- /dev/null +++ b/deploy/example/storageclass-nfs.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: synology-nfs-storage +provisioner: csi.san.synology.com +parameters: + protocol: "nfs" # required for nfs protocol + mountPermissions: '0777' + # dsm: "1.1.1.1" + # location: '/volume1' +mountOptions: + - nfsvers=4 #3,4,4.1 +reclaimPolicy: Delete +allowVolumeExpansion: true diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index 02d40fd..92ea88a 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: v1.1.3 +appVersion: v1.2.0 name: synology-csi description: A Helm chart for the Synology CSI Driver keywords: diff --git a/deploy/kubernetes/v1.19/controller.yml b/deploy/kubernetes/v1.19/controller.yml index e0af4ad..d6a2eef 100644 --- a/deploy/kubernetes/v1.19/controller.yml +++ b/deploy/kubernetes/v1.19/controller.yml @@ -144,7 +144,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: synology/synology-csi:v1.1.3 + image: synology/synology-csi:v1.2.0 args: - --nodeid=NotUsed - --endpoint=$(CSI_ENDPOINT) diff --git a/deploy/kubernetes/v1.19/node.yml b/deploy/kubernetes/v1.19/node.yml index 6c41392..f136078 100644 --- a/deploy/kubernetes/v1.19/node.yml +++ b/deploy/kubernetes/v1.19/node.yml @@ -86,7 +86,7 @@ spec: securityContext: privileged: true imagePullPolicy: IfNotPresent - image: synology/synology-csi:v1.1.3 + image: synology/synology-csi:v1.2.0 args: - --nodeid=$(KUBE_NODE_NAME) - --endpoint=$(CSI_ENDPOINT) diff --git a/deploy/kubernetes/v1.19/snapshotter/snapshotter.yaml b/deploy/kubernetes/v1.19/snapshotter/snapshotter.yaml index f219d86..a66ef1a 100644 --- a/deploy/kubernetes/v1.19/snapshotter/snapshotter.yaml +++ b/deploy/kubernetes/v1.19/snapshotter/snapshotter.yaml @@ -81,7 +81,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: synology/synology-csi:v1.1.3 + image: synology/synology-csi:v1.2.0 args: - --nodeid=NotUsed - --endpoint=$(CSI_ENDPOINT) diff --git a/deploy/kubernetes/v1.20/controller.yml b/deploy/kubernetes/v1.20/controller.yml index 6d212e5..fb3bebd 100644 --- a/deploy/kubernetes/v1.20/controller.yml +++ b/deploy/kubernetes/v1.20/controller.yml @@ -144,7 +144,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: synology/synology-csi:v1.1.3 + image: synology/synology-csi:v1.2.0 args: - --nodeid=NotUsed - --endpoint=$(CSI_ENDPOINT) diff --git a/deploy/kubernetes/v1.20/node.yml b/deploy/kubernetes/v1.20/node.yml index 6c41392..f136078 100644 --- a/deploy/kubernetes/v1.20/node.yml +++ b/deploy/kubernetes/v1.20/node.yml @@ -86,7 +86,7 @@ spec: securityContext: privileged: true imagePullPolicy: IfNotPresent - image: synology/synology-csi:v1.1.3 + image: synology/synology-csi:v1.2.0 args: - --nodeid=$(KUBE_NODE_NAME) - --endpoint=$(CSI_ENDPOINT) diff --git a/deploy/kubernetes/v1.20/snapshotter/snapshotter.yaml b/deploy/kubernetes/v1.20/snapshotter/snapshotter.yaml index e5d1feb..1e10c6b 100644 --- a/deploy/kubernetes/v1.20/snapshotter/snapshotter.yaml +++ b/deploy/kubernetes/v1.20/snapshotter/snapshotter.yaml @@ -81,7 +81,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: synology/synology-csi:v1.1.3 + image: synology/synology-csi:v1.2.0 args: - --nodeid=NotUsed - --endpoint=$(CSI_ENDPOINT) diff --git a/pkg/driver/controllerserver.go b/pkg/driver/controllerserver.go index a60ffd5..5d2918f 100644 --- a/pkg/driver/controllerserver.go +++ b/pkg/driver/controllerserver.go @@ -69,6 +69,18 @@ func (cs *controllerServer) isVolumeAccessModeSupport(mode csi.VolumeCapability_ return false } +func parseNfsVesrion(ops []string) string { + for _, op := range ops { + if strings.HasPrefix(op, "nfsvers") { + kvpair := strings.Split(op, "=") + if len(kvpair) == 2 { + return kvpair[1] + } + } + } + return "" +} + func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { sizeInByte, err := getSizeByCapacityRange(req.GetCapacityRange()) volName, volCap := req.GetName(), req.GetVolumeCapabilities() @@ -89,7 +101,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if volCap == nil { return nil, status.Errorf(codes.InvalidArgument, "No volume capabilities are provided") } - + var mountOptions []string for _, cap := range volCap { accessMode := cap.GetAccessMode().GetMode() @@ -102,6 +114,10 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } else if accessMode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { multiSession = true } + + if mount := cap.GetMount(); mount != nil { + mountOptions = mount.GetMountFlags() + } } if volContentSrc != nil { @@ -129,8 +145,15 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } // not needed during CreateVolume method - // used only in NodeStageVolume though VolumeContext + // used only in NodeStageVolume through VolumeContext formatOptions := params["formatOptions"] + mountPermissions := params["mountPermissions"] + // check mountPermissions valid + if mountPermissions != "" { + if _, err := strconv.ParseUint(mountPermissions, 8, 32); err != nil { + return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("invalid mountPermissions %s in storage class", mountPermissions)) + } + } lunDescription := "" if _, ok := params["csi.storage.k8s.io/pvc/name"]; ok { @@ -141,6 +164,11 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol lunDescription = pvcNamespace + "/" + pvcName } + nfsVer := parseNfsVesrion(mountOptions) + if nfsVer != "" && !isNfsVersionAllowed(nfsVer) { + return nil, status.Errorf(codes.InvalidArgument, "Unsupported nfsvers: %s", nfsVer) + } + spec := &models.CreateK8sVolumeSpec{ DsmIp: params["dsm"], K8sVolumeName: volName, @@ -156,6 +184,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol SourceSnapshotId: srcSnapshotId, SourceVolumeId: srcVolumeId, Protocol: protocol, + NfsVersion: nfsVer, } // idempotency @@ -172,7 +201,8 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } if (k8sVolume.Protocol == utils.ProtocolIscsi && k8sVolume.SizeInBytes != sizeInByte) || - (k8sVolume.Protocol == utils.ProtocolSmb && utils.BytesToMB(k8sVolume.SizeInBytes) != utils.BytesToMBCeil(sizeInByte)) { + (k8sVolume.Protocol == utils.ProtocolSmb && utils.BytesToMB(k8sVolume.SizeInBytes) != utils.BytesToMBCeil(sizeInByte)) || + (k8sVolume.Protocol == utils.ProtocolNfs && utils.BytesToMB(k8sVolume.SizeInBytes) != utils.BytesToMBCeil(sizeInByte)) { return nil, status.Errorf(codes.AlreadyExists, "Already existing volume name with different capacity") } @@ -182,10 +212,12 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol CapacityBytes: k8sVolume.SizeInBytes, ContentSource: volContentSrc, VolumeContext: map[string]string{ - "dsm": k8sVolume.DsmIp, - "protocol": k8sVolume.Protocol, - "source": k8sVolume.Source, - "formatOptions": formatOptions, + "dsm": k8sVolume.DsmIp, + "protocol": k8sVolume.Protocol, + "source": k8sVolume.Source, + "formatOptions": formatOptions, + "mountPermissions": mountPermissions, + "baseDir": k8sVolume.BaseDir, }, }, }, nil diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 1859b3a..df22b9c 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -25,12 +25,13 @@ import ( const ( DriverName = "csi.san.synology.com" // CSI dirver name - DriverVersion = "1.1.3" + DriverVersion = "1.2.0" ) var ( MultipathEnabled = true - supportedProtocolList = []string{utils.ProtocolIscsi, utils.ProtocolSmb} + supportedProtocolList = []string{utils.ProtocolIscsi, utils.ProtocolSmb, utils.ProtocolNfs} + allowedNfsVersionList = []string{"3", "4", "4.0", "4.1"} ) type IDriver interface { @@ -139,3 +140,7 @@ func (d *Driver) getVolumeCapabilityAccessModes() []*csi.VolumeCapability_Access func isProtocolSupport(protocol string) bool { return utils.SliceContains(supportedProtocolList, protocol) } + +func isNfsVersionAllowed(ver string) bool { + return utils.SliceContains(allowedNfsVersionList, ver) +} diff --git a/pkg/driver/nfs_utils.go b/pkg/driver/nfs_utils.go new file mode 100644 index 0000000..9cacab8 --- /dev/null +++ b/pkg/driver/nfs_utils.go @@ -0,0 +1,40 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "os" + log "github.com/sirupsen/logrus" +) + +// chmodIfPermissionMismatch only perform chmod when permission mismatches +func chmodIfPermissionMismatch(targetPath string, mode os.FileMode) error { + info, err := os.Lstat(targetPath) + if err != nil { + return err + } + perm := info.Mode() & os.ModePerm + if perm != mode { + log.Infof("chmod targetPath(%s, mode:0%o) with permissions(0%o)", targetPath, info.Mode(), mode) + if err := os.Chmod(targetPath, mode); err != nil { + return err + } + } else { + log.Infof("skip chmod on targetPath(%s) since mode is already 0%o)", targetPath, info.Mode()) + } + return nil +} \ No newline at end of file diff --git a/pkg/driver/nodeserver.go b/pkg/driver/nodeserver.go index 6ec97e7..5aac8ce 100644 --- a/pkg/driver/nodeserver.go +++ b/pkg/driver/nodeserver.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -111,6 +112,21 @@ func getVolumeMountPath(iscsiDevPaths []string) string { return path } +func createTargetMountPathNFS(mounter mount.Interface, mountPath string, mountPermissionsUint uint64) (bool, error) { + notMount, err := mounter.IsLikelyNotMountPoint(mountPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(mountPath, os.FileMode(mountPermissionsUint)); err != nil { + return notMount, err + } + notMount = true + } else { + return false, err + } + } + return notMount, nil +} + func createTargetMountPath(mounter mount.Interface, mountPath string, isBlock bool) (bool, error) { notMount, err := mount.IsNotMountPoint(mounter, mountPath) if err != nil { @@ -262,6 +278,48 @@ func (ns *nodeServer) mountSensitiveWithRetry(sourcePath string, targetPath stri return nil } + +func (ns *nodeServer) setNFSVolumePrivilege(sourcePath string, hostname string, authType utils.AuthType) error { + // NFSTODO: fix the parsing rule + s := strings.Split(strings.TrimPrefix(sourcePath, "//"), "/") + if len(s) != 2 { + return fmt.Errorf("Failed to parse dsmIp and shareName from source path") + } + dsmIp, shareName := s[0], s[1] + + dsm, err := ns.dsmService.GetDsm(dsmIp) + if err != nil { + return fmt.Errorf("Failed to get DSM[%s]", dsmIp) + } + + priv := webapi.SharePrivilege{ + ShareName: shareName, + Rule: []webapi.PrivilegeRule{ + { + Async: true, + Client: hostname, + Crossmnt: true, + Insecure: true, + Privilege: string(authType), + RootSquash: "root", + SecurityFlavor: webapi.SecurityFlavor{ + Kerbros: false, + KerbrosIntegrity: false, + KerbrosPrivacy: false, + Sys: true, + }, + }, + }, + } + + err = dsm.ShareNfsPrivilegeSave(priv) + if err != nil { + log.Printf("Failed to save share NFS privilege. Priv:%v. %v", priv, err) + return err + } + return nil +} + func (ns *nodeServer) setSMBVolumePermission(sourcePath string, userName string, authType utils.AuthType) error { s := strings.Split(strings.TrimPrefix(sourcePath, "//"), "/") if len(s) != 2 { @@ -391,6 +449,13 @@ func (ns *nodeServer) nodeStageSMBVolume(ctx context.Context, spec *models.NodeS return &csi.NodeStageVolumeResponse{}, nil } +func (ns *nodeServer) nodeStageNFSVolume(ctx context.Context, spec *models.NodeStageVolumeSpec) (*csi.NodeStageVolumeResponse, error) { + if err := ns.setNFSVolumePrivilege(spec.Source, "*", utils.AuthTypeReadWrite); err != nil { //NFSTODO: get workernode IP instead of * + return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to set NFS privilege rule, source: %s, err: %v", spec.Source, err)) + } + return &csi.NodeStageVolumeResponse{}, nil +} + func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { volumeId, stagingTargetPath, volumeCapability := req.GetVolumeId(), req.GetStagingTargetPath(), req.GetVolumeCapability() @@ -416,6 +481,8 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol switch req.VolumeContext["protocol"] { case utils.ProtocolSmb: return ns.nodeStageSMBVolume(ctx, spec, req.GetSecrets()) + case utils.ProtocolNfs: + return ns.nodeStageNFSVolume(ctx, spec) default: return ns.nodeStageISCSIVolume(ctx, spec) } @@ -461,7 +528,71 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis isBlock := req.GetVolumeCapability().GetBlock() != nil // raw block, only for iscsi protocol fsType := req.GetVolumeCapability().GetMount().GetFsType() + options := []string{} + if req.GetReadonly() { + options = append(options, "ro") + } + + // nfs + if req.VolumeContext["protocol"] == utils.ProtocolNfs { + options = append(options, req.GetVolumeCapability().GetMount().GetMountFlags()...) + + var server, baseDir string //NFSTODO: subDir + var mountPermissionsUint uint64 = 0750 // default + for k, v := range req.GetVolumeContext() { + switch k { + case "dsm": + server = v + case "baseDir": + baseDir = v + case "mountPermissions": + if v != "" { + var err error + mountPermissionsUint, err = strconv.ParseUint(v, 8, 32) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("invalid mountPermissions %s", v)) + } + } + } + } + + if server == "" || baseDir == "" { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid inputs: server(dsm) and baseDir are required.")) + } + source := fmt.Sprintf("%s:%s", server, baseDir) + + notMount, err := createTargetMountPathNFS(ns.Mounter.Interface, targetPath, mountPermissionsUint) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + if !notMount { + log.Infof("NodePublishVolume: %s is already mounted", targetPath) + return &csi.NodePublishVolumeResponse{}, nil + } + log.Debugf("NodePublishVolume: volumeId(%v) source(%s) targetPath(%s) mountflags(%v)", volumeId, source, targetPath, options) + err = ns.Mounter.Mount(source, targetPath, "nfs", options) + if err != nil { + if os.IsPermission(err) { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + if strings.Contains(err.Error(), "invalid argument") { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + if mountPermissionsUint > 0 { + if err := chmodIfPermissionMismatch(targetPath, os.FileMode(mountPermissionsUint)); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + + log.Debugf("NFS volume(%s) mount %s on %s succeeded", volumeId, source, targetPath) + return &csi.NodePublishVolumeResponse{}, nil + } + + // iscsi & smb notMount, err := createTargetMountPath(ns.Mounter.Interface, targetPath, isBlock) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -470,10 +601,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return &csi.NodePublishVolumeResponse{}, nil } - options := []string{"bind"} - if req.GetReadonly() { - options = append(options, "ro") - } + options = append(options, "bind") switch req.VolumeContext["protocol"] { case utils.ProtocolSmb: @@ -574,7 +702,7 @@ func (ns *nodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVo fmt.Sprintf("Volume[%s] does not exist on the %s", volumeId, volumePath)) } - if k8sVolume.Protocol == utils.ProtocolSmb { + if k8sVolume.Protocol == utils.ProtocolSmb || k8sVolume.Protocol == utils.ProtocolNfs { return &csi.NodeGetVolumeStatsResponse{ Usage: []*csi.VolumeUsage{ &csi.VolumeUsage{ @@ -635,7 +763,7 @@ func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume[%s] is not found", volumeId)) } - if k8sVolume.Protocol == utils.ProtocolSmb { + if k8sVolume.Protocol == utils.ProtocolSmb || k8sVolume.Protocol == utils.ProtocolNfs { return &csi.NodeExpandVolumeResponse{ CapacityBytes: sizeInByte}, nil } diff --git a/pkg/dsm/service/dsm.go b/pkg/dsm/service/dsm.go index 122a545..b2d7eb2 100644 --- a/pkg/dsm/service/dsm.go +++ b/pkg/dsm/service/dsm.go @@ -100,7 +100,7 @@ func (service *DsmService) ListDsmVolumes(ip string) ([]webapi.VolInfo, error) { return allVolInfos, nil } -func (service *DsmService) getFirstAvailableVolume(dsm *webapi.DSM, sizeInBytes int64) (webapi.VolInfo, error) { +func (service *DsmService) getFirstAvailableVolume(dsm *webapi.DSM, sizeInBytes int64, protocol string) (webapi.VolInfo, error) { volInfos, err := dsm.VolumeList() if err != nil { return webapi.VolInfo{}, err @@ -121,6 +121,10 @@ func (service *DsmService) getFirstAvailableVolume(dsm *webapi.DSM, sizeInBytes if volInfo.Container == "external" && volInfo.Location == "sata" { continue } + + if volInfo.FsType == models.FsTypeExt4 && (protocol == utils.ProtocolSmb || protocol == utils.ProtocolNfs) { + continue + } return volInfo, nil } return webapi.VolInfo{}, fmt.Errorf("Cannot find any available volume") @@ -201,7 +205,7 @@ func (service *DsmService) createMappingTarget(dsm *webapi.DSM, spec *models.Cre func (service *DsmService) createVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error) { // 1. Find a available location if spec.Location == "" { - vol, err := service.getFirstAvailableVolume(dsm, spec.Size) + vol, err := service.getFirstAvailableVolume(dsm, spec.Size, spec.Protocol) if err != nil { return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get available location, err: %v", err)) @@ -372,16 +376,25 @@ func (service *DsmService) createVolumeByVolume(dsm *webapi.DSM, spec *models.Cr return DsmLunToK8sVolume(dsm.Ip, lunInfo, targetInfo), nil } -func DsmShareToK8sVolume(dsmIp string, info webapi.ShareInfo) *models.K8sVolumeRespSpec { +func DsmShareToK8sVolume(dsmIp string, info webapi.ShareInfo, protocol string) *models.K8sVolumeRespSpec { + var source, baseDir string + if protocol == utils.ProtocolSmb { + source = "//" + dsmIp + "/" + info.Name + } else if protocol == utils.ProtocolNfs { + source = "//" + dsmIp + "/" + info.Name + baseDir = info.VolPath + "/" + info.Name + } + return &models.K8sVolumeRespSpec{ DsmIp: dsmIp, VolumeId: info.Uuid, SizeInBytes: utils.MBToBytes(info.QuotaValueInMB), Location: info.VolPath, Name: info.Name, - Source: "//" + dsmIp + "/" + info.Name, - Protocol: utils.ProtocolSmb, + Source: source, + Protocol: protocol, Share: info, + BaseDir: baseDir, } } @@ -399,6 +412,45 @@ func DsmLunToK8sVolume(dsmIp string, info webapi.LunInfo, targetInfo webapi.Targ } } +func isNfsVersionSupport(dsm *webapi.DSM, nfsVersion string) bool { + major := 0 + minor := 0 + + info, err := dsm.NfsGet() + if err != nil { + return false + } + + if nfsVersion == "" { + major = info.SupportMajorVer + minor = info.SupportMinorVer + } else if nfsVersion == "3" { + major = 3 + } else if nfsVersion == "4" || nfsVersion == "4.0" || nfsVersion == "4.1" { + major = 4 + if nfsVersion == "4.1" { + minor = 1 + } + } else { + log.Infof("Input nfsVersion = %s, not supported!", nfsVersion) + return false + } + + if major > info.SupportMajorVer || (major == info.SupportMajorVer && minor > info.SupportMinorVer) { + log.Infof("Dsm NFS version not supported") + return false + } + + // enable the highest NFS version the DSM supports + if err := dsm.NfsSet(true, (info.SupportMajorVer == 4), info.SupportMinorVer); err != nil { + log.Errorf("[%s] Failed to enable nfs: %v\n", dsm.Ip, err) + return false + } + + return true +} + + func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error) { if spec.SourceVolumeId != "" { /* Create volume by exists volume (Clone) */ @@ -414,7 +466,7 @@ func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (*mode if spec.Protocol == utils.ProtocolIscsi { return service.createVolumeByVolume(dsm, spec, k8sVolume.Lun) - } else if spec.Protocol == utils.ProtocolSmb { + } else if spec.Protocol == utils.ProtocolSmb || spec.Protocol == utils.ProtocolNfs { return service.createSMBVolumeByVolume(dsm, spec, k8sVolume.Share) } return nil, status.Error(codes.InvalidArgument, "Unknown protocol") @@ -438,7 +490,10 @@ func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (*mode snapshot.RootPath, spec.Location) return nil, status.Errorf(codes.InvalidArgument, msg) } - if spec.Protocol != snapshot.Protocol { + + log.Debugf("The source PVC protocol [%s] and the destination PVC protocol [%s]", snapshot.Protocol, spec.Protocol) + if (spec.Protocol == utils.ProtocolIscsi || snapshot.Protocol == utils.ProtocolIscsi) && + spec.Protocol != snapshot.Protocol { msg := fmt.Sprintf("The source PVC and destination PVCs shouldn't have different protocols. Source is %s, but new PVC is %s", snapshot.Protocol, spec.Protocol) return nil, status.Errorf(codes.InvalidArgument, msg) @@ -451,7 +506,7 @@ func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (*mode if spec.Protocol == utils.ProtocolIscsi { return service.createVolumeBySnapshot(dsm, spec, snapshot) - } else if spec.Protocol == utils.ProtocolSmb { + } else if spec.Protocol == utils.ProtocolSmb || spec.Protocol == utils.ProtocolNfs { return service.createSMBVolumeBySnapshot(dsm, spec, snapshot) } return nil, status.Error(codes.InvalidArgument, "Unknown protocol") @@ -468,7 +523,12 @@ func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (*mode if spec.Protocol == utils.ProtocolIscsi { k8sVolume, err = service.createVolumeByDsm(dsm, spec) } else if spec.Protocol == utils.ProtocolSmb { - k8sVolume, err = service.createSMBVolumeByDsm(dsm, spec) + k8sVolume, err = service.createSMBorNFSVolumeByDsm(dsm, spec, utils.ProtocolSmb) + } else if spec.Protocol == utils.ProtocolNfs { + if !isNfsVersionSupport(dsm, spec.NfsVersion) { + continue + } + k8sVolume, err = service.createSMBorNFSVolumeByDsm(dsm, spec, utils.ProtocolNfs) } if err != nil { @@ -494,7 +554,7 @@ func (service *DsmService) DeleteVolume(volId string) error { return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp)) } - if k8sVolume.Protocol == utils.ProtocolSmb { + if k8sVolume.Protocol == utils.ProtocolSmb || k8sVolume.Protocol == utils.ProtocolNfs { if err := dsm.ShareDelete(k8sVolume.Share.Name); err != nil { log.Errorf("[%s] Failed to delete Share(%s): %v", dsm.Ip, k8sVolume.Share.Name, err) return err @@ -616,7 +676,7 @@ func (service *DsmService) ExpandVolume(volId string, newSize int64) (*models.K8 return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp)) } - if k8sVolume.Protocol == utils.ProtocolSmb { + if k8sVolume.Protocol == utils.ProtocolSmb || k8sVolume.Protocol == utils.ProtocolNfs { newSizeInMB := utils.BytesToMBCeil(newSize) // round up to MB if err := dsm.SetShareQuota(k8sVolume.Share, newSizeInMB); err != nil { log.Errorf("[%s] Failed to set quota [%d (MB)] to Share [%s]: %v", @@ -674,7 +734,7 @@ func (service *DsmService) CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSp } return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Failed to get iscsi snapshot (%s). Not found", snapshotUuid)) - } else if k8sVolume.Protocol == utils.ProtocolSmb { + } else if k8sVolume.Protocol == utils.ProtocolSmb || k8sVolume.Protocol == utils.ProtocolNfs { snapshotSpec := webapi.ShareSnapshotCreateSpec{ ShareName: k8sVolume.Share.Name, Desc: models.ShareSnapshotDescPrefix + spec.SnapshotName, // limitations: don't change the desc by DSM @@ -692,7 +752,8 @@ func (service *DsmService) CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSp return snapshot, nil } } - return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Failed to get smb snapshot (%s, %s). Not found", snapshotTime, srcVolId)) + return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Failed to get %s snapshot (%s, %s). Not found", + k8sVolume.Protocol, snapshotTime, srcVolId)) } return nil, status.Error(codes.InvalidArgument, "Unsupported volume protocol") @@ -718,7 +779,7 @@ func (service *DsmService) DeleteSnapshot(snapshotUuid string) error { return err } - if snapshot.Protocol == utils.ProtocolSmb { + if snapshot.Protocol == utils.ProtocolSmb || snapshot.Protocol == utils.ProtocolNfs { if err := dsm.ShareSnapshotDelete(snapshot.Time, snapshot.ParentName); err != nil { if snapshot := service.getSMBSnapshot(snapshotUuid); snapshot == nil { // idempotency return nil diff --git a/pkg/dsm/service/share_volume.go b/pkg/dsm/service/share_volume.go index 4035d7c..e726d38 100644 --- a/pkg/dsm/service/share_volume.go +++ b/pkg/dsm/service/share_volume.go @@ -77,7 +77,7 @@ func (service *DsmService) createSMBVolumeBySnapshot(dsm *webapi.DSM, spec *mode log.Debugf("[%s] createSMBVolumeBySnapshot Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid); - return DsmShareToK8sVolume(dsm.Ip, shareInfo), nil + return DsmShareToK8sVolume(dsm.Ip, shareInfo, spec.Protocol), nil } func (service *DsmService) createSMBVolumeByVolume(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcShareInfo webapi.ShareInfo) (*models.K8sVolumeRespSpec, error) { @@ -124,15 +124,15 @@ func (service *DsmService) createSMBVolumeByVolume(dsm *webapi.DSM, spec *models log.Debugf("[%s] createSMBVolumeByVolume Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid); - return DsmShareToK8sVolume(dsm.Ip, shareInfo), nil + return DsmShareToK8sVolume(dsm.Ip, shareInfo, spec.Protocol), nil } -func (service *DsmService) createSMBVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error) { +func (service *DsmService) createSMBorNFSVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, protocol string) (*models.K8sVolumeRespSpec, error) { // TODO: Check if share name is allowable // 1. Find a available location if spec.Location == "" { - vol, err := service.getFirstAvailableVolume(dsm, spec.Size) + vol, err := service.getFirstAvailableVolume(dsm, spec.Size, spec.Protocol) if err != nil { return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get available location, err: %v", err)) } @@ -140,11 +140,15 @@ func (service *DsmService) createSMBVolumeByDsm(dsm *webapi.DSM, spec *models.Cr } // 2. Check if location exists - _, err := dsm.VolumeGet(spec.Location) + dsmVolInfo, err := dsm.VolumeGet(spec.Location) if err != nil { return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Unable to find location %s", spec.Location)) } + if dsmVolInfo.FsType == models.FsTypeExt4 { + return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Location: %s with ext4 fstype was not supported for creating smb/nfs protocol's K8s volume", spec.Location)) + } + // 3. Create Share sizeInMB := utils.BytesToMBCeil(spec.Size) shareSpec := webapi.ShareCreateSpec{ @@ -173,9 +177,9 @@ func (service *DsmService) createSMBVolumeByDsm(dsm *webapi.DSM, spec *models.Cr status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed Share with name: %s, err: %v", spec.ShareName, err)) } - log.Debugf("[%s] createSMBVolumeByDsm Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid) + log.Debugf("[%s] createSMBorNFSVolumeByDsm Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid) - return DsmShareToK8sVolume(dsm.Ip, shareInfo), nil + return DsmShareToK8sVolume(dsm.Ip, shareInfo, protocol), nil } func (service *DsmService) listSMBVolumes(dsmIp string) (infos []*models.K8sVolumeRespSpec) { @@ -198,7 +202,8 @@ func (service *DsmService) listSMBVolumes(dsmIp string) (infos []*models.K8sVolu if !strings.HasPrefix(share.Name, models.SharePrefix) { continue } - infos = append(infos, DsmShareToK8sVolume(dsm.Ip, share)) + //NFSTODO, if share has set nfs rule, deal it as NFS + infos = append(infos, DsmShareToK8sVolume(dsm.Ip, share, utils.ProtocolSmb)) } } diff --git a/pkg/dsm/webapi/share.go b/pkg/dsm/webapi/share.go index b1b73b5..1a7ddc3 100644 --- a/pkg/dsm/webapi/share.go +++ b/pkg/dsm/webapi/share.go @@ -76,6 +76,18 @@ type SharePermission struct { IsAdmin bool `json:"is_admin,omitempty"` // field for list } +type NfsInfo struct { + EnableNfs bool `json:"enable_nfs"` + EnableNfsV4 bool `json:"enable_nfs_v4"` + NfsV4Domain string `json:"nfs_v4_domain"` + ReadSize int `json:"read_size"` + SupportEncryptShare int `json:"support_encrypt_share"` + SupportMajorVer int `json:"support_major_ver"` + SupportMinorVer int `json:"support_minor_ver"` + UnixPriEnable bool `json:"unix_pri_enable"` + WriteSize int `json:"write_size"` +} + func shareErrCodeMapping(errCode int, oriErr error) error { switch errCode { case 402: // No such share @@ -361,3 +373,80 @@ func (dsm *DSM) SharePermissionList(shareName string, userGroupType string) ([]S return infos.Permissions, nil } +// ----------------------- FileServ NFS SharePrivilege APIs ----------------------- +type SecurityFlavor struct { + Kerbros bool `json:"kerberos"` + KerbrosIntegrity bool `json:"kerberos_integrity"` + KerbrosPrivacy bool `json:"kerberos_privacy"` + Sys bool `json:"sys"` +} + +type PrivilegeRule struct { + Async bool `json:"async"` + Client string `json:"client"` + Crossmnt bool `json:"crossmnt"` + Insecure bool `json:"insecure"` + Privilege string `json:"privilege"` + RootSquash string `json:"root_squash"` + SecurityFlavor SecurityFlavor `json:"security_flavor"` +} + +type SharePrivilege struct { + ShareName string `json:"share_name"` + Rule []PrivilegeRule `json:"rule"` +} + +func (dsm *DSM) ShareNfsPrivilegeSave(privilege SharePrivilege) error { + params := url.Values{} + params.Add("api", "SYNO.Core.FileServ.NFS.SharePrivilege") + params.Add("method", "save") + params.Add("share_name", strconv.Quote(privilege.ShareName)) + params.Add("version", "1") + + js, err := json.Marshal(privilege.Rule) + if err != nil { + return err + } + params.Add("rule", string(js)) + + _, err = dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi") + if err != nil { + return err + } + + return nil +} + +func (dsm *DSM) NfsGet() (NfsInfo, error) { + params := url.Values{} + params.Add("api", "SYNO.Core.FileServ.NFS") + params.Add("method", "get") + params.Add("version", "2") + + info := NfsInfo{} + _, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi") + if err != nil { + return NfsInfo{}, err + } + + return info, nil +} + +func (dsm *DSM) NfsSet(enableV3 bool, enableV4 bool, enabledMinorVer int) error { + params := url.Values{} + params.Add("api", "SYNO.Core.FileServ.NFS") + params.Add("method", "set") + params.Add("version", "2") + + params.Add("enable_nfs", strconv.FormatBool(enableV3)) + params.Add("enable_nfs_v4", strconv.FormatBool(enableV4)) + params.Add("enabled_minor_ver", strconv.Itoa(enabledMinorVer)) + + _, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi") + if err != nil { + return err + } + + return nil +} + diff --git a/pkg/models/dsm_req_spec.go b/pkg/models/dsm_req_spec.go index 3ef0571..354978b 100644 --- a/pkg/models/dsm_req_spec.go +++ b/pkg/models/dsm_req_spec.go @@ -23,6 +23,7 @@ type CreateK8sVolumeSpec struct { SourceSnapshotId string SourceVolumeId string Protocol string + NfsVersion string } type K8sVolumeRespSpec struct { @@ -36,6 +37,7 @@ type K8sVolumeRespSpec struct { Target webapi.TargetInfo Share webapi.ShareInfo Protocol string + BaseDir string } type K8sSnapshotRespSpec struct { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 167b518..f53505b 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -16,6 +16,7 @@ const ( ProtocolSmb = "smb" ProtocolIscsi = "iscsi" + ProtocolNfs = "nfs" ProtocolDefault = ProtocolIscsi AuthTypeReadWrite AuthType = "rw" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 77efe50..613db35 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,38 +1,12 @@ #!/bin/bash plugin_name="csi.san.synology.com" -min_support_minor=19 -max_support_minor=20 -deploy_k8s_version="v1".$min_support_minor SCRIPT_PATH="$(realpath "$0")" SOURCE_PATH="$(realpath "$(dirname "${SCRIPT_PATH}")"/../)" config_file="${SOURCE_PATH}/config/client-info.yml" plugin_dir="/var/lib/kubelet/plugins/$plugin_name" -parse_version(){ - ver=$(kubectl version | grep Server | awk '{print $3}') - major=$(echo "${ver##*v}" | cut -d'.' -f1) - minor=$(echo "${ver##*v}" | cut -d'.' -f2) - - if [[ "$major" != 1 ]]; then - echo "Version not supported: $ver" - exit 1 - fi - - case "$minor" in - 19|20) - deploy_k8s_version="v1".$minor - ;; - *) - if [[ $minor -lt $min_support_minor ]]; then - deploy_k8s_version="v1".$min_support_minor - else - deploy_k8s_version="v1".$max_support_minor - fi - ;; - esac - echo "Deploy Version: $deploy_k8s_version" -} +source "$SOURCE_PATH"/scripts/functions.sh # 1. Build csi_build(){ @@ -44,6 +18,7 @@ csi_build(){ csi_install(){ echo "==== Creates namespace and secrets, then installs synology-csi ====" parse_version + echo "Deploy Version: $deploy_k8s_version" kubectl create ns synology-csi kubectl create secret -n synology-csi generic client-info-secret --from-file="$config_file" diff --git a/scripts/functions.sh b/scripts/functions.sh new file mode 100644 index 0000000..12d3a3e --- /dev/null +++ b/scripts/functions.sh @@ -0,0 +1,29 @@ +#!/bin/bash +min_support_minor=19 +max_support_minor=20 +deploy_k8s_version="v1".$min_support_minor + +parse_version(){ + ver=$(kubectl version --output=json | awk -F'"' '/"serverVersion":/ {flag=1} flag && /"gitVersion":/ {print $(NF-1); flag=0}') + major=$(echo "${ver##*v}" | cut -d'.' -f1) + minor=$(echo "${ver##*v}" | cut -d'.' -f2) + + if [[ "$major" != 1 ]]; then + echo "Version not supported: $ver" + exit 1 + fi + + case "$minor" in + 19|20) + deploy_k8s_version="v1".$minor + ;; + *) + if [[ $minor -lt $min_support_minor ]]; then + deploy_k8s_version="v1".$min_support_minor + else + deploy_k8s_version="v1".$max_support_minor + fi + ;; + esac + echo "Current Server Version: $ver" +} \ No newline at end of file diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index c23288f..bc988af 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -9,33 +9,10 @@ deploy_k8s_version="v1".$min_support_minor SCRIPT_PATH="$(realpath "$0")" SOURCE_PATH="$(realpath "$(dirname "$SCRIPT_PATH")"/../)" -parse_version(){ - ver=$(kubectl version | grep Server | awk '{print $3}') - major=$(echo "${ver##*v}" | cut -d'.' -f1) - minor=$(echo "${ver##*v}" | cut -d'.' -f2) - - if [[ "$major" != 1 ]]; then - echo "Version not supported: $ver" - exit 1 - fi - - case "$minor" in - 19|20) - deploy_k8s_version="v1".$minor - ;; - *) - if [[ $minor -lt $min_support_minor ]]; then - deploy_k8s_version="v1".$min_support_minor - else - deploy_k8s_version="v1".$max_support_minor - fi - ;; - esac - echo "Uninstall Version: $deploy_k8s_version" -} - +source "$SOURCE_PATH"/scripts/functions.sh parse_version +echo "Uninstall Version: $deploy_k8s_version" kubectl delete -f "$SOURCE_PATH"/deploy/kubernetes/$deploy_k8s_version/snapshotter --ignore-not-found kubectl delete -f "$SOURCE_PATH"/deploy/kubernetes/$deploy_k8s_version --ignore-not-found echo "End of synology-csi uninstallation."