Support NFS mount options from backend config files in Kubernetes (#15)

CHANGELOG.md

@@ -16,6 +16,7 @@
 - **Kubernetes:** Added support for raw block volumes for iSCSI PVs.
 - **Kubernetes:** Added retry logic to installer for Kubernetes object creation.
 - **Kubernetes:** Updated etcd to v3.3.10 and client-go to v10.0.0.
+- **Kubernetes:** Trident now honors the nfsMountOptions parameter in ONTAP NAS backend config files.
 - **Behavioural change:** The Trident installer now automatically adds the backend used to provision the Trident volume in new installations.
 
 **Deprecations:**

cli/cmd/install.go

@@ -1228,7 +1228,8 @@ func createPV(sb *storage.Backend) error {
 	switch {
 	case volume.Config.AccessInfo.NfsAccessInfo.NfsServerIP != "":
 
-		pvYAML = k8sclient.GetNFSPVYAML(pvName, volumeSize, pvcName, TridentPodNamespace,
+		pvYAML = k8sclient.GetNFSPVYAML(
+			pvName, volumeSize, volume.Config.AccessInfo.MountOptions, pvcName, TridentPodNamespace,
 			volume.Config.AccessInfo.NfsAccessInfo.NfsServerIP,
 			volume.Config.AccessInfo.NfsAccessInfo.NfsPath,
 			appLabelValue)
@@ -1248,7 +1249,8 @@ func createPV(sb *storage.Backend) error {
 				return err
 			}
 
-			pvYAML = k8sclient.GetCHAPISCSIPVYAML(pvName, volumeSize, pvcName, TridentPodNamespace, secretName,
+			pvYAML = k8sclient.GetCHAPISCSIPVYAML(
+				pvName, volumeSize, volume.Config.AccessInfo.MountOptions, pvcName, TridentPodNamespace, secretName,
 				volume.Config.AccessInfo.IscsiAccessInfo.IscsiTargetPortal,
 				volume.Config.AccessInfo.IscsiAccessInfo.IscsiPortals,
 				volume.Config.AccessInfo.IscsiAccessInfo.IscsiTargetIQN,
@@ -1258,7 +1260,8 @@ func createPV(sb *storage.Backend) error {
 		} else {
 
 			// Not using CHAP
-			pvYAML = k8sclient.GetISCSIPVYAML(pvName, volumeSize, pvcName, TridentPodNamespace,
+			pvYAML = k8sclient.GetISCSIPVYAML(
+				pvName, volumeSize, volume.Config.AccessInfo.MountOptions, pvcName, TridentPodNamespace,
 				volume.Config.AccessInfo.IscsiAccessInfo.IscsiTargetPortal,
 				volume.Config.AccessInfo.IscsiAccessInfo.IscsiPortals,
 				volume.Config.AccessInfo.IscsiAccessInfo.IscsiTargetIQN,

cli/k8s_client/yaml_factory.go

@@ -2,6 +2,8 @@ package k8sclient
 
 import (
 	"encoding/base64"
+	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -671,7 +673,20 @@ spec:
   storageClassName: ''
 `
 
-func GetNFSPVYAML(pvName, size, pvcName, pvcNamespace, nfsServer, nfsPath, label string) string {
+func addMountOptionsToPVYAML(pvYAML, mountOptions string) string {
+
+	mountOptions = strings.TrimPrefix(mountOptions, "-o ")
+	if len(mountOptions) > 0 {
+		pvYAML += "  mountOptions:\n"
+		for _, option := range regexp.MustCompile(`\s*,\s*`).Split(mountOptions, -1) {
+			pvYAML += fmt.Sprintf("  - %s\n", option)
+		}
+	}
+
+	return pvYAML
+}
+
+func GetNFSPVYAML(pvName, size, mountOptions, pvcName, pvcNamespace, nfsServer, nfsPath, label string) string {
 
 	pvYAML := strings.Replace(persistentVolumeNFSYAMLTemplate, "{PV_NAME}", pvName, 1)
 	pvYAML = strings.Replace(pvYAML, "{SIZE}", size, 1)
@@ -680,6 +695,9 @@ func GetNFSPVYAML(pvName, size, pvcName, pvcNamespace, nfsServer, nfsPath, label
 	pvYAML = strings.Replace(pvYAML, "{SERVER}", nfsServer, 1)
 	pvYAML = strings.Replace(pvYAML, "{PATH}", nfsPath, 1)
 	pvYAML = strings.Replace(pvYAML, "{LABEL}", label, 1)
+
+	pvYAML = addMountOptionsToPVYAML(pvYAML, mountOptions)
+
 	return pvYAML
 }
 
@@ -706,7 +724,7 @@ spec:
     path: {PATH}
 `
 
-func GetISCSIPVYAML(pvName, size, pvcName, pvcNamespace, targetPortal string, portals []string,
+func GetISCSIPVYAML(pvName, size, mountOptions, pvcName, pvcNamespace, targetPortal string, portals []string,
 	iqn string, lun int32, label string) string {
 
 	pvYAML := strings.Replace(persistentVolumeISCSIYAMLTemplate, "{PV_NAME}", pvName, 1)
@@ -717,12 +735,16 @@ func GetISCSIPVYAML(pvName, size, pvcName, pvcNamespace, targetPortal string, po
 	pvYAML = strings.Replace(pvYAML, "{IQN}", iqn, 1)
 	pvYAML = strings.Replace(pvYAML, "{LUN}", strconv.FormatInt(int64(lun), 10), 1)
 	pvYAML = strings.Replace(pvYAML, "{LABEL}", label, 1)
+
 	if 0 != len(portals) {
 		pvYAML += "    portals:\n"
 		for _, portal := range portals {
 			pvYAML += "      - " + portal + "\n"
 		}
 	}
+
+	pvYAML = addMountOptionsToPVYAML(pvYAML, mountOptions)
+
 	return pvYAML
 }
 
@@ -752,7 +774,7 @@ spec:
     readOnly: false
 `
 
-func GetCHAPISCSIPVYAML(pvName, size, pvcName, pvcNamespace, secretName, targetPortal string,
+func GetCHAPISCSIPVYAML(pvName, size, mountOptions, pvcName, pvcNamespace, secretName, targetPortal string,
 	portals []string, iqn string, lun int32, label string) string {
 
 	pvYAML := strings.Replace(persistentVolumeCHAPISCSIYAMLTemplate, "{PV_NAME}", pvName, 1)
@@ -770,6 +792,9 @@ func GetCHAPISCSIPVYAML(pvName, size, pvcName, pvcNamespace, secretName, targetP
 			pvYAML += "      - " + portal + "\n"
 		}
 	}
+
+	pvYAML = addMountOptionsToPVYAML(pvYAML, mountOptions)
+
 	return pvYAML
 }
 

docs/kubernetes/operations/tasks/backends/ontap.rst

@@ -128,6 +128,7 @@ password                  Password to connect to the cluster/SVM
 storagePrefix             Prefix used when provisioning new volumes in the SVM                    "trident"
 limitAggregateUsage       Fail provisioning if usage is above this percentage                     "" (not enforced by default)
 limitVolumeSize           Fail provisioning if requested volume size is above this value          "" (not enforced by default)
+nfsMountOptions           Comma-separated list of NFS mount options (except ontap-san)            ""
 ========================= ======================================================================= ================================================
 
 A fully-qualified domain name (FQDN) can be specified for the managementLIF option. For the ontap-nas*
@@ -137,6 +138,12 @@ the SVM and to use iSCSI multipath. Specifying an IP address for the dataLIF for
 the driver to disable multipath and use only the specified address.  For the ontap-nas-economy driver,
 the limitVolumeSize option will also restrict the maximum size of the volumes it manages for qtrees.
 
+The nfsMountOptions parameter applies to all ONTAP drivers except ontap-san.  The mount options for Kubernetes
+persistent volumes are normally specified in storage classes, but if no mount options are specified in a storage
+class, Trident will fall back to using the mount options specified in the storage backend's config file.  If
+no mount options are specified in either the storage class or the config file, then Trident will not set any
+mount options on an associated persistent volume.
+
 You can control how each volume is provisioned by default using these options
 in a special section of the configuration. For an example, see the
 configuration examples below.
@@ -172,6 +179,7 @@ Example configuration
         "password": "secret",
         "limitAggregateUsage": "80%",
         "limitVolumeSize": "50Gi",
+        "nfsMountOptions": "nfsvers=4",
         "defaults": {
           "spaceReserve": "volume",
           "exportPolicy": "myk8scluster",

docs/kubernetes/troubleshooting.rst

@@ -19,6 +19,14 @@ Troubleshooting
   after a failed run. By default the script does not touch the etcd backing
   store, making it safe to uninstall and install again even in a running
   deployment.
+* If Trident fails to start and the logs from Trident's etcd container report "another
+  etcd process is running with the same data dir and holding the file lock" or similar,
+  then you may have stale NFSv3 locks held on the ONTAP storage system.  This situation
+  may be caused by an unclean shutdown of the Kubernetes node where Trident is running.
+  You can avoid this issue by enabling NFSv4 on your ONTAP SVM and setting
+  ``nfsMountOptions: "nfsvers=4"`` in the backend.json config file used during Trident
+  installation.  Furthermore, you should use ``kubectl drain`` or ``oc adm drain`` to
+  cleanly stop all pods on a Kubernetes node prior to powering it off.
 * After a successful install, if a PVC is stuck in the ``Pending`` phase,
   running ``kubectl describe pvc`` can provide additional information on why
   Trident failed to provsion a PV for this PVC.

frontend/csi/node_server.go

@@ -217,9 +217,9 @@ func (p *Plugin) nodePublishNFSVolume(
 	publishInfo := &utils.VolumePublishInfo{
 		Localhost:      true,
 		FilesystemType: "nfs",
-		MountOptions:   strings.Join(mountOptions, ","),
 	}
 
+	publishInfo.MountOptions = strings.Join(mountOptions, ",")
 	publishInfo.NfsServerIP = req.PublishInfo["nfsServerIp"]
 	publishInfo.NfsPath = req.PublishInfo["nfsPath"]
 

frontend/kubernetes/plugin.go

@@ -7,6 +7,7 @@ package kubernetes
 import (
 	"fmt"
 	"io/ioutil"
+	"regexp"
 	"strings"
 	"sync"
 
@@ -745,7 +746,7 @@ func (p *Plugin) createVolumeAndPV(uniqueName string, claim *v1.PersistentVolume
 	case kubeVersion.AtLeast(k8sutilversion.MustParseSemantic("v1.8.0")):
 		pv.Spec.StorageClassName = GetPersistentVolumeClaimClass(claim)
 		// Apply Storage Class mount options and reclaim policy
-		pv.Spec.MountOptions = p.storageClassCache[storageClass].MountOptions
+		pv.Spec.MountOptions = p.getPVMountOptions(p.storageClassCache[storageClass], vol)
 		pv.Spec.PersistentVolumeReclaimPolicy =
 			*p.storageClassCache[storageClass].PersistentVolumeReclaimPolicy
 	case kubeVersion.AtLeast(k8sutilversion.MustParseSemantic("v1.6.0")):
@@ -816,6 +817,17 @@ func (p *Plugin) createVolumeAndPV(uniqueName string, claim *v1.PersistentVolume
 	return
 }
 
+func (p *Plugin) getPVMountOptions(scSummary *StorageClassSummary, volume *storage.VolumeExternal) []string {
+
+	if scSummary != nil && len(scSummary.MountOptions) > 0 {
+		return scSummary.MountOptions
+	}
+	if volume != nil && volume.Config.AccessInfo.MountOptions != "" {
+		return regexp.MustCompile(`\s*,\s*`).Split(volume.Config.AccessInfo.MountOptions, -1)
+	}
+	return make([]string, 0)
+}
+
 func (p *Plugin) deleteVolumeAndPV(volume *v1.PersistentVolume) error {
 	err := p.orchestrator.DeleteVolume(volume.GetName())
 	if err != nil && !core.IsNotFoundError(err) {

storage_drivers/ontap/ontap_common.go

@@ -324,7 +324,8 @@ const DefaultUnixPermissions = "---rwxrwxrwx"
 const DefaultSnapshotDir = "false"
 const DefaultExportPolicy = "default"
 const DefaultSecurityStyle = "unix"
-const DefaultNfsMountOptions = "-o nfsvers=3"
+const DefaultNfsMountOptionsDocker = "-o nfsvers=3"
+const DefaultNfsMountOptionsKubernetes = ""
 const DefaultSplitOnClone = "false"
 const DefaultFileSystemType = "ext4"
 const DefaultEncryption = "false"
@@ -384,7 +385,12 @@ func PopulateConfigurationDefaults(config *drivers.OntapStorageDriverConfig) err
 	}
 
 	if config.NfsMountOptions == "" {
-		config.NfsMountOptions = DefaultNfsMountOptions
+		switch config.DriverContext {
+		case tridentconfig.ContextDocker:
+			config.NfsMountOptions = DefaultNfsMountOptionsDocker
+		default:
+			config.NfsMountOptions = DefaultNfsMountOptionsKubernetes
+		}
 	}
 
 	if config.SplitOnClone == "" {

storage_drivers/ontap/ontap_nas.go

@@ -386,6 +386,7 @@ func (d *NASStorageDriver) CreateFollowup(
 ) error {
 	volConfig.AccessInfo.NfsServerIP = d.Config.DataLIF
 	volConfig.AccessInfo.NfsPath = "/" + volConfig.InternalName
+	volConfig.AccessInfo.MountOptions = strings.TrimPrefix(d.Config.NfsMountOptions, "-o ")
 	volConfig.FileSystem = ""
 	return nil
 }

storage_drivers/ontap/ontap_nas_flexgroup.go

@@ -421,6 +421,7 @@ func (d *NASFlexGroupStorageDriver) CreateFollowup(
 ) error {
 	volConfig.AccessInfo.NfsServerIP = d.Config.DataLIF
 	volConfig.AccessInfo.NfsPath = "/" + volConfig.InternalName
+	volConfig.AccessInfo.MountOptions = strings.TrimPrefix(d.Config.NfsMountOptions, "-o ")
 	volConfig.FileSystem = ""
 	return nil
 }

storage_drivers/ontap/ontap_nas_qtree.go

@@ -1042,6 +1042,7 @@ func (d *NASQtreeStorageDriver) CreateFollowup(volConfig *storage.VolumeConfig)
 	// Set export path info on the volume config
 	volConfig.AccessInfo.NfsServerIP = d.Config.DataLIF
 	volConfig.AccessInfo.NfsPath = fmt.Sprintf("/%s/%s", flexvol, volConfig.InternalName)
+	volConfig.AccessInfo.MountOptions = strings.TrimPrefix(d.Config.NfsMountOptions, "-o ")
 
 	return nil
 }

utils/types.go

@@ -5,6 +5,7 @@ package utils
 type VolumeAccessInfo struct {
 	IscsiAccessInfo
 	NfsAccessInfo
+	MountOptions string `json:"mountOptions,omitempty"`
 }
 
 type IscsiAccessInfo struct {
@@ -31,7 +32,6 @@ type VolumePublishInfo struct {
 	HostIP         []string `json:"hostIP,omitempty"`
 	HostName       string   `json:"hostName,omitempty"`
 	FilesystemType string   `json:"fstype,omitempty"`
-	MountOptions   string   `json:"mountOptions,omitempty"`
 	UseCHAP        bool     `json:"useCHAP,omitempty"`
 	SharedTarget   bool     `json:"sharedTarget,omitempty"`
 	DevicePath     string   `json:"devicePath,omitempty"`