fix(resize): delegate RWX filesystem resize to share-manager

Signed-off-by: James Munson <[email protected]>

csi/deployment.go

@@ -418,6 +418,14 @@ func NewPluginDeployment(namespace, serviceAccount, nodeDriverRegistrarImage, li
 									Name:  "CSI_ENDPOINT",
 									Value: GetCSIEndpoint(),
 								},
+								{
+									Name: "POD_NAMESPACE",
+									ValueFrom: &corev1.EnvVarSource{
+										FieldRef: &corev1.ObjectFieldSelector{
+											FieldPath: "metadata.namespace",
+										},
+									},
+								},
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{

csi/node_server.go

@@ -16,18 +16,23 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
 	"k8s.io/mount-utils"
 
 	corev1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
 	utilexec "k8s.io/utils/exec"
 
 	"github.com/longhorn/longhorn-manager/csi/crypto"
+	"github.com/longhorn/longhorn-manager/engineapi"
 	"github.com/longhorn/longhorn-manager/types"
 
 	lhns "github.com/longhorn/go-common-libs/ns"
 
 	longhornclient "github.com/longhorn/longhorn-manager/client"
 	longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned"
 )
 
 const (
@@ -690,11 +695,66 @@ func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVo
 	}, nil
 }
 
+// NodeExpandShared Volume is designed to expand the file system in an RWX volume for ONLINE expansion.
+// It does so with a gRPC call into the share-manager pod.
+func (ns *NodeServer) NodeExpandSharedVolume(volumeName string) error {
+	log := ns.log.WithFields(logrus.Fields{"function": "NodeExpandSharedVolume"})
+
+	lhNamespace := os.Getenv(types.EnvPodNamespace)
+	if lhNamespace == "" {
+		return fmt.Errorf("failed to detect pod namespace, environment variable %v is missing", types.EnvPodNamespace)
+	}
+
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return errors.Wrap(err, "failed to get client config")
+	}
+
+	kubeClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		return errors.Wrap(err, "failed to get k8s client")
+	}
+
+	lhClient, err := lhclientset.NewForConfig(config)
+	if err != nil {
+		return errors.Wrap(err, "failed to get longhorn clientset")
+	}
+
+	sm, err := lhClient.LonghornV1beta2().ShareManagers(lhNamespace).Get(context.TODO(), volumeName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get ShareManager CR")
+	}
+
+	podName := types.GetShareManagerPodNameFromShareManagerName(sm.Name)
+	pod, err := kubeClient.CoreV1().Pods(lhNamespace).Get(context.TODO(), podName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get ShareManager pod")
+	}
+
+	client, err := engineapi.NewShareManagerClient(sm, pod)
+	if err != nil {
+		return errors.Wrapf(err, "failed to launch gRPC client for share manager before resizing volume %v", volumeName)
+	}
+	defer client.Close()
+
+	// Each node with a workload pod will send an RPC request.  The first will win, and the others are no-ops.
+	err = client.FilesystemResize()
+	if status.Code(err) == codes.Unimplemented {
+		// This is a downrev longhorn-share-manager image.  It will be necessary either to kill the share-manager pod
+		// and let it restart with the new image, or scale the workload down and back up to accomplish the same thing.
+		// It might be tempting to delete the pod here, but there will be one of these calls for each workload pod,
+		// and it would be messy to have multiple kill requests at once.  So the kill will need to be done by the user.
+		log.WithError(err).Warn("Share Manager image is down-rev, does not implement RPC FilesystemResize.  Share Manager pod must be restarted with current image.")
+	}
+
+	return err
+}
+
 // NodeExpandVolume is designed to expand the file system for ONLINE expansion,
 func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
 	log := ns.log.WithFields(logrus.Fields{"function": "NodeExpandVolume"})
 
-	log.Infof("NodeNodeExpandVolume is called with req %+v", req)
+	log.Infof("NodeExpandVolume is called with req %+v", req)
 
 	if req.CapacityRange == nil {
 		return nil, status.Error(codes.InvalidArgument, "capacity range missing in request")
@@ -729,6 +789,20 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV
 	if volume.State != string(longhorn.VolumeStateAttached) {
 		return nil, status.Errorf(codes.FailedPrecondition, "invalid state %v for volume %v node expansion", volume.State, volumeID)
 	}
+
+	if requiresSharedAccess(volume, volumeCapability) && !volume.Migratable {
+		if volume.AccessMode != string(longhorn.AccessModeReadWriteMany) {
+			return nil, status.Errorf(codes.FailedPrecondition, "volume %s requires shared access but is not marked for shared use", volumeID)
+		}
+
+		if err := ns.NodeExpandSharedVolume(volumeID); err != nil {
+			log.WithError(err).Info("NodeExpandSharedVolume returned error.")
+			return nil, err
+		}
+
+		return &csi.NodeExpandVolumeResponse{CapacityBytes: requestedSize}, nil
+	}
+
 	devicePath := volume.Controllers[0].Endpoint
 
 	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}

engineapi/share_manager.go

@@ -39,6 +39,10 @@ func (c *ShareManagerClient) FilesystemTrim(encryptedDevice bool) error {
 	return c.grpcClient.FilesystemTrim(encryptedDevice)
 }
 
+func (c *ShareManagerClient) FilesystemResize() error {
+	return c.grpcClient.FilesystemResize()
+}
+
 func (c *ShareManagerClient) Unmount() error {
 	return c.grpcClient.Unmount()
 }

go.mod

@@ -226,3 +226,7 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/longhorn/types => github.com/james-munson/types v0.0.0-20241106234456-f7f62e977be0
+
+replace github.com/longhorn/longhorn-share-manager => github.com/james-munson/longhorn-share-manager v1.4.0-rc1.0.20241111220442-c3b1b0d80797

go.sum

@@ -130,6 +130,10 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/james-munson/longhorn-share-manager v1.4.0-rc1.0.20241111220442-c3b1b0d80797 h1:lE4iOD+gHB5by2MniYHw7XH16+tFZqBrdRl34Ihtpsw=
+github.com/james-munson/longhorn-share-manager v1.4.0-rc1.0.20241111220442-c3b1b0d80797/go.mod h1:ziXm1g8gS3eRt/jLDJHQYNRf1yCK7n7y4uPxIy7Ull8=
+github.com/james-munson/types v0.0.0-20241106234456-f7f62e977be0 h1:RykCSQrMsQXzyjy99IXsgF+BVlvt3TOYfJBfuo+ObSk=
+github.com/james-munson/types v0.0.0-20241106234456-f7f62e977be0/go.mod h1:IpV+1bctQgBgp3brj0nsHmnBDFkd5IrzTgBtVAloJuw=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
@@ -171,10 +175,6 @@ github.com/longhorn/longhorn-engine v1.8.0-dev-20241117 h1:2sW/dPj69/XrMDjsh2LBZ
 github.com/longhorn/longhorn-engine v1.8.0-dev-20241117/go.mod h1:3VnMqI8YqJ76XdRvnMWJNp1U1ITAyOwOjKtbYPqLqQw=
 github.com/longhorn/longhorn-instance-manager v1.8.0-dev-20241117 h1:dReWXt6gh5mpL27fEhDFCLVK7iZwGWCNK3rH4BGP29M=
 github.com/longhorn/longhorn-instance-manager v1.8.0-dev-20241117/go.mod h1:HCh2gA5CNMpZ+b2j7uUgV8RMJ8BwB22cjiTbjWeQ5p4=
-github.com/longhorn/longhorn-share-manager v1.7.0-rc1 h1:LsSkSajhG8tCfORKKfwK+8XHVrT/8rI9DRWb7fuoVls=
-github.com/longhorn/longhorn-share-manager v1.7.0-rc1/go.mod h1:R6+NscPU4lAV5ueO7//lBCAO3en0aDbZi5KkkOSUJvk=
-github.com/longhorn/types v0.0.0-20241110123431-85dca7039c42 h1:zMSuk0V/sVfgvdd+CpxfqJAB0yKQaWFglOFCyrHU3xc=
-github.com/longhorn/types v0.0.0-20241110123431-85dca7039c42/go.mod h1:3A0asFlVzaBCvQHhqEY+HsnFnNBUFZ5onYKI2Qq9u74=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=