From 5d207bba10ffb112cf19e39d2452f045c77d4f9f Mon Sep 17 00:00:00 2001
From: Joe Webster <31218426+jwebster7@users.noreply.github.com>
Date: Wed, 13 Nov 2024 09:47:00 -0600
Subject: [PATCH] NVMe LUKS Support

---
 frontend/csi/controller_server.go             |  2 +-
 frontend/csi/node_server.go                   | 77 +++++++++++++++----
 storage_drivers/ontap/api/abstraction_rest.go |  6 +-
 storage_drivers/ontap/ontap_san.go            |  3 +-
 storage_drivers/ontap/ontap_san_nvme.go       | 36 ++++++++-
 storage_drivers/ontap/ontap_san_nvme_test.go  | 65 +++++++++++++++-
 utils/devices.go                              |  2 +-
 utils/devices_linux.go                        | 14 ++--
 utils/nvme.go                                 | 53 ++++++++++---
 utils/nvme_linux.go                           |  2 +-
 10 files changed, 218 insertions(+), 42 deletions(-)

diff --git a/frontend/csi/controller_server.go b/frontend/csi/controller_server.go
index 932212f1a..81e82ab44 100644
--- a/frontend/csi/controller_server.go
+++ b/frontend/csi/controller_server.go
@@ -1,4 +1,4 @@
-// Copyright 2022 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package csi
 
diff --git a/frontend/csi/node_server.go b/frontend/csi/node_server.go
index fbc50c2f7..8ca843b47 100644
--- a/frontend/csi/node_server.go
+++ b/frontend/csi/node_server.go
@@ -1536,7 +1536,7 @@ func (p *Plugin) nodeStageISCSIVolume(
 		}
 
 		var luksDevice models.LUKSDeviceInterface
-		luksDevice, err = utils.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
+		luksDevice, err = p.devices.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
 		if err != nil {
 			return err
 		}
@@ -1836,11 +1836,13 @@ func (p *Plugin) nodePublishISCSIVolume(
 			luksDevice, err = p.devices.NewLUKSDeviceFromMappingPath(ctx, devicePath,
 				req.VolumeContext["internalName"])
 		} else {
-			luksDevice, err = utils.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
+			luksDevice, err = p.devices.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
 		}
+
 		if err != nil {
 			return nil, status.Error(codes.Internal, err.Error())
 		}
+
 		err = ensureLUKSVolumePassphrase(ctx, p.restClient, luksDevice, req.GetVolumeId(), req.GetSecrets(), false)
 		if err != nil {
 			Logc(ctx).WithError(err).Error("Failed to ensure current LUKS passphrase.")
@@ -2441,7 +2443,7 @@ func (p *Plugin) nodeStageNVMeVolume(
 	ctx context.Context, req *csi.NodeStageVolumeRequest,
 	publishInfo *models.VolumePublishInfo,
 ) error {
-	isLUKS := utils.ParseBool(publishInfo.LUKSEncryption)
+	isLUKS := utils.ParseBool(req.PublishContext["LUKSEncryption"])
 	publishInfo.LUKSEncryption = strconv.FormatBool(isLUKS)
 	publishInfo.MountOptions = req.PublishContext["mountOptions"]
 	publishInfo.NVMeSubsystemNQN = req.PublishContext["nvmeSubsystemNqn"]
@@ -2449,8 +2451,10 @@ func (p *Plugin) nodeStageNVMeVolume(
 	publishInfo.NVMeTargetIPs = strings.Split(req.PublishContext["nvmeTargetIPs"], ",")
 	publishInfo.SANType = req.PublishContext["SANType"]
 
-	if err := utils.AttachNVMeVolumeRetry(ctx, req.VolumeContext["internalName"], "", publishInfo, nil,
-		utils.NVMeAttachTimeout); err != nil {
+	err := utils.AttachNVMeVolumeRetry(
+		ctx, req.VolumeContext["internalName"], "", publishInfo, req.GetSecrets(), utils.NVMeAttachTimeout,
+	)
+	if err != nil {
 		return err
 	}
 
@@ -2460,11 +2464,11 @@ func (p *Plugin) nodeStageNVMeVolume(
 	}
 
 	if isLUKS {
-		luksDevice, err := p.devices.NewLUKSDeviceFromMappingPath(ctx, publishInfo.DevicePath,
-			req.VolumeContext["internalName"])
+		luksDevice, err := p.devices.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
 		if err != nil {
 			return err
 		}
+
 		// Ensure we update the passphrase in case it has never been set before
 		err = ensureLUKSVolumePassphrase(ctx, p.restClient, luksDevice, volumeId, req.GetSecrets(), true)
 		if err != nil {
@@ -2500,12 +2504,42 @@ func (p *Plugin) nodeUnstageNVMeVolume(
 		publishInfo.NVMeNamespaceUUID)
 
 	// Get the device using 'nvme-cli' commands. Flush the device IOs.
+	// Proceed further with unstage flow, if device is not found.
 	nvmeDev, err := p.nvmeHandler.NewNVMeDevice(ctx, publishInfo.NVMeNamespaceUUID)
-	// Proceed further with Unstage flow, if 'device is not found'.
 	if err != nil && !errors.IsNotFoundError(err) {
 		return nil, fmt.Errorf("error while getting NVMe device, %v", err)
 	}
 
+	var devicePath string
+	if nvmeDev != nil {
+		devicePath = nvmeDev.GetPath()
+	}
+
+	var luksMapperPath string
+	if utils.ParseBool(publishInfo.LUKSEncryption) && devicePath != "" {
+		fields := LogFields{
+			"lunID":           publishInfo.IscsiLunNumber,
+			"publishedDevice": publishInfo.DevicePath,
+			"nvmeDevPath":     nvmeDev.GetPath(),
+		}
+
+		luksMapperPath, err = p.devices.GetLUKSDeviceForMultipathDevice(devicePath)
+		if err != nil {
+			return &csi.NodeUnstageVolumeResponse{}, err
+		}
+
+		// Ensure the LUKS device is closed if the luksMapperPath is set.
+		if luksMapperPath != "" {
+			if err = p.devices.EnsureLUKSDeviceClosedWithMaxWaitLimit(ctx, luksMapperPath); err != nil {
+				if !errors.IsMaxWaitExceededError(err) {
+					Logc(ctx).WithFields(fields).WithError(err).Error("Failed to close LUKS device.")
+					return &csi.NodeUnstageVolumeResponse{}, err
+				}
+				Logc(ctx).WithFields(fields).WithError(err).Debug("LUKS close wait time exceeded, continuing with device removal.")
+			}
+		}
+	}
+
 	if !nvmeDev.IsNil() {
 		// If device is found, proceed to flush and clean up.
 		err := nvmeDev.FlushDevice(ctx, p.unsafeDetach, force)
@@ -2580,6 +2614,20 @@ func (p *Plugin) nodeUnstageNVMeVolume(
 		return nil, status.Error(codes.Internal, errStr)
 	}
 
+	// If the luks device still exists, it means the device was unable to be closed prior to removing the block
+	// device. This can happen if the LUN was deleted or offline. It should be removable by this point.
+	// It needs to be removed prior to removing the 'unmappedMpathDevice' device below.
+	if luksMapperPath != "" {
+		// EnsureLUKSDeviceClosed will not return an error if the device is already closed or removed.
+		if err = p.devices.EnsureLUKSDeviceClosed(ctx, luksMapperPath); err != nil {
+			Logc(ctx).WithFields(LogFields{
+				"devicePath": luksMapperPath,
+			}).WithError(err).Warning("Unable to remove LUKS mapper device.")
+		}
+		// Clear the time duration for the LUKS device.
+		utils.LuksCloseDurations.RemoveDurationTracking(luksMapperPath)
+	}
+
 	// Delete the device info we saved to the volume tracking info path so unstage can succeed.
 	if err := p.nodeHelper.DeleteTrackingInfo(ctx, volumeId); err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -2608,22 +2656,25 @@ func (p *Plugin) nodePublishNVMeVolume(
 		publishInfo.MountOptions = utils.AppendToStringList(publishInfo.MountOptions, "ro", ",")
 	}
 
+	devicePath := publishInfo.DevicePath
 	if utils.ParseBool(publishInfo.LUKSEncryption) {
 		// Rotate the LUKS passphrase if needed, on failure, log and continue to publish
-		luksDevice, err := p.devices.NewLUKSDeviceFromMappingPath(ctx, publishInfo.DevicePath,
-			req.VolumeContext["internalName"])
+		luksDevice, err := p.devices.NewLUKSDevice(devicePath, req.VolumeContext["internalName"])
 		if err != nil {
 			return nil, status.Error(codes.Internal, err.Error())
 		}
+
 		err = ensureLUKSVolumePassphrase(ctx, p.restClient, luksDevice, req.GetVolumeId(), req.GetSecrets(), false)
 		if err != nil {
 			Logc(ctx).WithError(err).Error("Failed to ensure current LUKS passphrase.")
 		}
+
+		// At this point, we must reassign the device path to the luks mapper path for mounts to work.
+		devicePath = luksDevice.MappedDevicePath()
 	}
 
 	isRawBlock := publishInfo.FilesystemType == filesystem.Raw
 	if isRawBlock {
-
 		if len(publishInfo.MountOptions) > 0 {
 			publishInfo.MountOptions = utils.AppendToStringList(publishInfo.MountOptions, "bind", ",")
 		} else {
@@ -2631,13 +2682,13 @@ func (p *Plugin) nodePublishNVMeVolume(
 		}
 
 		// Place the block device at the target path for the raw-block.
-		err = p.mount.MountDevice(ctx, publishInfo.DevicePath, req.TargetPath, publishInfo.MountOptions, true)
+		err = p.mount.MountDevice(ctx, devicePath, req.TargetPath, publishInfo.MountOptions, true)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "unable to bind mount raw device; %s", err)
 		}
 	} else {
 		// Mount the device.
-		err = p.mount.MountDevice(ctx, publishInfo.DevicePath, req.TargetPath, publishInfo.MountOptions, false)
+		err = p.mount.MountDevice(ctx, devicePath, req.TargetPath, publishInfo.MountOptions, false)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "unable to mount device; %s", err)
 		}
diff --git a/storage_drivers/ontap/api/abstraction_rest.go b/storage_drivers/ontap/api/abstraction_rest.go
index 3baba2605..9d4b4145b 100644
--- a/storage_drivers/ontap/api/abstraction_rest.go
+++ b/storage_drivers/ontap/api/abstraction_rest.go
@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package api
 
@@ -3231,11 +3231,11 @@ func (d OntapAPIREST) NVMeSubsystemCreate(ctx context.Context, subsystemName str
 	fields := []string{"target_nqn"}
 	subsystem, err := d.api.NVMeSubsystemGetByName(ctx, subsystemName, fields)
 	if err != nil {
-		Logc(ctx).Infof("problem getting subsystem; %v", err)
+		Logc(ctx).Infof("Problem getting subsystem; %v", err)
 		return nil, err
 	}
 	if subsystem == nil {
-		Logc(ctx).Infof("subsystem doesn't exists, creating new subsystem %v now.", subsystemName)
+		Logc(ctx).Infof("Subsystem doesn't exist, creating new subsystem %v now.", subsystemName)
 		subsystem, err = d.api.NVMeSubsystemCreate(ctx, subsystemName)
 		if err != nil {
 			return nil, err
diff --git a/storage_drivers/ontap/ontap_san.go b/storage_drivers/ontap/ontap_san.go
index 3b9c778c7..08e7e1d3c 100644
--- a/storage_drivers/ontap/ontap_san.go
+++ b/storage_drivers/ontap/ontap_san.go
@@ -357,6 +357,7 @@ func (d *SANStorageDriver) Create(
 	if err != nil {
 		return fmt.Errorf("could not convert volume size %s: %v", volConfig.Size, err)
 	}
+
 	requestedSizeBytes, err := strconv.ParseUint(requestedSize, 10, 64)
 	if err != nil {
 		return fmt.Errorf("%v is an invalid volume size: %v", volConfig.Size, err)
@@ -367,8 +368,8 @@ func (d *SANStorageDriver) Create(
 	// part of the LUN but is not reported to the orchestrator.
 	reportedSize := lunSizeBytes
 	lunSizeBytes = incrementWithLUKSMetadataIfLUKSEnabled(ctx, lunSizeBytes, luksEncryption)
-
 	lunSize := strconv.FormatUint(lunSizeBytes, 10)
+
 	// Get the flexvol size based on the snapshot reserve
 	flexvolSize := drivers.CalculateVolumeSizeBytes(ctx, name, lunSizeBytes, snapshotReserveInt)
 	// Add extra 10% to the Flexvol to account for LUN metadata
diff --git a/storage_drivers/ontap/ontap_san_nvme.go b/storage_drivers/ontap/ontap_san_nvme.go
index 62ee05faa..5734fa257 100644
--- a/storage_drivers/ontap/ontap_san_nvme.go
+++ b/storage_drivers/ontap/ontap_san_nvme.go
@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package ontap
 
@@ -298,9 +298,19 @@ func (d *NVMeStorageDriver) Create(
 	if err != nil {
 		return fmt.Errorf("could not convert volume size %s: %v", volConfig.Size, err)
 	}
-	requestedSizeBytes, _ := strconv.ParseUint(requestedSize, 10, 64)
+
+	requestedSizeBytes, err := strconv.ParseUint(requestedSize, 10, 64)
+	if err != nil {
+		return fmt.Errorf("%v is an invalid volume size: %v", volConfig.Size, err)
+	}
 	namespaceSizeBytes := GetVolumeSize(requestedSizeBytes, storagePool.InternalAttributes()[Size])
+
+	// Add a constant overhead for LUKS volumes to account for LUKS metadata. This overhead is
+	// part of the LUN but is not reported to the orchestrator.
+	reportedSize := namespaceSizeBytes
+	namespaceSizeBytes = incrementWithLUKSMetadataIfLUKSEnabled(ctx, namespaceSizeBytes, luksEncryption)
 	namespaceSize := strconv.FormatUint(namespaceSizeBytes, 10)
+
 	// Get the FlexVol size based on the snapshot reserve.
 	flexVolSize := drivers.CalculateVolumeSizeBytes(ctx, name, namespaceSizeBytes, snapshotReserveInt)
 	// Add extra 10% to the FlexVol to account for Namespace metadata.
@@ -335,7 +345,7 @@ func (d *NVMeStorageDriver) Create(
 	}
 
 	// Update config to reflect values used to create volume.
-	volConfig.Size = strconv.FormatUint(namespaceSizeBytes, 10)
+	volConfig.Size = strconv.FormatUint(reportedSize, 10)
 	volConfig.SpaceReserve = spaceReserve
 	volConfig.SnapshotPolicy = snapshotPolicy
 	volConfig.SnapshotReserve = snapshotReserve
@@ -617,7 +627,9 @@ func (d *NVMeStorageDriver) Import(ctx context.Context, volConfig *storage.Volum
 	}
 
 	// Set the volume to LUKS if backend has LUKS true as default
-	volConfig.LUKSEncryption = d.Config.LUKSEncryption
+	if volConfig.LUKSEncryption == "" {
+		volConfig.LUKSEncryption = d.Config.LUKSEncryption
+	}
 
 	// Set the filesystem type to backend's, if it hasn't been set via annotation
 	// in the provided pvc during import.
@@ -649,6 +661,16 @@ func (d *NVMeStorageDriver) Import(ctx context.Context, volConfig *storage.Volum
 	// Use the Namespace size
 	volConfig.Size = nsInfo.Size
 
+	// If the import is a LUKS encrypted volume, then remove the LUKS metadata overhead from the reported
+	// size on the volConfig.
+	if utils.ParseBool(volConfig.LUKSEncryption) {
+		newSize, err := subtractUintFromSizeString(volConfig.Size, utils.LUKSMetadataSize)
+		if err != nil {
+			return err
+		}
+		volConfig.Size = newSize
+	}
+
 	// Rename the volume if Trident will manage its lifecycle
 	if !volConfig.ImportNotManaged {
 		err = d.API.VolumeRename(ctx, originalName, volConfig.InternalName)
@@ -1314,6 +1336,9 @@ func (d *NVMeStorageDriver) Resize(
 		return fmt.Errorf("requested size %d is less than existing volume size %d", requestedSizeBytes, nsSizeBytes)
 	}
 
+	// Add a constant overhead for LUKS volumes to account for LUKS metadata.
+	requestedSizeBytes = incrementWithLUKSMetadataIfLUKSEnabled(ctx, requestedSizeBytes, volConfig.LUKSEncryption)
+
 	snapshotReserveInt, err := getSnapshotReserveFromOntap(ctx, name, d.API.VolumeInfo)
 	if err != nil {
 		Logc(ctx).WithField("name", name).Errorf("Could not get the snapshot reserve percentage for volume.")
@@ -1376,6 +1401,9 @@ func (d *NVMeStorageDriver) Resize(
 		}
 	}
 
+	// LUKS metadata size is not reported so remove it from LUN size
+	requestedSizeBytes = decrementWithLUKSMetadataIfLUKSEnabled(ctx, requestedSizeBytes, volConfig.LUKSEncryption)
+
 	// Setting the new size in the volume config.
 	volConfig.Size = strconv.FormatUint(requestedSizeBytes, 10)
 	return nil
diff --git a/storage_drivers/ontap/ontap_san_nvme_test.go b/storage_drivers/ontap/ontap_san_nvme_test.go
index b1c9855f1..b2a4991bd 100644
--- a/storage_drivers/ontap/ontap_san_nvme_test.go
+++ b/storage_drivers/ontap/ontap_san_nvme_test.go
@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package ontap
 
@@ -6,6 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -18,6 +19,7 @@ import (
 	drivers "github.com/netapp/trident/storage_drivers"
 	"github.com/netapp/trident/storage_drivers/ontap/api"
 	"github.com/netapp/trident/storage_drivers/ontap/awsapi"
+	"github.com/netapp/trident/utils"
 	"github.com/netapp/trident/utils/errors"
 	"github.com/netapp/trident/utils/filesystem"
 	"github.com/netapp/trident/utils/models"
@@ -980,6 +982,21 @@ func TestNVMeCreate_NamespaceCreateAPIError(t *testing.T) {
 	assert.ErrorContains(t, err, "failed to create namespace")
 }
 
+func TestNVMeCreate_LUKSVolume(t *testing.T) {
+	d, mAPI := newNVMeDriverAndMockApi(t)
+	pool1, volConfig, volAttrs := getNVMeCreateArgs(d)
+
+	volConfig.LUKSEncryption = "true"
+	mAPI.EXPECT().VolumeExists(ctx, volConfig.InternalName).Return(false, nil)
+	mAPI.EXPECT().TieringPolicyValue(ctx).Return("TPolicy")
+	mAPI.EXPECT().VolumeCreate(ctx, gomock.Any()).Return(nil)
+	mAPI.EXPECT().NVMeNamespaceCreate(ctx, gomock.Any()).Return("nsUUID", nil)
+
+	err := d.Create(ctx, volConfig, pool1, volAttrs)
+
+	assert.NoError(t, err, "Failed to create NVMe volume.")
+}
+
 func TestNVMeCreate_Success(t *testing.T) {
 	d, mAPI := newNVMeDriverAndMockApi(t)
 	pool1, volConfig, volAttrs := getNVMeCreateArgs(d)
@@ -2248,6 +2265,52 @@ func TestImport(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestImport_LUKSNamespace(t *testing.T) {
+	d, mAPI := newNVMeDriverAndMockApi(t)
+	_, volConfig, _ := getNVMeCreateArgs(d)
+	originalName := "fakeOriginalName"
+	vol := &api.Volume{Aggregates: []string{"data"}}
+	ns := &api.NVMeNamespace{
+		Name: "/vol/cloneVol1/namespace0",
+		Size: "20GB",
+		UUID: "fakeUUID",
+	}
+	ns.State = "online"
+
+	vol.Comment = "fakeComment"
+	volConfig.LUKSEncryption = "true"
+	volConfig.ImportNotManaged = true
+	volConfig.Size = "20GB"
+	mAPI.EXPECT().VolumeInfo(ctx, gomock.Any()).Return(vol, nil)
+	mAPI.EXPECT().NVMeNamespaceGetByName(ctx, "/vol/"+originalName+"/*").Return(ns, nil)
+	mAPI.EXPECT().NVMeIsNamespaceMapped(ctx, "", ns.UUID).Return(false, nil)
+	// mAPI.EXPECT().VolumeRename(ctx, originalName, volConfig.InternalName).Return(nil)
+
+	beforeLUKSOverheadBytesStr, err := utils.ConvertSizeToBytes(volConfig.Size)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+	beforeLUKSOverhead, err := strconv.ParseUint(beforeLUKSOverheadBytesStr, 10, 64)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+
+	err = d.Import(ctx, volConfig, originalName)
+
+	afterLUKSOverheadBytesStr, err := utils.ConvertSizeToBytes(volConfig.Size)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+	afterLUKSOverhead, err := strconv.ParseUint(afterLUKSOverheadBytesStr, 10, 64)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+
+	assert.NoError(t, err)
+	assert.Less(t, afterLUKSOverhead, beforeLUKSOverhead)
+	assert.Equal(t, beforeLUKSOverhead, incrementWithLUKSMetadataIfLUKSEnabled(ctx, afterLUKSOverhead, "true"))
+}
+
 func TestImport_NameTemplate(t *testing.T) {
 	d, mAPI := newNVMeDriverAndMockApi(t)
 	_, volConfig, _ := getNVMeCreateArgs(d)
diff --git a/utils/devices.go b/utils/devices.go
index a08a2b629..a607aba8b 100644
--- a/utils/devices.go
+++ b/utils/devices.go
@@ -1071,7 +1071,7 @@ func GetLUKSDeviceForMultipathDevice(multipathDevice string) (string, error) {
 	// Get holder of mpath device
 	dirents, err := os.ReadDir(fmt.Sprintf("/sys/block/%s/holders/", dmDevice))
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to get holders directory entries; %v", err)
 	}
 
 	if len(dirents) == 0 {
diff --git a/utils/devices_linux.go b/utils/devices_linux.go
index 306301a02..3241caeb4 100644
--- a/utils/devices_linux.go
+++ b/utils/devices_linux.go
@@ -24,9 +24,10 @@ import (
 )
 
 const (
-	luksCommandTimeout time.Duration = time.Second * 30
-	luksCypherMode                   = "aes-xts-plain64"
-	luksType                         = "luks2"
+	luksCommandTimeout = time.Second * 30
+	luksCypherMode     = "aes-xts-plain64"
+	luksType           = "luks2"
+
 	// Return code for "no permission (bad passphrase)" from cryptsetup command
 	luksCryptsetupBadPassphraseReturnCode = 2
 
@@ -46,7 +47,8 @@ var (
 	beforeBlockDeviceFlushBuffer              = fiji.Register("beforeBlockDeviceFlushBuffer", "devices_linux")
 	beforeCryptSetupFormat                    = fiji.Register("beforeCryptSetupFormat", "node_server")
 	duringOpenBeforeCryptSetupOpen            = fiji.Register("duringOpenBeforeCryptSetupOpen", "devices_linux")
-	duringRotatePassphraseBeforeLuksKeyChange = fiji.Register("duringRotatePassphraseBeforeLuksKeyChange", "devices_linux")
+	duringRotatePassphraseBeforeLuksKeyChange = fiji.Register("duringRotatePassphraseBeforeLuksKeyChange",
+		"devices_linux")
 
 	osFs = afero.NewOsFs()
 )
@@ -360,8 +362,8 @@ func EnsureLUKSDeviceClosed(ctx context.Context, devicePath string) error {
 		Logc(ctx).WithFields(LogFields{
 			"device": devicePath,
 			"error":  err.Error(),
-		}).Debug("Failed to stat device")
-		return fmt.Errorf("could not stat device: %s %v.", devicePath, err)
+		}).Debug("Failed to stat device.")
+		return fmt.Errorf("could not stat device: %s; %v", devicePath, err)
 	}
 	Logc(ctx).WithFields(LogFields{
 		"device": devicePath,
diff --git a/utils/nvme.go b/utils/nvme.go
index 2454adad5..fccf208e5 100644
--- a/utils/nvme.go
+++ b/utils/nvme.go
@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package utils
 
@@ -280,6 +280,8 @@ func AttachNVMeVolumeRetry(
 func AttachNVMeVolume(
 	ctx context.Context, name, mountpoint string, publishInfo *models.VolumePublishInfo, secrets map[string]string,
 ) error {
+	Logc(ctx).Debug(">>>> nvme.AttachNVMeVolume")
+	defer Logc(ctx).Debug("<<<< nvme.AttachNVMeVolume")
 	nvmeHandler := NewNVMeHandler()
 	nvmeSubsys := nvmeHandler.NewNVMeSubsystem(ctx, publishInfo.NVMeSubsystemNQN)
 	connectionStatus := nvmeSubsys.GetConnectionStatus()
@@ -298,32 +300,61 @@ func AttachNVMeVolume(
 	devPath := nvmeDev.GetPath()
 	publishInfo.DevicePath = devPath
 
-	if err = NVMeMountVolume(ctx, name, mountpoint, publishInfo); err != nil {
+	if err = NVMeMountVolume(ctx, name, mountpoint, publishInfo, secrets); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func NVMeMountVolume(ctx context.Context, name, mountpoint string, publishInfo *models.VolumePublishInfo) error {
+func NVMeMountVolume(
+	ctx context.Context, name, mountpoint string, publishInfo *models.VolumePublishInfo, secrets map[string]string,
+) error {
+	Logc(ctx).Debug(">>>> nvme.NVMeMountVolume")
+	defer Logc(ctx).Debug("<<<< nvme.NVMeMountVolume")
+
+	// This is the raw device path for a nvme namespace.
+	devicePath := publishInfo.DevicePath
+
+	// Format and open a LUKS device if LUKS Encryption is set to true.
+	var luksFormatted bool
+	var err error
+	isLUKSDevice := ParseBool(publishInfo.LUKSEncryption)
+	if isLUKSDevice {
+		luksDevice, _ := NewLUKSDevice(devicePath, name)
+		luksFormatted, err = EnsureLUKSDeviceMappedOnHost(ctx, luksDevice, name, secrets)
+		if err != nil {
+			return err
+		}
+		devicePath = luksDevice.MappedDevicePath()
+	}
+
+	if isLUKSDevice && !luksFormatted {
+		Logc(ctx).Errorf("Unable to identify if luks device.", devicePath)
+		return err
+	}
+
+	// No filesystem work is required for raw block; return early.
 	if publishInfo.FilesystemType == filesystem.Raw {
 		return nil
 	}
-	devicePath := publishInfo.DevicePath
 
 	existingFstype, err := getDeviceFSType(ctx, devicePath)
 	if err != nil {
 		return err
 	}
 	if existingFstype == "" {
-		if unformatted, err := isDeviceUnformatted(ctx, devicePath); err != nil {
-			Logc(ctx).WithField("device",
-				devicePath).Errorf("Unable to identify if the device is not formatted; err: %v", err)
-			return err
-		} else if !unformatted {
-			Logc(ctx).WithField("device", devicePath).Errorf("Device is not not formatted; err: %v", err)
-			return fmt.Errorf("device %v is not unformatted", devicePath)
+		if !isLUKSDevice {
+			if unformatted, err := isDeviceUnformatted(ctx, devicePath); err != nil {
+				Logc(ctx).WithField("device",
+					devicePath).Errorf("Unable to identify if the device is not formatted; err: %v", err)
+				return err
+			} else if !unformatted {
+				Logc(ctx).WithField("device", devicePath).Errorf("Device is not not formatted; err: %v", err)
+				return fmt.Errorf("device %v is not unformatted", devicePath)
+			}
 		}
+
 		Logc(ctx).WithFields(LogFields{"volume": name, "fstype": publishInfo.FilesystemType}).Debug("Formatting LUN.")
 		err := fsClient.FormatVolume(ctx, devicePath, publishInfo.FilesystemType, publishInfo.FormatOptions)
 		if err != nil {
diff --git a/utils/nvme_linux.go b/utils/nvme_linux.go
index b614d6919..d6b0ae6fd 100644
--- a/utils/nvme_linux.go
+++ b/utils/nvme_linux.go
@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package utils