From 786353ed3b5814280caa8b6e5d031d779e0299b3 Mon Sep 17 00:00:00 2001 From: Chris Oo Date: Mon, 26 Oct 2020 18:46:45 +0000 Subject: [PATCH] MdeModulePkg: Add alternative queue sizes in nvme driver to support specific devices Add a new PCD that toggles between the old driver behavior and new behavior that uses a different hardcoded queue size to support specific devices. Default behavior should not change. MdeModulePkg/NvmExpressDxe: Improve NVMe controller init robustness It has been observed that some faulty NVMe devices may return invalid values. The code in NvmExpressDxe recognizes the controller is not responding correctly and issues an ASSERT() often in DEBUG builds or a reset in RELEASE builds. The following changes are made to NvmeControllerInit() to prevent a faulty NVMe device from halting the overall boot: 1. Check the Vendor ID and Device ID to verify they are read properly and if not, return EFI_DEVICE_ERROR 2. Replace the ASSERT() when Memory Page Size Minimum (Cap.Mpsmin) with an error message and return EFI_DEVICE_ERROR Signed-off-by: Michael Kubacki MdeModulePkg/NvmExpressDxe: Correct function parameter modifer Updates the `Cap` parameter for `ReadNvmeControllerCapabilities()` to be `OUT` since the buffer pointed to is written within the function. Signed-off-by: Michael Kubacki --- .../Bus/Pci/NvmExpressDxe/NvmExpress.c | 43 +++++- .../Bus/Pci/NvmExpressDxe/NvmExpress.h | 15 ++ .../Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf | 5 + .../Bus/Pci/NvmExpressDxe/NvmExpressHci.c | 132 +++++++++++++----- .../Pci/NvmExpressDxe/NvmExpressPassthru.c | 53 +++++-- MdeModulePkg/MdeModulePkg.dec | 10 ++ 6 files changed, 209 insertions(+), 49 deletions(-) diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c index dea14f1a446..2541a3b0705 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c @@ -555,6 +555,9 @@ ProcessAsyncTaskList ( EFI_BLOCK_IO2_TOKEN *Token; BOOLEAN HasNewItem; EFI_STATUS Status; + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + UINT16 QueueSize = PcdGetBool (PcdSupportAlternativeQueueSize) ? + NVME_ALTERNATIVE_MAX_QUEUE_SIZE : NVME_ASYNC_CCQ_SIZE; Private = (NVME_CONTROLLER_PRIVATE_DATA *)Context; QueueId = 2; @@ -697,7 +700,8 @@ ProcessAsyncTaskList ( } Private->CqHdbl[QueueId].Cqh++; - if (Private->CqHdbl[QueueId].Cqh > MIN (NVME_ASYNC_CCQ_SIZE, Private->Cap.Mqes)) { + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + if (Private->CqHdbl[QueueId].Cqh > MIN (QueueSize, Private->Cap.Mqes)) { Private->CqHdbl[QueueId].Cqh = 0; Private->Pt[QueueId] ^= 1; } @@ -930,6 +934,9 @@ NvmExpressDriverBindingStart ( EFI_PHYSICAL_ADDRESS MappedAddr; UINTN Bytes; EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *Passthru; + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + UINTN QueuePageCount = PcdGetBool (PcdSupportAlternativeQueueSize) ? + NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES : 6; DEBUG ((DEBUG_INFO, "NvmExpressDriverBindingStart: start\n")); @@ -1001,7 +1008,13 @@ NvmExpressDriverBindingStart ( DEBUG ((DEBUG_WARN, "NvmExpressDriverBindingStart: failed to enable 64-bit DMA (%r)\n", Status)); } + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + + // + // Depending on PCD disablement, either support the default or alternative + // queue sizes. // + // Default: // 6 x 4kB aligned buffers will be carved out of this buffer. // 1st 4kB boundary is the start of the admin submission queue. // 2nd 4kB boundary is the start of the admin completion queue. @@ -1012,11 +1025,22 @@ NvmExpressDriverBindingStart ( // // Allocate 6 pages of memory, then map it for bus master read and write. // + // Alternative: + // 15 x 4kB aligned buffers will be carved out of this buffer. + // 1st 4kB boundary is the start of the admin submission queue. + // 5th 4kB boundary is the start of the admin completion queue. + // 6th 4kB boundary is the start of I/O submission queue #1. + // 10th 4kB boundary is the start of I/O completion queue #1. + // 11th 4kB boundary is the start of I/O submission queue #2. + // 15th 4kB boundary is the start of I/O completion queue #2. + // + // Allocate 15 pages of memory, then map it for bus master read and write. + // Status = PciIo->AllocateBuffer ( PciIo, AllocateAnyPages, EfiBootServicesData, - 6, + QueuePageCount, (VOID **)&Private->Buffer, 0 ); @@ -1024,7 +1048,8 @@ NvmExpressDriverBindingStart ( goto Exit; } - Bytes = EFI_PAGES_TO_SIZE (6); + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + Bytes = EFI_PAGES_TO_SIZE (QueuePageCount); Status = PciIo->Map ( PciIo, EfiPciIoOperationBusMasterCommonBuffer, @@ -1034,7 +1059,8 @@ NvmExpressDriverBindingStart ( &Private->Mapping ); - if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) { + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (QueuePageCount))) { goto Exit; } @@ -1144,7 +1170,8 @@ NvmExpressDriverBindingStart ( } if ((Private != NULL) && (Private->Buffer != NULL)) { - PciIo->FreeBuffer (PciIo, 6, Private->Buffer); + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + PciIo->FreeBuffer (PciIo, QueuePageCount, Private->Buffer); } if ((Private != NULL) && (Private->ControllerData != NULL)) { @@ -1220,6 +1247,9 @@ NvmExpressDriverBindingStop ( EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru; BOOLEAN IsEmpty; EFI_TPL OldTpl; + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + UINT16 QueuePageCount = PcdGetBool (PcdSupportAlternativeQueueSize) ? + NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES : 6; if (NumberOfChildren == 0) { Status = gBS->OpenProtocol ( @@ -1266,7 +1296,8 @@ NvmExpressDriverBindingStop ( } if (Private->Buffer != NULL) { - Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer); + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + Private->PciIo->FreeBuffer (Private->PciIo, QueuePageCount, Private->Buffer); } FreePool (Private->ControllerData); diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h index ec66de83723..fb8b13a8377 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h @@ -81,6 +81,21 @@ extern EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiV #define NVME_MAX_QUEUES 3 // Number of queues supported by the driver +// MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + +// +// NVMe DXE to accommodate hardware which requires queue size 255. +// Driver supports queue size up to 255 (4 page SQ buffer). +// DXE driver creates queue size MIN(Cap.Mqes, NVME_MAX_QUEUE_SIZE) for all queues. +// Driver allocates queue buffer to support 255 max queue size. +// Each submission queue buffer is allocated as 64B * 256 = 4 * 4kB = 4 pages. +// Each completion queue buffer is allocated as 16B * 256 = 4kB = 1 page. +// +#define NVME_ALTERNATIVE_MAX_QUEUE_SIZE 255 +#define NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES NVME_MAX_QUEUES * 5 + +// MU_CHANGE [END] + #define NVME_CONTROLLER_ID 0 // diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf index a1e6821883d..bb61e842ae0 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf @@ -76,6 +76,11 @@ gEfiDriverSupportedEfiVersionProtocolGuid ## PRODUCES gEfiResetNotificationProtocolGuid ## CONSUMES +[Pcd] + ## MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize ## CONSUMES + ## MU_CHANGE [END] + # [Event] # EVENT_TYPE_RELATIVE_TIMER ## SOMETIMES_CONSUMES # diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c index 4327afc1794..afd3dd5429e 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c @@ -30,8 +30,10 @@ UINTN mNvmeControllerNumber = 0; **/ EFI_STATUS ReadNvmeControllerCapabilities ( - IN NVME_CONTROLLER_PRIVATE_DATA *Private, - IN NVME_CAP *Cap + // MU_CHANGE [BEGIN] - Correct Cap parameter modifier + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + OUT NVME_CAP *Cap + // MU_CHANGE [END] - Correct Cap parameter modifier ) { EFI_PCI_IO_PROTOCOL *PciIo; @@ -606,16 +608,19 @@ NvmeCreateIoCompletionQueue ( CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; CommandPacket.QueueType = NVME_ADMIN_QUEUE; - if (Index == 1) { + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes); + } else if (Index == 1) { QueueSize = NVME_CCQ_SIZE; + } else if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { + QueueSize = NVME_ASYNC_CCQ_SIZE; } else { - if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { - QueueSize = NVME_ASYNC_CCQ_SIZE; - } else { - QueueSize = Private->Cap.Mqes; - } + QueueSize = Private->Cap.Mqes; } + // MU_CHANGE [END] + CrIoCq.Qid = Index; CrIoCq.Qsize = QueueSize; CrIoCq.Pc = 1; @@ -678,16 +683,19 @@ NvmeCreateIoSubmissionQueue ( CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; CommandPacket.QueueType = NVME_ADMIN_QUEUE; - if (Index == 1) { - QueueSize = NVME_CSQ_SIZE; + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes); + } else if (Index == 1) { + QueueSize = NVME_CCQ_SIZE; + } else if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { + QueueSize = NVME_ASYNC_CCQ_SIZE; } else { - if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) { - QueueSize = NVME_ASYNC_CSQ_SIZE; - } else { - QueueSize = Private->Cap.Mqes; - } + QueueSize = Private->Cap.Mqes; } + // MU_CHANGE [END] + CrIoSq.Qid = Index; CrIoSq.Qsize = QueueSize; CrIoSq.Pc = 1; @@ -732,13 +740,36 @@ NvmeControllerInit ( NVME_AQA Aqa; NVME_ASQ Asq; NVME_ACQ Acq; + UINT16 VidDid[2]; // MU_CHANGE - Improve NVMe controller init robustness UINT8 Sn[21]; UINT8 Mn[41]; + // MU_CHANGE [BEGIN] - Improve NVMe controller init robustness + PciIo = Private->PciIo; + + // + // Verify the controller is still accessible + // + Status = PciIo->Pci.Read ( + PciIo, + EfiPciIoWidthUint16, + PCI_VENDOR_ID_OFFSET, + ARRAY_SIZE (VidDid), + VidDid + ); + if (EFI_ERROR (Status)) { + ASSERT_EFI_ERROR (Status); + return EFI_DEVICE_ERROR; + } + + if ((VidDid[0] == 0xFFFF) || (VidDid[1] == 0xFFFF)) { + return EFI_DEVICE_ERROR; + } + // // Enable this controller. // - PciIo = Private->PciIo; + // MU_CHANGE [END] - Improve NVMe controller init robustness Status = PciIo->Attributes ( PciIo, EfiPciIoAttributeOperationSupported, @@ -777,7 +808,17 @@ NvmeControllerInit ( // // Currently the driver only supports 4k page size. // - ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); + + // MU_CHANGE [BEGIN] - Improve NVMe controller init robustness + + // Currently, this means Cap.Mpsmin must be zero for an EFI_PAGE_SHIFT size of 12. + // ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); + if ((Private->Cap.Mpsmin + 12) > EFI_PAGE_SHIFT) { + DEBUG ((DEBUG_ERROR, "NvmeControllerInit: Mpsmin is larger than expected (0x%02x).\n", Private->Cap.Mpsmin)); + return EFI_DEVICE_ERROR; + } + + // MU_CHANGE [END] - Improve NVMe controller init robustness Private->Cid[0] = 0; Private->Cid[1] = 0; @@ -802,10 +843,12 @@ NvmeControllerInit ( // // set number of entries admin submission & completion queues. // - Aqa.Asqs = NVME_ASQ_SIZE; + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + Aqa.Asqs = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : NVME_ASQ_SIZE; Aqa.Rsvd1 = 0; - Aqa.Acqs = NVME_ACQ_SIZE; + Aqa.Acqs = PcdGetBool (PcdSupportAlternativeQueueSize) ? MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) : NVME_ACQ_SIZE; Aqa.Rsvd2 = 0; + // MU_CHANGE [END] // // Address of admin submission queue. @@ -815,24 +858,47 @@ NvmeControllerInit ( // // Address of admin completion queue. // - Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + Acq = (UINT64)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE) & ~0xFFF; + } else { + Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; + } // // Address of I/O submission & completion queue. // - ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); - Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); - Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); - Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); - Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); - Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); - Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); - Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); - Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); - Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); - Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); - Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); - Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (NVME_ALTERNATIVE_TOTAL_QUEUE_BUFFER_IN_PAGES)); + Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); + Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); + Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); + Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 9 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 9 * EFI_PAGE_SIZE); + Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 10 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 10 * EFI_PAGE_SIZE); + Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 14 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 14 * EFI_PAGE_SIZE); + } else { + ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); + Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); + Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); + Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); + Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); + Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); + Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); + Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + } + + // MU_CHANGE [END] DEBUG ((DEBUG_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer)); DEBUG ((DEBUG_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs)); diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c index 01133faf568..0a6ef3a641c 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c @@ -553,7 +553,15 @@ NvmExpressPassThru ( Prp = NULL; TimerEvent = NULL; Status = EFI_SUCCESS; - QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1; + + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + QueueSize = MIN (NVME_ALTERNATIVE_MAX_QUEUE_SIZE, Private->Cap.Mqes) + 1; + } else { + QueueSize = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1; + } + + // MU_CHANGE [END] if (Packet->QueueType == NVME_ADMIN_QUEUE) { QueueId = 0; @@ -581,6 +589,14 @@ NvmExpressPassThru ( return EFI_INVALID_PARAMETER; } + // MU_CHANGE - Support alternative hardware queue sizes in NVME driver + // + // Nvme DXE driver polls phase bit for CQe completion. + // Explicitly assign phase bit with the bit before completion. + // A flipped phase bit will be assigned by device upon CQe completes. + // + Cq->Pt = Private->Pt[QueueId]; + ZeroMem (Sq, sizeof (NVME_SQ)); Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode; Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation; @@ -723,16 +739,23 @@ NvmExpressPassThru ( Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15; } - // - // Ring the submission queue doorbell. - // - if ((Event != NULL) && (QueueId != 0)) { - Private->SqTdbl[QueueId].Sqt = - (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize; + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + Private->SqTdbl[QueueId].Sqt = (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize; } else { - Private->SqTdbl[QueueId].Sqt ^= 1; + // + // Ring the submission queue doorbell. + // + if ((Event != NULL) && (QueueId != 0)) { + Private->SqTdbl[QueueId].Sqt = + (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize; + } else { + Private->SqTdbl[QueueId].Sqt ^= 1; + } } + // MU_CHANGE [END] + Data = ReadUnaligned32 ((UINT32 *)&Private->SqTdbl[QueueId]); Status = PciIo->Mem.Write ( PciIo, @@ -867,10 +890,20 @@ NvmExpressPassThru ( goto EXIT; } - if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) { - Private->Pt[QueueId] ^= 1; + // MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + if (PcdGetBool (PcdSupportAlternativeQueueSize)) { + Private->CqHdbl[QueueId].Cqh = (Private->CqHdbl[QueueId].Cqh + 1) % QueueSize; + if (Private->CqHdbl[QueueId].Cqh == 0) { + Private->Pt[QueueId] ^= 1; + } + } else { + if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) { + Private->Pt[QueueId] ^= 1; + } } + // MU_CHANGE [END] + Data = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]); PreviousStatus = Status; Status = PciIo->Mem.Write ( diff --git a/MdeModulePkg/MdeModulePkg.dec b/MdeModulePkg/MdeModulePkg.dec index aedc6dd9713..ae42d8c6ddc 100644 --- a/MdeModulePkg/MdeModulePkg.dec +++ b/MdeModulePkg/MdeModulePkg.dec @@ -1246,6 +1246,16 @@ # @Prompt Delay access XHCI register after it issues HCRST (us) gEfiMdeModulePkgTokenSpaceGuid.PcdDelayXhciHCReset|2000|UINT16|0x30001060 + # MU_CHANGE TCBZ1086 [BEGIN] - Mitigate potential system brick due to uefi MemoryTypeInformation var changes + ## Maximum total pages that can be reserved by MemoryTypeInfo buckets. + # Size in pages. Current default is roughly 1.6GB. + gEfiMdeModulePkgTokenSpaceGuid.PcdMaxMemoryTypeInfoPages |0x60000|UINT32|0x40000150 + # MU_CHANGE TCBZ1086 [END] + + # MU_CHANGE [BEGIN] - Support alternative hardware queue sizes in NVME driver + gEfiMdeModulePkgTokenSpaceGuid.PcdSupportAlternativeQueueSize|FALSE|BOOLEAN|0x40000151 + # MU_CHANGE [END] + # MU_CHANGE [BEGIN] - Support indefinite boot retries # # Some platforms require that all EfiLoadOptions are retried until one of the options # # succeeds. When True, this Pcd will force Bds to retry all the valid EfiLoadOptions