Skip to content

Commit 1368f0e

Browse files
ashishsinghajgarver
authored andcommitted
feat(smmuv3): add cache maintenance
add cache maintenance support for: - the buffers shared between cpu and device - the queues and ste entries shared between cpu and smmu Signed-off-by: Ashish Singhal <[email protected]> Tested-by: Jake Garver <[email protected]> Reviewed-by: Jake Garver <[email protected]>
1 parent 11ffcb1 commit 1368f0e

File tree

3 files changed

+40
-17
lines changed

3 files changed

+40
-17
lines changed

Silicon/NVIDIA/Drivers/SmmuDriverDxe/SmmuDriverDxe.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <Library/BaseMemoryLib.h>
1616
#include <Library/UefiBootServicesTableLib.h>
1717
#include <Library/MemoryAllocationLib.h>
18+
#include <Library/CacheMaintenanceLib.h>
1819
#include <Protocol/PciIo.h>
1920
#include <Library/SmmuLib.h>
2021
#include <Protocol/SmmuV3Protocol.h>
@@ -289,6 +290,9 @@ IoMmuMap (
289290
(VOID *)(UINTN)MapInfo->HostAddress,
290291
MapInfo->NumberOfBytes
291292
);
293+
294+
// Perform cache maintenance on the destination (device memory)
295+
WriteBackDataCacheRange ((VOID *)(UINTN)MapInfo->DeviceAddress, MapInfo->NumberOfBytes);
292296
}
293297
} else {
294298
MapInfo->DeviceAddress = MapInfo->HostAddress;
@@ -351,6 +355,9 @@ IoMmuUnmap (
351355
if ((MapInfo->Operation == EdkiiIoMmuOperationBusMasterWrite) ||
352356
(MapInfo->Operation == EdkiiIoMmuOperationBusMasterWrite64))
353357
{
358+
// Invalidate the cache for the source (device memory)
359+
InvalidateDataCacheRange ((VOID *)(UINTN)MapInfo->DeviceAddress, MapInfo->NumberOfBytes);
360+
354361
CopyMem (
355362
(VOID *)(UINTN)MapInfo->HostAddress,
356363
(VOID *)(UINTN)MapInfo->DeviceAddress,

Silicon/NVIDIA/Drivers/SmmuDriverDxe/SmmuDriverDxe.inf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
UefiDriverEntryPoint
3636
DebugLib
3737
MemoryAllocationLib
38+
CacheMaintenanceLib
3839
PcdLib
3940
SmmuLib
4041

Silicon/NVIDIA/Drivers/SmmuV3Dxe/SmmuV3Dxe.c

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -423,13 +423,16 @@ SetupSmmuV3Cmdq (
423423
DEBUG ((DEBUG_INFO, "%a: Total CMDQ entries: %d\n", __FUNCTION__, (1 << Private->Features.CmdqEntriesLog2)));
424424

425425
QBase = (EFI_PHYSICAL_ADDRESS)AllocateAlignedPages (EFI_SIZE_TO_PAGES (CmdqSize), CmdqSize);
426-
ZeroMem ((VOID *)QBase, EFI_PAGES_TO_SIZE (EFI_SIZE_TO_PAGES (CmdqSize)));
427-
428426
if (!QBase) {
429427
DEBUG ((DEBUG_ERROR, "%a: Failed to allocate memory for CMDQ\n", __FUNCTION__));
430428
return EFI_OUT_OF_RESOURCES;
431429
}
432430

431+
ZeroMem ((VOID *)QBase, EFI_PAGES_TO_SIZE (EFI_SIZE_TO_PAGES (CmdqSize)));
432+
433+
// Perform cache maintenance to ensure SMMU sees the updated memory
434+
WriteBackDataCacheRange ((VOID *)QBase, EFI_PAGES_TO_SIZE (EFI_SIZE_TO_PAGES (CmdqSize)));
435+
433436
DEBUG ((DEBUG_INFO, "%a: Memory allocated at %lx for CMDQ\n", __FUNCTION__, QBase));
434437
Private->CmdQueue.QBase = QBase;
435438

@@ -448,6 +451,9 @@ SetupSmmuV3Cmdq (
448451
MmioWrite32 (Private->CmdQueue.ConsRegBase, 0);
449452
MmioWrite32 (Private->CmdQueue.ProdRegBase, 0);
450453

454+
// Ensure memory ordering
455+
ArmDataSynchronizationBarrier ();
456+
451457
return EFI_SUCCESS;
452458
}
453459

@@ -481,13 +487,16 @@ SetupSmmuV3Evtq (
481487
DEBUG ((DEBUG_INFO, "%a: Total EVTQ entries: %d\n", __FUNCTION__, (1 << Private->Features.EvtqEntriesLog2)));
482488

483489
QBase = (EFI_PHYSICAL_ADDRESS)AllocateAlignedPages (EFI_SIZE_TO_PAGES (EvtqSize), EvtqSize);
484-
ZeroMem ((VOID *)QBase, EFI_PAGES_TO_SIZE (EFI_SIZE_TO_PAGES (EvtqSize)));
485-
486490
if (!QBase) {
487491
DEBUG ((DEBUG_ERROR, "%a: Failed to allocate memory for EVTQ\n", __FUNCTION__));
488492
return EFI_OUT_OF_RESOURCES;
489493
}
490494

495+
ZeroMem ((VOID *)QBase, EFI_PAGES_TO_SIZE (EFI_SIZE_TO_PAGES (EvtqSize)));
496+
497+
// Perform cache maintenance to ensure SMMU sees the updated memory
498+
WriteBackDataCacheRange ((VOID *)QBase, EFI_PAGES_TO_SIZE (EFI_SIZE_TO_PAGES (EvtqSize)));
499+
491500
DEBUG ((DEBUG_INFO, "%a: Memory allocated at %lx for EVTQ\n", __FUNCTION__, QBase));
492501
Private->EvtQueue.QBase = QBase;
493502

@@ -506,6 +515,9 @@ SetupSmmuV3Evtq (
506515
MmioWrite32 (Private->EvtQueue.ConsRegBase, 0);
507516
MmioWrite32 (Private->EvtQueue.ProdRegBase, 0);
508517

518+
// Ensure memory ordering
519+
ArmDataSynchronizationBarrier ();
520+
509521
return EFI_SUCCESS;
510522
}
511523

@@ -559,7 +571,10 @@ WriteSte (
559571
SteEntry[Index] = SteData[Index];
560572
}
561573

562-
// Ensure written data (STE) is observable to SMMU controller by performing DSB
574+
// Perform cache maintenance to ensure SMMU sees the updated STE
575+
WriteBackDataCacheRange ((VOID *)SteEntry, SMMU_V3_STRTAB_ENTRY_SIZE);
576+
577+
// Ensure memory ordering
563578
ArmDataSynchronizationBarrier ();
564579
}
565580

@@ -839,6 +854,9 @@ PushEntryToCmdq (
839854
CmdqEntry[Index] = CmdDword[Index];
840855
}
841856

857+
// Perform cache maintenance to ensure SMMU sees the updated command
858+
WriteBackDataCacheRange ((VOID *)CmdqEntry, SMMU_V3_CMD_SIZE);
859+
842860
// Ensure data is observable to SMMU
843861
ArmDataSynchronizationBarrier ();
844862
}
@@ -1723,13 +1741,11 @@ CreateStage2Ste (
17231741
return EFI_OUT_OF_RESOURCES;
17241742
}
17251743

1726-
// Invalidate cache before page table setup to ensure working with
1727-
// clean memory state and prevent stale cache entries from affecting
1728-
// page table population
1729-
InvalidateDataCacheRange ((VOID *)Ttbr, EFI_PAGE_SIZE);
1730-
17311744
ZeroMem ((VOID *)Ttbr, EFI_PAGE_SIZE);
17321745

1746+
// Perform cache maintenance to ensure SMMU sees the updated page table
1747+
WriteBackDataCacheRange ((VOID *)Ttbr, EFI_PAGE_SIZE);
1748+
17331749
CopyMem (SteS2TtbAddr, &Ttbr, sizeof (EFI_PHYSICAL_ADDRESS));
17341750

17351751
SteCount = (1 << Private->Features.StreamNBits);
@@ -1743,6 +1759,9 @@ CreateStage2Ste (
17431759

17441760
Ste[3] |= BIT_FIELD_SET (TtbrTemp, SMMU_V3_STE_S2TTB_MASK, SMMU_V3_STE_S2TTB_SHIFT);
17451761

1762+
// Perform cache maintenance to ensure SMMU sees the updated STE
1763+
WriteBackDataCacheRange ((VOID *)Ste, SMMU_V3_STRTAB_ENTRY_SIZE);
1764+
17461765
return EFI_SUCCESS;
17471766
}
17481767

@@ -1889,7 +1908,7 @@ InstallBlockPte (
18891908
DEBUG ((DEBUG_INFO, "%a: L%u entry address: 0x%lx\n", __FUNCTION__, Level, (UINT64)Pte));
18901909
DEBUG ((DEBUG_INFO, "%a: L%u entry value: 0x%lx\n", __FUNCTION__, Level, *Pte));
18911910

1892-
InvalidateDataCacheRange ((VOID *)Pte, sizeof (*Pte));
1911+
WriteBackDataCacheRange ((VOID *)Pte, sizeof (*Pte));
18931912

18941913
DEBUG ((DEBUG_INFO, "%a: Exit Success\n", __FUNCTION__));
18951914
return EFI_SUCCESS;
@@ -1929,7 +1948,7 @@ InstallTablePte (
19291948
DEBUG ((DEBUG_INFO, "%a: L%u entry address: 0x%lx\n", __FUNCTION__, Level, (UINT64)TtPte));
19301949
DEBUG ((DEBUG_INFO, "%a: L%u entry value: 0x%lx\n", __FUNCTION__, Level, *TtPte));
19311950

1932-
InvalidateDataCacheRange ((VOID *)TtPte, sizeof (*TtPte));
1951+
WriteBackDataCacheRange ((VOID *)TtPte, sizeof (*TtPte));
19331952

19341953
DEBUG ((DEBUG_INFO, "%a: Exit Success\n", __FUNCTION__));
19351954
return EFI_SUCCESS;
@@ -2151,12 +2170,8 @@ SmmuV3EnableProtection (
21512170
break;
21522171
}
21532172

2154-
// Invalidate cache before page table setup to ensure working with
2155-
// clean memory state and prevent stale cache entries from affecting
2156-
// page table population
2157-
InvalidateDataCacheRange ((VOID *)TtNext, EFI_PAGE_SIZE);
2158-
21592173
ZeroMem ((VOID *)TtNext, EFI_PAGE_SIZE);
2174+
WriteBackDataCacheRange ((VOID *)TtNext, EFI_PAGE_SIZE);
21602175

21612176
DEBUG ((DEBUG_INFO, "%a: Allocated new translation table at:0x%p\n", __FUNCTION__, (VOID *)TtNext));
21622177
Status = InstallTablePte (TtNext, Lvl, TtPte);

0 commit comments

Comments
 (0)