From b30e046cfdcc883c627a1d0ede7e2d10d5c1a22c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 14 Mar 2025 20:09:28 -0700 Subject: [PATCH 01/66] iommu/amd: Explicitly bail from enable_iommus_vapic() when in legacy mode Bail early from enable_iommus_vapic() if IOMMUs are configured for either of the legacy modes, as it's absurdly difficult to see that iommu_ga_log_enable() is guaranteed to fail because iommu_init_ga_log() skips allocating the ga_log. Opportunistically have iommu_ga_log_enable() WARN if it's called without IOMMUs being configured to support AVIC/vAPIC. Cc: Suravee Suthikulpanit Signed-off-by: Sean Christopherson Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 3bdb380d23e9..16a4567984b8 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -972,8 +972,8 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) { int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; - if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) - return 0; + if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) + return -EINVAL; iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE); if (!iommu->ga_log) @@ -3022,8 +3022,10 @@ static void enable_iommus_vapic(void) return; } - if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && - !check_feature(FEATURE_GAM_VAPIC)) { + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) + return; + + if (!check_feature(FEATURE_GAM_VAPIC)) { amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; return; } From 4bf53c2d0c08bbdaa32f2114281f1ddab61902bf Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 14 Mar 2025 20:21:48 -0700 Subject: [PATCH 02/66] iommu/amd: Fix a stale comment about which legacy mode is user visible Update a stale comment about which of the legacy modes is visible to the user, i.e. can be forced via amd_iommu_intr=legacy. Fixes: b74aa02d7a30 ("iommu/amd: Fix legacy interrupt remapping for x2APIC-enabled system") Signed-off-by: Sean Christopherson Reviewed-by: Vasant Hegde Reviewed-by: Wei Wang Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f9f718087893..c726d115939a 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -948,12 +948,13 @@ static inline int get_hpet_devid(int id) } enum amd_iommu_intr_mode_type { - AMD_IOMMU_GUEST_IR_LEGACY, - - /* This mode is not visible to users. It is used when - * we cannot fully enable vAPIC and fallback to only support - * legacy interrupt remapping via 128-bit IRTE. + /* + * The legacy format mode is not visible to users to prevent the user + * from crashing x2APIC systems, which for all intents and purposes + * require 128-bit IRTEs. The legacy format will be forced as needed + * when hardware doesn't support 128-bit IRTEs. */ + AMD_IOMMU_GUEST_IR_LEGACY, AMD_IOMMU_GUEST_IR_LEGACY_GA, AMD_IOMMU_GUEST_IR_VAPIC, }; From 45ccfdd3b577f3f0a9e86fe3f37132798dfbb2a1 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Tue, 31 Mar 2026 23:25:50 +0800 Subject: [PATCH 03/66] iommu/amd: Remove dead code for exclusion ranges in IVMD Exclusion ranges in IVMD are treated as unity mappings with r&w permissions since commit 0bbe4ced53e3 ("iommu/amd: Fix the overwritten field in IVMD header"), using 1:1 mappings. And IOMMU Exclusion Base & Range Limit Registers (0x0020/0x0028) are actually no longer used. As a result, several definitions and code paths became dead code: - exclusion_start/exclusion_length in struct amd_iommu are never assigned, - iommu_set_exclusion_range() always returns 0 now. - MMIO_EXCL_ENABLE_MASK & MMIO_EXCL_ALLOW_MASK are only used by the removed iommu_set_exclusion_range(). - DEV_ENTRY_EX is no longer set in any DTE. - IOMMU_UNITY_MAP_FLAG_EXCL_RANGE is no longer set, thus the IOMMU_RESV_RESERVED branch in amd_iommu_get_resv_regions() is no longer reachable. Just remove all of the dead code. No functional change. Signed-off-by: Yu Zhang Reviewed-by: Suravee Suthikulpanit Reviewed-by: Jacob Pan Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 12 ------------ drivers/iommu/amd/init.c | 23 ----------------------- drivers/iommu/amd/iommu.c | 3 --- 3 files changed, 38 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index c726d115939a..9fc9e8f0887a 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -51,10 +51,6 @@ #define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT) #define MMIO_MSI_NUM(x) ((x) & 0x1f) -/* Flag masks for the AMD IOMMU exclusion range */ -#define MMIO_EXCL_ENABLE_MASK 0x01ULL -#define MMIO_EXCL_ALLOW_MASK 0x02ULL - /* Used offsets into the MMIO space */ #define MMIO_DEV_TABLE_OFFSET 0x0000 #define MMIO_CMD_BUF_OFFSET 0x0008 @@ -231,7 +227,6 @@ #define DEV_ENTRY_IR 0x3d #define DEV_ENTRY_IW 0x3e #define DEV_ENTRY_NO_PAGE_FAULT 0x62 -#define DEV_ENTRY_EX 0x67 #define DEV_ENTRY_SYSMGT1 0x68 #define DEV_ENTRY_SYSMGT2 0x69 #define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40) @@ -389,8 +384,6 @@ #define IOMMU_PROT_IR 0x01 #define IOMMU_PROT_IW 0x02 -#define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE (1 << 2) - /* IOMMU capabilities */ #define IOMMU_CAP_IOTLB 24 #define IOMMU_CAP_NPCACHE 26 @@ -685,11 +678,6 @@ struct amd_iommu { /* pci domain of this IOMMU */ struct amd_iommu_pci_seg *pci_seg; - /* start of exclusion range of that IOMMU */ - u64 exclusion_start; - /* length of exclusion range of that IOMMU */ - u64 exclusion_length; - /* command buffer virtual address */ u8 *cmd_buf; u32 cmd_buf_head; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 16a4567984b8..d146fa2f57ae 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -355,28 +355,6 @@ static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) * ****************************************************************************/ -/* - * This function set the exclusion range in the IOMMU. DMA accesses to the - * exclusion range are passed through untranslated - */ -static void iommu_set_exclusion_range(struct amd_iommu *iommu) -{ - u64 start = iommu->exclusion_start & PAGE_MASK; - u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK; - u64 entry; - - if (!iommu->exclusion_start) - return; - - entry = start | MMIO_EXCL_ENABLE_MASK; - memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, - &entry, sizeof(entry)); - - entry = limit; - memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, - &entry, sizeof(entry)); -} - static void iommu_set_cwwb_range(struct amd_iommu *iommu) { u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem); @@ -2905,7 +2883,6 @@ static void early_enable_iommu(struct amd_iommu *iommu) iommu_init_flags(iommu); iommu_set_device_table(iommu); iommu_enable_command_buffer(iommu); - iommu_set_exclusion_range(iommu); iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f78e23f03938..f1333071da10 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3082,9 +3082,6 @@ static void amd_iommu_get_resv_regions(struct device *dev, prot |= IOMMU_READ; if (entry->prot & IOMMU_PROT_IW) prot |= IOMMU_WRITE; - if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE) - /* Exclusion range */ - type = IOMMU_RESV_RESERVED; region = iommu_alloc_resv_region(entry->address_start, length, prot, type, From 5beda8cadb1f072140e58b1edb7604444a42d955 Mon Sep 17 00:00:00 2001 From: Fu Hao Date: Tue, 7 Apr 2026 16:25:40 +0800 Subject: [PATCH 04/66] iommu/amd: Add support for Hygon family 18h model 4h IOAPIC The SB IOAPIC is on the device 0xb from Hygon family 18h model 4h. Signed-off-by: Fu Hao Tested-by: Tingyin Duan Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index d146fa2f57ae..16073b5614b9 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3089,6 +3089,9 @@ static void __init free_iommu_resources(void) /* SB IOAPIC is always on this device in AMD systems */ #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0)) +/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */ +#define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0)) + static bool __init check_ioapic_information(void) { const char *fw_bug = FW_BUG; @@ -3114,7 +3117,12 @@ static bool __init check_ioapic_information(void) pr_err("%s: IOAPIC[%d] not in IVRS table\n", fw_bug, id); ret = false; - } else if (devid == IOAPIC_SB_DEVID) { + } else if (devid == IOAPIC_SB_DEVID || + (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON && + boot_cpu_data.x86 == 0x18 && + boot_cpu_data.x86_model >= 0x4 && + boot_cpu_data.x86_model <= 0xf && + devid == IOAPIC_SB_DEVID_FAM18H_M4H)) { has_sb_ioapic = true; ret = true; } From 90d50734815a227fa26563ec6ee3388a69d35843 Mon Sep 17 00:00:00 2001 From: Benjamin Gaignard Date: Wed, 15 Apr 2026 09:23:38 +0200 Subject: [PATCH 05/66] dt-bindings: iommu: verisilicon: Add binding for VSI IOMMU Add a device tree binding for the Verisilicon (VSI) IOMMU. This IOMMU sits in front of hardware encoder and decoder blocks on SoCs using Verisilicon IP, such as the Rockchip RK3588. Signed-off-by: Benjamin Gaignard Reviewed-by: Conor Dooley Signed-off-by: Joerg Roedel --- .../bindings/iommu/verisilicon,iommu.yaml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml diff --git a/Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml b/Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml new file mode 100644 index 000000000000..d3ce9e603b61 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/verisilicon,iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Verisilicon IOMMU + +maintainers: + - Benjamin Gaignard + +description: |+ + A Versilicon iommu translates io virtual addresses to physical addresses for + its associated video decoder. + +properties: + compatible: + items: + - const: rockchip,rk3588-av1-iommu + - const: verisilicon,iommu-1.2 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: Core clock + - description: Interface clock + + clock-names: + items: + - const: core + - const: iface + + "#iommu-cells": + const: 0 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - "#iommu-cells" + +additionalProperties: false + +examples: + - | + #include + #include + + bus { + #address-cells = <2>; + #size-cells = <2>; + + iommu@fdca0000 { + compatible = "rockchip,rk3588-av1-iommu","verisilicon,iommu-1.2"; + reg = <0x0 0xfdca0000 0x0 0x600>; + interrupts = ; + clocks = <&cru ACLK_AV1>, <&cru PCLK_AV1>; + clock-names = "core", "iface"; + #iommu-cells = <0>; + }; + }; From 917ace84b7702ab067572e3e9bff03a4e4dce7b9 Mon Sep 17 00:00:00 2001 From: Benjamin Gaignard Date: Wed, 15 Apr 2026 09:23:39 +0200 Subject: [PATCH 06/66] iommu: Add verisilicon IOMMU driver The Verisilicon IOMMU hardware block can be found in combination with Verisilicon hardware video codecs (encoders or decoders) on different SoCs. Enable it will allow us to use non contiguous memory allocators for Verisilicon video codecs. Signed-off-by: Benjamin Gaignard Signed-off-by: Joerg Roedel --- MAINTAINERS | 8 + drivers/iommu/Kconfig | 11 + drivers/iommu/Makefile | 1 + drivers/iommu/vsi-iommu.c | 796 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 816 insertions(+) create mode 100644 drivers/iommu/vsi-iommu.c diff --git a/MAINTAINERS b/MAINTAINERS index b2040011a386..dc593cd7c69b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -27952,6 +27952,14 @@ F: drivers/media/v4l2-core/v4l2-isp.c F: include/media/v4l2-isp.h F: include/uapi/linux/media/v4l2-isp.h +VERISILICON IOMMU DRIVER +M: Benjamin Gaignard +L: iommu@lists.linux.dev +S: Maintained +F: Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml +F: drivers/iommu/vsi-iommu.c +F: include/linux/vsi-iommu.h + VF610 NAND DRIVER M: Stefan Agner L: linux-mtd@lists.infradead.org diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f86262b11416..18d3d68af7cd 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -384,6 +384,17 @@ config SPRD_IOMMU Say Y here if you want to use the multimedia devices listed above. +config VSI_IOMMU + tristate "Verisilicon IOMMU Support" + depends on (ARCH_ROCKCHIP && ARM64) || COMPILE_TEST + select IOMMU_API + help + Support for IOMMUs used by Verisilicon sub-systems like video + decoders or encoder hardware blocks. + + Say Y here if you want to use this IOMMU in front of these + hardware blocks. + config IOMMU_DEBUG_PAGEALLOC bool "Debug IOMMU mappings against page allocations" depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 0275821f4ef9..887af357a7c9 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -36,4 +36,5 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o +obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o diff --git a/drivers/iommu/vsi-iommu.c b/drivers/iommu/vsi-iommu.c new file mode 100644 index 000000000000..5d0721bd2c7a --- /dev/null +++ b/drivers/iommu/vsi-iommu.c @@ -0,0 +1,796 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Collabora Ltd. + * + * IOMMU API for Verisilicon + * + * Module Authors: Yandong Lin + * Simon Xue + * Benjamin Gaignard + * + * This hardware block is using a 2 pages tables allocation structure. + * That make very similar to Rockhip iommu hardware blocks but it has + * it own driver because the registers offset and configuration bits + * are completely different. An additional reason is that this hardware + * has been developed by Verisilicon to be used by their hardware video + * decoders and not for a general purpose like Rockchip iommus. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iommu-pages.h" + +struct vsi_iommu { + struct device *dev; + void __iomem *regs; + struct clk_bulk_data *clocks; + int num_clocks; + struct iommu_device iommu; + struct list_head node; /* entry in vsi_iommu_domain.iommus */ + struct iommu_domain *domain; /* domain to which iommu is attached */ + spinlock_t lock; /* lock to protect vsi_iommu fields */ + int irq; + bool enable; +}; + +struct vsi_iommu_domain { + struct list_head iommus; + struct device *dev; + u32 *dt; + dma_addr_t dt_dma; + struct iommu_domain domain; + u64 *pta; + dma_addr_t pta_dma; + spinlock_t lock; /* lock to protect vsi_iommu_domain fields */ +}; + +static struct iommu_domain vsi_identity_domain; + +#define NUM_DT_ENTRIES 1024 +#define NUM_PT_ENTRIES 1024 + +#define SPAGE_SIZE BIT(12) + +/* vsi iommu regs address */ +#define VSI_MMU_CONFIG1_BASE 0x1ac +#define VSI_MMU_AHB_EXCEPTION_BASE 0x380 +#define VSI_MMU_AHB_CONTROL_BASE 0x388 +#define VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE 0x38C + +/* MMU register offsets */ +#define VSI_MMU_FLUSH_BASE 0x184 +#define VSI_MMU_BIT_FLUSH BIT(4) + +#define VSI_MMU_PAGE_FAULT_ADDR 0x380 +#define VSI_MMU_STATUS_BASE 0x384 /* IRQ status */ + +#define VSI_MMU_BIT_ENABLE BIT(0) + +#define VSI_MMU_OUT_OF_BOUND BIT(28) +/* Irq mask */ +#define VSI_MMU_IRQ_MASK 0x7 + +#define VSI_DTE_PT_ADDRESS_MASK 0xffffffc0 +#define VSI_DTE_PT_VALID BIT(0) + +#define VSI_PAGE_DESC_LO_MASK 0xfffff000 +#define VSI_PAGE_DESC_HI_MASK GENMASK_ULL(39, 32) +#define VSI_PAGE_DESC_HI_SHIFT (32 - 4) + +static inline phys_addr_t vsi_dte_pt_address(u32 dte) +{ + return (phys_addr_t)dte & VSI_DTE_PT_ADDRESS_MASK; +} + +static inline u32 vsi_mk_dte(u32 dte) +{ + return (phys_addr_t)dte | VSI_DTE_PT_VALID; +} + +#define VSI_PTE_PAGE_WRITABLE BIT(2) +#define VSI_PTE_PAGE_VALID BIT(0) + +static inline phys_addr_t vsi_pte_page_address(u64 pte) +{ + return ((pte << VSI_PAGE_DESC_HI_SHIFT) & VSI_PAGE_DESC_HI_MASK) | + (pte & VSI_PAGE_DESC_LO_MASK); +} + +static u32 vsi_mk_pte(phys_addr_t page, int prot) +{ + u32 flags = 0; + + flags |= (prot & IOMMU_WRITE) ? VSI_PTE_PAGE_WRITABLE : 0; + + page = (page & VSI_PAGE_DESC_LO_MASK) | + ((page & VSI_PAGE_DESC_HI_MASK) >> VSI_PAGE_DESC_HI_SHIFT); + + return page | flags | VSI_PTE_PAGE_VALID; +} + +#define VSI_DTE_PT_VALID BIT(0) + +static inline bool vsi_dte_is_pt_valid(u32 dte) +{ + return dte & VSI_DTE_PT_VALID; +} + +static inline bool vsi_pte_is_page_valid(u32 pte) +{ + return pte & VSI_PTE_PAGE_VALID; +} + +static u32 vsi_mk_pte_invalid(u32 pte) +{ + return pte & ~VSI_PTE_PAGE_VALID; +} + +#define VSI_MASTER_TLB_MASK GENMASK_ULL(31, 10) +/* mode 0 : 4k */ +#define VSI_PTA_4K_MODE 0 + +static u64 vsi_mk_pta(dma_addr_t dt_dma) +{ + u64 val = (dt_dma & VSI_MASTER_TLB_MASK) | VSI_PTA_4K_MODE; + + return val; +} + +static struct vsi_iommu_domain *to_vsi_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct vsi_iommu_domain, domain); +} + +static inline void vsi_table_flush(struct vsi_iommu_domain *vsi_domain, dma_addr_t dma, + unsigned int count) +{ + size_t size = count * sizeof(u32); /* count of u32 entry */ + + dma_sync_single_for_device(vsi_domain->dev, dma, size, DMA_TO_DEVICE); +} + +#define VSI_IOVA_DTE_MASK 0xffc00000 +#define VSI_IOVA_DTE_SHIFT 22 +#define VSI_IOVA_PTE_MASK 0x003ff000 +#define VSI_IOVA_PTE_SHIFT 12 +#define VSI_IOVA_PAGE_MASK 0x00000fff +#define VSI_IOVA_PAGE_SHIFT 0 + +static u32 vsi_iova_dte_index(u32 iova) +{ + return (iova & VSI_IOVA_DTE_MASK) >> VSI_IOVA_DTE_SHIFT; +} + +static u32 vsi_iova_pte_index(u32 iova) +{ + return (iova & VSI_IOVA_PTE_MASK) >> VSI_IOVA_PTE_SHIFT; +} + +static u32 vsi_iova_page_offset(u32 iova) +{ + return (iova & VSI_IOVA_PAGE_MASK) >> VSI_IOVA_PAGE_SHIFT; +} + +static irqreturn_t vsi_iommu_irq(int irq, void *dev_id) +{ + struct vsi_iommu *iommu = dev_id; + unsigned long flags; + dma_addr_t iova; + u32 status; + + if (pm_runtime_resume_and_get(iommu->dev) < 0) + return IRQ_NONE; + + spin_lock_irqsave(&iommu->lock, flags); + + status = readl(iommu->regs + VSI_MMU_STATUS_BASE); + if (status & VSI_MMU_IRQ_MASK) { + dev_err(iommu->dev, "unexpected int_status=%08x\n", status); + iova = readl(iommu->regs + VSI_MMU_PAGE_FAULT_ADDR); + report_iommu_fault(iommu->domain, iommu->dev, iova, status); + } + writel(0, iommu->regs + VSI_MMU_STATUS_BASE); + + spin_unlock_irqrestore(&iommu->lock, flags); + pm_runtime_put_autosuspend(iommu->dev); + + return IRQ_HANDLED; +} + +static struct vsi_iommu *vsi_iommu_get_from_dev(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct device *iommu_dev = bus_find_device_by_fwnode(&platform_bus_type, + fwspec->iommu_fwnode); + + put_device(iommu_dev); + + return iommu_dev ? dev_get_drvdata(iommu_dev) : NULL; +} + +static struct iommu_domain *vsi_iommu_domain_alloc_paging(struct device *dev) +{ + struct vsi_iommu *iommu = dev_iommu_priv_get(dev); + struct vsi_iommu_domain *vsi_domain; + + vsi_domain = kzalloc(sizeof(*vsi_domain), GFP_KERNEL); + if (!vsi_domain) + return NULL; + + vsi_domain->dev = iommu->dev; + spin_lock_init(&vsi_domain->lock); + + /* + * iommu use a 2 level pagetable. + * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. + * Allocate one 4 KiB page for each table. + */ + vsi_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, + SPAGE_SIZE); + if (!vsi_domain->dt) + goto err_free_domain; + + vsi_domain->dt_dma = dma_map_single(vsi_domain->dev, vsi_domain->dt, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(vsi_domain->dev, vsi_domain->dt_dma)) { + dev_err(dev, "DMA map error for DT\n"); + goto err_free_dt; + } + + vsi_domain->pta = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, + SPAGE_SIZE); + if (!vsi_domain->pta) + goto err_unmap_dt; + + vsi_domain->pta[0] = vsi_mk_pta(vsi_domain->dt_dma); + vsi_domain->pta_dma = dma_map_single(vsi_domain->dev, vsi_domain->pta, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(vsi_domain->dev, vsi_domain->pta_dma)) { + dev_err(dev, "DMA map error for PTA\n"); + goto err_free_pta; + } + + INIT_LIST_HEAD(&vsi_domain->iommus); + + vsi_domain->domain.geometry.aperture_start = 0; + vsi_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); + vsi_domain->domain.geometry.force_aperture = true; + vsi_domain->domain.pgsize_bitmap = SZ_4K; + + return &vsi_domain->domain; + +err_free_pta: + iommu_free_pages(vsi_domain->pta); +err_unmap_dt: + dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); +err_free_dt: + iommu_free_pages(vsi_domain->dt); +err_free_domain: + kfree(vsi_domain); + + return NULL; +} + +static phys_addr_t vsi_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + phys_addr_t pt_phys, phys = 0; + unsigned long flags; + u32 dte, pte; + u32 *page_table; + + spin_lock_irqsave(&vsi_domain->lock, flags); + dte = vsi_domain->dt[vsi_iova_dte_index(iova)]; + if (!vsi_dte_is_pt_valid(dte)) + goto unlock; + + pt_phys = vsi_dte_pt_address(dte); + page_table = (u32 *)phys_to_virt(pt_phys); + pte = page_table[vsi_iova_pte_index(iova)]; + if (!vsi_pte_is_page_valid(pte)) + goto unlock; + + phys = vsi_pte_page_address(pte) + vsi_iova_page_offset(iova); + +unlock: + spin_unlock_irqrestore(&vsi_domain->lock, flags); + return phys; +} + +static size_t vsi_iommu_unmap_iova(struct vsi_iommu_domain *vsi_domain, + u32 *pte_addr, dma_addr_t pte_dma, + size_t size) +{ + unsigned int pte_count; + unsigned int pte_total = size / SPAGE_SIZE; + + for (pte_count = 0; + pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) { + u32 pte = pte_addr[pte_count]; + + if (!vsi_pte_is_page_valid(pte)) + break; + + pte_addr[pte_count] = vsi_mk_pte_invalid(pte); + } + + vsi_table_flush(vsi_domain, pte_dma, pte_total); + + return pte_count * SPAGE_SIZE; +} + +static int vsi_iommu_map_iova(struct vsi_iommu_domain *vsi_domain, u32 *pte_addr, + dma_addr_t pte_dma, dma_addr_t iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned int pte_count; + unsigned int pte_total = size / SPAGE_SIZE; + + for (pte_count = 0; + pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) { + u32 pte = pte_addr[pte_count]; + + if (vsi_pte_is_page_valid(pte)) + return (pte_count - 1) * SPAGE_SIZE; + + pte_addr[pte_count] = vsi_mk_pte(paddr, prot); + + paddr += SPAGE_SIZE; + } + + vsi_table_flush(vsi_domain, pte_dma, pte_total); + + return 0; +} + +static void vsi_iommu_flush_tlb(struct iommu_domain *domain) +{ + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + struct list_head *pos; + + list_for_each(pos, &vsi_domain->iommus) { + struct vsi_iommu *iommu; + + iommu = list_entry(pos, struct vsi_iommu, node); + if (!iommu) + continue; + + if (pm_runtime_get(iommu->dev) < 0) + continue; + + spin_lock(&iommu->lock); + + if (iommu->enable) { + writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE); + writel(0, iommu->regs + VSI_MMU_FLUSH_BASE); + } + + spin_unlock(&iommu->lock); + + pm_runtime_put_autosuspend(iommu->dev); + } +} + +static size_t vsi_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, + size_t size, size_t count, struct iommu_iotlb_gather *gather) +{ + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; + unsigned long flags; + phys_addr_t pt_phys; + u32 dte; + u32 *pte_addr; + size_t unmap_size = 0; + + spin_lock_irqsave(&vsi_domain->lock, flags); + + dte = vsi_domain->dt[vsi_iova_dte_index(iova)]; + /* Just return 0 if iova is unmapped */ + if (!vsi_dte_is_pt_valid(dte)) + goto unlock; + + pt_phys = vsi_dte_pt_address(dte); + pte_addr = (u32 *)phys_to_virt(pt_phys) + vsi_iova_pte_index(iova); + pte_dma = pt_phys + vsi_iova_pte_index(iova) * sizeof(u32); + unmap_size = vsi_iommu_unmap_iova(vsi_domain, pte_addr, pte_dma, size); + if (!unmap_size) + goto unlock; + + vsi_iommu_flush_tlb(domain); +unlock: + spin_unlock_irqrestore(&vsi_domain->lock, flags); + + return unmap_size; +} + +static u32 *vsi_dte_get_page_table(struct vsi_iommu_domain *vsi_domain, + dma_addr_t iova, gfp_t gfp) +{ + u32 *page_table, *dte_addr; + u32 dte_index, dte; + phys_addr_t pt_phys; + dma_addr_t pt_dma; + gfp_t flags; + + dte_index = vsi_iova_dte_index(iova); + dte_addr = &vsi_domain->dt[dte_index]; + dte = *dte_addr; + if (vsi_dte_is_pt_valid(dte)) + goto done; + + /* Do not allow to sleep while allocating the buffer */ + flags = (gfp & ~GFP_KERNEL) | GFP_ATOMIC | GFP_DMA32; + page_table = iommu_alloc_pages_sz(flags, PAGE_SIZE); + if (!page_table) + return ERR_PTR(-ENOMEM); + + pt_dma = dma_map_single(vsi_domain->dev, page_table, PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(vsi_domain->dev, pt_dma)) { + dev_err(vsi_domain->dev, "DMA mapping error while allocating page table\n"); + iommu_free_pages(page_table); + return ERR_PTR(-ENOMEM); + } + + dte = vsi_mk_dte(pt_dma); + *dte_addr = dte; + + vsi_table_flush(vsi_domain, + vsi_domain->dt_dma + dte_index * sizeof(u32), 1); +done: + pt_phys = vsi_dte_pt_address(dte); + return (u32 *)phys_to_virt(pt_phys); +} + +static int vsi_iommu_map(struct iommu_domain *domain, unsigned long _iova, + phys_addr_t paddr, size_t size, size_t count, + int prot, gfp_t gfp, size_t *mapped) +{ + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; + u32 *page_table, *pte_addr; + u32 dte, pte_index; + unsigned long flags; + int ret; + + spin_lock_irqsave(&vsi_domain->lock, flags); + + page_table = vsi_dte_get_page_table(vsi_domain, iova, gfp); + if (IS_ERR(page_table)) { + spin_unlock_irqrestore(&vsi_domain->lock, flags); + return PTR_ERR(page_table); + } + + dte = vsi_domain->dt[vsi_iova_dte_index(iova)]; + pte_index = vsi_iova_pte_index(iova); + pte_addr = &page_table[pte_index]; + pte_dma = vsi_dte_pt_address(dte) + pte_index * sizeof(u32); + ret = vsi_iommu_map_iova(vsi_domain, pte_addr, pte_dma, iova, + paddr, size, prot); + if (!ret) + *mapped = size; + + vsi_iommu_flush_tlb(domain); + + spin_unlock_irqrestore(&vsi_domain->lock, flags); + + return ret; +} + +static void vsi_iommu_disable(struct vsi_iommu *iommu) +{ + writel(0, iommu->regs + VSI_MMU_AHB_CONTROL_BASE); + iommu->enable = false; +} + +static int vsi_iommu_identity_attach(struct iommu_domain *domain, + struct device *dev, struct iommu_domain *old) +{ + struct vsi_iommu *iommu = dev_iommu_priv_get(dev); + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + unsigned long flags; + int ret; + + ret = pm_runtime_resume_and_get(iommu->dev); + if (ret < 0) + return ret; + + spin_lock_irqsave(&vsi_domain->lock, flags); + spin_lock(&iommu->lock); + if (iommu->domain == domain) + goto unlock; + + vsi_iommu_disable(iommu); + list_del_init(&iommu->node); + + iommu->domain = domain; + +unlock: + spin_unlock(&iommu->lock); + spin_unlock_irqrestore(&vsi_domain->lock, flags); + pm_runtime_put_autosuspend(iommu->dev); + return 0; +} + +static const struct iommu_domain_ops vsi_identity_ops = { + .attach_dev = vsi_iommu_identity_attach, +}; + +static struct iommu_domain vsi_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &vsi_identity_ops, +}; + +static void vsi_iommu_enable(struct vsi_iommu *iommu, struct iommu_domain *domain) +{ + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + + if (domain == &vsi_identity_domain) + return; + + writel(vsi_domain->pta_dma, iommu->regs + VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE); + writel(VSI_MMU_OUT_OF_BOUND, iommu->regs + VSI_MMU_CONFIG1_BASE); + writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_EXCEPTION_BASE); + writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_CONTROL_BASE); + iommu->enable = true; +} + +static int vsi_iommu_attach_device(struct iommu_domain *domain, + struct device *dev, struct iommu_domain *old) +{ + struct vsi_iommu *iommu = dev_iommu_priv_get(dev); + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + unsigned long flags; + int ret = 0; + + ret = pm_runtime_resume_and_get(iommu->dev); + if (ret < 0) + return ret; + + spin_lock_irqsave(&vsi_domain->lock, flags); + spin_lock(&iommu->lock); + + vsi_iommu_enable(iommu, domain); + writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE); + writel(0, iommu->regs + VSI_MMU_FLUSH_BASE); + + list_del_init(&iommu->node); + list_add_tail(&iommu->node, &vsi_domain->iommus); + + iommu->domain = domain; + + spin_unlock(&iommu->lock); + spin_unlock_irqrestore(&vsi_domain->lock, flags); + pm_runtime_put_autosuspend(iommu->dev); + return ret; +} + +static void vsi_iommu_domain_free(struct iommu_domain *domain) +{ + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); + unsigned long flags; + int i; + + spin_lock_irqsave(&vsi_domain->lock, flags); + + WARN_ON(!list_empty(&vsi_domain->iommus)); + + for (i = 0; i < NUM_DT_ENTRIES; i++) { + u32 dte = vsi_domain->dt[i]; + + if (vsi_dte_is_pt_valid(dte)) { + phys_addr_t pt_phys = vsi_dte_pt_address(dte); + u32 *page_table = phys_to_virt(pt_phys); + + dma_unmap_single(vsi_domain->dev, pt_phys, + SPAGE_SIZE, DMA_TO_DEVICE); + iommu_free_pages(page_table); + } + } + + dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); + iommu_free_pages(vsi_domain->dt); + + dma_unmap_single(vsi_domain->dev, vsi_domain->pta_dma, + SPAGE_SIZE, DMA_TO_DEVICE); + iommu_free_pages(vsi_domain->pta); + + spin_unlock_irqrestore(&vsi_domain->lock, flags); + + kfree(vsi_domain); +} + +static struct iommu_device *vsi_iommu_probe_device(struct device *dev) +{ + struct vsi_iommu *iommu = vsi_iommu_get_from_dev(dev); + struct device_link *link; + + link = device_link_add(dev, iommu->dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + if (!link) + dev_err(dev, "Unable to link %s\n", dev_name(iommu->dev)); + + dev_iommu_priv_set(dev, iommu); + return &iommu->iommu; +} + +static void vsi_iommu_release_device(struct device *dev) +{ + struct vsi_iommu *iommu = dev_iommu_priv_get(dev); + + device_link_remove(dev, iommu->dev); +} + +static int vsi_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args) +{ + return iommu_fwspec_add_ids(dev, args->args, 1); +} + +static const struct iommu_ops vsi_iommu_ops = { + .identity_domain = &vsi_identity_domain, + .release_domain = &vsi_identity_domain, + .domain_alloc_paging = vsi_iommu_domain_alloc_paging, + .of_xlate = vsi_iommu_of_xlate, + .probe_device = vsi_iommu_probe_device, + .release_device = vsi_iommu_release_device, + .device_group = generic_single_device_group, + .owner = THIS_MODULE, + .default_domain_ops = &(const struct iommu_domain_ops) { + .attach_dev = vsi_iommu_attach_device, + .map_pages = vsi_iommu_map, + .unmap_pages = vsi_iommu_unmap, + .iova_to_phys = vsi_iommu_iova_to_phys, + .free = vsi_iommu_domain_free, + } +}; + +static const struct of_device_id vsi_iommu_dt_ids[] = { + { + .compatible = "verisilicon,iommu-1.2", + }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, vsi_iommu_dt_ids); + +static int vsi_iommu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vsi_iommu *iommu; + int err; + + iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); + if (!iommu) + return -ENOMEM; + + iommu->dev = dev; + spin_lock_init(&iommu->lock); + INIT_LIST_HEAD(&iommu->node); + + iommu->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(iommu->regs)) + return -ENOMEM; + + iommu->num_clocks = devm_clk_bulk_get_all(dev, &iommu->clocks); + if (iommu->num_clocks < 0) + return iommu->num_clocks; + + err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks); + if (err) + return err; + + iommu->irq = platform_get_irq(pdev, 0); + if (iommu->irq < 0) + return iommu->irq; + + err = devm_request_irq(iommu->dev, iommu->irq, vsi_iommu_irq, + IRQF_SHARED, dev_name(dev), iommu); + if (err) + goto err_unprepare_clocks; + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + platform_set_drvdata(pdev, iommu); + + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + + err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); + if (err) + goto err_runtime_disable; + + err = iommu_device_register(&iommu->iommu, &vsi_iommu_ops, dev); + if (err) + goto err_remove_sysfs; + + return 0; + +err_remove_sysfs: + iommu_device_sysfs_remove(&iommu->iommu); +err_runtime_disable: + pm_runtime_disable(dev); +err_unprepare_clocks: + clk_bulk_unprepare(iommu->num_clocks, iommu->clocks); + return err; +} + +static void vsi_iommu_shutdown(struct platform_device *pdev) +{ + struct vsi_iommu *iommu = platform_get_drvdata(pdev); + + disable_irq(iommu->irq); + pm_runtime_force_suspend(&pdev->dev); +} + +static int __maybe_unused vsi_iommu_suspend(struct device *dev) +{ + struct vsi_iommu *iommu = dev_get_drvdata(dev); + + vsi_iommu_disable(iommu); + + clk_bulk_disable(iommu->num_clocks, iommu->clocks); + + return 0; +} + +static int __maybe_unused vsi_iommu_resume(struct device *dev) +{ + struct vsi_iommu *iommu = dev_get_drvdata(dev); + unsigned long flags; + int ret; + + ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks); + if (ret) + return ret; + + if (iommu->domain) { + struct vsi_iommu_domain *vsi_domain = to_vsi_domain(iommu->domain); + + spin_lock_irqsave(&vsi_domain->lock, flags); + spin_lock(&iommu->lock); + vsi_iommu_enable(iommu, iommu->domain); + spin_unlock(&iommu->lock); + spin_unlock_irqrestore(&vsi_domain->lock, flags); + } + + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(vsi_iommu_pm_ops, + vsi_iommu_suspend, vsi_iommu_resume, + NULL); + +static struct platform_driver rockchip_vsi_iommu_driver = { + .probe = vsi_iommu_probe, + .shutdown = vsi_iommu_shutdown, + .driver = { + .name = "vsi_iommu", + .of_match_table = vsi_iommu_dt_ids, + .pm = pm_sleep_ptr(&vsi_iommu_pm_ops), + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(rockchip_vsi_iommu_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Benjamin Gaignard "); +MODULE_DESCRIPTION("Verisilicon IOMMU driver"); From 6ddfbec80077eca7b7e5a4298750d3dac82997ff Mon Sep 17 00:00:00 2001 From: Benjamin Gaignard Date: Wed, 15 Apr 2026 09:23:40 +0200 Subject: [PATCH 07/66] arm64: dts: rockchip: Add verisilicon IOMMU node on RK3588 Add the device tree node for the Verisilicon IOMMU present in the RK3588 SoC. This IOMMU handles address translation for the VPU hardware blocks. Signed-off-by: Benjamin Gaignard Signed-off-by: Joerg Roedel --- arch/arm64/boot/dts/rockchip/rk3588-base.dtsi | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index 4fb8888c281c..b78347a83ed9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1428,6 +1428,17 @@ av1d: video-codec@fdc70000 { clock-names = "aclk", "hclk"; power-domains = <&power RK3588_PD_AV1>; resets = <&cru SRST_A_AV1>, <&cru SRST_P_AV1>, <&cru SRST_A_AV1_BIU>, <&cru SRST_P_AV1_BIU>; + iommus = <&av1d_mmu>; + }; + + av1d_mmu: iommu@fdca0000 { + compatible = "rockchip,rk3588-av1-iommu", "verisilicon,iommu-1.2"; + reg = <0x0 0xfdca0000 0x0 0x600>; + interrupts = ; + clocks = <&cru ACLK_AV1>, <&cru PCLK_AV1>; + clock-names = "core", "iface"; + #iommu-cells = <0>; + power-domains = <&power RK3588_PD_AV1>; }; vop: vop@fdd90000 { From f196a86687974cfcc1e8cade99ffca4605141860 Mon Sep 17 00:00:00 2001 From: Fangyu Yu Date: Fri, 17 Apr 2026 22:07:45 +0800 Subject: [PATCH 08/66] iommu/riscv: Advertise Svpbmt support to generic page table The RISC-V IOMMU can optionally support Svpbmt page-based memory types in its page table format. When present,the generic page table code can use this capability to encode memory attributes (e.g. MMIO vs normal memory) in PTEs. Signed-off-by: Fangyu Yu Reviewed-by: Jason Gunthorpe Reviewed-by: Anup Patel Reviewed-by: Guo Ren Reviewed-by: Nutty Liu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 2 ++ include/linux/generic_pt/common.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index a31f50bbad35..6c324f9fdc53 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -1268,6 +1268,8 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | BIT(PT_FEAT_RISCV_SVNAPOT_64K); + if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT) + cfg.common.features |= BIT(PT_FEAT_RISCV_SVPBMT); domain->riscvpt.iommu.nid = dev_to_node(iommu->dev); domain->domain.ops = &riscv_iommu_paging_domain_ops; diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index fc5d0b5edadc..2683e5b38998 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h @@ -188,6 +188,10 @@ enum { * Support the 64k contiguous page size following the Svnapot extension. */ PT_FEAT_RISCV_SVNAPOT_64K = PT_FEAT_FMT_START, + /* + * Support Svpbmt extension: encode page-based memory type (PBMT) in PTEs. + */ + PT_FEAT_RISCV_SVPBMT, }; From 6c21eb174c6c7aebdecaf5c39e3100e6beb35faa Mon Sep 17 00:00:00 2001 From: Fangyu Yu Date: Fri, 17 Apr 2026 22:07:46 +0800 Subject: [PATCH 09/66] iommupt: Encode IOMMU_MMIO/IOMMU_CACHE via RISC-V Svpbmt bits When the RISC-V IOMMU page table format support Svpbmt, PBMT provides a way to tag mappings with page-based memory types. Encode memory type via PBMT in RISC-V IOMMU PTEs: - IOMMU_MMIO -> PBMT=IO - !IOMMU_MMIO && !IOMMU_CACHE -> PBMT=NC - otherwise -> PBMT=Normal (PBMT=0) Only touch PBMT when PT_FEAT_RISCV_SVPBMT is advertised. Signed-off-by: Fangyu Yu Reviewed-by: Jason Gunthorpe Reviewed-by: Anup Patel Reviewed-by: Guo Ren Reviewed-by: Nutty Liu Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/fmt/riscv.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/generic_pt/fmt/riscv.h b/drivers/iommu/generic_pt/fmt/riscv.h index a7fef6266a36..2648bb57953e 100644 --- a/drivers/iommu/generic_pt/fmt/riscv.h +++ b/drivers/iommu/generic_pt/fmt/riscv.h @@ -64,6 +64,8 @@ enum { RISCVPT_PPN64 = GENMASK_ULL(53, 10), RISCVPT_PPN64_64K = GENMASK_ULL(53, 14), RISCVPT_PBMT = GENMASK_ULL(62, 61), + RISCVPT_NC = BIT(61), + RISCVPT_IO = BIT(62), RISCVPT_N = BIT_ULL(63), /* Svnapot encodings for ppn[0] */ @@ -201,7 +203,8 @@ static inline void riscvpt_attr_from_entry(const struct pt_state *pts, { attrs->descriptor_bits = pts->entry & (RISCVPT_R | RISCVPT_W | RISCVPT_X | RISCVPT_U | - RISCVPT_G | RISCVPT_A | RISCVPT_D); + RISCVPT_G | RISCVPT_A | RISCVPT_D | RISCVPT_NC | + RISCVPT_IO); } #define pt_attr_from_entry riscvpt_attr_from_entry @@ -237,6 +240,12 @@ static inline int riscvpt_iommu_set_prot(struct pt_common *common, pte |= RISCVPT_R; if (!(iommu_prot & IOMMU_NOEXEC)) pte |= RISCVPT_X; + if (common->features & BIT(PT_FEAT_RISCV_SVPBMT)) { + if (iommu_prot & IOMMU_MMIO) + pte |= RISCVPT_IO; + else if (!(iommu_prot & IOMMU_CACHE)) + pte |= RISCVPT_NC; + } /* Caller must specify a supported combination of flags */ if (unlikely((pte & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) == 0)) From c6580699ffcd44941eaec895d6b5cc2c338d27bd Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 21 Apr 2026 18:00:38 +0800 Subject: [PATCH 10/66] iommu/amd: Remove unused IOMMU_IN/OUT_ADDR_BIT_SIZE macros The macros IOMMU_IN/OUT_ADDR_BIT_SIZE became unused after commit 2fdf6db436e3 ("iommu/amd: Remove AMD io_pgtable support"), which removed the last references in io_pgtable.c and io_pgtable_v2.c. Remove them to clean up unused definitions. Signed-off-by: Wei Wang Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 9fc9e8f0887a..96038b2e24d5 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -300,9 +300,6 @@ #define GA_GUEST_NR 0x1 -#define IOMMU_IN_ADDR_BIT_SIZE 52 -#define IOMMU_OUT_ADDR_BIT_SIZE 52 - /* * This bitmap is used to advertise the page sizes our hardware support * to the IOMMU core, which will then use this information to split From e861aaea2f478d23e9ee05edd4fd403f40ac75fa Mon Sep 17 00:00:00 2001 From: Fangyu Yu Date: Tue, 12 May 2026 22:43:30 +0800 Subject: [PATCH 11/66] iommupt: Fixup build warning by using BIT_ULL() for RISCVPT_NC/IO Fix build warning on 32-bit configurations by using BIT_ULL() for RISCVPT_NC and RISCVPT_IO. Fixes: 6c21eb174c6c ("iommupt: Encode IOMMU_MMIO/IOMMU_CACHE via RISC-V Svpbmt bits") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202605121350.wZxB51k0-lkp@intel.com/ Signed-off-by: Fangyu Yu Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/fmt/riscv.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/generic_pt/fmt/riscv.h b/drivers/iommu/generic_pt/fmt/riscv.h index 2648bb57953e..ae9a76514416 100644 --- a/drivers/iommu/generic_pt/fmt/riscv.h +++ b/drivers/iommu/generic_pt/fmt/riscv.h @@ -64,8 +64,8 @@ enum { RISCVPT_PPN64 = GENMASK_ULL(53, 10), RISCVPT_PPN64_64K = GENMASK_ULL(53, 14), RISCVPT_PBMT = GENMASK_ULL(62, 61), - RISCVPT_NC = BIT(61), - RISCVPT_IO = BIT(62), + RISCVPT_NC = BIT_ULL(61), + RISCVPT_IO = BIT_ULL(62), RISCVPT_N = BIT_ULL(63), /* Svnapot encodings for ppn[0] */ From 74c9d82c7e3fe9e5845103fe83d711d89b169ab1 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:00 -0300 Subject: [PATCH 12/66] iommu: Split the kdoc comment for struct iommu_iotlb_gather Use in-line member documentation and add some small clarifications to the members. This is preparation to add more members. - Note that pgsize is only used by arm-smmuv3 - Note that freelist is only used by iommupt - Reword queued to emphasize the flush-all behavior Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e587d4ac4d33..a9e89911c90f 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -345,12 +345,6 @@ struct iommu_pages_list { /** * struct iommu_iotlb_gather - Range information for a pending IOTLB flush * - * @start: IOVA representing the start of the range to be flushed - * @end: IOVA representing the end of the range to be flushed (inclusive) - * @pgsize: The interval at which to perform the flush - * @freelist: Removed pages to free after sync - * @queued: Indicates that the flush will be queued - * * This structure is intended to be updated by multiple calls to the * ->unmap() function in struct iommu_ops before eventually being passed * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after @@ -359,10 +353,24 @@ struct iommu_pages_list { * later instead of ->iotlb_sync(), so drivers may optimise accordingly. */ struct iommu_iotlb_gather { + /** @start: IOVA representing the start of the range to be flushed */ unsigned long start; + /** + * @end: IOVA representing the end of the range to be + * flushed (inclusive) + */ unsigned long end; + /** + * @pgsize: The interval at which to perform the flush, only used + * by arm-smmu-v3 + */ size_t pgsize; + /** + * @freelist: Removed pages to free after sync, only used by + * iommupt + */ struct iommu_pages_list freelist; + /** @queued: True if the gather will be completed with a flush all */ bool queued; }; From f0e4b7f5f87578cdd2d0f1e866183adc10d33b85 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:01 -0300 Subject: [PATCH 13/66] iommupt: Add struct iommupt_pending_gather Add a struct to keep track of all the things that are pending to be merged into the gather. The way gather merging works, the pending range is checked against the current gather, and the current gather can be flushed before the pending things are added. Thus, if new things have to be recorded in the gather they need to be kept in the pending struct until after the gather is optionally flushed. The next patch adds new items to the gather and the pending struct. Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 75 ++++++++++++++++++----------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 19b6daf88f2a..5ec135cf43e2 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -40,15 +40,27 @@ static void flush_writes_item(const struct pt_state *pts) PT_ITEM_WORD_SIZE); } -static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, - struct pt_iommu *iommu_table, pt_vaddr_t iova, - pt_vaddr_t len, - struct iommu_pages_list *free_list) +struct iommupt_pending_gather { + struct iommu_iotlb_gather *iotlb_gather; + struct iommu_pages_list free_list; +}; + +static void gather_add_table(struct iommupt_pending_gather *pending, + const struct pt_state *pts, + struct pt_table_p *table) { + iommu_pages_list_add(&pending->free_list, table); +} + +static void gather_range_pending(struct iommupt_pending_gather *pending, + struct pt_iommu *iommu_table, pt_vaddr_t iova, + pt_vaddr_t len) +{ + struct iommu_iotlb_gather *iotlb_gather = pending->iotlb_gather; struct pt_common *common = common_from_iommu(iommu_table); if (pt_feature(common, PT_FEAT_DMA_INCOHERENT)) - iommu_pages_stop_incoherent_list(free_list, + iommu_pages_stop_incoherent_list(&pending->free_list, iommu_table->iommu_device); /* @@ -72,7 +84,8 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, iommu_iotlb_gather_add_range(iotlb_gather, iova, len); } - iommu_pages_list_splice(free_list, &iotlb_gather->freelist); + iommu_pages_list_splice(&pending->free_list, &iotlb_gather->freelist); + INIT_LIST_HEAD(&pending->free_list.pages); } #define DOMAIN_NS(op) CONCATENATE(CONCATENATE(pt_iommu_, PTPFX), op) @@ -341,7 +354,7 @@ static int __maybe_unused NS(set_dirty)(struct pt_iommu *iommu_table, } struct pt_iommu_collect_args { - struct iommu_pages_list free_list; + struct iommupt_pending_gather pending; /* Fail if any OAs are within the range */ u8 check_mapped : 1; }; @@ -358,7 +371,8 @@ static int __collect_tables(struct pt_range *range, void *arg, for_each_pt_level_entry(&pts) { if (pts.type == PT_ENTRY_TABLE) { - iommu_pages_list_add(&collect->free_list, pts.table_lower); + gather_add_table(&collect->pending, &pts, + pts.table_lower); ret = pt_descend(&pts, arg, __collect_tables); if (ret) return ret; @@ -493,15 +507,18 @@ static int clear_contig(const struct pt_state *start_pts, struct pt_range range = *start_pts->range; struct pt_state pts = pt_init(&range, start_pts->level, start_pts->table); - struct pt_iommu_collect_args collect = { .check_mapped = true }; + struct pt_iommu_collect_args collect = { + .check_mapped = true, + .pending.iotlb_gather = iotlb_gather, + .pending.free_list = IOMMU_PAGES_LIST_INIT( + collect.pending.free_list), + }; int ret; pts.index = start_pts->index; pts.end_index = start_pts->index + step; for (; _pt_iter_load(&pts); pt_next_entry(&pts)) { if (pts.type == PT_ENTRY_TABLE) { - collect.free_list = - IOMMU_PAGES_LIST_INIT(collect.free_list); ret = pt_walk_descend_all(&pts, __collect_tables, &collect); if (ret) @@ -514,12 +531,11 @@ static int clear_contig(const struct pt_state *start_pts, pt_clear_entries(&pts, ilog2(1)); flush_writes_item(&pts); - iommu_pages_list_add(&collect.free_list, - pt_table_ptr(&pts)); - gather_range_pages( - iotlb_gather, iommu_table, range.va, - log2_to_int(pt_table_item_lg2sz(&pts)), - &collect.free_list); + gather_add_table(&collect.pending, &pts, + pts.table_lower); + gather_range_pending( + &collect.pending, iommu_table, range.va, + log2_to_int(pt_table_item_lg2sz(&pts))); } else if (pts.type != PT_ENTRY_EMPTY) { return -EADDRINUSE; } @@ -962,7 +978,7 @@ static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, } struct pt_unmap_args { - struct iommu_pages_list free_list; + struct iommupt_pending_gather pending; pt_vaddr_t unmapped; }; @@ -1025,8 +1041,8 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg, * succeed in clearing the lower table levels. */ if (fully_covered) { - iommu_pages_list_add(&unmap->free_list, - pts.table_lower); + gather_add_table(&unmap->pending, &pts, + pts.table_lower); pt_clear_entries(&pts, ilog2(1)); if (pts.index < flush_start_index) flush_start_index = pts.index; @@ -1065,8 +1081,11 @@ static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, dma_addr_t len, struct iommu_iotlb_gather *iotlb_gather) { - struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT( - unmap.free_list) }; + struct pt_unmap_args unmap = { + .pending.iotlb_gather = iotlb_gather, + .pending.free_list = IOMMU_PAGES_LIST_INIT( + unmap.pending.free_list), + }; struct pt_range range; int ret; @@ -1076,8 +1095,7 @@ static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, pt_walk_range(&range, __unmap_range, &unmap); - gather_range_pages(iotlb_gather, iommu_table, iova, unmap.unmapped, - &unmap.free_list); + gather_range_pending(&unmap.pending, iommu_table, iova, unmap.unmapped); return unmap.unmapped; } @@ -1111,10 +1129,11 @@ static void NS(deinit)(struct pt_iommu *iommu_table) struct pt_common *common = common_from_iommu(iommu_table); struct pt_range range = pt_all_range(common); struct pt_iommu_collect_args collect = { - .free_list = IOMMU_PAGES_LIST_INIT(collect.free_list), + .pending.free_list = IOMMU_PAGES_LIST_INIT( + collect.pending.free_list), }; - iommu_pages_list_add(&collect.free_list, range.top_table); + iommu_pages_list_add(&collect.pending.free_list, range.top_table); pt_walk_range(&range, __collect_tables, &collect); /* @@ -1122,9 +1141,9 @@ static void NS(deinit)(struct pt_iommu *iommu_table) * and invalidated any caching referring to this memory. */ if (pt_feature(common, PT_FEAT_DMA_INCOHERENT)) - iommu_pages_stop_incoherent_list(&collect.free_list, + iommu_pages_stop_incoherent_list(&collect.pending.free_list, iommu_table->iommu_device); - iommu_put_pages_list(&collect.free_list); + iommu_put_pages_list(&collect.pending.free_list); } static const struct pt_iommu_ops NS(ops) = { From 89792629d4b260104555f9719ac84c1afc9a2012 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:02 -0300 Subject: [PATCH 14/66] iommupt: Add PT_FEAT_DETAILED_GATHER Generating the ARM SMMUv3 and RISC-V invalidation commands optimally requires some additional details from iommupt: - leaf_levels_bitmap is used to compute the ARM Range Invalidation Table Top Level hint - leaf_levels_bitmap is also used to compute the stride when generating single invalidations to invalidate once per leaf - table_levels_bitmap also computes the ARM TTL for future cases when there are no leaves Put these under a feature since only two drivers need to calculate them. This is also useful for the coming kunit iotlb invalidation test to know more about what invalidation is happening. Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 23 ++++++++++++++++++++++ include/linux/generic_pt/common.h | 5 +++++ include/linux/iommu.h | 30 ++++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 5ec135cf43e2..61c6d79c712c 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -43,6 +43,8 @@ static void flush_writes_item(const struct pt_state *pts) struct iommupt_pending_gather { struct iommu_iotlb_gather *iotlb_gather; struct iommu_pages_list free_list; + u8 leaf_levels_bitmap; + u8 table_levels_bitmap; }; static void gather_add_table(struct iommupt_pending_gather *pending, @@ -50,6 +52,17 @@ static void gather_add_table(struct iommupt_pending_gather *pending, struct pt_table_p *table) { iommu_pages_list_add(&pending->free_list, table); + if (pts_feature(pts, PT_FEAT_DETAILED_GATHER)) + pending->table_levels_bitmap |= BIT(pts->level); +} + +static void gather_add_leaf(struct iommupt_pending_gather *pending, + const struct pt_state *pts) +{ + if (!pts_feature(pts, PT_FEAT_DETAILED_GATHER)) + return; + + pending->leaf_levels_bitmap |= BIT(pts->level); } static void gather_range_pending(struct iommupt_pending_gather *pending, @@ -86,6 +99,15 @@ static void gather_range_pending(struct iommupt_pending_gather *pending, iommu_pages_list_splice(&pending->free_list, &iotlb_gather->freelist); INIT_LIST_HEAD(&pending->free_list.pages); + + if (pt_feature(common, PT_FEAT_DETAILED_GATHER)) { + iotlb_gather->pt.leaf_levels_bitmap |= + pending->leaf_levels_bitmap; + iotlb_gather->pt.table_levels_bitmap |= + pending->table_levels_bitmap; + pending->leaf_levels_bitmap = 0; + pending->table_levels_bitmap = 0; + } } #define DOMAIN_NS(op) CONCATENATE(CONCATENATE(pt_iommu_, PTPFX), op) @@ -1059,6 +1081,7 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg, */ num_contig_lg2 = pt_entry_num_contig_lg2(&pts); pt_clear_entries(&pts, num_contig_lg2); + gather_add_leaf(&unmap->pending, &pts); num_oas += log2_to_int(num_contig_lg2); if (pts.index < flush_start_index) flush_start_index = pts.index; diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index 2683e5b38998..07ef1c8341a4 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h @@ -134,6 +134,11 @@ enum pt_features { * significant amount of page table. */ PT_FEAT_FLUSH_RANGE_NO_GAPS, + /** + * @PT_FEAT_DETAILED_GATHER: Fill in the struct iommu_iotlb_gather pt + * sub structure with information about which levels were changed. + */ + PT_FEAT_DETAILED_GATHER, /* private: */ PT_FEAT_FMT_START, }; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index a9e89911c90f..bf8a77a164e4 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -360,11 +360,31 @@ struct iommu_iotlb_gather { * flushed (inclusive) */ unsigned long end; - /** - * @pgsize: The interval at which to perform the flush, only used - * by arm-smmu-v3 - */ - size_t pgsize; + + union { + /** + * @pgsize: The interval at which to perform the flush, only + * used by arm-smmu-v3 + */ + size_t pgsize; + struct { + /** + * @pt.leaf_levels_bitmap: Bitmap of generic_pt + * levels where leaf entries were unmapped. Bit 0 + * means the leaf only level. If 0 no leafs + * were unmapped. + */ + u8 leaf_levels_bitmap; + /** + * @pt.table_levels_bitmap: Bitmap of generic_pt levels + * of table entries that were removed. Bit 0 is never + * set, bit 1 means a table of all leafs was removed. + * When freelist is empty this must be 0. + */ + u8 table_levels_bitmap; + } pt; + }; + /** * @freelist: Removed pages to free after sync, only used by * iommupt From db1aad42ca8b37437846078dfc405abda7868261 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:03 -0300 Subject: [PATCH 15/66] iommu/riscv: Enable PT_FEAT_DETAILED_GATHER and pass gather to iotlb_inval RISC-V can use the information from PT_FEAT_DETAILED_GATHER to compute the best stride to generate the single TLB invalidations. Pass the gather down to the lower functions and create a full-range gather for the flush-all callback. Reviewed-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/fmt/iommu_riscv64.c | 4 +- drivers/iommu/riscv/iommu.c | 44 +++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c index cbf60fffa9bf..b18fc4d109f5 100644 --- a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c +++ b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c @@ -6,6 +6,8 @@ #define PT_FMT_VARIANT 64 #define PT_SUPPORTED_FEATURES \ (BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | \ - BIT(PT_FEAT_RISCV_SVNAPOT_64K)) + BIT(PT_FEAT_RISCV_SVNAPOT_64K) | \ + BIT(PT_FEAT_DETAILED_GATHER)) +#define PT_FORCE_ENABLED_FEATURES BIT(PT_FEAT_DETAILED_GATHER) #include "iommu_template.h" diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 6c324f9fdc53..8ea085263f44 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -929,12 +929,28 @@ static void riscv_iommu_bond_unlink(struct riscv_iommu_domain *domain, #define RISCV_IOMMU_IOTLB_INVAL_LIMIT (2 << 20) static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, - unsigned long start, unsigned long end) + struct iommu_iotlb_gather *gather) { + unsigned long start; + unsigned long end; struct riscv_iommu_bond *bond; struct riscv_iommu_device *iommu, *prev; struct riscv_iommu_command cmd; + /* + * When non-leaf page table entries were changed, the base spec + * requires a full PSCID invalidation (AV=0) since there is no + * way to do targeted non-leaf invalidation without the NL + * extension. Force global invalidation to preserve correctness. + */ + if (gather->pt.table_levels_bitmap) { + start = 0; + end = ULONG_MAX; + } else { + start = gather->start; + end = gather->end; + } + /* * For each IOMMU linked with this protection domain (via bonds->dev), * an IOTLB invaliation command will be submitted and executed. @@ -1145,8 +1161,14 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu, static void riscv_iommu_iotlb_flush_all(struct iommu_domain *iommu_domain) { struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); + struct iommu_iotlb_gather gather = { + .start = 0, + .end = ULONG_MAX, + .pt.leaf_levels_bitmap = 0xFF, + .pt.table_levels_bitmap = 0xFE, + }; - riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX); + riscv_iommu_iotlb_inval(domain, &gather); } static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain, @@ -1154,19 +1176,8 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain, { struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); - if (iommu_pages_list_empty(&gather->freelist)) { - riscv_iommu_iotlb_inval(domain, gather->start, gather->end); - } else { - /* - * In 1.0 spec version, the smallest scope we can use to - * invalidate all levels of page table (i.e. leaf and non-leaf) - * is an invalidate-all-PSCID IOTINVAL.VMA with AV=0. - * This will be updated with hardware support for - * capability.NL (non-leaf) IOTINVAL command. - */ - riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX); - iommu_put_pages_list(&gather->freelist); - } + riscv_iommu_iotlb_inval(domain, gather); + iommu_put_pages_list(&gather->freelist); } static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain) @@ -1267,7 +1278,8 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) */ cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | - BIT(PT_FEAT_RISCV_SVNAPOT_64K); + BIT(PT_FEAT_RISCV_SVNAPOT_64K) | + BIT(PT_FEAT_DETAILED_GATHER); if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT) cfg.common.features |= BIT(PT_FEAT_RISCV_SVPBMT); domain->riscvpt.iommu.nid = dev_to_node(iommu->dev); From bb62adcf451336301f75763d3a4851febf1dd9ba Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:04 -0300 Subject: [PATCH 16/66] iommu/riscv: Compute best stride for single invalidation Replace the per-page IOTLB invalidation loop with stride-based invalidation that uses the level bitmaps from iommu_iotlb_gather. Pre-calculate the invalidation information before running over the bonds loop as it is the same for every entry. The lowest set bit in the PT_FEAT_DETAILED_GATHER bitmaps indicates the stride. This design ignores the SVNAPOT contiguous pages on the assumption that they still have to be individually invalidated like ARM requires, though it is not clear from the spec. Replace the 2M cutoff for global invalidation with a 512 command limit. This is the same for a 4k stride and now scales with the stride size. Reviewed-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 103 +++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 8ea085263f44..719d3b475d61 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -920,22 +920,58 @@ static void riscv_iommu_bond_unlink(struct riscv_iommu_domain *domain, } } -/* - * Send IOTLB.INVAL for whole address space for ranges larger than 2MB. - * This limit will be replaced with range invalidations, if supported by - * the hardware, when RISC-V IOMMU architecture specification update for - * range invalidations update will be available. - */ -#define RISCV_IOMMU_IOTLB_INVAL_LIMIT (2 << 20) +struct riscv_iommu_tlbi { + u64 start; + u64 last; + bool non_leaf; + struct { + bool use_global; + u8 stride_lg2; + unsigned int num; + } single; +}; -static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, - struct iommu_iotlb_gather *gather) +static void riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi *tlbi, + struct iommu_iotlb_gather *gather) +{ + u8 combined = gather->pt.leaf_levels_bitmap | + gather->pt.table_levels_bitmap; + u64 num; + + tlbi->non_leaf = gather->pt.table_levels_bitmap != 0; + tlbi->start = gather->start; + tlbi->last = gather->end; + + /* No level information available */ + if (!combined) { + tlbi->single.use_global = true; + return; + } + + /* + * Calculate stride from the lowest changed level. RISC-V uses 4KiB + * granule with 9 bits per level. + */ + tlbi->single.stride_lg2 = 9 * __ffs(combined) + 12; + num = (tlbi->last - tlbi->start + 1) >> tlbi->single.stride_lg2; + if (!num || num > 512) { + tlbi->single.use_global = true; + } else { + tlbi->single.num = num; + tlbi->single.use_global = false; + } +} + +static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu, + int pscid, + struct riscv_iommu_tlbi *tlbi) { - unsigned long start; - unsigned long end; - struct riscv_iommu_bond *bond; - struct riscv_iommu_device *iommu, *prev; struct riscv_iommu_command cmd; + unsigned long iova; + unsigned int i; + + riscv_iommu_cmd_inval_vma(&cmd); + riscv_iommu_cmd_inval_set_pscid(&cmd, pscid); /* * When non-leaf page table entries were changed, the base spec @@ -943,13 +979,28 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, * way to do targeted non-leaf invalidation without the NL * extension. Force global invalidation to preserve correctness. */ - if (gather->pt.table_levels_bitmap) { - start = 0; - end = ULONG_MAX; - } else { - start = gather->start; - end = gather->end; + if (tlbi->single.use_global || tlbi->non_leaf) + goto global; + + iova = tlbi->start; + for (i = 0; i < tlbi->single.num; i++) { + riscv_iommu_cmd_inval_set_addr(&cmd, iova); + riscv_iommu_cmd_send(iommu, &cmd); + iova += 1ULL << tlbi->single.stride_lg2; } + return; +global: + riscv_iommu_cmd_send(iommu, &cmd); +} + +static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + struct riscv_iommu_device *iommu, *prev; + struct riscv_iommu_bond *bond; + struct riscv_iommu_tlbi tlbi; + + riscv_iommu_tlbi_calc(&tlbi, gather); /* * For each IOMMU linked with this protection domain (via bonds->dev), @@ -990,19 +1041,7 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, if (iommu == prev) continue; - riscv_iommu_cmd_inval_vma(&cmd); - riscv_iommu_cmd_inval_set_pscid(&cmd, domain->pscid); - if (end - start < RISCV_IOMMU_IOTLB_INVAL_LIMIT - 1) { - unsigned long iova = start; - - do { - riscv_iommu_cmd_inval_set_addr(&cmd, iova); - riscv_iommu_cmd_send(iommu, &cmd); - } while (!check_add_overflow(iova, PAGE_SIZE, &iova) && - iova < end); - } else { - riscv_iommu_cmd_send(iommu, &cmd); - } + riscv_iommu_iotlb_inval_iommu(iommu, domain->pscid, &tlbi); prev = iommu; } From e4084c6bbb42b0ef2dbfceca70513cc1f49aff61 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:05 -0300 Subject: [PATCH 17/66] iommu/riscv: Add RISCV_IOMMU_CAPABILITIES_NL Non-leaf invalidation allows the single invalidate command to also clear the walk cache. If NL is available, set the NL bit if the gather indicates tables have been changed. The stride is already calculated properly. Reviewed-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu-bits.h | 7 +++++++ drivers/iommu/riscv/iommu.c | 12 +++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h index 29a0040b1c32..f01b49ac8155 100644 --- a/drivers/iommu/riscv/iommu-bits.h +++ b/drivers/iommu/riscv/iommu-bits.h @@ -63,6 +63,7 @@ #define RISCV_IOMMU_CAPABILITIES_PD8 BIT_ULL(38) #define RISCV_IOMMU_CAPABILITIES_PD17 BIT_ULL(39) #define RISCV_IOMMU_CAPABILITIES_PD20 BIT_ULL(40) +#define RISCV_IOMMU_CAPABILITIES_NL BIT_ULL(42) /** * enum riscv_iommu_igs_settings - Interrupt Generation Support Settings @@ -473,6 +474,7 @@ struct riscv_iommu_command { #define RISCV_IOMMU_CMD_IOTINVAL_PSCV BIT_ULL(32) #define RISCV_IOMMU_CMD_IOTINVAL_GV BIT_ULL(33) #define RISCV_IOMMU_CMD_IOTINVAL_GSCID GENMASK_ULL(59, 44) +#define RISCV_IOMMU_CMD_IOTINVAL_NL BIT_ULL(34) /* dword1[61:10] is the 4K-aligned page address */ #define RISCV_IOMMU_CMD_IOTINVAL_ADDR GENMASK_ULL(61, 10) @@ -724,6 +726,11 @@ static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cm cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV; } +static inline void riscv_iommu_cmd_inval_set_nl(struct riscv_iommu_command *cmd) +{ + cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_NL; +} + static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd, int pscid) { diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 719d3b475d61..165ced993756 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -966,6 +966,8 @@ static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu, int pscid, struct riscv_iommu_tlbi *tlbi) { + bool use_nl = tlbi->non_leaf && + (iommu->caps & RISCV_IOMMU_CAPABILITIES_NL); struct riscv_iommu_command cmd; unsigned long iova; unsigned int i; @@ -974,17 +976,17 @@ static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu, riscv_iommu_cmd_inval_set_pscid(&cmd, pscid); /* - * When non-leaf page table entries were changed, the base spec - * requires a full PSCID invalidation (AV=0) since there is no - * way to do targeted non-leaf invalidation without the NL - * extension. Force global invalidation to preserve correctness. + * If non-leaf entries were changed and the IOMMU doesn't + * support NL, we must fall back to global invalidation (AV=0). */ - if (tlbi->single.use_global || tlbi->non_leaf) + if (tlbi->single.use_global || (tlbi->non_leaf && !use_nl)) goto global; iova = tlbi->start; for (i = 0; i < tlbi->single.num; i++) { riscv_iommu_cmd_inval_set_addr(&cmd, iova); + if (use_nl) + riscv_iommu_cmd_inval_set_nl(&cmd); riscv_iommu_cmd_send(iommu, &cmd); iova += 1ULL << tlbi->single.stride_lg2; } From 835d06ee7ef0c2fc2adcf5bae5355c8bad900c21 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:06 -0300 Subject: [PATCH 18/66] iommu/riscv: Include the dword number in RISCV_IOMMU_CMD macros The command queue entry format is 128 bits. Follow the pattern of the other drivers and encode the 64 bit dword number in the macro itself. RISC-V further has similarly named macros that are not field layout macros, but field content macros which won't get a new number. Overall this is clearer to understand the code and check for errors like using the wrong macro in the wrong spot. Signed-off-by: Jason Gunthorpe Reviewed-by: Tomasz Jeznach Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu-bits.h | 102 +++++++++++++++---------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h index f01b49ac8155..8c60780363da 100644 --- a/drivers/iommu/riscv/iommu-bits.h +++ b/drivers/iommu/riscv/iommu-bits.h @@ -461,32 +461,32 @@ struct riscv_iommu_command { }; /* Fields on dword0, common for all commands */ -#define RISCV_IOMMU_CMD_OPCODE GENMASK_ULL(6, 0) -#define RISCV_IOMMU_CMD_FUNC GENMASK_ULL(9, 7) +#define RISCV_IOMMU_CMD0_OPCODE GENMASK_ULL(6, 0) +#define RISCV_IOMMU_CMD0_FUNC GENMASK_ULL(9, 7) /* 3.1.1 IOMMU Page-table cache invalidation */ /* Fields on dword0 */ #define RISCV_IOMMU_CMD_IOTINVAL_OPCODE 1 #define RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA 0 #define RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA 1 -#define RISCV_IOMMU_CMD_IOTINVAL_AV BIT_ULL(10) -#define RISCV_IOMMU_CMD_IOTINVAL_PSCID GENMASK_ULL(31, 12) -#define RISCV_IOMMU_CMD_IOTINVAL_PSCV BIT_ULL(32) -#define RISCV_IOMMU_CMD_IOTINVAL_GV BIT_ULL(33) -#define RISCV_IOMMU_CMD_IOTINVAL_GSCID GENMASK_ULL(59, 44) -#define RISCV_IOMMU_CMD_IOTINVAL_NL BIT_ULL(34) +#define RISCV_IOMMU_CMD0_IOTINVAL_AV BIT_ULL(10) +#define RISCV_IOMMU_CMD0_IOTINVAL_PSCID GENMASK_ULL(31, 12) +#define RISCV_IOMMU_CMD0_IOTINVAL_PSCV BIT_ULL(32) +#define RISCV_IOMMU_CMD0_IOTINVAL_GV BIT_ULL(33) +#define RISCV_IOMMU_CMD0_IOTINVAL_GSCID GENMASK_ULL(59, 44) +#define RISCV_IOMMU_CMD0_IOTINVAL_NL BIT_ULL(34) /* dword1[61:10] is the 4K-aligned page address */ -#define RISCV_IOMMU_CMD_IOTINVAL_ADDR GENMASK_ULL(61, 10) +#define RISCV_IOMMU_CMD1_IOTINVAL_ADDR GENMASK_ULL(61, 10) /* 3.1.2 IOMMU Command Queue Fences */ /* Fields on dword0 */ #define RISCV_IOMMU_CMD_IOFENCE_OPCODE 2 #define RISCV_IOMMU_CMD_IOFENCE_FUNC_C 0 -#define RISCV_IOMMU_CMD_IOFENCE_AV BIT_ULL(10) -#define RISCV_IOMMU_CMD_IOFENCE_WSI BIT_ULL(11) -#define RISCV_IOMMU_CMD_IOFENCE_PR BIT_ULL(12) -#define RISCV_IOMMU_CMD_IOFENCE_PW BIT_ULL(13) -#define RISCV_IOMMU_CMD_IOFENCE_DATA GENMASK_ULL(63, 32) +#define RISCV_IOMMU_CMD0_IOFENCE_AV BIT_ULL(10) +#define RISCV_IOMMU_CMD0_IOFENCE_WSI BIT_ULL(11) +#define RISCV_IOMMU_CMD0_IOFENCE_PR BIT_ULL(12) +#define RISCV_IOMMU_CMD0_IOFENCE_PW BIT_ULL(13) +#define RISCV_IOMMU_CMD0_IOFENCE_DATA GENMASK_ULL(63, 32) /* dword1 is the address, word-size aligned and shifted to the right by two bits. */ /* 3.1.3 IOMMU Directory cache invalidation */ @@ -494,9 +494,9 @@ struct riscv_iommu_command { #define RISCV_IOMMU_CMD_IODIR_OPCODE 3 #define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT 0 #define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT 1 -#define RISCV_IOMMU_CMD_IODIR_PID GENMASK_ULL(31, 12) -#define RISCV_IOMMU_CMD_IODIR_DV BIT_ULL(33) -#define RISCV_IOMMU_CMD_IODIR_DID GENMASK_ULL(63, 40) +#define RISCV_IOMMU_CMD0_IODIR_PID GENMASK_ULL(31, 12) +#define RISCV_IOMMU_CMD0_IODIR_DV BIT_ULL(33) +#define RISCV_IOMMU_CMD0_IODIR_DID GENMASK_ULL(63, 40) /* dword1 is reserved for standard use */ /* 3.1.4 IOMMU PCIe ATS */ @@ -504,25 +504,25 @@ struct riscv_iommu_command { #define RISCV_IOMMU_CMD_ATS_OPCODE 4 #define RISCV_IOMMU_CMD_ATS_FUNC_INVAL 0 #define RISCV_IOMMU_CMD_ATS_FUNC_PRGR 1 -#define RISCV_IOMMU_CMD_ATS_PID GENMASK_ULL(31, 12) -#define RISCV_IOMMU_CMD_ATS_PV BIT_ULL(32) -#define RISCV_IOMMU_CMD_ATS_DSV BIT_ULL(33) -#define RISCV_IOMMU_CMD_ATS_RID GENMASK_ULL(55, 40) -#define RISCV_IOMMU_CMD_ATS_DSEG GENMASK_ULL(63, 56) +#define RISCV_IOMMU_CMD0_ATS_PID GENMASK_ULL(31, 12) +#define RISCV_IOMMU_CMD0_ATS_PV BIT_ULL(32) +#define RISCV_IOMMU_CMD0_ATS_DSV BIT_ULL(33) +#define RISCV_IOMMU_CMD0_ATS_RID GENMASK_ULL(55, 40) +#define RISCV_IOMMU_CMD0_ATS_DSEG GENMASK_ULL(63, 56) /* dword1 is the ATS payload, two different payload types for INVAL and PRGR */ /* ATS.INVAL payload*/ -#define RISCV_IOMMU_CMD_ATS_INVAL_G BIT_ULL(0) +#define RISCV_IOMMU_CMD1_ATS_INVAL_G BIT_ULL(0) /* Bits 1 - 10 are zeroed */ -#define RISCV_IOMMU_CMD_ATS_INVAL_S BIT_ULL(11) -#define RISCV_IOMMU_CMD_ATS_INVAL_UADDR GENMASK_ULL(63, 12) +#define RISCV_IOMMU_CMD1_ATS_INVAL_S BIT_ULL(11) +#define RISCV_IOMMU_CMD1_ATS_INVAL_UADDR GENMASK_ULL(63, 12) /* ATS.PRGR payload */ /* Bits 0 - 31 are zeroed */ -#define RISCV_IOMMU_CMD_ATS_PRGR_PRG_INDEX GENMASK_ULL(40, 32) +#define RISCV_IOMMU_CMD1_ATS_PRGR_PRG_INDEX GENMASK_ULL(40, 32) /* Bits 41 - 43 are zeroed */ -#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44) -#define RISCV_IOMMU_CMD_ATS_PRGR_DST_ID GENMASK_ULL(63, 48) +#define RISCV_IOMMU_CMD1_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44) +#define RISCV_IOMMU_CMD1_ATS_PRGR_DST_ID GENMASK_ULL(63, 48) /** * struct riscv_iommu_fq_record - Fault/Event Queue Record @@ -713,8 +713,8 @@ struct riscv_iommu_msipte { static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd) { - cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) | - FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA); + cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) | + FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA); cmd->dword1 = 0; } @@ -722,72 +722,72 @@ static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cm u64 addr) { cmd->dword1 = - FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, PHYS_PFN(addr)); - cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV; + FIELD_PREP(RISCV_IOMMU_CMD1_IOTINVAL_ADDR, PHYS_PFN(addr)); + cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_AV; } static inline void riscv_iommu_cmd_inval_set_nl(struct riscv_iommu_command *cmd) { - cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_NL; + cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_NL; } static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd, int pscid) { - cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_PSCID, pscid) | - RISCV_IOMMU_CMD_IOTINVAL_PSCV; + cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IOTINVAL_PSCID, pscid) | + RISCV_IOMMU_CMD0_IOTINVAL_PSCV; } static inline void riscv_iommu_cmd_inval_set_gscid(struct riscv_iommu_command *cmd, int gscid) { - cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_GSCID, gscid) | - RISCV_IOMMU_CMD_IOTINVAL_GV; + cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IOTINVAL_GSCID, gscid) | + RISCV_IOMMU_CMD0_IOTINVAL_GV; } static inline void riscv_iommu_cmd_iofence(struct riscv_iommu_command *cmd) { - cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) | - FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) | - RISCV_IOMMU_CMD_IOFENCE_PR | RISCV_IOMMU_CMD_IOFENCE_PW; + cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) | + FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) | + RISCV_IOMMU_CMD0_IOFENCE_PR | RISCV_IOMMU_CMD0_IOFENCE_PW; cmd->dword1 = 0; } static inline void riscv_iommu_cmd_iofence_set_av(struct riscv_iommu_command *cmd, u64 addr, u32 data) { - cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) | - FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) | - FIELD_PREP(RISCV_IOMMU_CMD_IOFENCE_DATA, data) | - RISCV_IOMMU_CMD_IOFENCE_AV; + cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) | + FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) | + FIELD_PREP(RISCV_IOMMU_CMD0_IOFENCE_DATA, data) | + RISCV_IOMMU_CMD0_IOFENCE_AV; cmd->dword1 = addr >> 2; } static inline void riscv_iommu_cmd_iodir_inval_ddt(struct riscv_iommu_command *cmd) { - cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) | - FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT); + cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) | + FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT); cmd->dword1 = 0; } static inline void riscv_iommu_cmd_iodir_inval_pdt(struct riscv_iommu_command *cmd) { - cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) | - FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT); + cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) | + FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT); cmd->dword1 = 0; } static inline void riscv_iommu_cmd_iodir_set_did(struct riscv_iommu_command *cmd, unsigned int devid) { - cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_DID, devid) | - RISCV_IOMMU_CMD_IODIR_DV; + cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IODIR_DID, devid) | + RISCV_IOMMU_CMD0_IODIR_DV; } static inline void riscv_iommu_cmd_iodir_set_pid(struct riscv_iommu_command *cmd, unsigned int pasid) { - cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_PID, pasid); + cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IODIR_PID, pasid); } #endif /* _RISCV_IOMMU_BITS_H_ */ From 082ad5ed078516972571060b54643a0777486071 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 8 May 2026 11:53:07 -0300 Subject: [PATCH 19/66] iommu/riscv: Add NAPOT range invalidation support Use the RISC-V IOMMU Address Range Invalidation extension (capabilities.S, spec section 9.3) to invalidate an IOVA range with a single IOTINVAL.VMA command using NAPOT-encoded addressing. One iommu_iotlb_gather maps to one NAPOT invalidation command. The smallest power-of-two aligned range covering the gather is used since over-invalidation is always safe. S and NL seem to be orthogonal in the spec, so if NL is not supported then global invalidation is probably always going to happen as wiping a large range without a table change is not common. Reviewed-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Tested-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu-bits.h | 18 +++++++++++++ drivers/iommu/riscv/iommu.c | 43 +++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h index 8c60780363da..f2ef9bd3cde9 100644 --- a/drivers/iommu/riscv/iommu-bits.h +++ b/drivers/iommu/riscv/iommu-bits.h @@ -64,6 +64,7 @@ #define RISCV_IOMMU_CAPABILITIES_PD17 BIT_ULL(39) #define RISCV_IOMMU_CAPABILITIES_PD20 BIT_ULL(40) #define RISCV_IOMMU_CAPABILITIES_NL BIT_ULL(42) +#define RISCV_IOMMU_CAPABILITIES_S BIT_ULL(43) /** * enum riscv_iommu_igs_settings - Interrupt Generation Support Settings @@ -475,6 +476,7 @@ struct riscv_iommu_command { #define RISCV_IOMMU_CMD0_IOTINVAL_GV BIT_ULL(33) #define RISCV_IOMMU_CMD0_IOTINVAL_GSCID GENMASK_ULL(59, 44) #define RISCV_IOMMU_CMD0_IOTINVAL_NL BIT_ULL(34) +#define RISCV_IOMMU_CMD1_IOTINVAL_S BIT_ULL(9) /* dword1[61:10] is the 4K-aligned page address */ #define RISCV_IOMMU_CMD1_IOTINVAL_ADDR GENMASK_ULL(61, 10) @@ -731,6 +733,22 @@ static inline void riscv_iommu_cmd_inval_set_nl(struct riscv_iommu_command *cmd) cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_NL; } +/* + * Set NAPOT-encoded address for range invalidation (S=1). + * sz_lg2: log2 of total range in bytes, must be >= 13 (8KiB, 2 pages). + * addr must be naturally aligned to 2^sz_lg2. + */ +static inline void riscv_iommu_cmd_inval_set_napot( + struct riscv_iommu_command *cmd, u64 addr, unsigned int sz_lg2) +{ + u64 pfn = addr >> 12; + + pfn |= BIT_U64(sz_lg2 - 13) - 1; + cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD1_IOTINVAL_ADDR, pfn) | + RISCV_IOMMU_CMD1_IOTINVAL_S; + cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_AV; +} + static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd, int pscid) { diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 165ced993756..cec3ddd7ab10 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -929,6 +929,10 @@ struct riscv_iommu_tlbi { u8 stride_lg2; unsigned int num; } single; + struct { + u8 sz_lg2; + u64 addr; + } range; }; static void riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi *tlbi, @@ -945,9 +949,23 @@ static void riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi *tlbi, /* No level information available */ if (!combined) { tlbi->single.use_global = true; + tlbi->range.sz_lg2 = 0; return; } + /* + * Calculate the smallest NAPOT range containing [start, last]. + * NAPOT encoding requires a power-of-two sized, naturally aligned + * range. Over-invalidation is always safe. + */ + tlbi->range.sz_lg2 = fls64(tlbi->start ^ tlbi->last); + if (unlikely(tlbi->range.sz_lg2 >= 64)) { + tlbi->single.use_global = true; + tlbi->range.sz_lg2 = 0; + return; + } + tlbi->range.addr = tlbi->start & ~(BIT_U64(tlbi->range.sz_lg2) - 1); + /* * Calculate stride from the lowest changed level. RISC-V uses 4KiB * granule with 9 bits per level. @@ -969,7 +987,6 @@ static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu, bool use_nl = tlbi->non_leaf && (iommu->caps & RISCV_IOMMU_CAPABILITIES_NL); struct riscv_iommu_command cmd; - unsigned long iova; unsigned int i; riscv_iommu_cmd_inval_vma(&cmd); @@ -979,16 +996,30 @@ static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu, * If non-leaf entries were changed and the IOMMU doesn't * support NL, we must fall back to global invalidation (AV=0). */ - if (tlbi->single.use_global || (tlbi->non_leaf && !use_nl)) + if (tlbi->non_leaf && !use_nl) goto global; - iova = tlbi->start; - for (i = 0; i < tlbi->single.num; i++) { - riscv_iommu_cmd_inval_set_addr(&cmd, iova); + if (iommu->caps & RISCV_IOMMU_CAPABILITIES_S && + tlbi->range.sz_lg2 >= 13) { + riscv_iommu_cmd_inval_set_napot(&cmd, tlbi->range.addr, + tlbi->range.sz_lg2); if (use_nl) riscv_iommu_cmd_inval_set_nl(&cmd); riscv_iommu_cmd_send(iommu, &cmd); - iova += 1ULL << tlbi->single.stride_lg2; + } else { + unsigned long iova; + + if (tlbi->single.use_global) + goto global; + + iova = tlbi->start; + for (i = 0; i < tlbi->single.num; i++) { + riscv_iommu_cmd_inval_set_addr(&cmd, iova); + if (use_nl) + riscv_iommu_cmd_inval_set_nl(&cmd); + riscv_iommu_cmd_send(iommu, &cmd); + iova += 1ULL << tlbi->single.stride_lg2; + } } return; global: From 04f1f4bf30b8b00e9eca7143dea094ccc2ec478e Mon Sep 17 00:00:00 2001 From: Ankit Soni Date: Mon, 13 Apr 2026 14:45:18 +0000 Subject: [PATCH 20/66] iommu_pt: Fix pgsize_bitmap calculation in get_info for smaller vasz's To properly enforce the domain VA limit, clamp pgsize_bitmap using the requested max_vasz_lg2 in get_info(). Apply the same VA limit as get_info() in the kunit possible_sizes test so assertions stay consistent with the domain bitmap. Suggested-by: Jason Gunthorpe Signed-off-by: Ankit Soni Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 8 ++++++-- drivers/iommu/generic_pt/kunit_generic_pt.h | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 19b6daf88f2a..1765986e24e8 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -1102,8 +1102,12 @@ static void NS(get_info)(struct pt_iommu *iommu_table, pgsize_bitmap |= pt_possible_sizes(&pts); } - /* Hide page sizes larger than the maximum OA */ - info->pgsize_bitmap = oalog2_mod(pgsize_bitmap, common->max_oasz_lg2); + /* + * Hide page sizes larger than the maximum. -1 because a whole table + * pgsize is not allowed + */ + info->pgsize_bitmap = log2_mod(pgsize_bitmap, common->max_vasz_lg2 - 1); + info->pgsize_bitmap = oalog2_mod(info->pgsize_bitmap, common->max_oasz_lg2); } static void NS(deinit)(struct pt_iommu *iommu_table) diff --git a/drivers/iommu/generic_pt/kunit_generic_pt.h b/drivers/iommu/generic_pt/kunit_generic_pt.h index 374e475f591e..ef2c90b6d6af 100644 --- a/drivers/iommu/generic_pt/kunit_generic_pt.h +++ b/drivers/iommu/generic_pt/kunit_generic_pt.h @@ -438,6 +438,9 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts, { unsigned int num_items_lg2 = safe_pt_num_items_lg2(pts); pt_vaddr_t pgsize_bitmap = pt_possible_sizes(pts); + /* Matches get_info() */ + pt_vaddr_t limited_pgsize_bitmap = + log2_mod(pgsize_bitmap, pts->range->common->max_vasz_lg2 - 1); unsigned int isz_lg2 = pt_table_item_lg2sz(pts); if (!pt_can_have_leaf(pts)) { @@ -448,7 +451,8 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts, /* No bits for sizes that would be outside this table */ KUNIT_ASSERT_EQ(test, log2_mod(pgsize_bitmap, isz_lg2), 0); KUNIT_ASSERT_EQ( - test, fvalog2_div(pgsize_bitmap, num_items_lg2 + isz_lg2), 0); + test, + fvalog2_div(limited_pgsize_bitmap, num_items_lg2 + isz_lg2), 0); /* * Non contiguous must be supported. AMDv1 has a HW bug where it does @@ -463,8 +467,8 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts, /* A contiguous entry should not span the whole table */ if (num_items_lg2 + isz_lg2 != PT_VADDR_MAX_LG2) KUNIT_ASSERT_FALSE( - test, - pgsize_bitmap & log2_to_int(num_items_lg2 + isz_lg2)); + test, limited_pgsize_bitmap & + log2_to_int(num_items_lg2 + isz_lg2)); } static void test_entry_possible_sizes(struct kunit *test) From 5240dab55b51c5f14448643ad044a5643f5b569b Mon Sep 17 00:00:00 2001 From: Ankit Soni Date: Mon, 13 Apr 2026 14:45:19 +0000 Subject: [PATCH 21/66] iommu_pt: support small VA for AMDv1 When hardware/VM request a small VA limit, the generic page-table code clears PT_FEAT_DYNAMIC_TOP. This later causes domain initialization to fail with -EOPNOTSUPP. Remove the clearing so init succeeds when the VA fits in the starting level and no top-level growth is needed. Signed-off-by: Ankit Soni Reviewed-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 1765986e24e8..512c03f72d92 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -1149,10 +1149,6 @@ static int pt_init_common(struct pt_common *common) if (PT_WARN_ON(top_range.top_level > PT_MAX_TOP_LEVEL)) return -EINVAL; - if (top_range.top_level == PT_MAX_TOP_LEVEL || - common->max_vasz_lg2 == top_range.max_vasz_lg2) - common->features &= ~BIT(PT_FEAT_DYNAMIC_TOP); - if (top_range.max_vasz_lg2 == PT_VADDR_MAX_LG2) common->features |= BIT(PT_FEAT_FULL_VA); From d9f759704b546a46649f8c51d5e9b63e96afdbae Mon Sep 17 00:00:00 2001 From: Ankit Soni Date: Mon, 13 Apr 2026 14:45:20 +0000 Subject: [PATCH 22/66] iommu_pt: add kunit config for 32-bit VA (amdv1_cfg_1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add test coverage for small VAs (32‑bit) starting at level 2 by enabling the AMDv1 KUnit configuration. This limits level expansion because the starting level can accommodate only the maximum virtual address requested. Reviewed-by: Jason Gunthorpe Signed-off-by: Ankit Soni Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/fmt/amdv1.h | 1 + drivers/iommu/generic_pt/kunit_iommu_pt.h | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/generic_pt/fmt/amdv1.h b/drivers/iommu/generic_pt/fmt/amdv1.h index 8d11b08291d7..1426a6758ff2 100644 --- a/drivers/iommu/generic_pt/fmt/amdv1.h +++ b/drivers/iommu/generic_pt/fmt/amdv1.h @@ -404,6 +404,7 @@ amdv1pt_iommu_fmt_hw_info(struct pt_iommu_amdv1 *table, static const struct pt_iommu_amdv1_cfg amdv1_kunit_fmt_cfgs[] = { /* Matches what io_pgtable does */ [0] = { .starting_level = 2 }, + [1] = { .starting_level = 2, .common.hw_max_vasz_lg2 = 32 }, }; #define kunit_fmt_cfgs amdv1_kunit_fmt_cfgs enum { KUNIT_FMT_FEATURES = 0 }; diff --git a/drivers/iommu/generic_pt/kunit_iommu_pt.h b/drivers/iommu/generic_pt/kunit_iommu_pt.h index e8a63c8ea850..ece1c9b8c55d 100644 --- a/drivers/iommu/generic_pt/kunit_iommu_pt.h +++ b/drivers/iommu/generic_pt/kunit_iommu_pt.h @@ -112,8 +112,9 @@ static void test_increase_level(struct kunit *test) if (IS_32BIT) kunit_skip(test, "Unable to test on 32bit"); - KUNIT_ASSERT_GT(test, common->max_vasz_lg2, - pt_top_range(common).max_vasz_lg2); + if (common->max_vasz_lg2 <= pt_top_range(common).max_vasz_lg2) + kunit_skip(test, + "max_vasz_lg2 fits in starting level, no growth possible"); /* Add every possible level to the max */ while (common->max_vasz_lg2 != pt_top_range(common).max_vasz_lg2) { From 33730f06881640894d2b4228b04d4f3c7351f6c0 Mon Sep 17 00:00:00 2001 From: Ankit Soni Date: Mon, 13 Apr 2026 14:45:21 +0000 Subject: [PATCH 23/66] iommu/amd: Adhere to IVINFO[VASIZE] for address limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ACPI IVRS IVHD’s IVINFO field reports the maximum virtual address size (VASIZE) supported by the IOMMU. The AMD IOMMU driver currently caps this with pagetable level reported by EFR[HATS] when configuring paging domains (hw_max_vasz_lg2). On systems where firmware or VM advertises smaller or different limits, the driver may over-advertise capabilities and create domains outside the hardware’s actual bounds. Reviewed-by: Jason Gunthorpe Reviewed-by: Vasant Hegde Signed-off-by: Ankit Soni Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/amd_iommu_types.h | 1 + drivers/iommu/amd/init.c | 13 +++++++++---- drivers/iommu/amd/iommu.c | 3 +-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 834d8fabfba3..af720bf14914 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -44,7 +44,7 @@ int amd_iommu_enable_faulting(unsigned int cpu); extern int amd_iommu_guest_ir; extern enum protection_domain_mode amd_iommu_pgtable; extern int amd_iommu_gpt_level; -extern u8 amd_iommu_hpt_level; +extern u8 amd_iommu_hpt_vasize; extern unsigned long amd_iommu_pgsize_bitmap; extern bool amd_iommu_hatdis; diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f9f718087893..315c77e5adb2 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -400,6 +400,7 @@ #define IOMMU_IVINFO_OFFSET 36 #define IOMMU_IVINFO_EFRSUP BIT(0) #define IOMMU_IVINFO_DMA_REMAP BIT(1) +#define IOMMU_IVINFO_VASIZE GENMASK_ULL(21, 15) /* IOMMU Feature Reporting Field (for IVHD type 10h */ #define IOMMU_FEAT_GASUP_SHIFT 6 diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 3bdb380d23e9..b4d810a1cf33 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -155,8 +155,8 @@ bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1; -/* Host page table level */ -u8 amd_iommu_hpt_level; +/* Virtual address size */ +u8 amd_iommu_hpt_vasize; /* Guest page table level */ int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; @@ -3202,7 +3202,7 @@ static int __init early_amd_iommu_init(void) struct acpi_table_header *ivrs_base; int ret; acpi_status status; - u8 efr_hats; + u8 efr_hats, max_vasize; if (!amd_iommu_detected) return -ENODEV; @@ -3232,6 +3232,10 @@ static int __init early_amd_iommu_init(void) ivinfo_init(ivrs_base); + max_vasize = FIELD_GET(IOMMU_IVINFO_VASIZE, amd_iommu_ivinfo); + if (!max_vasize) + max_vasize = 64; + amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); @@ -3254,7 +3258,8 @@ static int __init early_amd_iommu_init(void) * efr[HATS] bits specify the maximum host translation level * supported, with LEVEL 4 being initial max level. */ - amd_iommu_hpt_level = efr_hats + PAGE_MODE_4_LEVEL; + amd_iommu_hpt_vasize = min_t(unsigned int, max_vasize, + (efr_hats + PAGE_MODE_4_LEVEL - 1) * 9 + 21); } else { pr_warn_once(FW_BUG "Disable host address translation due to invalid translation level (%#x).\n", efr_hats); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f78e23f03938..65082b9e4b52 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2714,8 +2714,7 @@ static struct iommu_domain *amd_iommu_domain_alloc_paging_v1(struct device *dev, else cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); - cfg.common.hw_max_vasz_lg2 = - min(64, (amd_iommu_hpt_level - 1) * 9 + 21); + cfg.common.hw_max_vasz_lg2 = amd_iommu_hpt_vasize; cfg.common.hw_max_oasz_lg2 = 52; cfg.starting_level = 2; domain->domain.ops = &amdv1_ops; From 4315401bad65582b28d99fb56fb816797fce9fc3 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Wed, 13 May 2026 21:52:01 +0000 Subject: [PATCH 24/66] iommu/io-pgtable-arm: Use consistent sizes for page allocation and freeing At the moment we use alloc_size to allocate memory but then there is a logical error where we just size in the error and free path, which might be smaller. Also we size to do DMA-API operations, which is OK, but confusing. Instead of this error-prone handling, just set size to alloc_size and use it everywhere. Signed-off-by: Mostafa Saleh Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 0208e5897c29..0cbe545c491d 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -253,7 +253,6 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, void *cookie) { struct device *dev = cfg->iommu_dev; - size_t alloc_size; dma_addr_t dma; void *pages; @@ -261,12 +260,11 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, * For very small starting-level translation tables the HW requires a * minimum alignment of at least 64 to cover all cases. */ - alloc_size = max(size, 64); + size = max(size, 64); if (cfg->alloc) - pages = cfg->alloc(cookie, alloc_size, gfp); + pages = cfg->alloc(cookie, size, gfp); else - pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, - alloc_size); + pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, size); if (!pages) return NULL; @@ -303,6 +301,9 @@ static void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg, void *cookie) { + /* See __arm_lpae_alloc_pages(). */ + size = max(size, 64); + if (!cfg->coherent_walk) dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), size, DMA_TO_DEVICE); From d996116cde972756e8415d6e070b960f6f6a8f6c Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Wed, 13 May 2026 21:52:02 +0000 Subject: [PATCH 25/66] iommu/io-pgtable-arm: Rework to use the iommu-pages API Update the io-pgtable-arm allocator to use the iommu-pages API. Replace the DMA API usage from __arm_lpae_alloc_pages() with iommu_pages_start_incoherent() and from __arm_lpae_free_pages() with iommu_pages_free_incoherent(). Since the iommu-pages API relies on metadata stored in the struct page during iommu_alloc_pages_node_sz(), it cannot be used safely with memory allocated via the custom cfg->alloc (which may not be backed by pages). So, isolate that logic and keep it as it. Suggested-by: Jason Gunthorpe Signed-off-by: Mostafa Saleh Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 79 ++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 0cbe545c491d..86b23aa04324 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -248,24 +248,15 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages) return (dma_addr_t)virt_to_phys(pages); } -static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, - struct io_pgtable_cfg *cfg, - void *cookie) +static void *__arm_lpae_cfg_alloc(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg, + void *cookie) { struct device *dev = cfg->iommu_dev; dma_addr_t dma; void *pages; - /* - * For very small starting-level translation tables the HW requires a - * minimum alignment of at least 64 to cover all cases. - */ - size = max(size, 64); - if (cfg->alloc) - pages = cfg->alloc(cookie, size, gfp); - else - pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, size); - + pages = cfg->alloc(cookie, size, gfp); if (!pages) return NULL; @@ -289,14 +280,55 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); out_free: - if (cfg->free) - cfg->free(cookie, pages, size); - else - iommu_free_pages(pages); - + cfg->free(cookie, pages, size); return NULL; } +static void __arm_lpae_cfg_free(void *pages, size_t size, + struct io_pgtable_cfg *cfg, + void *cookie) +{ + if (!cfg->coherent_walk) + dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), + size, DMA_TO_DEVICE); + + cfg->free(cookie, pages, size); +} + +static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg, + void *cookie) +{ + struct device *dev = cfg->iommu_dev; + void *pages; + + /* + * For very small starting-level translation tables the HW requires a + * minimum alignment of at least 64 to cover all cases. + */ + size = max(size, 64); + + if (cfg->alloc) + return __arm_lpae_cfg_alloc(size, gfp, cfg, cookie); + + pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, size); + if (!pages) + return NULL; + + if (!cfg->coherent_walk) { + int ret = iommu_pages_start_incoherent(pages, dev); + + if (ret) { + if (ret == -EOPNOTSUPP) + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + iommu_free_pages(pages); + return NULL; + } + } + + return pages; +} + static void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg, void *cookie) @@ -304,12 +336,13 @@ static void __arm_lpae_free_pages(void *pages, size_t size, /* See __arm_lpae_alloc_pages(). */ size = max(size, 64); - if (!cfg->coherent_walk) - dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), - size, DMA_TO_DEVICE); + if (cfg->free) { + __arm_lpae_cfg_free(pages, size, cfg, cookie); + return; + } - if (cfg->free) - cfg->free(cookie, pages, size); + if (!cfg->coherent_walk) + iommu_pages_free_incoherent(pages, cfg->iommu_dev); else iommu_free_pages(pages); } From adb77e757680dcfa79ccc554ec51d42cf56ef69a Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Wed, 13 May 2026 21:52:03 +0000 Subject: [PATCH 26/66] iommu/io-pgtable-arm: Use address conversion consistently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use consistent address conversions in the driver: - virt_to_phys(): For all virtual to physical address conversion, convert __pa users as we don’t need to rely on it type casting. - phys_to_virt(): For all physical to virtual address conversion, similarly, convert __va users. That changes nothing at all. However, it will be useful when compiling this file for the KVM hypervisor as it can cleanly replace virt_to_phys/phys_to_virt Signed-off-by: Mostafa Saleh Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 86b23aa04324..476c0e25631a 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -143,7 +143,7 @@ #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL /* IOPTE accessors */ -#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) +#define iopte_deref(pte, d) phys_to_virt(iopte_to_paddr(pte, d)) #define iopte_type(pte) \ (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) @@ -429,7 +429,7 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, arm_lpae_iopte old, new; struct io_pgtable_cfg *cfg = &data->iop.cfg; - new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE; + new = paddr_to_iopte(virt_to_phys(table), data) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) new |= ARM_LPAE_PTE_NSTABLE; From a6e1618a65d453d4e739e5898c662ccb1e3de6c0 Mon Sep 17 00:00:00 2001 From: Bibek Kumar Patro Date: Thu, 23 Apr 2026 15:32:30 +0530 Subject: [PATCH 27/66] iommu/arm-smmu-qcom: Fix fastrpc compatible string in ACTLR client match table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The qcom_smmu_actlr_client_of_match table contained "qcom,fastrpc" as the compatible string for applying ACTLR prefetch settings to FastRPC devices. However, "qcom,fastrpc" is the compatible string for the parent rpmsg channel node, which is not an IOMMU client — it carries no "iommus" property in the device tree and is never attached to an SMMU context bank. The actual IOMMU clients are the compute context bank (CB) child nodes, which use the compatible string "qcom,fastrpc-compute-cb". These nodes carry the "iommus" property and are probed by fastrpc_cb_driver via fastrpc_cb_probe(), which sets up the DMA mask and IOMMU mappings for each FastRPC session. The device tree structure is: fastrpc { compatible = "qcom,fastrpc"; /* rpmsg channel, no iommus */ ... compute-cb@3 { compatible = "qcom,fastrpc-compute-cb"; iommus = <&apps_smmu 0x1823 0x0>; /* actual IOMMU client */ }; }; Since qcom_smmu_set_actlr_dev() calls of_match_device() against the device being attached to the SMMU context bank, the "qcom,fastrpc" entry was never matching any device. As a result, the ACTLR prefetch settings (PREFETCH_DEEP | CPRE | CMTLB) were silently never applied for FastRPC compute context banks. Fix this by replacing "qcom,fastrpc" with "qcom,fastrpc-compute-cb" in the match table so that the ACTLR settings are correctly applied to the compute CB devices that are the true IOMMU clients. Assisted-by: Claude:claude-sonnet-4-6 Fixes: 3e35c3e725de ("iommu/arm-smmu: Add ACTLR data and support for qcom_smmu_500") Reviewed-by: Dmitry Baryshkov Reviewed-by: Shawn Guo Signed-off-by: Bibek Kumar Patro Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index edd41b5a3b6a..2d006049dd61 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -39,7 +39,7 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,adreno-smmu", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, - { .compatible = "qcom,fastrpc", + { .compatible = "qcom,fastrpc-compute-cb", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,qcm2290-mdss", .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, From c3f9dabf58bbbd594d9476e241074b607680d89f Mon Sep 17 00:00:00 2001 From: Mukesh Ojha Date: Mon, 27 Apr 2026 23:19:15 +0530 Subject: [PATCH 28/66] dt-bindings: arm-smmu: qcom: Add Hawi compatible for Application processor Commit 5e8323c3d528 ("dt-bindings: arm-smmu: qcom: Add compatible for Hawi SoC") was intended for the APSS SMMU but was mistakenly placed under the Adreno GPU SMMU section. Since that compatible is also valid for the Hawi GPU SMMU, keep that commit as-is and add proper documentation for the Hawi APSS SMMU here. Signed-off-by: Mukesh Ojha Reviewed-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 06fb5c8e7547..07b71b347205 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -37,6 +37,7 @@ properties: - enum: - qcom,eliza-smmu-500 - qcom,glymur-smmu-500 + - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 - qcom,qcm2290-smmu-500 From 8bcad9e3a6742f15d3a8569a987b9ccbefeb4f55 Mon Sep 17 00:00:00 2001 From: Lokanadha M R Date: Tue, 28 Apr 2026 15:13:50 +0530 Subject: [PATCH 29/66] iommu/arm-smmu-qcom: Add glymur MDSS to ACTLR client table Add qcom,glymur-mdss to the qcom_smmu_actlr_client_of_match[] table to configure the SMMU-500 context bank for the display subsystem (MDSS) on the Glymur platform. The settings disable the next-page prefetcher while keeping macro TLB caching enabled. Without this entry, qcom_smmu_set_actlr_dev() finds no match for the glymur MDSS device and leaves the context bank ACTLR at its reset value. Signed-off-by: Lokanadha M R Reviewed-by: Pankaj Patil Reviewed-by: Dmitry Baryshkov Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 2d006049dd61..5747b908e49a 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -41,6 +41,8 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,fastrpc-compute-cb", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, + { .compatible = "qcom,glymur-mdss", + .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, { .compatible = "qcom,qcm2290-mdss", .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, { .compatible = "qcom,sa8775p-mdss", From 8a0aab012b52b5c044ca3bec523e91f95dd40765 Mon Sep 17 00:00:00 2001 From: Anna Maniscalco Date: Thu, 7 May 2026 17:43:15 +0200 Subject: [PATCH 30/66] iommu: arm-smmu-qcom: Ensure smmu is powered up in set_ttbr0_cfg arm_smmu_write_context_bank() assumes it is being called with RPM active, but it turns out that is not guaranteed in the path from qcom_adreno_smmu_set_ttbr0_cfg(), so it's possible for the register writes to get lost when configuring the context bank while the GPU is idle, leading to page faults later. Add the RPM calls here to make sure the SMMU is active before we touch it. Signed-off-by: Anna Maniscalco Reviewed-by: Rob Clark Reviewed-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 5747b908e49a..e2c914fccd6f 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -261,6 +261,7 @@ static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie, struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; + int ret; /* The domain must have split pagetables already enabled */ if (cb->tcr[0] & ARM_SMMU_TCR_EPD1) @@ -290,8 +291,16 @@ static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie, cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid); } + ret = pm_runtime_resume_and_get(smmu_domain->smmu->dev); + if (ret < 0) { + dev_err(smmu_domain->smmu->dev, "failed to get runtime PM: %d\n", ret); + return -ENODEV; + } + arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx); + pm_runtime_put_autosuspend(smmu_domain->smmu->dev); + return 0; } From 5091bfe5d4c674990e7272ad35616915802842f9 Mon Sep 17 00:00:00 2001 From: Komal Bajaj Date: Thu, 30 Apr 2026 17:54:44 +0530 Subject: [PATCH 31/66] dt-bindings: arm-smmu: qcom: Add compatible for Qualcomm Shikra SoC Qualcomm Shikra SoC includes an apps SMMU that implements arm,mmu-500, which is used to translate device-visible virtual addresses to physical addresses. Add compatible for it. Signed-off-by: Komal Bajaj Acked-by: Rob Herring (Arm) Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 07b71b347205..62631b912fd2 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -56,6 +56,7 @@ properties: - qcom,sdx55-smmu-500 - qcom,sdx65-smmu-500 - qcom,sdx75-smmu-500 + - qcom,shikra-smmu-500 - qcom,sm6115-smmu-500 - qcom,sm6125-smmu-500 - qcom,sm6350-smmu-500 From 2b9f593dad4666998d2440f4a8a6905c90b805dd Mon Sep 17 00:00:00 2001 From: Leo Jiang Date: Sat, 9 May 2026 15:07:33 +0800 Subject: [PATCH 32/66] iommu/arm-smmu-v3: Limit queue allocation retry boundary to PAGE_SIZE Stop retrying queue allocation when qsz reaches PAGE_SIZE. Signed-off-by: Leo Jiang Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index e8d7dbe495f0..4a0b15b1d4d2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -4418,7 +4418,7 @@ int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, qsz = ((1 << q->llq.max_n_shift) * dwords) << 3; q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL); - if (q->base || qsz < PAGE_SIZE) + if (q->base || qsz <= PAGE_SIZE) break; q->llq.max_n_shift--; From 23bc2dd17b2000912b7ee3a92090ec67e4e572d7 Mon Sep 17 00:00:00 2001 From: Akhil P Oommen Date: Wed, 13 May 2026 00:51:20 +0530 Subject: [PATCH 33/66] dt-bindings: arm-smmu: Update the description for Glymur GPU SMMU Add the interconnects property to the common SMMU properties and extend the sm8750 clock description section to also cover Glymur since it uses the same single "hlos" vote clock. Signed-off-by: Akhil P Oommen Acked-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 62631b912fd2..67c7a9207c98 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -568,7 +568,9 @@ allOf: properties: compatible: items: - - const: qcom,sm8750-smmu-500 + - enum: + - qcom,glymur-smmu-500 + - qcom,sm8750-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 - const: arm,mmu-500 From 75949eb0265387ba004744320e9bfbc0b918be5e Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 19 May 2026 09:41:00 +0200 Subject: [PATCH 34/66] dt-bindings: arm-smmu: Constrain clocks for newer Qualcomm variants Many of SMMU on Qualcomm SoCs come in two flavors using the same front compatible but a bit different fallback: 1. For application processor, usually without any controllable clocks, 2. For the Adreno GPU, with some controllable clock(s) and using additionally qcom,adreno-smmu fallback compatible. Add missing constraints for Glymur SMMU on Adreno GPU and several other Qualcomm SMMUs for application processors, to restrict the clocks property to a specific length. Acked-by: Rob Herring (Arm) Reviewed-by: Shawn Guo Signed-off-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- .../devicetree/bindings/iommu/arm,smmu.yaml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 67c7a9207c98..25fd3efa2420 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -599,6 +599,7 @@ allOf: - qcom,sdm845-smmu-500 - qcom,sdx55-smmu-500 - qcom,sdx65-smmu-500 + - qcom,sdx75-smmu-500 - qcom,sm6350-smmu-500 - qcom,sm6375-smmu-500 then: @@ -606,6 +607,39 @@ allOf: clock-names: false clocks: false + # Disallow clocks for all other platforms where specific compatible is used + # with different fallbacks and only one combination has no clocks + - if: + properties: + compatible: + items: + - enum: + - qcom,eliza-smmu-500 + - qcom,glymur-smmu-500 + - qcom,kaanapali-smmu-500 + - qcom,milos-smmu-500 + - qcom,qcs615-smmu-500 + - qcom,qcs8300-smmu-500 + - qcom,sa8775p-smmu-500 + - qcom,sm6115-smmu-500 + - qcom,sm6125-smmu-500 + - qcom,sm6350-smmu-500 + - qcom,sm6375-smmu-500 + - qcom,sm8150-smmu-500 + - qcom,sm8250-smmu-500 + - qcom,sm8350-smmu-500 + - qcom,sm8450-smmu-500 + - qcom,sm8550-smmu-500 + - qcom,sm8650-smmu-500 + - qcom,sm8750-smmu-500 + - qcom,x1e80100-smmu-500 + - const: qcom,smmu-500 + - const: arm,mmu-500 + then: + properties: + clock-names: false + clocks: false + - if: properties: compatible: From bf00a29234a46d10b3f6890343f7a6755e5809aa Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:40 -0300 Subject: [PATCH 35/66] iommu/arm-smmu-v3: Add struct arm_smmu_cmd to represent the HW format command Like STE/CD, add a wrapper struct around the u64 array to represent the already FIELD_PREP'd command data. Unlike the STE/CD this is a u64 because the command submission path will have the swap to le64. This makes the API clearer when a u64 is referring to a formatted command and makes the following changes easier to follow. Move the command constants out of the struct and into an enum alongside the rest of the constants defining the HW format so the entire HW format is self contained and independent of struct arm_smmu_cmdq_ent. Reviewed-by: Samiullah Khawaja Reviewed-by: Mostafa Saleh Reviewed-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 48 ++++++++++++--------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ef42df4753ec..092179f689e9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -390,6 +390,10 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid) #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG +struct arm_smmu_cmd { + u64 data[CMDQ_ENT_DWORDS]; +}; + /* * This is used to size the command queue and therefore must be at least * BITS_PER_LONG so that the valid_map works correctly (it relies on the @@ -426,6 +430,8 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid) #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0) #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12) +#define ATC_INV_SIZE_ALL 52 + #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) @@ -447,6 +453,28 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid) #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32) #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2) +enum arm_smmu_cmdq_opcode { + CMDQ_OP_PREFETCH_CFG = 0x1, + CMDQ_OP_CFGI_STE = 0x3, + CMDQ_OP_CFGI_ALL = 0x4, + CMDQ_OP_CFGI_CD = 0x5, + CMDQ_OP_CFGI_CD_ALL = 0x6, + CMDQ_OP_TLBI_NH_ALL = 0x10, + CMDQ_OP_TLBI_NH_ASID = 0x11, + CMDQ_OP_TLBI_NH_VA = 0x12, + CMDQ_OP_TLBI_NH_VAA = 0x13, + CMDQ_OP_TLBI_EL2_ALL = 0x20, + CMDQ_OP_TLBI_EL2_ASID = 0x21, + CMDQ_OP_TLBI_EL2_VA = 0x22, + CMDQ_OP_TLBI_S12_VMALL = 0x28, + CMDQ_OP_TLBI_S2_IPA = 0x2a, + CMDQ_OP_TLBI_NSNH_ALL = 0x30, + CMDQ_OP_ATC_INV = 0x40, + CMDQ_OP_PRI_RESP = 0x41, + CMDQ_OP_RESUME = 0x44, + CMDQ_OP_CMD_SYNC = 0x46, +}; + /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) @@ -520,15 +548,10 @@ struct arm_smmu_cmdq_ent { /* Command-specific fields */ union { - #define CMDQ_OP_PREFETCH_CFG 0x1 struct { u32 sid; } prefetch; - #define CMDQ_OP_CFGI_STE 0x3 - #define CMDQ_OP_CFGI_ALL 0x4 - #define CMDQ_OP_CFGI_CD 0x5 - #define CMDQ_OP_CFGI_CD_ALL 0x6 struct { u32 sid; u32 ssid; @@ -538,16 +561,6 @@ struct arm_smmu_cmdq_ent { }; } cfgi; - #define CMDQ_OP_TLBI_NH_ALL 0x10 - #define CMDQ_OP_TLBI_NH_ASID 0x11 - #define CMDQ_OP_TLBI_NH_VA 0x12 - #define CMDQ_OP_TLBI_NH_VAA 0x13 - #define CMDQ_OP_TLBI_EL2_ALL 0x20 - #define CMDQ_OP_TLBI_EL2_ASID 0x21 - #define CMDQ_OP_TLBI_EL2_VA 0x22 - #define CMDQ_OP_TLBI_S12_VMALL 0x28 - #define CMDQ_OP_TLBI_S2_IPA 0x2a - #define CMDQ_OP_TLBI_NSNH_ALL 0x30 struct { u8 num; u8 scale; @@ -559,8 +572,6 @@ struct arm_smmu_cmdq_ent { u64 addr; } tlbi; - #define CMDQ_OP_ATC_INV 0x40 - #define ATC_INV_SIZE_ALL 52 struct { u32 sid; u32 ssid; @@ -569,7 +580,6 @@ struct arm_smmu_cmdq_ent { bool global; } atc; - #define CMDQ_OP_PRI_RESP 0x41 struct { u32 sid; u32 ssid; @@ -577,14 +587,12 @@ struct arm_smmu_cmdq_ent { enum pri_resp resp; } pri; - #define CMDQ_OP_RESUME 0x44 struct { u32 sid; u16 stag; u8 resp; } resume; - #define CMDQ_OP_CMD_SYNC 0x46 struct { u64 msiaddr; } sync; From f59c5b6858d8a0baa8a0895ec78457ae64770d82 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:41 -0300 Subject: [PATCH 36/66] iommu/arm-smmu-v3: Use the HW arm_smmu_cmd in cmdq selection functions Start removing the use of struct arm_smmu_cmdq_ent, starting at the lower levels of the call chain. Change the functions that determine what cmdq to issue the batch to into using struct arm_smmu_cmd directly. Reviewed-by: Pranjal Shrivastava Reviewed-by: Mostafa Saleh Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 47 ++++++++++++------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 8 ++-- .../iommu/arm/arm-smmu-v3/tegra241-cmdqv.c | 8 ++-- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4a0b15b1d4d2..866471c7684f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -371,12 +371,12 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) } static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_ent *ent) + struct arm_smmu_cmd *cmd) { struct arm_smmu_cmdq *cmdq = NULL; if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq) - cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent); + cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, cmd); return cmdq ?: &smmu->cmdq; } @@ -924,16 +924,16 @@ static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent, bool sync) { - u64 cmd[CMDQ_ENT_DWORDS]; + struct arm_smmu_cmd cmd; - if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) { + if (unlikely(arm_smmu_cmdq_build_cmd(cmd.data, ent))) { dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", ent->opcode); return -EINVAL; } return arm_smmu_cmdq_issue_cmdlist( - smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync); + smmu, arm_smmu_get_cmdq(smmu, &cmd), cmd.data, 1, sync); } static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, @@ -948,42 +948,55 @@ static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu, return __arm_smmu_cmdq_issue_cmd(smmu, ent, true); } +static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd) +{ + cmds->num = 0; + cmds->cmdq = arm_smmu_get_cmdq(smmu, cmd); +} + static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmdq_ent *ent) { - cmds->num = 0; - cmds->cmdq = arm_smmu_get_cmdq(smmu, ent); + struct arm_smmu_cmd cmd; + + arm_smmu_cmdq_build_cmd(cmd.data, ent); + arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); } static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmdq_ent *cmd) + struct arm_smmu_cmdq_ent *ent) { - bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd); bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) && (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC); + struct arm_smmu_cmd cmd; + bool unsupported_cmd; int index; + if (unlikely(arm_smmu_cmdq_build_cmd(cmd.data, ent))) { + dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", + ent->opcode); + return; + } + + unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, &cmd); if (force_sync || unsupported_cmd) { arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, cmds->num, true); - arm_smmu_cmdq_batch_init(smmu, cmds, cmd); + arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); } if (cmds->num == CMDQ_BATCH_ENTRIES) { arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, cmds->num, false); - arm_smmu_cmdq_batch_init(smmu, cmds, cmd); + arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); } index = cmds->num * CMDQ_ENT_DWORDS; - if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) { - dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", - cmd->opcode); - return; - } - + memcpy(&cmds->cmds[index], cmd.data, sizeof(cmd.data)); cmds->num++; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 092179f689e9..6d73f6b63e64 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -641,13 +641,13 @@ struct arm_smmu_cmdq { atomic_long_t *valid_map; atomic_t owner_prod; atomic_t lock; - bool (*supports_cmd)(struct arm_smmu_cmdq_ent *ent); + bool (*supports_cmd)(struct arm_smmu_cmd *cmd); }; static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq, - struct arm_smmu_cmdq_ent *ent) + struct arm_smmu_cmd *cmd) { - return cmdq->supports_cmd ? cmdq->supports_cmd(ent) : true; + return cmdq->supports_cmd ? cmdq->supports_cmd(cmd) : true; } struct arm_smmu_cmdq_batch { @@ -815,7 +815,7 @@ struct arm_smmu_impl_ops { void (*device_remove)(struct arm_smmu_device *smmu); int (*init_structures)(struct arm_smmu_device *smmu); struct arm_smmu_cmdq *(*get_secondary_cmdq)( - struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent); + struct arm_smmu_device *smmu, struct arm_smmu_cmd *cmd); /* * An implementation should define its own type other than the default * IOMMU_HW_INFO_TYPE_ARM_SMMUV3. And it must validate the input @type diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 83f6e9f6c51d..b4d8c1f2fd38 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -367,9 +367,9 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) /* Command Queue Function */ -static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent *ent) +static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmd *cmd) { - switch (ent->opcode) { + switch (FIELD_GET(CMDQ_0_OP, cmd->data[0])) { case CMDQ_OP_TLBI_NH_ASID: case CMDQ_OP_TLBI_NH_VA: case CMDQ_OP_ATC_INV: @@ -381,7 +381,7 @@ static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent *ent) static struct arm_smmu_cmdq * tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_ent *ent) + struct arm_smmu_cmd *cmd) { struct tegra241_cmdqv *cmdqv = container_of(smmu, struct tegra241_cmdqv, smmu); @@ -409,7 +409,7 @@ tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu, return NULL; /* Unsupported CMD goes for smmu->cmdq pathway */ - if (!arm_smmu_cmdq_supports_cmd(&vcmdq->cmdq, ent)) + if (!arm_smmu_cmdq_supports_cmd(&vcmdq->cmdq, cmd)) return NULL; return &vcmdq->cmdq; } From d455e3a7bf0a7499ec8bcf1b43b1ee14e6e35876 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:42 -0300 Subject: [PATCH 37/66] iommu/arm-smmu-v3: Use the HW arm_smmu_cmd in cmdq submission functions Continue removing struct arm_smmu_cmdq_ent in favour of the HW based struct arm_smmu_cmd. Switch the lower level issue commands to work on the native struct by lifting arm_smmu_cmdq_build_cmd() into all the callers. Following patches will revise each of the arm_smmu_cmdq_build_cmd() call sites to replace it with the HW struct. Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 53 ++++++++++++--------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 866471c7684f..fbf56c64da7f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -921,31 +921,23 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, } static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_ent *ent, + struct arm_smmu_cmd *cmd, bool sync) { - struct arm_smmu_cmd cmd; - - if (unlikely(arm_smmu_cmdq_build_cmd(cmd.data, ent))) { - dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", - ent->opcode); - return -EINVAL; - } - return arm_smmu_cmdq_issue_cmdlist( - smmu, arm_smmu_get_cmdq(smmu, &cmd), cmd.data, 1, sync); + smmu, arm_smmu_get_cmdq(smmu, cmd), cmd->data, 1, sync); } static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_ent *ent) + struct arm_smmu_cmd *cmd) { - return __arm_smmu_cmdq_issue_cmd(smmu, ent, false); + return __arm_smmu_cmdq_issue_cmd(smmu, cmd, false); } static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_ent *ent) + struct arm_smmu_cmd *cmd) { - return __arm_smmu_cmdq_issue_cmd(smmu, ent, true); + return __arm_smmu_cmdq_issue_cmd(smmu, cmd, true); } static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu, @@ -1013,6 +1005,7 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused struct arm_smmu_cmdq_ent cmd = {0}; struct arm_smmu_master *master = dev_iommu_priv_get(dev); int sid = master->streams[0].id; + struct arm_smmu_cmd hw_cmd; if (WARN_ON(!master->stall_enabled)) return; @@ -1032,7 +1025,9 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused break; } - arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); + arm_smmu_cmdq_build_cmd(hw_cmd.data, &cmd); + arm_smmu_cmdq_issue_cmd(master->smmu, &hw_cmd); + /* * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP. * RESUME consumption guarantees that the stalled transaction will be @@ -1861,14 +1856,16 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) { struct arm_smmu_ste_writer *ste_writer = container_of(writer, struct arm_smmu_ste_writer, writer); - struct arm_smmu_cmdq_ent cmd = { + struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_CFGI_STE, .cfgi = { .sid = ste_writer->sid, .leaf = true, }, }; + struct arm_smmu_cmd cmd; + arm_smmu_cmdq_build_cmd(cmd.data, &ent); arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd); } @@ -1896,11 +1893,13 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, /* It's likely that we'll want to use the new STE soon */ if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { struct arm_smmu_cmdq_ent - prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, + prefetch_ent = { .opcode = CMDQ_OP_PREFETCH_CFG, .prefetch = { .sid = sid, } }; + struct arm_smmu_cmd prefetch_cmd; + arm_smmu_cmdq_build_cmd(prefetch_cmd.data, &prefetch_ent); arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); } } @@ -2328,7 +2327,7 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) evt[1] & PRIQ_1_ADDR_MASK); if (last) { - struct arm_smmu_cmdq_ent cmd = { + struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_PRI_RESP, .substream_valid = ssv, .pri = { @@ -2338,7 +2337,9 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) .resp = PRI_RESP_DENY, }, }; + struct arm_smmu_cmd cmd; + arm_smmu_cmdq_build_cmd(cmd.data, &ent); arm_smmu_cmdq_issue_cmd(smmu, &cmd); } } @@ -3446,6 +3447,7 @@ arm_smmu_install_new_domain_invs(struct arm_smmu_attach_state *state) static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv) { struct arm_smmu_cmdq_ent cmd = {}; + struct arm_smmu_cmd hw_cmd; switch (inv->type) { case INV_TYPE_S1_ASID: @@ -3460,7 +3462,8 @@ static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv) } cmd.opcode = inv->nsize_opcode; - arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &cmd); + arm_smmu_cmdq_build_cmd(hw_cmd.data, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &hw_cmd); } /* Should be installed after arm_smmu_install_ste_for_dev() */ @@ -4823,7 +4826,8 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; u32 reg, enables; - struct arm_smmu_cmdq_ent cmd; + struct arm_smmu_cmdq_ent ent; + struct arm_smmu_cmd cmd; /* Clear CR0 and sync (disables SMMU and queue processing) */ reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); @@ -4870,16 +4874,19 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) } /* Invalidate any cached configuration */ - cmd.opcode = CMDQ_OP_CFGI_ALL; + ent.opcode = CMDQ_OP_CFGI_ALL; + arm_smmu_cmdq_build_cmd(cmd.data, &ent); arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); /* Invalidate any stale TLB entries */ if (smmu->features & ARM_SMMU_FEAT_HYP) { - cmd.opcode = CMDQ_OP_TLBI_EL2_ALL; + ent.opcode = CMDQ_OP_TLBI_EL2_ALL; + arm_smmu_cmdq_build_cmd(cmd.data, &ent); arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); } - cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; + ent.opcode = CMDQ_OP_TLBI_NSNH_ALL; + arm_smmu_cmdq_build_cmd(cmd.data, &ent); arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); /* Event queue */ From 27e02ca615528866ba98166888cd701b299c0b33 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:43 -0300 Subject: [PATCH 38/66] iommu/arm-smmu-v3: Convert arm_smmu_cmdq_batch cmds to struct arm_smmu_cmd Convert the batch's type to also get the remaining helper functions to use the new type and complete replacing naked u64s with the new struct. The low-level queue_write()/queue_read()/queue_remove_raw() functions remain u64-based since they are shared by event and PRI queues which have different entry sizes. Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- .../arm/arm-smmu-v3/arm-smmu-v3-iommufd.c | 24 +++--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 74 ++++++++++--------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 +- .../iommu/arm/arm-smmu-v3/tegra241-cmdqv.c | 8 +- 4 files changed, 58 insertions(+), 53 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index ddae0b07c76b..1e9f7d2de344 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -300,7 +300,7 @@ static int arm_vsmmu_vsid_to_sid(struct arm_vsmmu *vsmmu, u32 vsid, u32 *sid) /* This is basically iommu_viommu_arm_smmuv3_invalidate in u64 for conversion */ struct arm_vsmmu_invalidation_cmd { union { - u64 cmd[2]; + struct arm_smmu_cmd cmd; struct iommu_viommu_arm_smmuv3_invalidate ucmd; }; }; @@ -316,32 +316,32 @@ static int arm_vsmmu_convert_user_cmd(struct arm_vsmmu *vsmmu, struct arm_vsmmu_invalidation_cmd *cmd) { /* Commands are le64 stored in u64 */ - cmd->cmd[0] = le64_to_cpu(cmd->ucmd.cmd[0]); - cmd->cmd[1] = le64_to_cpu(cmd->ucmd.cmd[1]); + cmd->cmd.data[0] = le64_to_cpu(cmd->ucmd.cmd[0]); + cmd->cmd.data[1] = le64_to_cpu(cmd->ucmd.cmd[1]); - switch (cmd->cmd[0] & CMDQ_0_OP) { + switch (cmd->cmd.data[0] & CMDQ_0_OP) { case CMDQ_OP_TLBI_NSNH_ALL: /* Convert to NH_ALL */ - cmd->cmd[0] = CMDQ_OP_TLBI_NH_ALL | + cmd->cmd.data[0] = CMDQ_OP_TLBI_NH_ALL | FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid); - cmd->cmd[1] = 0; + cmd->cmd.data[1] = 0; break; case CMDQ_OP_TLBI_NH_VA: case CMDQ_OP_TLBI_NH_VAA: case CMDQ_OP_TLBI_NH_ALL: case CMDQ_OP_TLBI_NH_ASID: - cmd->cmd[0] &= ~CMDQ_TLBI_0_VMID; - cmd->cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid); + cmd->cmd.data[0] &= ~CMDQ_TLBI_0_VMID; + cmd->cmd.data[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid); break; case CMDQ_OP_ATC_INV: case CMDQ_OP_CFGI_CD: case CMDQ_OP_CFGI_CD_ALL: { - u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd[0]); + u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd.data[0]); if (arm_vsmmu_vsid_to_sid(vsmmu, vsid, &sid)) return -EIO; - cmd->cmd[0] &= ~CMDQ_CFGI_0_SID; - cmd->cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + cmd->cmd.data[0] &= ~CMDQ_CFGI_0_SID; + cmd->cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); break; } default: @@ -386,7 +386,7 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu, continue; /* FIXME always uses the main cmdq rather than trying to group by type */ - ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, last->cmd, + ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &last->cmd, cur - last, true); if (ret) { cur--; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index fbf56c64da7f..a3ce3c11c4d1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -268,9 +268,12 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) } /* High-level queue accessors */ -static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) +static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, + struct arm_smmu_cmdq_ent *ent) { - memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT); + u64 *cmd = cmd_out->data; + + memset(cmd_out, 0, sizeof(*cmd_out)); cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); switch (ent->opcode) { @@ -390,7 +393,8 @@ static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu, return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV; } -static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, +static void arm_smmu_cmdq_build_sync_cmd(struct arm_smmu_cmd *cmd, + struct arm_smmu_device *smmu, struct arm_smmu_cmdq *cmdq, u32 prod) { struct arm_smmu_queue *q = &cmdq->q; @@ -409,7 +413,8 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, arm_smmu_cmdq_build_cmd(cmd, &ent); if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) - u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS); + u64p_replace_bits(&cmd->data[0], CMDQ_SYNC_0_CS_NONE, + CMDQ_SYNC_0_CS); } void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, @@ -422,9 +427,8 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout", }; struct arm_smmu_queue *q = &cmdq->q; - int i; - u64 cmd[CMDQ_ENT_DWORDS]; + struct arm_smmu_cmd cmd; u32 cons = readl_relaxed(q->cons_reg); u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons); struct arm_smmu_cmdq_ent cmd_sync = { @@ -457,17 +461,18 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, * We may have concurrent producers, so we need to be careful * not to touch any of the shadow cmdq state. */ - queue_read(cmd, Q_ENT(q, cons), q->ent_dwords); + queue_read(cmd.data, Q_ENT(q, cons), q->ent_dwords); dev_err(smmu->dev, "skipping command in error state:\n"); - for (i = 0; i < ARRAY_SIZE(cmd); ++i) - dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); + for (i = 0; i < ARRAY_SIZE(cmd.data); ++i) + dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd.data[i]); /* Convert the erroneous command into a CMD_SYNC */ - arm_smmu_cmdq_build_cmd(cmd, &cmd_sync); + arm_smmu_cmdq_build_cmd(&cmd, &cmd_sync); if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) - u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS); + u64p_replace_bits(&cmd.data[0], CMDQ_SYNC_0_CS_NONE, + CMDQ_SYNC_0_CS); - queue_write(Q_ENT(q, cons), cmd, q->ent_dwords); + queue_write(Q_ENT(q, cons), cmd.data, q->ent_dwords); } static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) @@ -767,7 +772,8 @@ static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu, return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq); } -static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds, +static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, + struct arm_smmu_cmd *cmds, u32 prod, int n) { int i; @@ -777,10 +783,9 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds, }; for (i = 0; i < n; ++i) { - u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS]; - prod = queue_inc_prod_n(&llq, i); - queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS); + queue_write(Q_ENT(&cmdq->q, prod), cmds[i].data, + ARRAY_SIZE(cmds[i].data)); } } @@ -801,10 +806,11 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds, * CPU will appear before any of the commands from the other CPU. */ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq *cmdq, u64 *cmds, int n, + struct arm_smmu_cmdq *cmdq, + struct arm_smmu_cmd *cmds, int n, bool sync) { - u64 cmd_sync[CMDQ_ENT_DWORDS]; + struct arm_smmu_cmd cmd_sync; u32 prod; unsigned long flags; bool owner; @@ -847,8 +853,9 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); if (sync) { prod = queue_inc_prod_n(&llq, n); - arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod); - queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); + arm_smmu_cmdq_build_sync_cmd(&cmd_sync, smmu, cmdq, prod); + queue_write(Q_ENT(&cmdq->q, prod), cmd_sync.data, + ARRAY_SIZE(cmd_sync.data)); /* * In order to determine completion of our CMD_SYNC, we must @@ -925,7 +932,7 @@ static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, bool sync) { return arm_smmu_cmdq_issue_cmdlist( - smmu, arm_smmu_get_cmdq(smmu, cmd), cmd->data, 1, sync); + smmu, arm_smmu_get_cmdq(smmu, cmd), cmd, 1, sync); } static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, @@ -954,7 +961,7 @@ static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu, { struct arm_smmu_cmd cmd; - arm_smmu_cmdq_build_cmd(cmd.data, ent); + arm_smmu_cmdq_build_cmd(&cmd, ent); arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); } @@ -966,9 +973,8 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC); struct arm_smmu_cmd cmd; bool unsupported_cmd; - int index; - if (unlikely(arm_smmu_cmdq_build_cmd(cmd.data, ent))) { + if (unlikely(arm_smmu_cmdq_build_cmd(&cmd, ent))) { dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", ent->opcode); return; @@ -987,9 +993,7 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); } - index = cmds->num * CMDQ_ENT_DWORDS; - memcpy(&cmds->cmds[index], cmd.data, sizeof(cmd.data)); - cmds->num++; + cmds->cmds[cmds->num++] = cmd; } static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, @@ -1025,7 +1029,7 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused break; } - arm_smmu_cmdq_build_cmd(hw_cmd.data, &cmd); + arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd); arm_smmu_cmdq_issue_cmd(master->smmu, &hw_cmd); /* @@ -1865,7 +1869,7 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) }; struct arm_smmu_cmd cmd; - arm_smmu_cmdq_build_cmd(cmd.data, &ent); + arm_smmu_cmdq_build_cmd(&cmd, &ent); arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd); } @@ -1899,7 +1903,7 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, } }; struct arm_smmu_cmd prefetch_cmd; - arm_smmu_cmdq_build_cmd(prefetch_cmd.data, &prefetch_ent); + arm_smmu_cmdq_build_cmd(&prefetch_cmd, &prefetch_ent); arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); } } @@ -2339,7 +2343,7 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) }; struct arm_smmu_cmd cmd; - arm_smmu_cmdq_build_cmd(cmd.data, &ent); + arm_smmu_cmdq_build_cmd(&cmd, &ent); arm_smmu_cmdq_issue_cmd(smmu, &cmd); } } @@ -3462,7 +3466,7 @@ static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv) } cmd.opcode = inv->nsize_opcode; - arm_smmu_cmdq_build_cmd(hw_cmd.data, &cmd); + arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd); arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &hw_cmd); } @@ -4875,18 +4879,18 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Invalidate any cached configuration */ ent.opcode = CMDQ_OP_CFGI_ALL; - arm_smmu_cmdq_build_cmd(cmd.data, &ent); + arm_smmu_cmdq_build_cmd(&cmd, &ent); arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); /* Invalidate any stale TLB entries */ if (smmu->features & ARM_SMMU_FEAT_HYP) { ent.opcode = CMDQ_OP_TLBI_EL2_ALL; - arm_smmu_cmdq_build_cmd(cmd.data, &ent); + arm_smmu_cmdq_build_cmd(&cmd, &ent); arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); } ent.opcode = CMDQ_OP_TLBI_NSNH_ALL; - arm_smmu_cmdq_build_cmd(cmd.data, &ent); + arm_smmu_cmdq_build_cmd(&cmd, &ent); arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); /* Event queue */ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 6d73f6b63e64..1fe6917448b7 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -651,7 +651,7 @@ static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq, } struct arm_smmu_cmdq_batch { - u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; + struct arm_smmu_cmd cmds[CMDQ_BATCH_ENTRIES]; struct arm_smmu_cmdq *cmdq; int num; }; @@ -1148,7 +1148,8 @@ void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master, const struct arm_smmu_ste *target); int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq *cmdq, u64 *cmds, int n, + struct arm_smmu_cmdq *cmdq, + struct arm_smmu_cmd *cmds, int n, bool sync); #ifdef CONFIG_ARM_SMMU_V3_SVA diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index b4d8c1f2fd38..67be62a6e764 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -427,16 +427,16 @@ tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu, static void tegra241_vcmdq_hw_flush_timeout(struct tegra241_vcmdq *vcmdq) { struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu; - u64 cmd_sync[CMDQ_ENT_DWORDS] = {}; + struct arm_smmu_cmd cmd_sync = {}; - cmd_sync[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) | - FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE); + cmd_sync.data[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) | + FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE); /* * It does not hurt to insert another CMD_SYNC, taking advantage of the * arm_smmu_cmdq_issue_cmdlist() that waits for the CMD_SYNC completion. */ - arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, cmd_sync, 1, true); + arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &cmd_sync, 1, true); } /* This function is for LVCMDQ, so @vcmdq must not be unmapped yet */ From c5758947cb7b375303cf8f6f7600a445cb992e7d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:44 -0300 Subject: [PATCH 39/66] iommu/arm-smmu-v3: Remove CMDQ_OP_CFGI_CD_ALL from arm_smmu_cmdq_build_cmd() Nothing uses this, just dead code. Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index a3ce3c11c4d1..908c7e8a20b2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -290,9 +290,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); break; - case CMDQ_OP_CFGI_CD_ALL: - cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); - break; case CMDQ_OP_CFGI_ALL: /* Cover the entire SID range */ cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); From 6e771be45e8a49ae3099607850f3965cfcf952e4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:45 -0300 Subject: [PATCH 40/66] iommu/arm-smmu-v3: Directly encode simple commands Add make functions to build commands for CMDQ_OP_TLBI_EL2_ALL CMDQ_OP_TLBI_NSNH_ALL CMDQ_OP_CFGI_ALL CMDQ_OP_PREFETCH_CFG CMDQ_OP_CFGI_STE CMDQ_OP_CFGI_CD CMDQ_OP_RESUME CMDQ_OP_PRI_RESP Convert all of these call sites to use the make function instead of going through arm_smmu_cmdq_build_cmd(). Use a #define so the general pattern is always: arm_smmu_cmdq_issue_cmd(smmu, arm_smmu_make_cmd_XX(..)); Add arm_smmu_cmdq_batch_add_cmd() which takes struct arm_smmu_cmd directly to match the new flow. Reviewed-by: Pranjal Shrivastava Reviewed-by: Mostafa Saleh Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 213 +++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 109 +++++++--- 2 files changed, 151 insertions(+), 171 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 908c7e8a20b2..340b60d1d107 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -277,23 +277,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); switch (ent->opcode) { - case CMDQ_OP_TLBI_EL2_ALL: - case CMDQ_OP_TLBI_NSNH_ALL: - break; - case CMDQ_OP_PREFETCH_CFG: - cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); - break; - case CMDQ_OP_CFGI_CD: - cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid); - fallthrough; - case CMDQ_OP_CFGI_STE: - cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); - cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); - break; - case CMDQ_OP_CFGI_ALL: - /* Cover the entire SID range */ - cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); - break; case CMDQ_OP_TLBI_NH_VA: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); fallthrough; @@ -333,26 +316,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK; break; - case CMDQ_OP_PRI_RESP: - cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); - cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid); - cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid); - cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid); - switch (ent->pri.resp) { - case PRI_RESP_DENY: - case PRI_RESP_FAIL: - case PRI_RESP_SUCC: - break; - default: - return -EINVAL; - } - cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp); - break; - case CMDQ_OP_RESUME: - cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid); - cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp); - cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag); - break; case CMDQ_OP_CMD_SYNC: if (ent->sync.msiaddr) { cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); @@ -924,25 +887,24 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, return ret; } -static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, - struct arm_smmu_cmd *cmd, - bool sync) +static int arm_smmu_cmdq_issue_cmd_p(struct arm_smmu_device *smmu, + struct arm_smmu_cmd *cmd, bool sync) { return arm_smmu_cmdq_issue_cmdlist( smmu, arm_smmu_get_cmdq(smmu, cmd), cmd, 1, sync); } -static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, - struct arm_smmu_cmd *cmd) -{ - return __arm_smmu_cmdq_issue_cmd(smmu, cmd, false); -} +#define arm_smmu_cmdq_issue_cmd(smmu, cmd) \ + ({ \ + struct arm_smmu_cmd __cmd = cmd; \ + arm_smmu_cmdq_issue_cmd_p(smmu, &__cmd, false); \ + }) -static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu, - struct arm_smmu_cmd *cmd) -{ - return __arm_smmu_cmdq_issue_cmd(smmu, cmd, true); -} +#define arm_smmu_cmdq_issue_cmd_with_sync(smmu, cmd) \ + ({ \ + struct arm_smmu_cmd __cmd = cmd; \ + arm_smmu_cmdq_issue_cmd_p(smmu, &__cmd, true); \ + }) static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, @@ -962,14 +924,41 @@ static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu, arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); } +static void arm_smmu_cmdq_batch_add_cmd_p(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmd *cmd) +{ + bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) && + (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC); + bool unsupported_cmd; + + unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd); + if (force_sync || unsupported_cmd) { + arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, + cmds->num, true); + arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd); + } + + if (cmds->num == CMDQ_BATCH_ENTRIES) { + arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, + cmds->num, false); + arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd); + } + + cmds->cmds[cmds->num++] = *cmd; +} + +#define arm_smmu_cmdq_batch_add_cmd(smmu, cmds, cmd) \ + ({ \ + struct arm_smmu_cmd __cmd = cmd; \ + arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &__cmd); \ + }) + static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmdq_ent *ent) { - bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) && - (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC); struct arm_smmu_cmd cmd; - bool unsupported_cmd; if (unlikely(arm_smmu_cmdq_build_cmd(&cmd, ent))) { dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", @@ -977,20 +966,7 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, return; } - unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, &cmd); - if (force_sync || unsupported_cmd) { - arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, - cmds->num, true); - arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); - } - - if (cmds->num == CMDQ_BATCH_ENTRIES) { - arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, - cmds->num, false); - arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); - } - - cmds->cmds[cmds->num++] = cmd; + arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &cmd); } static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, @@ -1003,32 +979,29 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused, struct iommu_page_response *resp) { - struct arm_smmu_cmdq_ent cmd = {0}; struct arm_smmu_master *master = dev_iommu_priv_get(dev); - int sid = master->streams[0].id; - struct arm_smmu_cmd hw_cmd; + u8 resume_resp; if (WARN_ON(!master->stall_enabled)) return; - cmd.opcode = CMDQ_OP_RESUME; - cmd.resume.sid = sid; - cmd.resume.stag = resp->grpid; switch (resp->code) { case IOMMU_PAGE_RESP_INVALID: case IOMMU_PAGE_RESP_FAILURE: - cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; + resume_resp = CMDQ_RESUME_0_RESP_ABORT; break; case IOMMU_PAGE_RESP_SUCCESS: - cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; + resume_resp = CMDQ_RESUME_0_RESP_RETRY; break; default: + resume_resp = CMDQ_RESUME_0_RESP_TERM; break; } - arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd); - arm_smmu_cmdq_issue_cmd(master->smmu, &hw_cmd); - + arm_smmu_cmdq_issue_cmd(master->smmu, + arm_smmu_make_cmd_resume(master->streams[0].id, + resp->grpid, + resume_resp)); /* * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP. * RESUME consumption guarantees that the stalled transaction will be @@ -1552,19 +1525,14 @@ static void arm_smmu_sync_cd(struct arm_smmu_master *master, size_t i; struct arm_smmu_cmdq_batch cmds; struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_cmdq_ent cmd = { - .opcode = CMDQ_OP_CFGI_CD, - .cfgi = { - .ssid = ssid, - .leaf = leaf, - }, - }; + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_cfgi_cd(0, ssid, leaf); - arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd); - for (i = 0; i < master->num_streams; i++) { - cmd.cfgi.sid = master->streams[i].id; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); - } + arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd); + for (i = 0; i < master->num_streams; i++) + arm_smmu_cmdq_batch_add_cmd( + smmu, &cmds, + arm_smmu_make_cmd_cfgi_cd(master->streams[i].id, ssid, + leaf)); arm_smmu_cmdq_batch_submit(smmu, &cmds); } @@ -1857,17 +1825,10 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) { struct arm_smmu_ste_writer *ste_writer = container_of(writer, struct arm_smmu_ste_writer, writer); - struct arm_smmu_cmdq_ent ent = { - .opcode = CMDQ_OP_CFGI_STE, - .cfgi = { - .sid = ste_writer->sid, - .leaf = true, - }, - }; - struct arm_smmu_cmd cmd; - arm_smmu_cmdq_build_cmd(&cmd, &ent); - arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync( + writer->master->smmu, + arm_smmu_make_cmd_cfgi_ste(ste_writer->sid, true)); } static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = { @@ -1892,17 +1853,9 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data); /* It's likely that we'll want to use the new STE soon */ - if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { - struct arm_smmu_cmdq_ent - prefetch_ent = { .opcode = CMDQ_OP_PREFETCH_CFG, - .prefetch = { - .sid = sid, - } }; - struct arm_smmu_cmd prefetch_cmd; - - arm_smmu_cmdq_build_cmd(&prefetch_cmd, &prefetch_ent); - arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); - } + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) + arm_smmu_cmdq_issue_cmd(smmu, + arm_smmu_make_cmd_prefetch_cfg(sid)); } void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) @@ -2327,22 +2280,10 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", evt[1] & PRIQ_1_ADDR_MASK); - if (last) { - struct arm_smmu_cmdq_ent ent = { - .opcode = CMDQ_OP_PRI_RESP, - .substream_valid = ssv, - .pri = { - .sid = sid, - .ssid = ssid, - .grpid = grpid, - .resp = PRI_RESP_DENY, - }, - }; - struct arm_smmu_cmd cmd; - - arm_smmu_cmdq_build_cmd(&cmd, &ent); - arm_smmu_cmdq_issue_cmd(smmu, &cmd); - } + if (last) + arm_smmu_cmdq_issue_cmd( + smmu, arm_smmu_make_cmd_pri_resp(sid, ssid, ssv, grpid, + PRI_RESP_DENY)); } static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) @@ -3464,7 +3405,7 @@ static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv) cmd.opcode = inv->nsize_opcode; arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd); - arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &hw_cmd); + arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, hw_cmd); } /* Should be installed after arm_smmu_install_ste_for_dev() */ @@ -4827,8 +4768,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; u32 reg, enables; - struct arm_smmu_cmdq_ent ent; - struct arm_smmu_cmd cmd; /* Clear CR0 and sync (disables SMMU and queue processing) */ reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); @@ -4875,20 +4814,16 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) } /* Invalidate any cached configuration */ - ent.opcode = CMDQ_OP_CFGI_ALL; - arm_smmu_cmdq_build_cmd(&cmd, &ent); - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync(smmu, arm_smmu_make_cmd_cfgi_all()); /* Invalidate any stale TLB entries */ if (smmu->features & ARM_SMMU_FEAT_HYP) { - ent.opcode = CMDQ_OP_TLBI_EL2_ALL; - arm_smmu_cmdq_build_cmd(&cmd, &ent); - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync( + smmu, arm_smmu_make_cmd_op(CMDQ_OP_TLBI_EL2_ALL)); } - ent.opcode = CMDQ_OP_TLBI_NSNH_ALL; - arm_smmu_cmdq_build_cmd(&cmd, &ent); - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync( + smmu, arm_smmu_make_cmd_op(CMDQ_OP_TLBI_NSNH_ALL)); /* Event queue */ writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 1fe6917448b7..10b3d95d9ee6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -437,6 +437,12 @@ struct arm_smmu_cmd { #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12) +enum pri_resp { + PRI_RESP_DENY = 0, + PRI_RESP_FAIL = 1, + PRI_RESP_SUCC = 2, +}; + #define CMDQ_RESUME_0_RESP_TERM 0UL #define CMDQ_RESUME_0_RESP_RETRY 1UL #define CMDQ_RESUME_0_RESP_ABORT 2UL @@ -475,6 +481,77 @@ enum arm_smmu_cmdq_opcode { CMDQ_OP_CMD_SYNC = 0x46, }; +static inline struct arm_smmu_cmd +arm_smmu_make_cmd_op(enum arm_smmu_cmdq_opcode op) +{ + struct arm_smmu_cmd cmd = {}; + + cmd.data[0] = FIELD_PREP(CMDQ_0_OP, op); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_all(void) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_ALL); + + cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_prefetch_cfg(u32 sid) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_PREFETCH_CFG); + + cmd.data[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, sid); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_ste(u32 sid, bool leaf) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_STE); + + cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, leaf); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_cd(u32 sid, u32 ssid, + bool leaf) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_CD); + + cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid) | + FIELD_PREP(CMDQ_CFGI_0_SSID, ssid); + cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, leaf); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_resume(u32 sid, u16 stag, + u8 resp) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_RESUME); + + cmd.data[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, sid) | + FIELD_PREP(CMDQ_RESUME_0_RESP, resp); + cmd.data[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, stag); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid, + bool ssv, + u16 grpid, + enum pri_resp resp) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_PRI_RESP); + + cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssv) | + FIELD_PREP(CMDQ_PRI_0_SID, sid) | + FIELD_PREP(CMDQ_PRI_0_SSID, ssid); + cmd.data[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, grpid) | + FIELD_PREP(CMDQ_PRI_1_RESP, resp); + return cmd; +} + /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) @@ -535,12 +612,6 @@ enum arm_smmu_cmdq_opcode { #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 -enum pri_resp { - PRI_RESP_DENY = 0, - PRI_RESP_FAIL = 1, - PRI_RESP_SUCC = 2, -}; - struct arm_smmu_cmdq_ent { /* Common fields */ u8 opcode; @@ -548,19 +619,6 @@ struct arm_smmu_cmdq_ent { /* Command-specific fields */ union { - struct { - u32 sid; - } prefetch; - - struct { - u32 sid; - u32 ssid; - union { - bool leaf; - u8 span; - }; - } cfgi; - struct { u8 num; u8 scale; @@ -580,19 +638,6 @@ struct arm_smmu_cmdq_ent { bool global; } atc; - struct { - u32 sid; - u32 ssid; - u16 grpid; - enum pri_resp resp; - } pri; - - struct { - u32 sid; - u16 stag; - u8 resp; - } resume; - struct { u64 msiaddr; } sync; From 2eedb906f9c605aa789758f9aa172a930c88944f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:46 -0300 Subject: [PATCH 41/66] iommu/arm-smmu-v3: Directly encode CMDQ_OP_ATC_INV Add a new command make function and convert all the places using ATC_INV. Split out full invalidation to directly make the cmd instead of overloading size=0 to mean full invalidation. In section "3.9.1 ATS Interface" of F.b the specification says: When the SMMU returns an ATS Translation Completion for a request that had a PASID, the Global bit of the Translation Completion Data Entry must be zero. Even though it faithfully forwards the G bit through to the ATS invalidation command there is no way to create G mappings so there is never any need to send a G invalidation. Thus don't expose global in the new helpers and leave CMDQ_ATC_0_GLOBAL unused. Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 59 ++++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 27 +++++++--- 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 340b60d1d107..d30bac229d68 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -308,14 +308,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, case CMDQ_OP_TLBI_EL2_ASID: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); break; - case CMDQ_OP_ATC_INV: - cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); - cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global); - cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid); - cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid); - cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); - cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK; - break; case CMDQ_OP_CMD_SYNC: if (ent->sync.msiaddr) { cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); @@ -2371,9 +2363,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) return IRQ_WAKE_THREAD; } -static void -arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, - struct arm_smmu_cmdq_ent *cmd) +static struct arm_smmu_cmd +arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) { size_t log2_span; size_t span_mask; @@ -2395,17 +2386,6 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, * This has the unpleasant side-effect of invalidating all PASID-tagged * ATC entries within the address range. */ - *cmd = (struct arm_smmu_cmdq_ent) { - .opcode = CMDQ_OP_ATC_INV, - .substream_valid = (ssid != IOMMU_NO_PASID), - .atc.ssid = ssid, - }; - - if (!size) { - cmd->atc.size = ATC_INV_SIZE_ALL; - return; - } - page_start = iova >> inval_grain_shift; page_end = (iova + size - 1) >> inval_grain_shift; @@ -2434,24 +2414,25 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, page_start &= ~span_mask; - cmd->atc.addr = page_start << inval_grain_shift; - cmd->atc.size = log2_span; + return arm_smmu_make_cmd_atc_inv(sid, ssid, + page_start << inval_grain_shift, + log2_span); } static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, ioasid_t ssid) { int i; - struct arm_smmu_cmdq_ent cmd; + struct arm_smmu_cmd cmd; struct arm_smmu_cmdq_batch cmds; - arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd); - - arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd); - for (i = 0; i < master->num_streams; i++) { - cmd.atc.sid = master->streams[i].id; - arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd); - } + cmd = arm_smmu_make_cmd_atc_inv_all(0, IOMMU_NO_PASID); + arm_smmu_cmdq_batch_init_cmd(master->smmu, &cmds, &cmd); + for (i = 0; i < master->num_streams; i++) + arm_smmu_cmdq_batch_add_cmd( + master->smmu, &cmds, + arm_smmu_make_cmd_atc_inv_all(master->streams[i].id, + ssid)); return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); } @@ -2650,14 +2631,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); break; case INV_TYPE_ATS: - arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd); - cmd.atc.sid = cur->id; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + arm_smmu_cmdq_batch_add_cmd( + smmu, &cmds, + arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid, + iova, size)); break; case INV_TYPE_ATS_FULL: - arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd); - cmd.atc.sid = cur->id; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + arm_smmu_cmdq_batch_add_cmd( + smmu, &cmds, + arm_smmu_make_cmd_atc_inv_all(cur->id, + IOMMU_NO_PASID)); break; default: WARN_ON_ONCE(1); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 10b3d95d9ee6..194f73cabef5 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -552,6 +552,25 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid, return cmd; } +static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv(u32 sid, u32 ssid, + u64 addr, u8 size) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_ATC_INV); + + cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssid != IOMMU_NO_PASID) | + FIELD_PREP(CMDQ_ATC_0_SSID, ssid) | + FIELD_PREP(CMDQ_ATC_0_SID, sid); + cmd.data[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, size) | + (addr & CMDQ_ATC_1_ADDR_MASK); + return cmd; +} + +static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid, + u32 ssid) +{ + return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL); +} + /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) @@ -630,14 +649,6 @@ struct arm_smmu_cmdq_ent { u64 addr; } tlbi; - struct { - u32 sid; - u32 ssid; - u64 addr; - u8 size; - bool global; - } atc; - struct { u64 msiaddr; } sync; From c3f84707ad4fe9bf579e7fb0785f155a8a52c7e8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:47 -0300 Subject: [PATCH 42/66] iommu/arm-smmu-v3: Directly encode CMDQ_OP_SYNC Change the flow so the caller controls the CS field and remove the weird u64p_replace_bits() thing to override it. Reviewed-by: Pranjal Shrivastava Reviewed-by: Mostafa Saleh Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 42 ++++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 16 ++++++-- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d30bac229d68..5fbde147f771 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -308,16 +308,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, case CMDQ_OP_TLBI_EL2_ASID: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); break; - case CMDQ_OP_CMD_SYNC: - if (ent->sync.msiaddr) { - cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); - cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; - } else { - cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV); - } - cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH); - cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); - break; default: return -ENOENT; } @@ -350,23 +340,24 @@ static void arm_smmu_cmdq_build_sync_cmd(struct arm_smmu_cmd *cmd, struct arm_smmu_cmdq *cmdq, u32 prod) { struct arm_smmu_queue *q = &cmdq->q; - struct arm_smmu_cmdq_ent ent = { - .opcode = CMDQ_OP_CMD_SYNC, - }; + u64 msiaddr = 0; + unsigned int cs; /* * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI * payload, so the write will zero the entire command on that platform. */ - if (smmu->options & ARM_SMMU_OPT_MSIPOLL) { - ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * - q->ent_dwords * 8; + if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) { + cs = CMDQ_SYNC_0_CS_NONE; + } else if (smmu->options & ARM_SMMU_OPT_MSIPOLL) { + cs = CMDQ_SYNC_0_CS_IRQ; + msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * + q->ent_dwords * 8; + } else { + cs = CMDQ_SYNC_0_CS_SEV; } - arm_smmu_cmdq_build_cmd(cmd, &ent); - if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) - u64p_replace_bits(&cmd->data[0], CMDQ_SYNC_0_CS_NONE, - CMDQ_SYNC_0_CS); + *cmd = arm_smmu_make_cmd_sync(cs, msiaddr); } void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, @@ -383,9 +374,6 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, struct arm_smmu_cmd cmd; u32 cons = readl_relaxed(q->cons_reg); u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons); - struct arm_smmu_cmdq_ent cmd_sync = { - .opcode = CMDQ_OP_CMD_SYNC, - }; dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown"); @@ -419,10 +407,10 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd.data[i]); /* Convert the erroneous command into a CMD_SYNC */ - arm_smmu_cmdq_build_cmd(&cmd, &cmd_sync); - if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) - u64p_replace_bits(&cmd.data[0], CMDQ_SYNC_0_CS_NONE, - CMDQ_SYNC_0_CS); + cmd = arm_smmu_make_cmd_sync( + arm_smmu_cmdq_needs_busy_polling(smmu, cmdq) ? + CMDQ_SYNC_0_CS_NONE : CMDQ_SYNC_0_CS_SEV, + 0); queue_write(Q_ENT(q, cons), cmd.data, q->ent_dwords); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 194f73cabef5..538380de7d48 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -571,6 +571,18 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid, return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL); } +static inline struct arm_smmu_cmd arm_smmu_make_cmd_sync(unsigned int cs, + u64 msiaddr) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CMD_SYNC); + + cmd.data[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, cs) | + FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); + cmd.data[1] |= msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; + return cmd; +} + /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) @@ -648,10 +660,6 @@ struct arm_smmu_cmdq_ent { u8 tg; u64 addr; } tlbi; - - struct { - u64 msiaddr; - } sync; }; }; From be0d0b8588613e27f7c41f6e7a176842135427f4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 13 May 2026 20:57:48 -0300 Subject: [PATCH 43/66] iommu/arm-smmu-v3: Directly encode TLBI commands TLBI is more complicated than all the other commands because the invalidation loop builds a template command from the struct arm_smmu_inv which is then expanded into many TLBI commands for the invalidation. Reviewed-by: Mostafa Saleh Reviewed-by: Pranjal Shrivastava Tested-by: Pranjal Shrivastava Tested-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Reviewed-by: Nicolin Chen Tested-by: Samiullah Khawaja Tested-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 170 +++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 35 ++-- 2 files changed, 71 insertions(+), 134 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 5fbde147f771..8ce3e801eda3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -268,53 +268,6 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) } /* High-level queue accessors */ -static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out, - struct arm_smmu_cmdq_ent *ent) -{ - u64 *cmd = cmd_out->data; - - memset(cmd_out, 0, sizeof(*cmd_out)); - cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); - - switch (ent->opcode) { - case CMDQ_OP_TLBI_NH_VA: - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); - fallthrough; - case CMDQ_OP_TLBI_EL2_VA: - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); - cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); - cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); - cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); - cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; - break; - case CMDQ_OP_TLBI_S2_IPA: - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); - cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); - cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); - cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg); - cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; - break; - case CMDQ_OP_TLBI_NH_ASID: - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); - fallthrough; - case CMDQ_OP_TLBI_NH_ALL: - case CMDQ_OP_TLBI_S12_VMALL: - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); - break; - case CMDQ_OP_TLBI_EL2_ASID: - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); - break; - default: - return -ENOENT; - } - - return 0; -} - static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu, struct arm_smmu_cmd *cmd) { @@ -894,16 +847,6 @@ static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu, cmds->cmdq = arm_smmu_get_cmdq(smmu, cmd); } -static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmdq_ent *ent) -{ - struct arm_smmu_cmd cmd; - - arm_smmu_cmdq_build_cmd(&cmd, ent); - arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd); -} - static void arm_smmu_cmdq_batch_add_cmd_p(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd) @@ -934,21 +877,6 @@ static void arm_smmu_cmdq_batch_add_cmd_p(struct arm_smmu_device *smmu, arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &__cmd); \ }) -static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, - struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmdq_ent *ent) -{ - struct arm_smmu_cmd cmd; - - if (unlikely(arm_smmu_cmdq_build_cmd(&cmd, ent))) { - dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", - ent->opcode); - return; - } - - arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &cmd); -} - static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds) { @@ -2450,12 +2378,14 @@ static void arm_smmu_tlb_inv_context(void *cookie) static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmdq_ent *cmd, + struct arm_smmu_cmd *cmd, bool leaf, unsigned long iova, size_t size, size_t granule, size_t pgsize) { unsigned long end = iova + size, num_pages = 0, tg = pgsize; + u64 orig_data0 = cmd->data[0]; size_t inv_range = granule; + u8 ttl = 0, tg_enc = 0; if (WARN_ON_ONCE(!size)) return; @@ -2464,7 +2394,7 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, num_pages = size >> tg; /* Convert page size of 12,14,16 (log2) to 1,2,3 */ - cmd->tlbi.tg = (tg - 10) / 2; + tg_enc = (tg - 10) / 2; /* * Determine what level the granule is at. For non-leaf, both @@ -2474,8 +2404,8 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, * want to use a range command, so avoid the SVA corner case * where both scale and num could be 0 as well. */ - if (cmd->tlbi.leaf) - cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); + if (leaf) + ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1) num_pages++; } @@ -2493,11 +2423,13 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, /* Determine the power of 2 multiple number of pages */ scale = __ffs(num_pages); - cmd->tlbi.scale = scale; /* Determine how many chunks of 2^scale size we have */ num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; - cmd->tlbi.num = num - 1; + + cmd->data[0] = orig_data0 | + FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) | + FIELD_PREP(CMDQ_TLBI_0_SCALE, scale); /* range is num * 2^scale * pgsize */ inv_range = num << (scale + tg); @@ -2506,8 +2438,17 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, num_pages -= num << scale; } - cmd->tlbi.addr = iova; - arm_smmu_cmdq_batch_add(smmu, cmds, cmd); + /* + * IPA has fewer bits than VA, but they are reserved in the + * command and something would be very broken if iova had them + * set. + */ + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) | + FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | + FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | + (iova & ~GENMASK_U64(11, 0)); + + arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd); iova += inv_range; } } @@ -2538,19 +2479,22 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, /* Used by non INV_TYPE_ATS* invalidations */ static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmdq_ent *cmd, + struct arm_smmu_cmd *cmd, + bool leaf, unsigned long iova, size_t size, unsigned int granule) { if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) { - cmd->opcode = inv->nsize_opcode; - arm_smmu_cmdq_batch_add(inv->smmu, cmds, cmd); + struct arm_smmu_cmd nsize_cmd = *cmd; + + u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode, + CMDQ_0_OP); + arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd); return; } - cmd->opcode = inv->size_opcode; - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, iova, size, granule, - inv->pgsize); + arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, + iova, size, granule, inv->pgsize); } static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, @@ -2585,38 +2529,39 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, break; while (cur != end) { struct arm_smmu_device *smmu = cur->smmu; - struct arm_smmu_cmdq_ent cmd = { - /* - * Pick size_opcode to run arm_smmu_get_cmdq(). This can - * be changed to nsize_opcode, which would result in the - * same CMDQ pointer. - */ - .opcode = cur->size_opcode, - }; + /* + * Pick size_opcode to run arm_smmu_get_cmdq(). This can + * be changed to nsize_opcode, which would result in the + * same CMDQ pointer. + */ + struct arm_smmu_cmd cmd = + arm_smmu_make_cmd_op(cur->size_opcode); struct arm_smmu_inv *next; if (!cmds.num) - arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd); + arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd); switch (cur->type) { case INV_TYPE_S1_ASID: - cmd.tlbi.asid = cur->id; - cmd.tlbi.leaf = leaf; - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size, - granule); + cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, + cur->id, 0); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf, + iova, size, granule); break; case INV_TYPE_S2_VMID: - cmd.tlbi.vmid = cur->id; - cmd.tlbi.leaf = leaf; - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size, - granule); + cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, + 0, cur->id); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf, + iova, size, granule); break; case INV_TYPE_S2_VMID_S1_CLEAR: /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ if (arm_smmu_inv_size_too_big(cur->smmu, size, granule)) break; - cmd.tlbi.vmid = cur->id; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + arm_smmu_cmdq_batch_add_cmd( + smmu, &cmds, + arm_smmu_make_cmd_tlbi(cur->size_opcode, 0, + cur->id)); break; case INV_TYPE_ATS: arm_smmu_cmdq_batch_add_cmd( @@ -3359,24 +3304,21 @@ arm_smmu_install_new_domain_invs(struct arm_smmu_attach_state *state) static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv) { - struct arm_smmu_cmdq_ent cmd = {}; - struct arm_smmu_cmd hw_cmd; - switch (inv->type) { case INV_TYPE_S1_ASID: - cmd.tlbi.asid = inv->id; + arm_smmu_cmdq_issue_cmd_with_sync( + inv->smmu, + arm_smmu_make_cmd_tlbi(inv->nsize_opcode, inv->id, 0)); break; case INV_TYPE_S2_VMID: /* S2_VMID using nsize_opcode covers S2_VMID_S1_CLEAR */ - cmd.tlbi.vmid = inv->id; + arm_smmu_cmdq_issue_cmd_with_sync( + inv->smmu, + arm_smmu_make_cmd_tlbi(inv->nsize_opcode, 0, inv->id)); break; default: return; } - - cmd.opcode = inv->nsize_opcode; - arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd); - arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, hw_cmd); } /* Should be installed after arm_smmu_install_ste_for_dev() */ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 538380de7d48..16353596e08a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -583,6 +583,21 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_sync(unsigned int cs, return cmd; } +/* + * TLBI commands - the non-sized variants just need opcode + asid/vmid. + * For sized variants the caller sets up data[0] with the immutable fields + * (opcode + asid/vmid) and the range loop fills in per-iteration fields. + */ +static inline struct arm_smmu_cmd +arm_smmu_make_cmd_tlbi(enum arm_smmu_cmdq_opcode op, u16 asid, u16 vmid) +{ + struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(op); + + cmd.data[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, asid) | + FIELD_PREP(CMDQ_TLBI_0_VMID, vmid); + return cmd; +} + /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) @@ -643,26 +658,6 @@ static inline struct arm_smmu_cmd arm_smmu_make_cmd_sync(unsigned int cs, #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 -struct arm_smmu_cmdq_ent { - /* Common fields */ - u8 opcode; - bool substream_valid; - - /* Command-specific fields */ - union { - struct { - u8 num; - u8 scale; - u16 asid; - u16 vmid; - bool leaf; - u8 ttl; - u8 tg; - u64 addr; - } tlbi; - }; -}; - struct arm_smmu_ll_queue { union { u64 val; From 74fa4c177ad09800b007cba043370c887bb1b4e3 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Sun, 3 May 2026 06:54:12 -0700 Subject: [PATCH 44/66] iommu/arm-smmu-v3-sva: Enable Hardware Access and Hardware Dirty bits HTTU is introduced by utilizing the Dirty Bit Modifier (DBM) in the PTE. When kernel maps a clean but writable page, it will set PTE_READONLY and PTE_DBM (aka PTE_WRITE) at the same time. When a write occurs, an HTTU- capable MMU will automatically clear the PTE_RDONLY bit without software intervention. On the other hand, SMMU has the same HTTU feature, yet it is not enabled in the SVA CD. As a result, SMMU will not clear the PTE_RDONLY bit while sharing the CPU page table, resulting in unnecessary stalls. Thus, enable CTXDESC_CD_0_TCR_HA and CTXDESC_CD_0_TCR_HD in the SVA CD. Suggested-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index f1f8e01a7e91..1ed8a6f29dc4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -92,6 +92,16 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & CTXDESC_CD_1_TTB0_MASK); + + /* + * Enable Hardware Access and Dirty updates (DBM) if supported. + * This is safe to enable by default, as PTE_WRITE and PTE_DBM + * share the same bit. + */ + if (master->smmu->features & ARM_SMMU_FEAT_HA) + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA); + if (master->smmu->features & ARM_SMMU_FEAT_HD) + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD); } else { target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); From 283d245468a2b61c41aa8b582f25ed5615d1c304 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Sun, 17 May 2026 12:29:25 +0000 Subject: [PATCH 45/66] iommu/amd: Fix premature break in init_iommu_one() In init_iommu_one(), when processing IOMMU EFR attributes, the code checks whether GASUP is enabled. If GASUP is not enabled, the code falls back to legacy guest IR mode and then breaks out of the switch statement. This break incorrectly skips the subsequent initialization steps that follow the GASUP check. These initializations are independent of GASUP support and must always be performed. Fix this by replacing the early break with a conditional else block, ensuring that the XTSUP check is only skipped when GASUP is not available. Fixes: a44092e326d4 ("iommu/amd: Use IVHD EFR for early initialization of IOMMU features") Reported-by: Sudheer Dantuluri Tested-by: Dheeraj Kumar Srivastava Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 16073b5614b9..a7dfbd515863 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1917,12 +1917,11 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, /* XT and GAM require GA mode. */ if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; - break; + } else { + if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) + amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; } - if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) - amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; - if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { pr_warn_once("Host Address Translation is not supported.\n"); amd_iommu_hatdis = true; From 80b0d3546ce18c5bbae7cebec014beb2fbe81580 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 19 May 2026 22:36:51 +0200 Subject: [PATCH 46/66] iommu: vsi: avoid -Wformat-security warning When -Wformat-security is enabled, it catches a call to iommu_device_sysfs_add() that passes a string variable in place of a format: drivers/iommu/vsi-iommu.c: In function 'vsi_iommu_probe': drivers/iommu/vsi-iommu.c:717:9: error: format not a string literal and no format arguments [-Werror=format-security] 717 | err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); | ^~~ Pass this indirectly using "%s" as the format instead. Fixes: 917ace84b770 ("iommu: Add verisilicon IOMMU driver") Signed-off-by: Arnd Bergmann Reviewed-by: Benjamin Gaignard Signed-off-by: Joerg Roedel --- drivers/iommu/vsi-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/vsi-iommu.c b/drivers/iommu/vsi-iommu.c index 5d0721bd2c7a..23913720d484 100644 --- a/drivers/iommu/vsi-iommu.c +++ b/drivers/iommu/vsi-iommu.c @@ -714,7 +714,8 @@ static int vsi_iommu_probe(struct platform_device *pdev) pm_runtime_use_autosuspend(dev); pm_runtime_enable(dev); - err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); + err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, "%s", + dev_name(dev)); if (err) goto err_runtime_disable; From 3040784f872133836cfa63c58c7e0ca329476d25 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 20 May 2026 09:00:23 +0300 Subject: [PATCH 47/66] iommu/vsi: Use list_for_each_entry() Smatch complains about the NULL check on "iommu" because list_entry() can't be NULL. Clean up this code by using list_for_each_entry(). Signed-off-by: Dan Carpenter Reviewed-by: Benjamin Gaignard Signed-off-by: Joerg Roedel --- drivers/iommu/vsi-iommu.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/vsi-iommu.c b/drivers/iommu/vsi-iommu.c index 23913720d484..42c424496d07 100644 --- a/drivers/iommu/vsi-iommu.c +++ b/drivers/iommu/vsi-iommu.c @@ -366,15 +366,9 @@ static int vsi_iommu_map_iova(struct vsi_iommu_domain *vsi_domain, u32 *pte_addr static void vsi_iommu_flush_tlb(struct iommu_domain *domain) { struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain); - struct list_head *pos; - - list_for_each(pos, &vsi_domain->iommus) { - struct vsi_iommu *iommu; - - iommu = list_entry(pos, struct vsi_iommu, node); - if (!iommu) - continue; + struct vsi_iommu *iommu; + list_for_each_entry(iommu, &vsi_domain->iommus, node) { if (pm_runtime_get(iommu->dev) < 0) continue; From 91561e1dc94b8a33857370ef3c5b5523c4461d5b Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 21 May 2026 13:34:20 -0700 Subject: [PATCH 48/66] PCI: Add pci_ats_required() for CXL.cache capable devices Controlled by IOMMU drivers, ATS can be enabled "on demand", when a given PASID on a device is attached to an I/O page table. This is working, even when a device has no translation on its RID (i.e., RID is IOMMU bypassed). However, certain PCIe devices require non-PASID ATS on their RID even when the RID is IOMMU bypassed. Call this "ATS always on" in IOMMU term. For example, CXL spec r4.0 notes in sec 3.2.5.13 Memory Type on CXL.cache: "To source requests on CXL.cache, devices need to get the Host Physical Address (HPA) from the Host by means of an ATS request on CXL.io." In other words, the CXL.cache capability requires ATS; otherwise, it can't access host physical memory. Introduce a new pci_ats_required() helper for the IOMMU driver to scan a PCI device and shift ATS policies between "on demand" and "always on". Add the support for CXL.cache devices first. Pre-CXL devices will be added in quirks.c file. Note that pci_ats_required() validates against pci_ats_supported(), so we ensure that untrusted devices (e.g. external ports) will not be always on. This maintains the existing ATS security policy regarding potential side- channel attacks via ATS. Cc: linux-cxl@vger.kernel.org Suggested-by: Vikram Sethi Suggested-by: Jason Gunthorpe Reviewed-by: Jonathan Cameron Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Nirmoy Das Acked-by: Nirmoy Das Reviewed-by: Dave Jiang Acked-by: Bjorn Helgaas Signed-off-by: Nicolin Chen Reviewed-by: Yi Liu Signed-off-by: Joerg Roedel --- drivers/pci/ats.c | 46 +++++++++++++++++++++++++++++++++++ include/linux/pci-ats.h | 3 +++ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 50 insertions(+) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index ec6c8dbdc5e9..84cd06d74fc9 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -205,6 +205,52 @@ int pci_ats_page_aligned(struct pci_dev *pdev) return 0; } +/* + * CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on + * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host + * by means of an ATS request on CXL.io. + * + * In other words, CXL.cache devices cannot access host physical memory without + * ATS. + * + * Check Cache_Capable instead of Cache_Enable because CXL.cache may be enabled + * after the caller uses this to make its ATS decision. + */ +static bool pci_cxl_ats_required(struct pci_dev *pdev) +{ + int offset; + u16 cap; + + offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, + PCI_DVSEC_CXL_DEVICE); + if (!offset) + return false; + + if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap)) + return false; + + return cap & PCI_DVSEC_CXL_CACHE_CAPABLE; +} + +/** + * pci_ats_required - Whether the PCI device requires ATS + * @pdev: the PCI device + * + * Returns true, if the PCI device requires ATS for basic functional operation. + */ +bool pci_ats_required(struct pci_dev *pdev) +{ + if (!pci_ats_supported(pdev)) + return false; + + /* A VF inherits its PF's requirement for ATS function */ + if (pdev->is_virtfn) + pdev = pci_physfn(pdev); + + return pci_cxl_ats_required(pdev); +} +EXPORT_SYMBOL_GPL(pci_ats_required); + #ifdef CONFIG_PCI_PRI void pci_pri_init(struct pci_dev *pdev) { diff --git a/include/linux/pci-ats.h b/include/linux/pci-ats.h index 75c6c86cf09d..f3723b686129 100644 --- a/include/linux/pci-ats.h +++ b/include/linux/pci-ats.h @@ -12,6 +12,7 @@ int pci_prepare_ats(struct pci_dev *dev, int ps); void pci_disable_ats(struct pci_dev *dev); int pci_ats_queue_depth(struct pci_dev *dev); int pci_ats_page_aligned(struct pci_dev *dev); +bool pci_ats_required(struct pci_dev *dev); #else /* CONFIG_PCI_ATS */ static inline bool pci_ats_supported(struct pci_dev *d) { return false; } @@ -24,6 +25,8 @@ static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; } +static inline bool pci_ats_required(struct pci_dev *dev) +{ return false; } #endif /* CONFIG_PCI_ATS */ #ifdef CONFIG_PCI_PRI diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 14f634ab9350..6ac45be1008b 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -1349,6 +1349,7 @@ /* CXL r4.0, 8.1.3: PCIe DVSEC for CXL Device */ #define PCI_DVSEC_CXL_DEVICE 0 #define PCI_DVSEC_CXL_CAP 0xA +#define PCI_DVSEC_CXL_CACHE_CAPABLE _BITUL(0) #define PCI_DVSEC_CXL_MEM_CAPABLE _BITUL(2) #define PCI_DVSEC_CXL_HDM_COUNT __GENMASK(5, 4) #define PCI_DVSEC_CXL_CTRL 0xC From 073e6381252541fb2e0f3d36e96605b7f945b362 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 21 May 2026 13:34:21 -0700 Subject: [PATCH 49/66] PCI: Allow ATS to be always on for pre-CXL devices Some NVIDIA GPU/NIC devices, though they don't implement CXL config space, have many CXL-like properties. Call this kind "pre-CXL". Similar to CXL.cache capability, these pre-CXL devices also require the ATS function even when their RIDs are IOMMU bypassed, i.e. keep ATS "always on" v.s. "on demand" when a non-zero PASID line gets enabled in SVA use cases. Introduce pci_dev_specific_ats_required() quirk function to scan a list of IDs for these devices. Then, include it in pci_ats_required(). Suggested-by: Jason Gunthorpe Reviewed-by: Nirmoy Das Tested-by: Nirmoy Das Reviewed-by: Jonathan Cameron Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Dave Jiang Acked-by: Bjorn Helgaas Signed-off-by: Nicolin Chen Reviewed-by: Yi Liu Signed-off-by: Joerg Roedel --- drivers/pci/ats.c | 3 ++- drivers/pci/pci.h | 9 +++++++++ drivers/pci/quirks.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 84cd06d74fc9..96efa00d9743 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -247,7 +247,8 @@ bool pci_ats_required(struct pci_dev *pdev) if (pdev->is_virtfn) pdev = pci_physfn(pdev); - return pci_cxl_ats_required(pdev); + return pci_cxl_ats_required(pdev) || + pci_dev_specific_ats_required(pdev); } EXPORT_SYMBOL_GPL(pci_ats_required); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 4a14f88e543a..e8ad27abb1cf 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -1155,6 +1155,15 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, bool probe) } #endif +#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_PCI_ATS) +bool pci_dev_specific_ats_required(struct pci_dev *dev); +#else +static inline bool pci_dev_specific_ats_required(struct pci_dev *dev) +{ + return false; +} +#endif + #if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64) int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment, struct resource *res); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index caaed1a01dc0..c0242f3e9f06 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5715,6 +5715,48 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1457, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1459, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145a, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145c, quirk_intel_e2000_no_ats); + +static bool quirk_nvidia_gpu_ats_required(struct pci_dev *pdev) +{ + switch (pdev->device) { + case 0x2e00 ... 0x2e3f: /* GB20B */ + return true; + } + return false; +} + +static const struct pci_dev_ats_required { + u16 vendor; + u16 device; + bool (*ats_required)(struct pci_dev *dev); +} pci_dev_ats_required[] = { + /* NVIDIA GPUs */ + { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, quirk_nvidia_gpu_ats_required }, + /* NVIDIA CX10 Family NVlink-C2C */ + { PCI_VENDOR_ID_MELLANOX, 0x2101, NULL }, + { 0 } +}; + +/* + * Some NVIDIA devices do not implement CXL config space, but present as PCIe + * devices that can issue CXL-like cache operations like CXL.cache. Thus, they + * require ATS to obtain host physical addresses, like pci_cxl_ats_required(). + */ +bool pci_dev_specific_ats_required(struct pci_dev *pdev) +{ + const struct pci_dev_ats_required *i; + + for (i = pci_dev_ats_required; i->vendor; i++) { + if (i->vendor != pdev->vendor) + continue; + if (i->ats_required && i->ats_required(pdev)) + return true; + if (!i->ats_required && i->device == pdev->device) + return true; + } + + return false; +} #endif /* CONFIG_PCI_ATS */ /* Freescale PCIe doesn't support MSI in RC mode */ From 5ff98a0da6b48d8722eac0080e4185c417925d05 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 21 May 2026 13:34:22 -0700 Subject: [PATCH 50/66] iommu/arm-smmu-v3: Allow ATS to be always on When a device's default substream attaches to an identity domain, the SMMU driver currently sets the device's STE between two modes: Mode 1: Cfg=Translate, S1DSS=Bypass, EATS=1 Mode 2: Cfg=bypass (EATS is ignored by HW) When there is an active PASID (non-default substream), mode 1 is used. And when there is no PASID support or no active PASID, mode 2 is used. The driver will also downgrade an STE from mode 1 to mode 2, when the last active substream becomes inactive. However, there are PCIe devices that demand ATS to be always on. For these devices, their STEs have to use the mode 1 as HW ignores EATS with mode 2. Change the driver accordingly: - always use the mode 1 - never downgrade to mode 2 - allocate and retain a CD table (see note below) Note that these devices might not support PASID, i.e. doing non-PASID ATS. In such a case, the ssid_bits is set to 0. However, s1cdmax must be set to a !0 value in order to keep the S1DSS field effective. Thus, when a master requires ats_always_on, set its s1cdmax to at least 1, meaning that the CD table will have a dummy entry (SSID=1) that will never be used. Now for these devices, arm_smmu_cdtab_allocated() will always return true, v.s. false prior to this change. When its default substream is attached to an IDENTITY domain, its first CD is NULL in the table, which is a totally valid case. Thus, add "!master->ats_always_on" to the condition. Reviewed-by: Jonathan Cameron Tested-by: Nirmoy Das Acked-by: Nirmoy Das Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Dave Jiang Signed-off-by: Nicolin Chen Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 81 ++++++++++++++++++--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index e8d7dbe495f0..4afdb775e072 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1742,8 +1742,11 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) if (!arm_smmu_cdtab_allocated(&master->cd_table)) return; cdptr = arm_smmu_get_cd_ptr(master, ssid); - if (WARN_ON(!cdptr)) + if (!cdptr) { + /* Only ats_always_on allows a NULL CD on default substream */ + WARN_ON(!master->ats_always_on || ssid); return; + } arm_smmu_write_cd_entry(master, ssid, cdptr, &target); } @@ -1756,6 +1759,22 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; cd_table->s1cdmax = master->ssid_bits; + + /* + * When a device doesn't support PASID (non default SSID), ssid_bits is + * set to 0. This also sets S1CDMAX to 0, which disables the substreams + * and ignores the S1DSS field. + * + * On the other hand, if a device demands ATS to be always on even when + * its default substream is IOMMU bypassed, it has to use EATS that is + * only effective with an STE (CFG=S1translate, S1DSS=Bypass). For such + * use cases, S1CDMAX has to be !0, in order to make use of S1DSS/EATS. + * + * Set S1CDMAX no lower than 1. This would add a dummy substream in the + * CD table but it should never be used by an actual CD. + */ + if (master->ats_always_on) + cd_table->s1cdmax = max_t(u8, cd_table->s1cdmax, 1); max_contexts = 1 << cd_table->s1cdmax; if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || @@ -3854,9 +3873,12 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, if (!arm_smmu_ssids_in_use(&master->cd_table)) { struct iommu_domain *sid_domain = iommu_driver_get_domain_for_dev(master->dev); + bool ats_always_on = master->ats_always_on && + sid_domain->type != IOMMU_DOMAIN_BLOCKED; + bool downgrade = sid_domain->type == IOMMU_DOMAIN_IDENTITY || + sid_domain->type == IOMMU_DOMAIN_BLOCKED; - if (sid_domain->type == IOMMU_DOMAIN_IDENTITY || - sid_domain->type == IOMMU_DOMAIN_BLOCKED) + if (!ats_always_on && downgrade) sid_domain->ops->attach_dev(sid_domain, dev, sid_domain); } @@ -3875,6 +3897,8 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, .old_domain = old_domain, .ssid = IOMMU_NO_PASID, }; + bool ats_always_on = master->ats_always_on && + s1dss != STRTAB_STE_1_S1DSS_TERMINATE; /* * Do not allow any ASID to be changed while are working on the STE, @@ -3886,7 +3910,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, * If the CD table is not in use we can use the provided STE, otherwise * we use a cdtable STE with the provided S1DSS. */ - if (arm_smmu_ssids_in_use(&master->cd_table)) { + if (ats_always_on || arm_smmu_ssids_in_use(&master->cd_table)) { /* * If a CD table has to be present then we need to run with ATS * on because we have to assume a PASID is using ATS. For @@ -4215,6 +4239,44 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) kfree(master->build_invs); } +static int arm_smmu_master_prepare_ats(struct arm_smmu_master *master) +{ + bool s1p = master->smmu->features & ARM_SMMU_FEAT_TRANS_S1; + unsigned int stu = __ffs(master->smmu->pgsize_bitmap); + struct pci_dev *pdev; + int ret; + + if (!dev_is_pci(master->dev)) + return 0; + pdev = to_pci_dev(master->dev); + + if (!arm_smmu_ats_supported(master)) { + if (pci_ats_required(pdev)) { + dev_err_once(master->dev, "SMMU doesn't support ATS\n"); + return -EOPNOTSUPP; + } + return 0; + } + + ret = pci_prepare_ats(pdev, stu); + if (ret || !pci_ats_required(pdev)) + return ret; + + /* + * S1DSS is required for ATS to be always on for identity domain cases. + * However, the S1DSS field is ignored if !IDR0_S1P or !IDR1_SSIDSIZE. + */ + if (!s1p || !master->smmu->ssid_bits) { + dev_err_once(master->dev, + "SMMU doesn't support ATS to be always on\n"); + return -EOPNOTSUPP; + } + + master->ats_always_on = true; + + return arm_smmu_alloc_cd_tables(master); +} + static struct iommu_device *arm_smmu_probe_device(struct device *dev) { int ret; @@ -4263,14 +4325,15 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) smmu->features & ARM_SMMU_FEAT_STALL_FORCE) master->stall_enabled = true; - if (dev_is_pci(dev)) { - unsigned int stu = __ffs(smmu->pgsize_bitmap); - - pci_prepare_ats(to_pci_dev(dev), stu); - } + ret = arm_smmu_master_prepare_ats(master); + if (ret) + goto err_disable_pasid; return &smmu->iommu; +err_disable_pasid: + arm_smmu_disable_pasid(master); + arm_smmu_remove_master(master); err_free_master: kfree(master); return ERR_PTR(ret); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ef42df4753ec..8c3600f4364c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -943,6 +943,7 @@ struct arm_smmu_master { bool ats_enabled : 1; bool ste_ats_enabled : 1; bool stall_enabled; + bool ats_always_on; unsigned int ssid_bits; unsigned int iopf_refcount; }; From 8d4346ecd4950ae08cc76a6de327c264e846758c Mon Sep 17 00:00:00 2001 From: Simon Xue Date: Tue, 28 Apr 2026 18:05:31 +0200 Subject: [PATCH 51/66] iommu/rockchip: disable fetch dte time limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disable the Bit 31 of the AUTO_GATING iommu register, as it causes hangups with the RGA3 (Raster Graphics Acceleration 3) peripheral. The RGA3 register description of the TRM already states that the bit must be set to 1. The vendor kernel sets the bit unconditionally to 1 to fix VOP (Video Output Processor) screen black issues. This patch squashes the 2 vendor kernel commits with the following commit messages: Master fetch data and cpu update page table may work in parallel, may have the following procedure: master cpu fetch dte update page tabl | | (make dte invalid) <- zap iotlb entry | | fetch dte again (make dte invalid) <- zap iotlb entry | | fetch dte again (make dte invalid) <- zap iotlb entry | | fetch dte again (make iommu block) <- zap iotlb entry New iommu version has the above bug, if fetch dte consecutively four times, then it will be blocked. Fortunately, we can set bit 31 of register MMU_AUTO_GATING to 1 to make it work as old version which does not have this issue. This issue only appears on RV1126 so far, so make a workaround dedicated to "rockchip,rv1126" machine type. iommu/rockchip: fix vop blocked and screen black on RK356X and RK3588 RK3568 and RK3588 has the same issue as RV1126/RV1109 that caused by dte fetch time limit, So we can set BIT(31) of register 0x24 default to 1 as a workaround. Signed-off-by: Simon Xue Signed-off-by: Sven Püschel Acked-by: Heiko Stuebner Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 0013cf196c57..87ae036d6414 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -76,6 +76,8 @@ #define SPAGE_ORDER 12 #define SPAGE_SIZE (1 << SPAGE_ORDER) +#define DISABLE_FETCH_DTE_TIME_LIMIT BIT(31) + /* * Support mapping any size that fits in one page table: * 4 KiB to 4 MiB @@ -930,6 +932,7 @@ static int rk_iommu_enable(struct rk_iommu *iommu) struct iommu_domain *domain = iommu->domain; struct rk_iommu_domain *rk_domain = to_rk_domain(domain); int ret, i; + u32 auto_gate; ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks); if (ret) @@ -948,6 +951,11 @@ static int rk_iommu_enable(struct rk_iommu *iommu) rk_ops->mk_dtentries(rk_domain->dt_dma)); rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); + + /* Workaround for iommu blocked, BIT(31) default to 1 */ + auto_gate = rk_iommu_read(iommu->bases[i], RK_MMU_AUTO_GATING); + auto_gate |= DISABLE_FETCH_DTE_TIME_LIMIT; + rk_iommu_write(iommu->bases[i], RK_MMU_AUTO_GATING, auto_gate); } ret = rk_iommu_enable_paging(iommu); From 69fe699afe1afcb730164b86c228483c2da05f94 Mon Sep 17 00:00:00 2001 From: Weinan Liu Date: Thu, 28 May 2026 22:31:47 +0000 Subject: [PATCH 52/66] iommu/amd: Don't split flush for amd_iommu_domain_flush_all() We have observed multiple full invalidations occurring during device detach when we are done using the vfio-device. blocked_domain_attach_device() -> detach_device() -> amd_iommu_domain_flush_all() -> amd_iommu_domain_flush_pages(..., CMD_INV_IOMMU_ALL_PAGES_ADDRESS) while (size != 0) { -> __domain_flush_pages( flush_size /* power of 2 flush_size */) -> domain_flush_pages_v1() -> build_inv_iommu_pages() -> build_inv_address() } build_inv_address() will trigger a full invalidation if the chunk size > (1 << 51). Consequently, the guest will issue multiple full invalidations for a single call to amd_iommu_domain_flush_all() Without this patch, we will see 10 time instead of 1 time full invalidations for every amd_iommu_domain_flush_all(). Cc: stable@vger.kernel.org Fixes: a270be1b3fdf ("iommu/amd: Use only natural aligned flushes in a VM") Suggested-by: Josef Bacik Suggested-by: Jason Gunthorpe Signed-off-by: Weinan Liu Reviewed-by: Wei Wang Reviewed-by: Jason Gunthorpe Reviewed-by: Samiullah Khawaja Reviewed-by: Suravee Suthikulpanit Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f1333071da10..0c042668b32d 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1770,7 +1770,8 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, { lockdep_assert_held(&domain->lock); - if (likely(!amd_iommu_np_cache)) { + if (likely(!amd_iommu_np_cache) || + size >= (1ULL<<52)) { __domain_flush_pages(domain, address, size); /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ From 2a5b7d11ee28c036a9c94d66ba13d59ff07d068d Mon Sep 17 00:00:00 2001 From: Shawn Guo Date: Tue, 19 May 2026 09:39:50 +0800 Subject: [PATCH 53/66] dt-bindings: arm-smmu: Add compatible for Qualcomm Nord SoC Document Applications Processor Subsystem (APSS) SMMU on Qualcomm Nord SoC. Signed-off-by: Shawn Guo Reviewed-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 25fd3efa2420..a1b562c87858 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -40,6 +40,7 @@ properties: - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 + - qcom,nord-smmu-500 - qcom,qcm2290-smmu-500 - qcom,qcs615-smmu-500 - qcom,qcs8300-smmu-500 @@ -618,6 +619,7 @@ allOf: - qcom,glymur-smmu-500 - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 + - qcom,nord-smmu-500 - qcom,qcs615-smmu-500 - qcom,qcs8300-smmu-500 - qcom,sa8775p-smmu-500 From aab556d118fc02d75df3220aa3133242d5d1b151 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 20 May 2026 13:09:14 +0200 Subject: [PATCH 54/66] dt-bindings: arm-smmu: Correct and add constraints for Hawi, Shikra and Kaanapali Previous commit 75949eb02653 ("dt-bindings: arm-smmu: Constrain clocks for newer Qualcomm variants") duplicated constraints for qcom,sm6350-smmu-500 and qcom,sm6375-smmu-500 - these are already part of previous "if:" block. It also missed enforcing one clock for qcom,kaanapali-smmu-500 in GPU case and missed simultaneously added Shikra and Hawi. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index a1b562c87858..a701dec2fa0a 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -571,6 +571,8 @@ allOf: items: - enum: - qcom,glymur-smmu-500 + - qcom,hawi-smmu-500 + - qcom,kaanapali-smmu-500 - qcom,sm8750-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 @@ -617,16 +619,16 @@ allOf: - enum: - qcom,eliza-smmu-500 - qcom,glymur-smmu-500 + - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 - qcom,nord-smmu-500 - qcom,qcs615-smmu-500 - qcom,qcs8300-smmu-500 - qcom,sa8775p-smmu-500 + - qcom,shikra-smmu-500 - qcom,sm6115-smmu-500 - qcom,sm6125-smmu-500 - - qcom,sm6350-smmu-500 - - qcom,sm6375-smmu-500 - qcom,sm8150-smmu-500 - qcom,sm8250-smmu-500 - qcom,sm8350-smmu-500 From 534b5f98ab7319d8004bbc7dab6481462243e883 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 4 Jun 2026 14:03:06 +0800 Subject: [PATCH 55/66] iommu/vt-d: Avoid WARNING in sva unbind path The Intel IOMMU driver allows SVA on devices even if they do not support PCI/PRI. Commit 39c20c4e83b9 ("iommu/vt-d: Only handle IOPF for SVA when PRI is supported") modified the SVA bind path to allow this configuration by skipping IOPF enablement when PRI is missing. However, it failed to update the unbind path. This creates an imbalance: the unbind path attempts to disable IOPF for a device that never had it enabled, triggering a WARNING in intel_iommu_disable_iopf(): WARNING: drivers/iommu/intel/iommu.c:3475 at intel_iommu_disable_iopf+0x4f/0x90d Call Trace: blocking_domain_set_dev_pasid+0x50/0x70 iommu_detach_device_pasid+0x89/0xc0 iommu_sva_unbind_device+0x73/0x150 xe_vm_close_and_put+0x4d2/0x1200 [xe] Fix this by bypassing IOPF operations for SVA domains on non-PRI hardware in both the bind and unbind paths. Fixes: 39c20c4e83b9 ("iommu/vt-d: Only handle IOPF for SVA when PRI is supported") Cc: stable@vger.kernel.org Reported-by: Nareshkumar Gollakoti Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20260519052917.3729796-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.h | 11 +++++++++++ drivers/iommu/intel/svm.c | 12 ++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index ef145560aa98..775f1c4ae346 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1254,18 +1254,29 @@ void intel_iommu_disable_iopf(struct device *dev); static inline int iopf_for_domain_set(struct iommu_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); + if (!domain || !domain->iopf_handler) return 0; + /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ + if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported) + return 0; + return intel_iommu_enable_iopf(dev); } static inline void iopf_for_domain_remove(struct iommu_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); + if (!domain || !domain->iopf_handler) return; + if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported) + return; + intel_iommu_disable_iopf(dev); } diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 57cd1db7207a..fea10acd4f02 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -164,12 +164,9 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (IS_ERR(dev_pasid)) return PTR_ERR(dev_pasid); - /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ - if (info->pri_supported) { - ret = iopf_for_domain_replace(domain, old, dev); - if (ret) - goto out_remove_dev_pasid; - } + ret = iopf_for_domain_replace(domain, old, dev); + if (ret) + goto out_remove_dev_pasid; /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -184,8 +181,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return 0; out_unwind_iopf: - if (info->pri_supported) - iopf_for_domain_replace(old, domain, dev); + iopf_for_domain_replace(old, domain, dev); out_remove_dev_pasid: domain_remove_dev_pasid(domain, dev, pasid); return ret; From f46452c3df7a8d8a5addc0926e76ef19ea7da0a0 Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Thu, 4 Jun 2026 14:03:07 +0800 Subject: [PATCH 56/66] iommu/vt-d: Clear Present bit before tearing down scalable-mode context entry device_pasid_table_teardown() zeroes the 128-bit scalable-mode context entry with context_clear_entry() while the Present bit is still set. This creates a window where the hardware can fetch a torn entry, with some fields already zeroed while Present is still set, leading to unpredictable behavior or spurious faults. The context-cache invalidation is issued only after the entry has been zeroed, and intel_pasid_free_table() then frees the PASID directory pages, so the IOMMU can keep walking a stale Present=1 entry that points at freed memory. While x86 provides strong write ordering, the compiler may reorder the two 64-bit writes to the entry, and the hardware fetch is not guaranteed to be atomic with respect to multiple CPU writes. Commit c1e4f1dccbe9d ("iommu/vt-d: Clear Present bit before tearing down context entry") fixed this exact pattern in domain_context_clear_one() and the copied-context path, but device_pasid_table_teardown() was not converted. Align it with the "Guidance to Software for Invalidations" in the VT-d spec, Section 6.5.3.3, using the same ownership handshake as the sibling fix: clear only the Present bit, flush it to the IOMMU, perform the context-cache invalidation, and only then zero the rest of the entry. Fixes: 81e921fd32161 ("iommu/vt-d: Fix NULL domain on device release") Signed-off-by: Michael Bommarito Assisted-by: Claude:claude-opus-4-7 Link: https://lore.kernel.org/r/20260528025557.3209367-1-michael.bommarito@gmail.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 89541b74ab8c..40910dc7363b 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -748,10 +748,12 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data) From 214ffed161b48d5a40f654f23d0806b9ec704920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Grzelak?= Date: Thu, 4 Jun 2026 14:03:08 +0800 Subject: [PATCH 57/66] iommu/vt-d: Remove typo from pasid_pte_config_nested() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename pasid_pte_config_nestd() into pasid_pte_config_nested(). Do it to match other function names ending with _nested(). Signed-off-by: Michał Grzelak Link: https://lore.kernel.org/r/20260509174503.831134-1-michal.grzelak@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 40910dc7363b..81353fd46b37 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -618,11 +618,11 @@ void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu, intel_pasid_flush_present(iommu, dev, pasid, did, pte); } -static void pasid_pte_config_nestd(struct intel_iommu *iommu, - struct pasid_entry *pte, - struct iommu_hwpt_vtd_s1 *s1_cfg, - struct dmar_domain *s2_domain, - u16 did) +static void pasid_pte_config_nested(struct intel_iommu *iommu, + struct pasid_entry *pte, + struct iommu_hwpt_vtd_s1 *s1_cfg, + struct dmar_domain *s2_domain, + u16 did) { struct pt_iommu_vtdss_hw_info pt_info; @@ -720,7 +720,7 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, return -EBUSY; } - pasid_pte_config_nestd(iommu, pte, s1_cfg, s2_domain, did); + pasid_pte_config_nested(iommu, pte, s1_cfg, s2_domain, did); spin_unlock(&iommu->lock); pasid_flush_caches(iommu, pte, pasid, did); From c468814b3fd384225a10f0ad1d6905937e50b233 Mon Sep 17 00:00:00 2001 From: Guanghui Feng Date: Thu, 4 Jun 2026 14:03:09 +0800 Subject: [PATCH 58/66] iommu/vt-d: Improve IOMMU fault information In some environments, multiple PCIe segments exist, and PCIe device information needs to be differentiated and identified based on the segment. When an IOMMU fault event occurs, the IOMMU and device segment information should be output in detail in dmar_fault_do_one. Signed-off-by: Guanghui Feng Link: https://lore.kernel.org/r/20260528022943.1697564-1-guanghuifeng@linux.alibaba.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index d33c119a935e..767ec092accd 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1894,7 +1894,8 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, reason = dmar_get_fault_reason(fault_reason, &fault_type); if (fault_type == INTR_REMAP) { - pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n", + pr_err("[INTR-REMAP] Request device [%04x:%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n", + iommu->segment, source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr >> 48, fault_reason, reason); @@ -1903,14 +1904,16 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, } if (pasid == IOMMU_PASID_INVALID) - pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", + pr_err("[%s NO_PASID] Request device [%04x:%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", type ? "DMA Read" : "DMA Write", + iommu->segment, source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason); else - pr_err("[%s PASID 0x%x] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", + pr_err("[%s PASID 0x%x] Request device [%04x:%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", type ? "DMA Read" : "DMA Write", pasid, + iommu->segment, source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason); From 43bd9e6d5513cb1edbafdeef146a1edc3aaced56 Mon Sep 17 00:00:00 2001 From: Pranjal Shrivastava Date: Thu, 4 Jun 2026 14:03:10 +0800 Subject: [PATCH 59/66] iommu/vt-d: Fix RB-tree corruption in probe error path The info->node RB-tree member is zero-initialized via kzalloc. If a device does not support ATS, the device_rbtree_insert() call is skipped. If a subsequent probe step fails, the error path jumps to device_rbtree_remove(), which misinterprets the zeroed node as a tree root and corrupts the device RB-tree. Fix this by explicitly initializing the RB-node as empty using RB_CLEAR_NODE() during initialization and guarding the removal with RB_EMPTY_NODE(). Fixes: 4f1492efb495 ("iommu/vt-d: Revert ATS timing change to fix boot failure") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260525205628.CD4431F000E9@smtp.kernel.org/ Suggested-by: Baolu Lu Signed-off-by: Pranjal Shrivastava Link: https://lore.kernel.org/r/20260531170254.60493-2-praan@google.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index c3d18cd77d2f..2702e9aa2241 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -157,7 +157,10 @@ static void device_rbtree_remove(struct device_domain_info *info) unsigned long flags; spin_lock_irqsave(&iommu->device_rbtree_lock, flags); - rb_erase(&info->node, &iommu->device_rbtree); + if (!RB_EMPTY_NODE(&info->node)) { + rb_erase(&info->node, &iommu->device_rbtree); + RB_CLEAR_NODE(&info->node); + } spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); } @@ -3254,6 +3257,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) info->dev = dev; info->iommu = iommu; + RB_CLEAR_NODE(&info->node); if (dev_is_pci(dev)) { if (ecap_dev_iotlb_support(iommu->ecap) && pci_ats_supported(pdev) && From cff81c4af495ea0ecaffdc85e8f80a4f87ce41db Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Tue, 9 Jun 2026 12:56:04 -0700 Subject: [PATCH 60/66] iommu/apple-dart: correct CONFIG_PCIE_APPLE macro name in comment A comment in drivers/iommu/apple-dart.c incorrectly refers to CONFIG_PCI_APPLE instead of CONFIG_PCIE_APPLE. Correct it. Discovered while searching for CONFIG_* symbols referenced in code but not defined in any Kconfig file. Signed-off-by: Ethan Nelson-Moore Reviewed-by: Sven Peter Signed-off-by: Joerg Roedel --- drivers/iommu/apple-dart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 17bdadb6b504..b160fb464c5f 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -976,7 +976,7 @@ static int apple_dart_def_domain_type(struct device *dev) } #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR -/* Keep things compiling when CONFIG_PCI_APPLE isn't selected */ +/* Keep things compiling when CONFIG_PCIE_APPLE isn't selected */ #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0 #endif #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK) From 144c05d88c7f459c22abafe891ee149692f29734 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Mar 2026 12:23:55 -0300 Subject: [PATCH 61/66] iommu/amd: Simplify build_inv_address() This function is doing more work than it needs to: - iommu_num_pages() is pointless, the fls() is going to compute the required page size already. - It is easier to understand as sz_lg2, which is 12 if size is 4K, than msb_diff which is 11 if size is 4K. - Simplify the control flow to early exit on the out of range cases. - Use the usual last instead of end to signify an inclusive last address. - Use GENMASK to compute the 1's mask. - Use GENMASK to compute the address mask for the command layout, not PAGE_MASK. - Directly reference the spec language that defines the 52 bit limit. No functional change intended. Signed-off-by: Jason Gunthorpe Reviewed-by: Wei Wang Tested-by: Dheeraj Kumar Srivastava Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 50 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 0c042668b32d..09b22a1873eb 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1268,39 +1268,37 @@ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) */ static inline u64 build_inv_address(u64 address, size_t size) { - u64 pages, end, msb_diff; + u64 last = address + size - 1; + unsigned int sz_lg2; - pages = iommu_num_pages(address, size, PAGE_SIZE); - - if (pages == 1) - return address & PAGE_MASK; - - end = address + size - 1; + address &= GENMASK_U64(63, 12); + sz_lg2 = fls64(address ^ last); + if (sz_lg2 <= 12) + return address; /* - * msb_diff would hold the index of the most significant bit that - * flipped between the start and end. + * Encode sz_lg2 according to Table 14: Example Page Size Encodings + * + * See "Note *": + * Address bits 51:32 can be used to encode page sizes greater + * that 4 Gbytes. + * Which we take to mean that the highest page size has bit + * [51]=0, [50:12]=1 + * and that coding happens when sz_lg2 is 52. Fall back to full + * invalidation if the size is too big. + * */ - msb_diff = fls64(end ^ address) - 1; + if (unlikely(sz_lg2 > 52)) + return (CMD_INV_IOMMU_ALL_PAGES_ADDRESS & PAGE_MASK) | + CMD_INV_IOMMU_PAGES_SIZE_MASK; /* - * Bits 63:52 are sign extended. If for some reason bit 51 is different - * between the start and the end, invalidate everything. + * The sz_lg2 calculation with fls() ensures that: + * address & BIT(sz_lg2 - 1) == 0 + * Therefore only the 1's need to be added. 8KB requires no 1's */ - if (unlikely(msb_diff > 51)) { - address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - } else { - /* - * The msb-bit must be clear on the address. Just set all the - * lower bits. - */ - address |= (1ull << msb_diff) - 1; - } - - /* Clear bits 11:0 */ - address &= PAGE_MASK; - - /* Set the size bit - we flush more than one 4kb page */ + if (sz_lg2 > 13) + address |= GENMASK_U64(sz_lg2 - 2, 12); return address | CMD_INV_IOMMU_PAGES_SIZE_MASK; } From 2e43a291d7a79059ae0cc02be3d3931ae3dad242 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Mar 2026 12:23:56 -0300 Subject: [PATCH 62/66] iommu/amd: Pass last in through to build_inv_address() This is the trivial call chain below amd_iommu_domain_flush_pages(). Cases that are doing a full invalidate will pass a last of U64_MAX. This avoids converting between size and last, and type confusion with size_t, unsigned long and u64 all being used in different places along the driver's invalidation path. Consistently use u64 in the internals. Signed-off-by: Jason Gunthorpe Tested-by: Dheeraj Kumar Srivastava Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/iommu.c | 56 +++++++++++++++++------------------ drivers/iommu/amd/pasid.c | 2 +- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 834d8fabfba3..98f6da97df92 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -93,7 +93,7 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu); void amd_iommu_domain_flush_pages(struct protection_domain *domain, u64 address, size_t size); void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, - ioasid_t pasid, u64 address, size_t size); + ioasid_t pasid, u64 address, u64 last); #ifdef CONFIG_IRQ_REMAP int amd_iommu_create_irq_domain(struct amd_iommu *iommu); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 09b22a1873eb..ce3a8b05e4f1 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1266,9 +1266,8 @@ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) * Builds an invalidation address which is suitable for one page or multiple * pages. Sets the size bit (S) as needed is more than one page is flushed. */ -static inline u64 build_inv_address(u64 address, size_t size) +static inline u64 build_inv_address(u64 address, u64 last) { - u64 last = address + size - 1; unsigned int sz_lg2; address &= GENMASK_U64(63, 12); @@ -1303,10 +1302,10 @@ static inline u64 build_inv_address(u64 address, size_t size) } static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, - size_t size, u16 domid, + u64 last, u16 domid, ioasid_t pasid, bool gn) { - u64 inv_address = build_inv_address(address, size); + u64 inv_address = build_inv_address(address, last); memset(cmd, 0, sizeof(*cmd)); @@ -1323,10 +1322,10 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, } static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, - u64 address, size_t size, + u64 address, u64 last, ioasid_t pasid, bool gn) { - u64 inv_address = build_inv_address(address, size); + u64 inv_address = build_inv_address(address, last); memset(cmd, 0, sizeof(*cmd)); @@ -1524,7 +1523,7 @@ static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu) for (dom_id = 0; dom_id <= last_bdf; ++dom_id) { struct iommu_cmd cmd; - build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, + build_inv_iommu_pages(&cmd, 0, U64_MAX, dom_id, IOMMU_NO_PASID, false); iommu_queue_command(iommu, &cmd); } @@ -1536,14 +1535,14 @@ static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) { struct iommu_cmd cmd; - build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, + build_inv_iommu_pages(&cmd, 0, U64_MAX, dom_id, IOMMU_NO_PASID, false); iommu_queue_command(iommu, &cmd); iommu_completion_wait(iommu); } -static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, size_t size) +static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, u64 last) { int ret = 0; struct amd_iommu_viommu *aviommu; @@ -1560,7 +1559,7 @@ static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 add pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__, iommu->devid, gdom_info->hdom_id); - build_inv_iommu_pages(&cmd, address, size, gdom_info->hdom_id, + build_inv_iommu_pages(&cmd, address, last, gdom_info->hdom_id, IOMMU_NO_PASID, false); ret |= iommu_queue_command(iommu, &cmd); } @@ -1617,14 +1616,14 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu) * Command send function for flushing on-device TLB */ static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, - size_t size, ioasid_t pasid, bool gn) + u64 last, ioasid_t pasid, bool gn) { struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct iommu_cmd cmd; int qdep = dev_data->ats_qdep; build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, - size, pasid, gn); + last, pasid, gn); return iommu_queue_command(iommu, &cmd); } @@ -1668,7 +1667,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) if (dev_data->ats_enabled) { /* Invalidate the entire contents of an IOTLB */ - ret = device_flush_iotlb(dev_data, 0, ~0UL, + ret = device_flush_iotlb(dev_data, 0, U64_MAX, IOMMU_NO_PASID, false); } @@ -1676,7 +1675,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) } static int domain_flush_pages_v2(struct protection_domain *pdom, - u64 address, size_t size) + u64 address, u64 last) { struct iommu_dev_data *dev_data; struct iommu_cmd cmd; @@ -1687,7 +1686,7 @@ static int domain_flush_pages_v2(struct protection_domain *pdom, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); u16 domid = dev_data->gcr3_info.domid; - build_inv_iommu_pages(&cmd, address, size, + build_inv_iommu_pages(&cmd, address, last, domid, IOMMU_NO_PASID, true); ret |= iommu_queue_command(iommu, &cmd); @@ -1697,7 +1696,7 @@ static int domain_flush_pages_v2(struct protection_domain *pdom, } static int domain_flush_pages_v1(struct protection_domain *pdom, - u64 address, size_t size) + u64 address, u64 last) { struct pdom_iommu_info *pdom_iommu_info; struct iommu_cmd cmd; @@ -1706,7 +1705,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, lockdep_assert_held(&pdom->lock); - build_inv_iommu_pages(&cmd, address, size, + build_inv_iommu_pages(&cmd, address, last, pdom->id, IOMMU_NO_PASID, false); xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) { @@ -1726,7 +1725,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, * See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested() */ if (!list_empty(&pdom->viommu_list)) - ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, size); + ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, last); return ret; } @@ -1736,7 +1735,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, * It flushes range of PTEs of the domain. */ static void __domain_flush_pages(struct protection_domain *domain, - u64 address, size_t size) + u64 address, u64 last) { struct iommu_dev_data *dev_data; int ret = 0; @@ -1747,9 +1746,9 @@ static void __domain_flush_pages(struct protection_domain *domain, if (pdom_is_v2_pgtbl_mode(domain)) { gn = true; - ret = domain_flush_pages_v2(domain, address, size); + ret = domain_flush_pages_v2(domain, address, last); } else { - ret = domain_flush_pages_v1(domain, address, size); + ret = domain_flush_pages_v1(domain, address, last); } list_for_each_entry(dev_data, &domain->dev_list, list) { @@ -1757,7 +1756,7 @@ static void __domain_flush_pages(struct protection_domain *domain, if (!dev_data->ats_enabled) continue; - ret |= device_flush_iotlb(dev_data, address, size, pasid, gn); + ret |= device_flush_iotlb(dev_data, address, last, pasid, gn); } WARN_ON(ret); @@ -1770,7 +1769,7 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, if (likely(!amd_iommu_np_cache) || size >= (1ULL<<52)) { - __domain_flush_pages(domain, address, size); + __domain_flush_pages(domain, address, address + size - 1); /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ domain_flush_complete(domain); @@ -1807,7 +1806,7 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, flush_size = 1ul << min_alignment; - __domain_flush_pages(domain, address, flush_size); + __domain_flush_pages(domain, address, address + flush_size - 1); address += flush_size; size -= flush_size; } @@ -1824,17 +1823,17 @@ static void amd_iommu_domain_flush_all(struct protection_domain *domain) } void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, - ioasid_t pasid, u64 address, size_t size) + ioasid_t pasid, u64 address, u64 last) { struct iommu_cmd cmd; struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); - build_inv_iommu_pages(&cmd, address, size, + build_inv_iommu_pages(&cmd, address, last, dev_data->gcr3_info.domid, pasid, true); iommu_queue_command(iommu, &cmd); if (dev_data->ats_enabled) - device_flush_iotlb(dev_data, address, size, pasid, true); + device_flush_iotlb(dev_data, address, last, pasid, true); iommu_completion_wait(iommu); } @@ -1842,8 +1841,7 @@ void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, static void dev_flush_pasid_all(struct iommu_dev_data *dev_data, ioasid_t pasid) { - amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, - CMD_INV_IOMMU_ALL_PAGES_ADDRESS); + amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, U64_MAX); } int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag) diff --git a/drivers/iommu/amd/pasid.c b/drivers/iommu/amd/pasid.c index 67eace9809f1..d708c6532480 100644 --- a/drivers/iommu/amd/pasid.c +++ b/drivers/iommu/amd/pasid.c @@ -71,7 +71,7 @@ static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, for_each_pdom_dev_data(pdom_dev_data, sva_pdom) { amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data, pdom_dev_data->pasid, - start, end - start); + start, end - 1); } spin_unlock_irqrestore(&sva_pdom->lock, flags); From cc08ecaf8e584073600abdf71ea4213267387226 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Mar 2026 12:23:57 -0300 Subject: [PATCH 63/66] iommu/amd: Have amd_iommu_domain_flush_pages() use last Finish clearing out the size/last/end switching by converting amd_iommu_domain_flush_pages() to use last-based logic. This algorithm is simpler than the previous. Ultimately all this wants to do is select powers of two that are aligned to address and not longer than the distance to last. The new version is fully safe for size = U64_MAX and last = U64_MAX. Finally, the gather can be passed through natively without risking an overflow in (gather->end - gather->start + 1). Signed-off-by: Jason Gunthorpe Reviewed-by: Wei Wang Tested-by: Dheeraj Kumar Srivastava Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/iommu.c | 43 ++++++++++++----------------------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 98f6da97df92..f11e4aa66121 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -91,7 +91,7 @@ int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag); */ void amd_iommu_flush_all_caches(struct amd_iommu *iommu); void amd_iommu_domain_flush_pages(struct protection_domain *domain, - u64 address, size_t size); + u64 address, u64 last); void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, ioasid_t pasid, u64 address, u64 last); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index ce3a8b05e4f1..703cbb84953b 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1763,13 +1763,13 @@ static void __domain_flush_pages(struct protection_domain *domain, } void amd_iommu_domain_flush_pages(struct protection_domain *domain, - u64 address, size_t size) + u64 address, u64 last) { lockdep_assert_held(&domain->lock); if (likely(!amd_iommu_np_cache) || - size >= (1ULL<<52)) { - __domain_flush_pages(domain, address, address + size - 1); + unlikely(address == 0 && last == U64_MAX)) { + __domain_flush_pages(domain, address, last); /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ domain_flush_complete(domain); @@ -1787,28 +1787,17 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, * between the natural alignment of the address that we flush and the * greatest naturally aligned region that fits in the range. */ - while (size != 0) { - int addr_alignment = __ffs(address); - int size_alignment = __fls(size); - int min_alignment; - size_t flush_size; + while (address <= last) { + unsigned int sz_lg2 = ilog2(last - address + 1); + u64 flush_last; - /* - * size is always non-zero, but address might be zero, causing - * addr_alignment to be negative. As the casting of the - * argument in __ffs(address) to long might trim the high bits - * of the address on x86-32, cast to long when doing the check. - */ - if (likely((unsigned long)address != 0)) - min_alignment = min(addr_alignment, size_alignment); - else - min_alignment = size_alignment; + if (likely(address)) + sz_lg2 = min_t(unsigned int, sz_lg2, __ffs64(address)); - flush_size = 1ul << min_alignment; - - __domain_flush_pages(domain, address, address + flush_size - 1); - address += flush_size; - size -= flush_size; + flush_last = address + (1ULL << sz_lg2) - 1; + __domain_flush_pages(domain, address, flush_last); + if (check_add_overflow(flush_last, 1, &address)) + break; } /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ @@ -1818,8 +1807,7 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, /* Flush the whole IO/TLB for a given protection domain - including PDE */ static void amd_iommu_domain_flush_all(struct protection_domain *domain) { - amd_iommu_domain_flush_pages(domain, 0, - CMD_INV_IOMMU_ALL_PAGES_ADDRESS); + amd_iommu_domain_flush_pages(domain, 0, U64_MAX); } void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, @@ -2620,7 +2608,7 @@ static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom, return 0; spin_lock_irqsave(&domain->lock, flags); - amd_iommu_domain_flush_pages(domain, iova, size); + amd_iommu_domain_flush_pages(domain, iova, iova + size - 1); spin_unlock_irqrestore(&domain->lock, flags); return 0; } @@ -2642,8 +2630,7 @@ static void amd_iommu_iotlb_sync(struct iommu_domain *domain, unsigned long flags; spin_lock_irqsave(&dom->lock, flags); - amd_iommu_domain_flush_pages(dom, gather->start, - gather->end - gather->start + 1); + amd_iommu_domain_flush_pages(dom, gather->start, gather->end); spin_unlock_irqrestore(&dom->lock, flags); iommu_put_pages_list(&gather->freelist); } From 17149077e01c9f1a9171928d3ebdaaaf504577a9 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Mar 2026 12:23:58 -0300 Subject: [PATCH 64/66] iommu/amd: Make CMD_INV_IOMMU_ALL_PAGES_ADDRESS match the spec The spec in Table 14 defines the "Entire Cache" case as having the low 12 bits as zero. Indeed the command format doesn't even have the low 12 bits. Since there is only one user now, fix the constant to have 0 in the low 12 bits instead of 1 and remove the masking. Signed-off-by: Jason Gunthorpe Reviewed-by: Wei Wang Tested-by: Dheeraj Kumar Srivastava Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/iommu.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 96038b2e24d5..093643061597 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -217,7 +217,7 @@ #define PPR_STATUS_MASK 0xf #define PPR_STATUS_SHIFT 12 -#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL +#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7ffffffffffff000ULL /* macros and definitions for device table entries */ #define DEV_ENTRY_VALID 0x00 diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 703cbb84953b..15bd0f3e22cc 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1288,7 +1288,7 @@ static inline u64 build_inv_address(u64 address, u64 last) * */ if (unlikely(sz_lg2 > 52)) - return (CMD_INV_IOMMU_ALL_PAGES_ADDRESS & PAGE_MASK) | + return CMD_INV_IOMMU_ALL_PAGES_ADDRESS | CMD_INV_IOMMU_PAGES_SIZE_MASK; /* From e4f39d793123d1da3979ce6c749995f1eee337b5 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Mar 2026 12:23:59 -0300 Subject: [PATCH 65/66] iommu/amd: Control INVALIDATE_IOMMU_PAGES PDE from the gather Now that AMD uses iommupt, it is easy to make use of the PDE bit. If the gather has no free list then no page directory entries were changed. Pass GN/PDE through the invalidation call chain in a u32 flags field that is OR'd into data[2] and set it properly from the gather. Signed-off-by: Jason Gunthorpe Reviewed-by: Wei Wang Tested-by: Dheeraj Kumar Srivastava Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/iommu.c | 61 +++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f11e4aa66121..bfb2ab6d6f39 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -91,7 +91,7 @@ int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag); */ void amd_iommu_flush_all_caches(struct amd_iommu *iommu); void amd_iommu_domain_flush_pages(struct protection_domain *domain, - u64 address, u64 last); + u64 address, u64 last, u32 flags); void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, ioasid_t pasid, u64 address, u64 last); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 15bd0f3e22cc..379e55187727 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1302,8 +1302,8 @@ static inline u64 build_inv_address(u64 address, u64 last) } static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, - u64 last, u16 domid, - ioasid_t pasid, bool gn) + u64 last, u16 domid, ioasid_t pasid, + u32 flags) { u64 inv_address = build_inv_address(address, last); @@ -1312,12 +1312,9 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, cmd->data[1] |= domid; cmd->data[2] = lower_32_bits(inv_address); cmd->data[3] = upper_32_bits(inv_address); - /* PDE bit - we want to flush everything, not only the PTEs */ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; - if (gn) { + cmd->data[2] |= flags; + if (flags & CMD_INV_IOMMU_PAGES_GN_MASK) cmd->data[0] |= pasid; - cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; - } CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); } @@ -1524,7 +1521,8 @@ static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu) for (dom_id = 0; dom_id <= last_bdf; ++dom_id) { struct iommu_cmd cmd; build_inv_iommu_pages(&cmd, 0, U64_MAX, - dom_id, IOMMU_NO_PASID, false); + dom_id, IOMMU_NO_PASID, + CMD_INV_IOMMU_PAGES_PDE_MASK); iommu_queue_command(iommu, &cmd); } @@ -1536,13 +1534,15 @@ static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) struct iommu_cmd cmd; build_inv_iommu_pages(&cmd, 0, U64_MAX, - dom_id, IOMMU_NO_PASID, false); + dom_id, IOMMU_NO_PASID, + CMD_INV_IOMMU_PAGES_PDE_MASK); iommu_queue_command(iommu, &cmd); iommu_completion_wait(iommu); } -static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, u64 last) +static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, + u64 address, u64 last, u32 flags) { int ret = 0; struct amd_iommu_viommu *aviommu; @@ -1560,7 +1560,7 @@ static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 add pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__, iommu->devid, gdom_info->hdom_id); build_inv_iommu_pages(&cmd, address, last, gdom_info->hdom_id, - IOMMU_NO_PASID, false); + IOMMU_NO_PASID, flags); ret |= iommu_queue_command(iommu, &cmd); } xa_unlock(&aviommu->gdomid_array); @@ -1675,7 +1675,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) } static int domain_flush_pages_v2(struct protection_domain *pdom, - u64 address, u64 last) + u64 address, u64 last, u32 flags) { struct iommu_dev_data *dev_data; struct iommu_cmd cmd; @@ -1686,8 +1686,9 @@ static int domain_flush_pages_v2(struct protection_domain *pdom, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); u16 domid = dev_data->gcr3_info.domid; - build_inv_iommu_pages(&cmd, address, last, - domid, IOMMU_NO_PASID, true); + build_inv_iommu_pages(&cmd, address, last, domid, + IOMMU_NO_PASID, + flags | CMD_INV_IOMMU_PAGES_GN_MASK); ret |= iommu_queue_command(iommu, &cmd); } @@ -1696,7 +1697,7 @@ static int domain_flush_pages_v2(struct protection_domain *pdom, } static int domain_flush_pages_v1(struct protection_domain *pdom, - u64 address, u64 last) + u64 address, u64 last, u32 flags) { struct pdom_iommu_info *pdom_iommu_info; struct iommu_cmd cmd; @@ -1706,7 +1707,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, lockdep_assert_held(&pdom->lock); build_inv_iommu_pages(&cmd, address, last, - pdom->id, IOMMU_NO_PASID, false); + pdom->id, IOMMU_NO_PASID, flags); xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) { /* @@ -1725,7 +1726,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, * See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested() */ if (!list_empty(&pdom->viommu_list)) - ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, last); + ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, last, flags); return ret; } @@ -1735,7 +1736,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom, * It flushes range of PTEs of the domain. */ static void __domain_flush_pages(struct protection_domain *domain, - u64 address, u64 last) + u64 address, u64 last, u32 flags) { struct iommu_dev_data *dev_data; int ret = 0; @@ -1746,9 +1747,9 @@ static void __domain_flush_pages(struct protection_domain *domain, if (pdom_is_v2_pgtbl_mode(domain)) { gn = true; - ret = domain_flush_pages_v2(domain, address, last); + ret = domain_flush_pages_v2(domain, address, last, flags); } else { - ret = domain_flush_pages_v1(domain, address, last); + ret = domain_flush_pages_v1(domain, address, last, flags); } list_for_each_entry(dev_data, &domain->dev_list, list) { @@ -1763,13 +1764,13 @@ static void __domain_flush_pages(struct protection_domain *domain, } void amd_iommu_domain_flush_pages(struct protection_domain *domain, - u64 address, u64 last) + u64 address, u64 last, u32 flags) { lockdep_assert_held(&domain->lock); if (likely(!amd_iommu_np_cache) || unlikely(address == 0 && last == U64_MAX)) { - __domain_flush_pages(domain, address, last); + __domain_flush_pages(domain, address, last, flags); /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ domain_flush_complete(domain); @@ -1795,7 +1796,7 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, sz_lg2 = min_t(unsigned int, sz_lg2, __ffs64(address)); flush_last = address + (1ULL << sz_lg2) - 1; - __domain_flush_pages(domain, address, flush_last); + __domain_flush_pages(domain, address, flush_last, flags); if (check_add_overflow(flush_last, 1, &address)) break; } @@ -1807,7 +1808,8 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, /* Flush the whole IO/TLB for a given protection domain - including PDE */ static void amd_iommu_domain_flush_all(struct protection_domain *domain) { - amd_iommu_domain_flush_pages(domain, 0, U64_MAX); + amd_iommu_domain_flush_pages(domain, 0, U64_MAX, + CMD_INV_IOMMU_PAGES_PDE_MASK); } void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, @@ -1817,7 +1819,9 @@ void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); build_inv_iommu_pages(&cmd, address, last, - dev_data->gcr3_info.domid, pasid, true); + dev_data->gcr3_info.domid, pasid, + CMD_INV_IOMMU_PAGES_GN_MASK | + CMD_INV_IOMMU_PAGES_PDE_MASK); iommu_queue_command(iommu, &cmd); if (dev_data->ats_enabled) @@ -2608,7 +2612,8 @@ static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom, return 0; spin_lock_irqsave(&domain->lock, flags); - amd_iommu_domain_flush_pages(domain, iova, iova + size - 1); + amd_iommu_domain_flush_pages(domain, iova, iova + size - 1, + CMD_INV_IOMMU_PAGES_PDE_MASK); spin_unlock_irqrestore(&domain->lock, flags); return 0; } @@ -2630,7 +2635,9 @@ static void amd_iommu_iotlb_sync(struct iommu_domain *domain, unsigned long flags; spin_lock_irqsave(&dom->lock, flags); - amd_iommu_domain_flush_pages(dom, gather->start, gather->end); + amd_iommu_domain_flush_pages(dom, gather->start, gather->end, + iommu_pages_list_empty(&gather->freelist) ? + 0 : CMD_INV_IOMMU_PAGES_PDE_MASK); spin_unlock_irqrestore(&dom->lock, flags); iommu_put_pages_list(&gather->freelist); } From db50fb87015b955a5a0c155293b2dd40d63a3b9e Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Sat, 30 May 2026 07:28:52 -0400 Subject: [PATCH 66/66] iommu/dma-iommu: Fix wrong scatterlist length assignment in P2PDMA path In iommu_dma_map_sg(), when handling PCI P2PDMA cases, the DMA length of the current scatterlist segment `s` is incorrectly assigned from the head entry `sg->length` instead of the current entry `s->length`. This typo causes all P2PDMA segments in the scatterlist to inherit the length of the first segment, leading to corrupted DMA lengths for multi- segment scatterlists. Fix this by using `s->length` instead of `sg->length`. Fixes: a25e7962db ("PCI/P2PDMA: Refactor the p2pdma mapping helpers") Signed-off-by: Li RongQing Reviewed-by: Logan Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 54d96e847f16..e8d4c2dac4da 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1465,7 +1465,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, */ s->dma_address = pci_p2pdma_bus_addr_map( p2pdma_state.mem, sg_phys(s)); - sg_dma_len(s) = sg->length; + sg_dma_len(s) = s->length; sg_dma_mark_bus_address(s); continue; default: