mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-22 02:17:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'iommu-updates-v7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux

Browse Source 
Pull iommu updates from Joerg Roedel:
 "Core Code:

   - Fix dma-iommu scatterlist length handling in the P2PDMA path

   - Extend the generic IOMMU page-table code with detailed gather
     support for more precise invalidations

   - Add pending-gather tracking to generic page-table invalidation
     handling

   - Add support for smaller virtual address sizes in the generic AMDv1
     page-table format, including KUnit coverage

   - Fix page-size bitmap calculation for smaller VA configurations

   - Rework Arm io-pgtable allocation/freeing to consistently use the
     iommu-pages API and address-conversion helpers

   - Add PCI ATS infrastructure for devices that require ATS, including
     always-on ATS handling for pre-CXL devices

  AMD IOMMU:

   - Fix several IOTLB invalidation details, including PDE handling,
     flush-all behavior, and command address encoding

   - Honor IVINFO[VASIZE] when deriving address limits

   - Fix premature loop termination in init_iommu_one()

   - Add Hygon family 18h model 4h IOAPIC support

   - Clean up legacy-mode handling, stale comments, dead IVMD
     exclusion-range code, and unused address-size macros

  Arm SMMU / Arm SMMU v3:

   - SMMUv2:
      - Device-tree binding updates for Qualcomm Hawi, Nord and Shikra
        SoCs
      - Constrain the clocks which can be specified for recent Qualcomm
        SoCs
      - Fix broken compatible string for Qualcomm prefetcher
        configuration an add new entry for the Glymur MDSS
      - Ensure SMMU is powered-up when writing context bank for Adreno
        client

   - SMMUv3:
      - Fix off-by-one in queue allocation retry loop
      - Enable hardware update of access/dirty bits from the SMMU
      - Re-jig command construction to use separate inline helpers for
        each command type

  Intel VT-d:

   - Add the PCI segment number to DMA fault messages

   - Improve support for non-PRI mode SVA

   - Ensure atomicity during context entry teardown

   - Fix RB-tree corruption in the probe error path

  RISC-V IOMMU:

   - Add NAPOT range invalidation support

   - Use detailed gather information for invalidation decisions

   - Compute the best stride for single invalidations

   - Advertise Svpbmt support to the generic page-table code

   - Add capability definitions and clean up command macro encoding

  VeriSilicon IOMMU:

   - Add a new VeriSilicon IOMMU driver

   - Add devicetree binding documentation and MAINTAINERS coverage

   - Add the RK3588 VeriSilicon IOMMU node

   - Apply small cleanups and warning fixes in the new driver

  Rockchip IOMMU:

   - Disable the fetch DTE time limit

  Apple DART:

   - Correct a stale CONFIG_PCIE_APPLE macro name in a comment"

* tag 'iommu-updates-v7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux: (66 commits)
  iommu/dma-iommu: Fix wrong scatterlist length assignment in P2PDMA path
  iommu/amd: Control INVALIDATE_IOMMU_PAGES PDE from the gather
  iommu/amd: Make CMD_INV_IOMMU_ALL_PAGES_ADDRESS match the spec
  iommu/amd: Have amd_iommu_domain_flush_pages() use last
  iommu/amd: Pass last in through to build_inv_address()
  iommu/amd: Simplify build_inv_address()
  iommu/apple-dart: correct CONFIG_PCIE_APPLE macro name in comment
  iommu/vt-d: Fix RB-tree corruption in probe error path
  iommu/vt-d: Improve IOMMU fault information
  iommu/vt-d: Remove typo from pasid_pte_config_nested()
  iommu/vt-d: Clear Present bit before tearing down scalable-mode context entry
  iommu/vt-d: Avoid WARNING in sva unbind path
  dt-bindings: arm-smmu: Correct and add constraints for Hawi, Shikra and Kaanapali
  dt-bindings: arm-smmu: Add compatible for Qualcomm Nord SoC
  iommu/amd: Don't split flush for amd_iommu_domain_flush_all()
  iommu/rockchip: disable fetch dte time limit
  iommu/arm-smmu-v3: Allow ATS to be always on
  PCI: Allow ATS to be always on for pre-CXL devices
  PCI: Add pci_ats_required() for CXL.cache capable devices
  iommu/vsi: Use list_for_each_entry()
  ...
This commit is contained in:
Linus Torvalds
2026-06-17 12:24:50 -07:00
parent 4cc14386e3 dd8a3c6cd5
commit d076a8d3b9
42 changed files with 2055 additions and 766 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
Documentation/devicetree/bindings/iommu/arm,smmu.yaml
View File

@@ -37,8 +37,10 @@ properties:
- enum:
- qcom,eliza-smmu-500
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
- qcom,milos-smmu-500
- qcom,nord-smmu-500
- qcom,qcm2290-smmu-500
- qcom,qcs615-smmu-500
- qcom,qcs8300-smmu-500
@@ -55,6 +57,7 @@ properties:
- qcom,sdx55-smmu-500
- qcom,sdx65-smmu-500
- qcom,sdx75-smmu-500
- qcom,shikra-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6125-smmu-500
- qcom,sm6350-smmu-500
@@ -566,7 +569,11 @@ allOf:
properties:
compatible:
items:
- const: qcom,sm8750-smmu-500
- enum:
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
- qcom,sm8750-smmu-500
- const: qcom,adreno-smmu
- const: qcom,smmu-500
- const: arm,mmu-500
@@ -595,6 +602,7 @@ allOf:
- qcom,sdm845-smmu-500
- qcom,sdx55-smmu-500
- qcom,sdx65-smmu-500
- qcom,sdx75-smmu-500
- qcom,sm6350-smmu-500
- qcom,sm6375-smmu-500
then:
@@ -602,6 +610,40 @@ allOf:
clock-names: false
clocks: false
# Disallow clocks for all other platforms where specific compatible is used
# with different fallbacks and only one combination has no clocks
- if:
properties:
compatible:
items:
- enum:
- qcom,eliza-smmu-500
- qcom,glymur-smmu-500
- qcom,hawi-smmu-500
- qcom,kaanapali-smmu-500
- qcom,milos-smmu-500
- qcom,nord-smmu-500
- qcom,qcs615-smmu-500
- qcom,qcs8300-smmu-500
- qcom,sa8775p-smmu-500
- qcom,shikra-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6125-smmu-500
- qcom,sm8150-smmu-500
- qcom,sm8250-smmu-500
- qcom,sm8350-smmu-500
- qcom,sm8450-smmu-500
- qcom,sm8550-smmu-500
- qcom,sm8650-smmu-500
- qcom,sm8750-smmu-500
- qcom,x1e80100-smmu-500
- const: qcom,smmu-500
- const: arm,mmu-500
then:
properties:
clock-names: false
clocks: false
- if:
properties:
compatible:

71
Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml Normal file
View File

@@ -0,0 +1,71 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/iommu/verisilicon,iommu.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Verisilicon IOMMU
maintainers:
- Benjamin Gaignard <benjamin.gaignard@collabora.com>
description: |+
A Versilicon iommu translates io virtual addresses to physical addresses for
its associated video decoder.
properties:
compatible:
items:
- const: rockchip,rk3588-av1-iommu
- const: verisilicon,iommu-1.2
reg:
maxItems: 1
interrupts:
maxItems: 1
clocks:
items:
- description: Core clock
- description: Interface clock
clock-names:
items:
- const: core
- const: iface
"#iommu-cells":
const: 0
power-domains:
maxItems: 1
required:
- compatible
- reg
- interrupts
- clocks
- clock-names
- "#iommu-cells"
additionalProperties: false
examples:
- |
#include <dt-bindings/clock/rockchip,rk3588-cru.h>
#include <dt-bindings/interrupt-controller/arm-gic.h>
bus {
#address-cells = <2>;
#size-cells = <2>;
iommu@fdca0000 {
compatible = "rockchip,rk3588-av1-iommu","verisilicon,iommu-1.2";
reg = <0x0 0xfdca0000 0x0 0x600>;
interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>;
clocks = <&cru ACLK_AV1>, <&cru PCLK_AV1>;
clock-names = "core", "iface";
#iommu-cells = <0>;
};
};

8
MAINTAINERS
View File

@@ -28093,6 +28093,14 @@ F: drivers/media/v4l2-core/v4l2-isp.c
F: include/media/v4l2-isp.h
F: include/uapi/linux/media/v4l2-isp.h
VERISILICON IOMMU DRIVER
M: Benjamin Gaignard <benjamin.gaignard@collabora.com>
L: iommu@lists.linux.dev
S: Maintained
F: Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml
F: drivers/iommu/vsi-iommu.c
F: include/linux/vsi-iommu.h
VF610 NAND DRIVER
M: Stefan Agner <stefan@agner.ch>
L: linux-mtd@lists.infradead.org

11
arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
View File

@@ -1472,6 +1472,17 @@ av1d: video-codec@fdc70000 {
clock-names = "aclk", "hclk";
power-domains = <&power RK3588_PD_AV1>;
resets = <&cru SRST_A_AV1>, <&cru SRST_P_AV1>, <&cru SRST_A_AV1_BIU>, <&cru SRST_P_AV1_BIU>;
iommus = <&av1d_mmu>;
};
av1d_mmu: iommu@fdca0000 {
compatible = "rockchip,rk3588-av1-iommu", "verisilicon,iommu-1.2";
reg = <0x0 0xfdca0000 0x0 0x600>;
interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>;
clocks = <&cru ACLK_AV1>, <&cru PCLK_AV1>;
clock-names = "core", "iface";
#iommu-cells = <0>;
power-domains = <&power RK3588_PD_AV1>;
};
vicap: video-capture@fdce0000 {

11
drivers/iommu/Kconfig
View File

@@ -384,6 +384,17 @@ config SPRD_IOMMU
Say Y here if you want to use the multimedia devices listed above.
config VSI_IOMMU
tristate "Verisilicon IOMMU Support"
depends on (ARCH_ROCKCHIP && ARM64) || COMPILE_TEST
select IOMMU_API
help
Support for IOMMUs used by Verisilicon sub-systems like video
decoders or encoder hardware blocks.
Say Y here if you want to use this IOMMU in front of these
hardware blocks.
config IOMMU_DEBUG_PAGEALLOC
bool "Debug IOMMU mappings against page allocations"
depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION

1
drivers/iommu/Makefile
View File

@@ -36,4 +36,5 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o
obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o
obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
obj-$(CONFIG_APPLE_DART) += apple-dart.o
obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o
obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o

6
drivers/iommu/amd/amd_iommu.h
View File

@@ -44,7 +44,7 @@ int amd_iommu_enable_faulting(unsigned int cpu);
extern int amd_iommu_guest_ir;
extern enum protection_domain_mode amd_iommu_pgtable;
extern int amd_iommu_gpt_level;
extern u8 amd_iommu_hpt_level;
extern u8 amd_iommu_hpt_vasize;
extern unsigned long amd_iommu_pgsize_bitmap;
extern bool amd_iommu_hatdis;
@@ -91,9 +91,9 @@ int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag);
*/
void amd_iommu_flush_all_caches(struct amd_iommu *iommu);
void amd_iommu_domain_flush_pages(struct protection_domain *domain,
u64 address, size_t size);
u64 address, u64 last, u32 flags);
void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
ioasid_t pasid, u64 address, size_t size);
ioasid_t pasid, u64 address, u64 last);
#ifdef CONFIG_IRQ_REMAP
int amd_iommu_create_irq_domain(struct amd_iommu *iommu);

29
drivers/iommu/amd/amd_iommu_types.h
View File

@@ -51,10 +51,6 @@
#define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
#define MMIO_MSI_NUM(x) ((x) & 0x1f)
/* Flag masks for the AMD IOMMU exclusion range */
#define MMIO_EXCL_ENABLE_MASK 0x01ULL
#define MMIO_EXCL_ALLOW_MASK 0x02ULL
/* Used offsets into the MMIO space */
#define MMIO_DEV_TABLE_OFFSET 0x0000
#define MMIO_CMD_BUF_OFFSET 0x0008
@@ -221,7 +217,7 @@
#define PPR_STATUS_MASK 0xf
#define PPR_STATUS_SHIFT 12
#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL
#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7ffffffffffff000ULL
/* macros and definitions for device table entries */
#define DEV_ENTRY_VALID 0x00
@@ -231,7 +227,6 @@
#define DEV_ENTRY_IR 0x3d
#define DEV_ENTRY_IW 0x3e
#define DEV_ENTRY_NO_PAGE_FAULT 0x62
#define DEV_ENTRY_EX 0x67
#define DEV_ENTRY_SYSMGT1 0x68
#define DEV_ENTRY_SYSMGT2 0x69
#define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40)
@@ -305,9 +300,6 @@
#define GA_GUEST_NR 0x1
#define IOMMU_IN_ADDR_BIT_SIZE 52
#define IOMMU_OUT_ADDR_BIT_SIZE 52
/*
* This bitmap is used to advertise the page sizes our hardware support
* to the IOMMU core, which will then use this information to split
@@ -389,8 +381,6 @@
#define IOMMU_PROT_IR 0x01
#define IOMMU_PROT_IW 0x02
#define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE (1 << 2)
/* IOMMU capabilities */
#define IOMMU_CAP_IOTLB 24
#define IOMMU_CAP_NPCACHE 26
@@ -400,6 +390,7 @@
#define IOMMU_IVINFO_OFFSET 36
#define IOMMU_IVINFO_EFRSUP BIT(0)
#define IOMMU_IVINFO_DMA_REMAP BIT(1)
#define IOMMU_IVINFO_VASIZE GENMASK_ULL(21, 15)
/* IOMMU Feature Reporting Field (for IVHD type 10h */
#define IOMMU_FEAT_GASUP_SHIFT 6
@@ -685,11 +676,6 @@ struct amd_iommu {
/* pci domain of this IOMMU */
struct amd_iommu_pci_seg *pci_seg;
/* start of exclusion range of that IOMMU */
u64 exclusion_start;
/* length of exclusion range of that IOMMU */
u64 exclusion_length;
/* command buffer virtual address */
u8 *cmd_buf;
u32 cmd_buf_head;
@@ -948,12 +934,13 @@ static inline int get_hpet_devid(int id)
}
enum amd_iommu_intr_mode_type {
AMD_IOMMU_GUEST_IR_LEGACY,
/* This mode is not visible to users. It is used when
* we cannot fully enable vAPIC and fallback to only support
* legacy interrupt remapping via 128-bit IRTE.
/*
* The legacy format mode is not visible to users to prevent the user
* from crashing x2APIC systems, which for all intents and purposes
* require 128-bit IRTEs. The legacy format will be forced as needed
* when hardware doesn't support 128-bit IRTEs.
*/
AMD_IOMMU_GUEST_IR_LEGACY,
AMD_IOMMU_GUEST_IR_LEGACY_GA,
AMD_IOMMU_GUEST_IR_VAPIC,
};

63
drivers/iommu/amd/init.c
View File

@@ -155,8 +155,8 @@ bool amd_iommu_dump;
bool amd_iommu_irq_remap __read_mostly;
enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1;
/* Host page table level */
u8 amd_iommu_hpt_level;
/* Virtual address size */
u8 amd_iommu_hpt_vasize;
/* Guest page table level */
int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
@@ -355,28 +355,6 @@ static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
*
****************************************************************************/
/*
* This function set the exclusion range in the IOMMU. DMA accesses to the
* exclusion range are passed through untranslated
*/
static void iommu_set_exclusion_range(struct amd_iommu *iommu)
{
u64 start = iommu->exclusion_start & PAGE_MASK;
u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
u64 entry;
if (!iommu->exclusion_start)
return;
entry = start | MMIO_EXCL_ENABLE_MASK;
memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
&entry, sizeof(entry));
entry = limit;
memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
&entry, sizeof(entry));
}
static void iommu_set_cwwb_range(struct amd_iommu *iommu)
{
u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
@@ -972,8 +950,8 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
{
int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
return 0;
if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)))
return -EINVAL;
iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);
if (!iommu->ga_log)
@@ -1939,12 +1917,11 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
/* XT and GAM require GA mode. */
if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) {
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
break;
} else {
if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
}
if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) {
pr_warn_once("Host Address Translation is not supported.\n");
amd_iommu_hatdis = true;
@@ -2905,7 +2882,6 @@ static void early_enable_iommu(struct amd_iommu *iommu)
iommu_init_flags(iommu);
iommu_set_device_table(iommu);
iommu_enable_command_buffer(iommu);
iommu_set_exclusion_range(iommu);
iommu_enable_gt(iommu);
iommu_enable_ga(iommu);
iommu_enable_xt(iommu);
@@ -3022,8 +2998,10 @@ static void enable_iommus_vapic(void)
return;
}
if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
!check_feature(FEATURE_GAM_VAPIC)) {
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
return;
if (!check_feature(FEATURE_GAM_VAPIC)) {
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
return;
}
@@ -3110,6 +3088,9 @@ static void __init free_iommu_resources(void)
/* SB IOAPIC is always on this device in AMD systems */
#define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */
#define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0))
static bool __init check_ioapic_information(void)
{
const char *fw_bug = FW_BUG;
@@ -3135,7 +3116,12 @@ static bool __init check_ioapic_information(void)
pr_err("%s: IOAPIC[%d] not in IVRS table\n",
fw_bug, id);
ret = false;
} else if (devid == IOAPIC_SB_DEVID) {
} else if (devid == IOAPIC_SB_DEVID ||
(boot_cpu_data.x86_vendor == X86_VENDOR_HYGON &&
boot_cpu_data.x86 == 0x18 &&
boot_cpu_data.x86_model >= 0x4 &&
boot_cpu_data.x86_model <= 0xf &&
devid == IOAPIC_SB_DEVID_FAM18H_M4H)) {
has_sb_ioapic = true;
ret = true;
}
@@ -3202,7 +3188,7 @@ static int __init early_amd_iommu_init(void)
struct acpi_table_header *ivrs_base;
int ret;
acpi_status status;
u8 efr_hats;
u8 efr_hats, max_vasize;
if (!amd_iommu_detected)
return -ENODEV;
@@ -3232,6 +3218,10 @@ static int __init early_amd_iommu_init(void)
ivinfo_init(ivrs_base);
max_vasize = FIELD_GET(IOMMU_IVINFO_VASIZE, amd_iommu_ivinfo);
if (!max_vasize)
max_vasize = 64;
amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
@@ -3254,7 +3244,8 @@ static int __init early_amd_iommu_init(void)
* efr[HATS] bits specify the maximum host translation level
* supported, with LEVEL 4 being initial max level.
*/
amd_iommu_hpt_level = efr_hats + PAGE_MODE_4_LEVEL;
amd_iommu_hpt_vasize = min_t(unsigned int, max_vasize,
(efr_hats + PAGE_MODE_4_LEVEL - 1) * 9 + 21);
} else {
pr_warn_once(FW_BUG "Disable host address translation due to invalid translation level (%#x).\n",
efr_hats);

181
drivers/iommu/amd/iommu.c
View File

@@ -1270,69 +1270,63 @@ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
* Builds an invalidation address which is suitable for one page or multiple
* pages. Sets the size bit (S) as needed is more than one page is flushed.
*/
static inline u64 build_inv_address(u64 address, size_t size)
static inline u64 build_inv_address(u64 address, u64 last)
{
u64 pages, end, msb_diff;
unsigned int sz_lg2;
pages = iommu_num_pages(address, size, PAGE_SIZE);
if (pages == 1)
return address & PAGE_MASK;
end = address + size - 1;
address &= GENMASK_U64(63, 12);
sz_lg2 = fls64(address ^ last);
if (sz_lg2 <= 12)
return address;
/*
* msb_diff would hold the index of the most significant bit that
* flipped between the start and end.
* Encode sz_lg2 according to Table 14: Example Page Size Encodings
*
* See "Note *":
* Address bits 51:32 can be used to encode page sizes greater
* that 4 Gbytes.
* Which we take to mean that the highest page size has bit
* [51]=0, [50:12]=1
* and that coding happens when sz_lg2 is 52. Fall back to full
* invalidation if the size is too big.
*
*/
msb_diff = fls64(end ^ address) - 1;
if (unlikely(sz_lg2 > 52))
return CMD_INV_IOMMU_ALL_PAGES_ADDRESS |
CMD_INV_IOMMU_PAGES_SIZE_MASK;
/*
* Bits 63:52 are sign extended. If for some reason bit 51 is different
* between the start and the end, invalidate everything.
* The sz_lg2 calculation with fls() ensures that:
* address & BIT(sz_lg2 - 1) == 0
* Therefore only the 1's need to be added. 8KB requires no 1's
*/
if (unlikely(msb_diff > 51)) {
address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
} else {
/*
* The msb-bit must be clear on the address. Just set all the
* lower bits.
*/
address |= (1ull << msb_diff) - 1;
}
/* Clear bits 11:0 */
address &= PAGE_MASK;
/* Set the size bit - we flush more than one 4kb page */
if (sz_lg2 > 13)
address |= GENMASK_U64(sz_lg2 - 2, 12);
return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;
}
static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
size_t size, u16 domid,
ioasid_t pasid, bool gn)
u64 last, u16 domid, ioasid_t pasid,
u32 flags)
{
u64 inv_address = build_inv_address(address, size);
u64 inv_address = build_inv_address(address, last);
memset(cmd, 0, sizeof(*cmd));
cmd->data[1] |= domid;
cmd->data[2] = lower_32_bits(inv_address);
cmd->data[3] = upper_32_bits(inv_address);
/* PDE bit - we want to flush everything, not only the PTEs */
cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
if (gn) {
cmd->data[2] |= flags;
if (flags & CMD_INV_IOMMU_PAGES_GN_MASK)
cmd->data[0] |= pasid;
cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
}
CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
}
static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
u64 address, size_t size,
u64 address, u64 last,
ioasid_t pasid, bool gn)
{
u64 inv_address = build_inv_address(address, size);
u64 inv_address = build_inv_address(address, last);
memset(cmd, 0, sizeof(*cmd));
@@ -1530,8 +1524,9 @@ static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {
struct iommu_cmd cmd;
build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
dom_id, IOMMU_NO_PASID, false);
build_inv_iommu_pages(&cmd, 0, U64_MAX,
dom_id, IOMMU_NO_PASID,
CMD_INV_IOMMU_PAGES_PDE_MASK);
iommu_queue_command(iommu, &cmd);
}
@@ -1542,14 +1537,16 @@ static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)
{
struct iommu_cmd cmd;
build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
dom_id, IOMMU_NO_PASID, false);
build_inv_iommu_pages(&cmd, 0, U64_MAX,
dom_id, IOMMU_NO_PASID,
CMD_INV_IOMMU_PAGES_PDE_MASK);
iommu_queue_command(iommu, &cmd);
iommu_completion_wait(iommu);
}
static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 address, size_t size)
static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom,
u64 address, u64 last, u32 flags)
{
int ret = 0;
struct amd_iommu_viommu *aviommu;
@@ -1566,8 +1563,8 @@ static int iommu_flush_pages_v1_hdom_ids(struct protection_domain *pdom, u64 add
pr_debug("%s: iommu=%#x, hdom_id=%#x\n", __func__,
iommu->devid, gdom_info->hdom_id);
build_inv_iommu_pages(&cmd, address, size, gdom_info->hdom_id,
IOMMU_NO_PASID, false);
build_inv_iommu_pages(&cmd, address, last, gdom_info->hdom_id,
IOMMU_NO_PASID, flags);
ret |= iommu_queue_command(iommu, &cmd);
}
xa_unlock(&aviommu->gdomid_array);
@@ -1623,14 +1620,14 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu)
* Command send function for flushing on-device TLB
*/
static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address,
size_t size, ioasid_t pasid, bool gn)
u64 last, ioasid_t pasid, bool gn)
{
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
struct iommu_cmd cmd;
int qdep = dev_data->ats_qdep;
build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,
size, pasid, gn);
last, pasid, gn);
return iommu_queue_command(iommu, &cmd);
}
@@ -1674,7 +1671,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data)
if (dev_data->ats_enabled) {
/* Invalidate the entire contents of an IOTLB */
ret = device_flush_iotlb(dev_data, 0, ~0UL,
ret = device_flush_iotlb(dev_data, 0, U64_MAX,
IOMMU_NO_PASID, false);
}
@@ -1682,7 +1679,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data)
}
static int domain_flush_pages_v2(struct protection_domain *pdom,
u64 address, size_t size)
u64 address, u64 last, u32 flags)
{
struct iommu_dev_data *dev_data;
struct iommu_cmd cmd;
@@ -1693,8 +1690,9 @@ static int domain_flush_pages_v2(struct protection_domain *pdom,
struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
u16 domid = dev_data->gcr3_info.domid;
build_inv_iommu_pages(&cmd, address, size,
domid, IOMMU_NO_PASID, true);
build_inv_iommu_pages(&cmd, address, last, domid,
IOMMU_NO_PASID,
flags | CMD_INV_IOMMU_PAGES_GN_MASK);
ret |= iommu_queue_command(iommu, &cmd);
}
@@ -1703,7 +1701,7 @@ static int domain_flush_pages_v2(struct protection_domain *pdom,
}
static int domain_flush_pages_v1(struct protection_domain *pdom,
u64 address, size_t size)
u64 address, u64 last, u32 flags)
{
struct pdom_iommu_info *pdom_iommu_info;
struct iommu_cmd cmd;
@@ -1712,8 +1710,8 @@ static int domain_flush_pages_v1(struct protection_domain *pdom,
lockdep_assert_held(&pdom->lock);
build_inv_iommu_pages(&cmd, address, size,
pdom->id, IOMMU_NO_PASID, false);
build_inv_iommu_pages(&cmd, address, last,
pdom->id, IOMMU_NO_PASID, flags);
xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) {
/*
@@ -1732,7 +1730,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom,
* See drivers/iommu/amd/nested.c: amd_iommu_alloc_domain_nested()
*/
if (!list_empty(&pdom->viommu_list))
ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, size);
ret |= iommu_flush_pages_v1_hdom_ids(pdom, address, last, flags);
return ret;
}
@@ -1742,7 +1740,7 @@ static int domain_flush_pages_v1(struct protection_domain *pdom,
* It flushes range of PTEs of the domain.
*/
static void __domain_flush_pages(struct protection_domain *domain,
u64 address, size_t size)
u64 address, u64 last, u32 flags)
{
struct iommu_dev_data *dev_data;
int ret = 0;
@@ -1753,9 +1751,9 @@ static void __domain_flush_pages(struct protection_domain *domain,
if (pdom_is_v2_pgtbl_mode(domain)) {
gn = true;
ret = domain_flush_pages_v2(domain, address, size);
ret = domain_flush_pages_v2(domain, address, last, flags);
} else {
ret = domain_flush_pages_v1(domain, address, size);
ret = domain_flush_pages_v1(domain, address, last, flags);
}
list_for_each_entry(dev_data, &domain->dev_list, list) {
@@ -1763,19 +1761,20 @@ static void __domain_flush_pages(struct protection_domain *domain,
if (!dev_data->ats_enabled)
continue;
ret |= device_flush_iotlb(dev_data, address, size, pasid, gn);
ret |= device_flush_iotlb(dev_data, address, last, pasid, gn);
}
WARN_ON(ret);
}
void amd_iommu_domain_flush_pages(struct protection_domain *domain,
u64 address, size_t size)
u64 address, u64 last, u32 flags)
{
lockdep_assert_held(&domain->lock);
if (likely(!amd_iommu_np_cache)) {
__domain_flush_pages(domain, address, size);
if (likely(!amd_iommu_np_cache) ||
unlikely(address == 0 && last == U64_MAX)) {
__domain_flush_pages(domain, address, last, flags);
/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
domain_flush_complete(domain);
@@ -1793,28 +1792,17 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain,
* between the natural alignment of the address that we flush and the
* greatest naturally aligned region that fits in the range.
*/
while (size != 0) {
int addr_alignment = __ffs(address);
int size_alignment = __fls(size);
int min_alignment;
size_t flush_size;
while (address <= last) {
unsigned int sz_lg2 = ilog2(last - address + 1);
u64 flush_last;
/*
* size is always non-zero, but address might be zero, causing
* addr_alignment to be negative. As the casting of the
* argument in __ffs(address) to long might trim the high bits
* of the address on x86-32, cast to long when doing the check.
*/
if (likely((unsigned long)address != 0))
min_alignment = min(addr_alignment, size_alignment);
else
min_alignment = size_alignment;
if (likely(address))
sz_lg2 = min_t(unsigned int, sz_lg2, __ffs64(address));
flush_size = 1ul << min_alignment;
__domain_flush_pages(domain, address, flush_size);
address += flush_size;
size -= flush_size;
flush_last = address + (1ULL << sz_lg2) - 1;
__domain_flush_pages(domain, address, flush_last, flags);
if (check_add_overflow(flush_last, 1, &address))
break;
}
/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
@@ -1824,22 +1812,24 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain,
/* Flush the whole IO/TLB for a given protection domain - including PDE */
static void amd_iommu_domain_flush_all(struct protection_domain *domain)
{
amd_iommu_domain_flush_pages(domain, 0,
CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
amd_iommu_domain_flush_pages(domain, 0, U64_MAX,
CMD_INV_IOMMU_PAGES_PDE_MASK);
}
void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
ioasid_t pasid, u64 address, size_t size)
ioasid_t pasid, u64 address, u64 last)
{
struct iommu_cmd cmd;
struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
build_inv_iommu_pages(&cmd, address, size,
dev_data->gcr3_info.domid, pasid, true);
build_inv_iommu_pages(&cmd, address, last,
dev_data->gcr3_info.domid, pasid,
CMD_INV_IOMMU_PAGES_GN_MASK |
CMD_INV_IOMMU_PAGES_PDE_MASK);
iommu_queue_command(iommu, &cmd);
if (dev_data->ats_enabled)
device_flush_iotlb(dev_data, address, size, pasid, true);
device_flush_iotlb(dev_data, address, last, pasid, true);
iommu_completion_wait(iommu);
}
@@ -1847,8 +1837,7 @@ void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
static void dev_flush_pasid_all(struct iommu_dev_data *dev_data,
ioasid_t pasid)
{
amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0,
CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, U64_MAX);
}
int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
@@ -2627,7 +2616,8 @@ static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
return 0;
spin_lock_irqsave(&domain->lock, flags);
amd_iommu_domain_flush_pages(domain, iova, size);
amd_iommu_domain_flush_pages(domain, iova, iova + size - 1,
CMD_INV_IOMMU_PAGES_PDE_MASK);
spin_unlock_irqrestore(&domain->lock, flags);
return 0;
}
@@ -2649,8 +2639,9 @@ static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
unsigned long flags;
spin_lock_irqsave(&dom->lock, flags);
amd_iommu_domain_flush_pages(dom, gather->start,
gather->end - gather->start + 1);
amd_iommu_domain_flush_pages(dom, gather->start, gather->end,
iommu_pages_list_empty(&gather->freelist) ?
0 : CMD_INV_IOMMU_PAGES_PDE_MASK);
spin_unlock_irqrestore(&dom->lock, flags);
iommu_put_pages_list(&gather->freelist);
}
@@ -2718,8 +2709,7 @@ static struct iommu_domain *amd_iommu_domain_alloc_paging_v1(struct device *dev,
else
cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE);
cfg.common.hw_max_vasz_lg2 =
min(64, (amd_iommu_hpt_level - 1) * 9 + 21);
cfg.common.hw_max_vasz_lg2 = amd_iommu_hpt_vasize;
cfg.common.hw_max_oasz_lg2 = 52;
cfg.starting_level = 2;
domain->domain.ops = &amdv1_ops;
@@ -3086,9 +3076,6 @@ static void amd_iommu_get_resv_regions(struct device *dev,
prot |= IOMMU_READ;
if (entry->prot & IOMMU_PROT_IW)
prot |= IOMMU_WRITE;
if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)
/* Exclusion range */
type = IOMMU_RESV_RESERVED;
region = iommu_alloc_resv_region(entry->address_start,
length, prot, type,

2
drivers/iommu/amd/pasid.c
View File

@@ -71,7 +71,7 @@ static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
for_each_pdom_dev_data(pdom_dev_data, sva_pdom) {
amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data,
pdom_dev_data->pasid,
start, end - start);
start, end - 1);
}
spin_unlock_irqrestore(&sva_pdom->lock, flags);

2
drivers/iommu/apple-dart.c
View File

@@ -976,7 +976,7 @@ static int apple_dart_def_domain_type(struct device *dev)
}
#ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
/* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
/* Keep things compiling when CONFIG_PCIE_APPLE isn't selected */
#define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0
#endif
#define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)

24
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
View File

@@ -300,7 +300,7 @@ static int arm_vsmmu_vsid_to_sid(struct arm_vsmmu *vsmmu, u32 vsid, u32 *sid)
/* This is basically iommu_viommu_arm_smmuv3_invalidate in u64 for conversion */
struct arm_vsmmu_invalidation_cmd {
union {
u64 cmd[2];
struct arm_smmu_cmd cmd;
struct iommu_viommu_arm_smmuv3_invalidate ucmd;
};
};
@@ -316,32 +316,32 @@ static int arm_vsmmu_convert_user_cmd(struct arm_vsmmu *vsmmu,
struct arm_vsmmu_invalidation_cmd *cmd)
{
/* Commands are le64 stored in u64 */
cmd->cmd[0] = le64_to_cpu(cmd->ucmd.cmd[0]);
cmd->cmd[1] = le64_to_cpu(cmd->ucmd.cmd[1]);
cmd->cmd.data[0] = le64_to_cpu(cmd->ucmd.cmd[0]);
cmd->cmd.data[1] = le64_to_cpu(cmd->ucmd.cmd[1]);
switch (cmd->cmd[0] & CMDQ_0_OP) {
switch (cmd->cmd.data[0] & CMDQ_0_OP) {
case CMDQ_OP_TLBI_NSNH_ALL:
/* Convert to NH_ALL */
cmd->cmd[0] = CMDQ_OP_TLBI_NH_ALL |
cmd->cmd.data[0] = CMDQ_OP_TLBI_NH_ALL |
FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
cmd->cmd[1] = 0;
cmd->cmd.data[1] = 0;
break;
case CMDQ_OP_TLBI_NH_VA:
case CMDQ_OP_TLBI_NH_VAA:
case CMDQ_OP_TLBI_NH_ALL:
case CMDQ_OP_TLBI_NH_ASID:
cmd->cmd[0] &= ~CMDQ_TLBI_0_VMID;
cmd->cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
cmd->cmd.data[0] &= ~CMDQ_TLBI_0_VMID;
cmd->cmd.data[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
break;
case CMDQ_OP_ATC_INV:
case CMDQ_OP_CFGI_CD:
case CMDQ_OP_CFGI_CD_ALL: {
u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd[0]);
u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd.data[0]);
if (arm_vsmmu_vsid_to_sid(vsmmu, vsid, &sid))
return -EIO;
cmd->cmd[0] &= ~CMDQ_CFGI_0_SID;
cmd->cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
cmd->cmd.data[0] &= ~CMDQ_CFGI_0_SID;
cmd->cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
break;
}
default:
@@ -386,7 +386,7 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
continue;
/* FIXME always uses the main cmdq rather than trying to group by type */
ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, last->cmd,
ret = arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &last->cmd,
cur - last, true);
if (ret) {
cur--;

10
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
View File

@@ -92,6 +92,16 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) &
CTXDESC_CD_1_TTB0_MASK);
/*
* Enable Hardware Access and Dirty updates (DBM) if supported.
* This is safe to enable by default, as PTE_WRITE and PTE_DBM
* share the same bit.
*/
if (master->smmu->features & ARM_SMMU_FEAT_HA)
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA);
if (master->smmu->features & ARM_SMMU_FEAT_HD)
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD);
} else {
target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0);

558
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
View File

@@ -268,115 +268,13 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
}
/* High-level queue accessors */
static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
{
memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
switch (ent->opcode) {
case CMDQ_OP_TLBI_EL2_ALL:
case CMDQ_OP_TLBI_NSNH_ALL:
break;
case CMDQ_OP_PREFETCH_CFG:
cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
break;
case CMDQ_OP_CFGI_CD:
cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
fallthrough;
case CMDQ_OP_CFGI_STE:
cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
break;
case CMDQ_OP_CFGI_CD_ALL:
cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
break;
case CMDQ_OP_CFGI_ALL:
/* Cover the entire SID range */
cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
break;
case CMDQ_OP_TLBI_NH_VA:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
fallthrough;
case CMDQ_OP_TLBI_EL2_VA:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
break;
case CMDQ_OP_TLBI_S2_IPA:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
break;
case CMDQ_OP_TLBI_NH_ASID:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
fallthrough;
case CMDQ_OP_TLBI_NH_ALL:
case CMDQ_OP_TLBI_S12_VMALL:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
break;
case CMDQ_OP_TLBI_EL2_ASID:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
break;
case CMDQ_OP_ATC_INV:
cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
break;
case CMDQ_OP_PRI_RESP:
cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
switch (ent->pri.resp) {
case PRI_RESP_DENY:
case PRI_RESP_FAIL:
case PRI_RESP_SUCC:
break;
default:
return -EINVAL;
}
cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
break;
case CMDQ_OP_RESUME:
cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
break;
case CMDQ_OP_CMD_SYNC:
if (ent->sync.msiaddr) {
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
} else {
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
}
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
break;
default:
return -ENOENT;
}
return 0;
}
static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_ent *ent)
struct arm_smmu_cmd *cmd)
{
struct arm_smmu_cmdq *cmdq = NULL;
if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, cmd);
return cmdq ?: &smmu->cmdq;
}
@@ -390,26 +288,29 @@ static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
}
static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
static void arm_smmu_cmdq_build_sync_cmd(struct arm_smmu_cmd *cmd,
struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq, u32 prod)
{
struct arm_smmu_queue *q = &cmdq->q;
struct arm_smmu_cmdq_ent ent = {
.opcode = CMDQ_OP_CMD_SYNC,
};
u64 msiaddr = 0;
unsigned int cs;
/*
* Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
* payload, so the write will zero the entire command on that platform.
*/
if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
q->ent_dwords * 8;
if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) {
cs = CMDQ_SYNC_0_CS_NONE;
} else if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
cs = CMDQ_SYNC_0_CS_IRQ;
msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
q->ent_dwords * 8;
} else {
cs = CMDQ_SYNC_0_CS_SEV;
}
arm_smmu_cmdq_build_cmd(cmd, &ent);
if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
*cmd = arm_smmu_make_cmd_sync(cs, msiaddr);
}
void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
@@ -422,14 +323,10 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
[CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
};
struct arm_smmu_queue *q = &cmdq->q;
int i;
u64 cmd[CMDQ_ENT_DWORDS];
struct arm_smmu_cmd cmd;
u32 cons = readl_relaxed(q->cons_reg);
u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
struct arm_smmu_cmdq_ent cmd_sync = {
.opcode = CMDQ_OP_CMD_SYNC,
};
dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
@@ -457,17 +354,18 @@ void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
* We may have concurrent producers, so we need to be careful
* not to touch any of the shadow cmdq state.
*/
queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
queue_read(cmd.data, Q_ENT(q, cons), q->ent_dwords);
dev_err(smmu->dev, "skipping command in error state:\n");
for (i = 0; i < ARRAY_SIZE(cmd); ++i)
dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
for (i = 0; i < ARRAY_SIZE(cmd.data); ++i)
dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd.data[i]);
/* Convert the erroneous command into a CMD_SYNC */
arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
cmd = arm_smmu_make_cmd_sync(
arm_smmu_cmdq_needs_busy_polling(smmu, cmdq) ?
CMDQ_SYNC_0_CS_NONE : CMDQ_SYNC_0_CS_SEV,
0);
queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
queue_write(Q_ENT(q, cons), cmd.data, q->ent_dwords);
}
static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
@@ -767,7 +665,8 @@ static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
}
static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds,
u32 prod, int n)
{
int i;
@@ -777,10 +676,9 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
};
for (i = 0; i < n; ++i) {
u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
prod = queue_inc_prod_n(&llq, i);
queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
queue_write(Q_ENT(&cmdq->q, prod), cmds[i].data,
ARRAY_SIZE(cmds[i].data));
}
}
@@ -801,10 +699,11 @@ static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
* CPU will appear before any of the commands from the other CPU.
*/
int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync)
{
u64 cmd_sync[CMDQ_ENT_DWORDS];
struct arm_smmu_cmd cmd_sync;
u32 prod;
unsigned long flags;
bool owner;
@@ -847,8 +746,9 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
if (sync) {
prod = queue_inc_prod_n(&llq, n);
arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
arm_smmu_cmdq_build_sync_cmd(&cmd_sync, smmu, cmdq, prod);
queue_write(Q_ENT(&cmdq->q, prod), cmd_sync.data,
ARRAY_SIZE(cmd_sync.data));
/*
* In order to determine completion of our CMD_SYNC, we must
@@ -920,73 +820,63 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
return ret;
}
static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_ent *ent,
bool sync)
static int arm_smmu_cmdq_issue_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd, bool sync)
{
u64 cmd[CMDQ_ENT_DWORDS];
if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
ent->opcode);
return -EINVAL;
}
return arm_smmu_cmdq_issue_cmdlist(
smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
smmu, arm_smmu_get_cmdq(smmu, cmd), cmd, 1, sync);
}
static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_ent *ent)
{
return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
}
#define arm_smmu_cmdq_issue_cmd(smmu, cmd) \
({ \
struct arm_smmu_cmd __cmd = cmd; \
arm_smmu_cmdq_issue_cmd_p(smmu, &__cmd, false); \
})
static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_ent *ent)
{
return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
}
#define arm_smmu_cmdq_issue_cmd_with_sync(smmu, cmd) \
({ \
struct arm_smmu_cmd __cmd = cmd; \
arm_smmu_cmdq_issue_cmd_p(smmu, &__cmd, true); \
})
static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *ent)
static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd)
{
cmds->num = 0;
cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
cmds->cmdq = arm_smmu_get_cmdq(smmu, cmd);
}
static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *cmd)
static void arm_smmu_cmdq_batch_add_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd)
{
bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
int index;
bool unsupported_cmd;
unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
if (force_sync || unsupported_cmd) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, true);
arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd);
}
if (cmds->num == CMDQ_BATCH_ENTRIES) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, false);
arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd);
}
index = cmds->num * CMDQ_ENT_DWORDS;
if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
cmd->opcode);
return;
}
cmds->num++;
cmds->cmds[cmds->num++] = *cmd;
}
#define arm_smmu_cmdq_batch_add_cmd(smmu, cmds, cmd) \
({ \
struct arm_smmu_cmd __cmd = cmd; \
arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &__cmd); \
})
static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds)
{
@@ -997,29 +887,29 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
struct iommu_page_response *resp)
{
struct arm_smmu_cmdq_ent cmd = {0};
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
int sid = master->streams[0].id;
u8 resume_resp;
if (WARN_ON(!master->stall_enabled))
return;
cmd.opcode = CMDQ_OP_RESUME;
cmd.resume.sid = sid;
cmd.resume.stag = resp->grpid;
switch (resp->code) {
case IOMMU_PAGE_RESP_INVALID:
case IOMMU_PAGE_RESP_FAILURE:
cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
resume_resp = CMDQ_RESUME_0_RESP_ABORT;
break;
case IOMMU_PAGE_RESP_SUCCESS:
cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
resume_resp = CMDQ_RESUME_0_RESP_RETRY;
break;
default:
resume_resp = CMDQ_RESUME_0_RESP_TERM;
break;
}
arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
arm_smmu_cmdq_issue_cmd(master->smmu,
arm_smmu_make_cmd_resume(master->streams[0].id,
resp->grpid,
resume_resp));
/*
* Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
* RESUME consumption guarantees that the stalled transaction will be
@@ -1543,19 +1433,14 @@ static void arm_smmu_sync_cd(struct arm_smmu_master *master,
size_t i;
struct arm_smmu_cmdq_batch cmds;
struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_cmdq_ent cmd = {
.opcode = CMDQ_OP_CFGI_CD,
.cfgi = {
.ssid = ssid,
.leaf = leaf,
},
};
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_cfgi_cd(0, ssid, leaf);
arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
for (i = 0; i < master->num_streams; i++) {
cmd.cfgi.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
}
arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd);
for (i = 0; i < master->num_streams; i++)
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_make_cmd_cfgi_cd(master->streams[i].id, ssid,
leaf));
arm_smmu_cmdq_batch_submit(smmu, &cmds);
}
@@ -1742,8 +1627,11 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
if (!arm_smmu_cdtab_allocated(&master->cd_table))
return;
cdptr = arm_smmu_get_cd_ptr(master, ssid);
if (WARN_ON(!cdptr))
if (!cdptr) {
/* Only ats_always_on allows a NULL CD on default substream */
WARN_ON(!master->ats_always_on || ssid);
return;
}
arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
}
@@ -1756,6 +1644,22 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
cd_table->s1cdmax = master->ssid_bits;
/*
* When a device doesn't support PASID (non default SSID), ssid_bits is
* set to 0. This also sets S1CDMAX to 0, which disables the substreams
* and ignores the S1DSS field.
*
* On the other hand, if a device demands ATS to be always on even when
* its default substream is IOMMU bypassed, it has to use EATS that is
* only effective with an STE (CFG=S1translate, S1DSS=Bypass). For such
* use cases, S1CDMAX has to be !0, in order to make use of S1DSS/EATS.
*
* Set S1CDMAX no lower than 1. This would add a dummy substream in the
* CD table but it should never be used by an actual CD.
*/
if (master->ats_always_on)
cd_table->s1cdmax = max_t(u8, cd_table->s1cdmax, 1);
max_contexts = 1 << cd_table->s1cdmax;
if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
@@ -1848,15 +1752,10 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
{
struct arm_smmu_ste_writer *ste_writer =
container_of(writer, struct arm_smmu_ste_writer, writer);
struct arm_smmu_cmdq_ent cmd = {
.opcode = CMDQ_OP_CFGI_STE,
.cfgi = {
.sid = ste_writer->sid,
.leaf = true,
},
};
arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(
writer->master->smmu,
arm_smmu_make_cmd_cfgi_ste(ste_writer->sid, true));
}
static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
@@ -1881,15 +1780,9 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
/* It's likely that we'll want to use the new STE soon */
if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
struct arm_smmu_cmdq_ent
prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
.prefetch = {
.sid = sid,
} };
arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
}
if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
arm_smmu_cmdq_issue_cmd(smmu,
arm_smmu_make_cmd_prefetch_cfg(sid));
}
void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
@@ -2314,20 +2207,10 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
evt[1] & PRIQ_1_ADDR_MASK);
if (last) {
struct arm_smmu_cmdq_ent cmd = {
.opcode = CMDQ_OP_PRI_RESP,
.substream_valid = ssv,
.pri = {
.sid = sid,
.ssid = ssid,
.grpid = grpid,
.resp = PRI_RESP_DENY,
},
};
arm_smmu_cmdq_issue_cmd(smmu, &cmd);
}
if (last)
arm_smmu_cmdq_issue_cmd(
smmu, arm_smmu_make_cmd_pri_resp(sid, ssid, ssv, grpid,
PRI_RESP_DENY));
}
static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
@@ -2415,9 +2298,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
return IRQ_WAKE_THREAD;
}
static void
arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
struct arm_smmu_cmdq_ent *cmd)
static struct arm_smmu_cmd
arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size)
{
size_t log2_span;
size_t span_mask;
@@ -2439,17 +2321,6 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
* This has the unpleasant side-effect of invalidating all PASID-tagged
* ATC entries within the address range.
*/
*cmd = (struct arm_smmu_cmdq_ent) {
.opcode = CMDQ_OP_ATC_INV,
.substream_valid = (ssid != IOMMU_NO_PASID),
.atc.ssid = ssid,
};
if (!size) {
cmd->atc.size = ATC_INV_SIZE_ALL;
return;
}
page_start = iova >> inval_grain_shift;
page_end = (iova + size - 1) >> inval_grain_shift;
@@ -2478,24 +2349,25 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
page_start &= ~span_mask;
cmd->atc.addr = page_start << inval_grain_shift;
cmd->atc.size = log2_span;
return arm_smmu_make_cmd_atc_inv(sid, ssid,
page_start << inval_grain_shift,
log2_span);
}
static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
ioasid_t ssid)
{
int i;
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_cmd cmd;
struct arm_smmu_cmdq_batch cmds;
arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
}
cmd = arm_smmu_make_cmd_atc_inv_all(0, IOMMU_NO_PASID);
arm_smmu_cmdq_batch_init_cmd(master->smmu, &cmds, &cmd);
for (i = 0; i < master->num_streams; i++)
arm_smmu_cmdq_batch_add_cmd(
master->smmu, &cmds,
arm_smmu_make_cmd_atc_inv_all(master->streams[i].id,
ssid));
return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
}
@@ -2525,12 +2397,14 @@ static void arm_smmu_tlb_inv_context(void *cookie)
static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *cmd,
struct arm_smmu_cmd *cmd, bool leaf,
unsigned long iova, size_t size,
size_t granule, size_t pgsize)
{
unsigned long end = iova + size, num_pages = 0, tg = pgsize;
u64 orig_data0 = cmd->data[0];
size_t inv_range = granule;
u8 ttl = 0, tg_enc = 0;
if (WARN_ON_ONCE(!size))
return;
@@ -2539,7 +2413,7 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
num_pages = size >> tg;
/* Convert page size of 12,14,16 (log2) to 1,2,3 */
cmd->tlbi.tg = (tg - 10) / 2;
tg_enc = (tg - 10) / 2;
/*
* Determine what level the granule is at. For non-leaf, both
@@ -2549,8 +2423,8 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
* want to use a range command, so avoid the SVA corner case
* where both scale and num could be 0 as well.
*/
if (cmd->tlbi.leaf)
cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
if (leaf)
ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
num_pages++;
}
@@ -2568,11 +2442,13 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
/* Determine the power of 2 multiple number of pages */
scale = __ffs(num_pages);
cmd->tlbi.scale = scale;
/* Determine how many chunks of 2^scale size we have */
num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
cmd->tlbi.num = num - 1;
cmd->data[0] = orig_data0 |
FIELD_PREP(CMDQ_TLBI_0_NUM, num - 1) |
FIELD_PREP(CMDQ_TLBI_0_SCALE, scale);
/* range is num * 2^scale * pgsize */
inv_range = num << (scale + tg);
@@ -2581,8 +2457,17 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
num_pages -= num << scale;
}
cmd->tlbi.addr = iova;
arm_smmu_cmdq_batch_add(smmu, cmds, cmd);
/*
* IPA has fewer bits than VA, but they are reserved in the
* command and something would be very broken if iova had them
* set.
*/
cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) |
FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) |
FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) |
(iova & ~GENMASK_U64(11, 0));
arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd);
iova += inv_range;
}
}
@@ -2613,19 +2498,22 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
/* Used by non INV_TYPE_ATS* invalidations */
static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *cmd,
struct arm_smmu_cmd *cmd,
bool leaf,
unsigned long iova, size_t size,
unsigned int granule)
{
if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) {
cmd->opcode = inv->nsize_opcode;
arm_smmu_cmdq_batch_add(inv->smmu, cmds, cmd);
struct arm_smmu_cmd nsize_cmd = *cmd;
u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode,
CMDQ_0_OP);
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd);
return;
}
cmd->opcode = inv->size_opcode;
arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, iova, size, granule,
inv->pgsize);
arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf,
iova, size, granule, inv->pgsize);
}
static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
@@ -2660,48 +2548,51 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
break;
while (cur != end) {
struct arm_smmu_device *smmu = cur->smmu;
struct arm_smmu_cmdq_ent cmd = {
/*
* Pick size_opcode to run arm_smmu_get_cmdq(). This can
* be changed to nsize_opcode, which would result in the
* same CMDQ pointer.
*/
.opcode = cur->size_opcode,
};
/*
* Pick size_opcode to run arm_smmu_get_cmdq(). This can
* be changed to nsize_opcode, which would result in the
* same CMDQ pointer.
*/
struct arm_smmu_cmd cmd =
arm_smmu_make_cmd_op(cur->size_opcode);
struct arm_smmu_inv *next;
if (!cmds.num)
arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd);
switch (cur->type) {
case INV_TYPE_S1_ASID:
cmd.tlbi.asid = cur->id;
cmd.tlbi.leaf = leaf;
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size,
granule);
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
cur->id, 0);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf,
iova, size, granule);
break;
case INV_TYPE_S2_VMID:
cmd.tlbi.vmid = cur->id;
cmd.tlbi.leaf = leaf;
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size,
granule);
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
0, cur->id);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf,
iova, size, granule);
break;
case INV_TYPE_S2_VMID_S1_CLEAR:
/* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */
if (arm_smmu_inv_size_too_big(cur->smmu, size, granule))
break;
cmd.tlbi.vmid = cur->id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_make_cmd_tlbi(cur->size_opcode, 0,
cur->id));
break;
case INV_TYPE_ATS:
arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd);
cmd.atc.sid = cur->id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid,
iova, size));
break;
case INV_TYPE_ATS_FULL:
arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
cmd.atc.sid = cur->id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_make_cmd_atc_inv_all(cur->id,
IOMMU_NO_PASID));
break;
default:
WARN_ON_ONCE(1);
@@ -3432,22 +3323,21 @@ arm_smmu_install_new_domain_invs(struct arm_smmu_attach_state *state)
static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv)
{
struct arm_smmu_cmdq_ent cmd = {};
switch (inv->type) {
case INV_TYPE_S1_ASID:
cmd.tlbi.asid = inv->id;
arm_smmu_cmdq_issue_cmd_with_sync(
inv->smmu,
arm_smmu_make_cmd_tlbi(inv->nsize_opcode, inv->id, 0));
break;
case INV_TYPE_S2_VMID:
/* S2_VMID using nsize_opcode covers S2_VMID_S1_CLEAR */
cmd.tlbi.vmid = inv->id;
arm_smmu_cmdq_issue_cmd_with_sync(
inv->smmu,
arm_smmu_make_cmd_tlbi(inv->nsize_opcode, 0, inv->id));
break;
default:
return;
}
cmd.opcode = inv->nsize_opcode;
arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &cmd);
}
/* Should be installed after arm_smmu_install_ste_for_dev() */
@@ -3854,9 +3744,12 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
if (!arm_smmu_ssids_in_use(&master->cd_table)) {
struct iommu_domain *sid_domain =
iommu_driver_get_domain_for_dev(master->dev);
bool ats_always_on = master->ats_always_on &&
sid_domain->type != IOMMU_DOMAIN_BLOCKED;
bool downgrade = sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
sid_domain->type == IOMMU_DOMAIN_BLOCKED;
if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
sid_domain->type == IOMMU_DOMAIN_BLOCKED)
if (!ats_always_on && downgrade)
sid_domain->ops->attach_dev(sid_domain, dev,
sid_domain);
}
@@ -3875,6 +3768,8 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
.old_domain = old_domain,
.ssid = IOMMU_NO_PASID,
};
bool ats_always_on = master->ats_always_on &&
s1dss != STRTAB_STE_1_S1DSS_TERMINATE;
/*
* Do not allow any ASID to be changed while are working on the STE,
@@ -3886,7 +3781,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
* If the CD table is not in use we can use the provided STE, otherwise
* we use a cdtable STE with the provided S1DSS.
*/
if (arm_smmu_ssids_in_use(&master->cd_table)) {
if (ats_always_on || arm_smmu_ssids_in_use(&master->cd_table)) {
/*
* If a CD table has to be present then we need to run with ATS
* on because we have to assume a PASID is using ATS. For
@@ -4215,6 +4110,44 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master)
kfree(master->build_invs);
}
static int arm_smmu_master_prepare_ats(struct arm_smmu_master *master)
{
bool s1p = master->smmu->features & ARM_SMMU_FEAT_TRANS_S1;
unsigned int stu = __ffs(master->smmu->pgsize_bitmap);
struct pci_dev *pdev;
int ret;
if (!dev_is_pci(master->dev))
return 0;
pdev = to_pci_dev(master->dev);
if (!arm_smmu_ats_supported(master)) {
if (pci_ats_required(pdev)) {
dev_err_once(master->dev, "SMMU doesn't support ATS\n");
return -EOPNOTSUPP;
}
return 0;
}
ret = pci_prepare_ats(pdev, stu);
if (ret || !pci_ats_required(pdev))
return ret;
/*
* S1DSS is required for ATS to be always on for identity domain cases.
* However, the S1DSS field is ignored if !IDR0_S1P or !IDR1_SSIDSIZE.
*/
if (!s1p || !master->smmu->ssid_bits) {
dev_err_once(master->dev,
"SMMU doesn't support ATS to be always on\n");
return -EOPNOTSUPP;
}
master->ats_always_on = true;
return arm_smmu_alloc_cd_tables(master);
}
static struct iommu_device *arm_smmu_probe_device(struct device *dev)
{
int ret;
@@ -4263,14 +4196,15 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
master->stall_enabled = true;
if (dev_is_pci(dev)) {
unsigned int stu = __ffs(smmu->pgsize_bitmap);
pci_prepare_ats(to_pci_dev(dev), stu);
}
ret = arm_smmu_master_prepare_ats(master);
if (ret)
goto err_disable_pasid;
return &smmu->iommu;
err_disable_pasid:
arm_smmu_disable_pasid(master);
arm_smmu_remove_master(master);
err_free_master:
kfree(master);
return ERR_PTR(ret);
@@ -4418,7 +4352,7 @@ int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
GFP_KERNEL);
if (q->base || qsz < PAGE_SIZE)
if (q->base || qsz <= PAGE_SIZE)
break;
q->llq.max_n_shift--;
@@ -4810,7 +4744,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
{
int ret;
u32 reg, enables;
struct arm_smmu_cmdq_ent cmd;
/* Clear CR0 and sync (disables SMMU and queue processing) */
reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
@@ -4857,17 +4790,16 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
}
/* Invalidate any cached configuration */
cmd.opcode = CMDQ_OP_CFGI_ALL;
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(smmu, arm_smmu_make_cmd_cfgi_all());
/* Invalidate any stale TLB entries */
if (smmu->features & ARM_SMMU_FEAT_HYP) {
cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(
smmu, arm_smmu_make_cmd_op(CMDQ_OP_TLBI_EL2_ALL));
}
cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(
smmu, arm_smmu_make_cmd_op(CMDQ_OP_TLBI_NSNH_ALL));
/* Event queue */
writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);

249
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
View File

@@ -390,6 +390,10 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
#define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
struct arm_smmu_cmd {
u64 data[CMDQ_ENT_DWORDS];
};
/*
* This is used to size the command queue and therefore must be at least
* BITS_PER_LONG so that the valid_map works correctly (it relies on the
@@ -426,11 +430,19 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
#define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
#define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
#define ATC_INV_SIZE_ALL 52
#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
enum pri_resp {
PRI_RESP_DENY = 0,
PRI_RESP_FAIL = 1,
PRI_RESP_SUCC = 2,
};
#define CMDQ_RESUME_0_RESP_TERM 0UL
#define CMDQ_RESUME_0_RESP_RETRY 1UL
#define CMDQ_RESUME_0_RESP_ABORT 2UL
@@ -447,6 +459,145 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
#define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
enum arm_smmu_cmdq_opcode {
CMDQ_OP_PREFETCH_CFG = 0x1,
CMDQ_OP_CFGI_STE = 0x3,
CMDQ_OP_CFGI_ALL = 0x4,
CMDQ_OP_CFGI_CD = 0x5,
CMDQ_OP_CFGI_CD_ALL = 0x6,
CMDQ_OP_TLBI_NH_ALL = 0x10,
CMDQ_OP_TLBI_NH_ASID = 0x11,
CMDQ_OP_TLBI_NH_VA = 0x12,
CMDQ_OP_TLBI_NH_VAA = 0x13,
CMDQ_OP_TLBI_EL2_ALL = 0x20,
CMDQ_OP_TLBI_EL2_ASID = 0x21,
CMDQ_OP_TLBI_EL2_VA = 0x22,
CMDQ_OP_TLBI_S12_VMALL = 0x28,
CMDQ_OP_TLBI_S2_IPA = 0x2a,
CMDQ_OP_TLBI_NSNH_ALL = 0x30,
CMDQ_OP_ATC_INV = 0x40,
CMDQ_OP_PRI_RESP = 0x41,
CMDQ_OP_RESUME = 0x44,
CMDQ_OP_CMD_SYNC = 0x46,
};
static inline struct arm_smmu_cmd
arm_smmu_make_cmd_op(enum arm_smmu_cmdq_opcode op)
{
struct arm_smmu_cmd cmd = {};
cmd.data[0] = FIELD_PREP(CMDQ_0_OP, op);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_all(void)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_ALL);
cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_prefetch_cfg(u32 sid)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_PREFETCH_CFG);
cmd.data[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, sid);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_ste(u32 sid, bool leaf)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_STE);
cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, leaf);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_cd(u32 sid, u32 ssid,
bool leaf)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_CD);
cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid) |
FIELD_PREP(CMDQ_CFGI_0_SSID, ssid);
cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, leaf);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_resume(u32 sid, u16 stag,
u8 resp)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_RESUME);
cmd.data[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, sid) |
FIELD_PREP(CMDQ_RESUME_0_RESP, resp);
cmd.data[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, stag);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid,
bool ssv,
u16 grpid,
enum pri_resp resp)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_PRI_RESP);
cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssv) |
FIELD_PREP(CMDQ_PRI_0_SID, sid) |
FIELD_PREP(CMDQ_PRI_0_SSID, ssid);
cmd.data[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, grpid) |
FIELD_PREP(CMDQ_PRI_1_RESP, resp);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv(u32 sid, u32 ssid,
u64 addr, u8 size)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_ATC_INV);
cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssid != IOMMU_NO_PASID) |
FIELD_PREP(CMDQ_ATC_0_SSID, ssid) |
FIELD_PREP(CMDQ_ATC_0_SID, sid);
cmd.data[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, size) |
(addr & CMDQ_ATC_1_ADDR_MASK);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_atc_inv_all(u32 sid,
u32 ssid)
{
return arm_smmu_make_cmd_atc_inv(sid, ssid, 0, ATC_INV_SIZE_ALL);
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_sync(unsigned int cs,
u64 msiaddr)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CMD_SYNC);
cmd.data[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, cs) |
FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH) |
FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
cmd.data[1] |= msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
return cmd;
}
/*
* TLBI commands - the non-sized variants just need opcode + asid/vmid.
* For sized variants the caller sets up data[0] with the immutable fields
* (opcode + asid/vmid) and the range loop fills in per-iteration fields.
*/
static inline struct arm_smmu_cmd
arm_smmu_make_cmd_tlbi(enum arm_smmu_cmdq_opcode op, u16 asid, u16 vmid)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(op);
cmd.data[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, asid) |
FIELD_PREP(CMDQ_TLBI_0_VMID, vmid);
return cmd;
}
/* Event queue */
#define EVTQ_ENT_SZ_SHIFT 5
#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
@@ -507,90 +658,6 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
#define MSI_IOVA_BASE 0x8000000
#define MSI_IOVA_LENGTH 0x100000
enum pri_resp {
PRI_RESP_DENY = 0,
PRI_RESP_FAIL = 1,
PRI_RESP_SUCC = 2,
};
struct arm_smmu_cmdq_ent {
/* Common fields */
u8 opcode;
bool substream_valid;
/* Command-specific fields */
union {
#define CMDQ_OP_PREFETCH_CFG 0x1
struct {
u32 sid;
} prefetch;
#define CMDQ_OP_CFGI_STE 0x3
#define CMDQ_OP_CFGI_ALL 0x4
#define CMDQ_OP_CFGI_CD 0x5
#define CMDQ_OP_CFGI_CD_ALL 0x6
struct {
u32 sid;
u32 ssid;
union {
bool leaf;
u8 span;
};
} cfgi;
#define CMDQ_OP_TLBI_NH_ALL 0x10
#define CMDQ_OP_TLBI_NH_ASID 0x11
#define CMDQ_OP_TLBI_NH_VA 0x12
#define CMDQ_OP_TLBI_NH_VAA 0x13
#define CMDQ_OP_TLBI_EL2_ALL 0x20
#define CMDQ_OP_TLBI_EL2_ASID 0x21
#define CMDQ_OP_TLBI_EL2_VA 0x22
#define CMDQ_OP_TLBI_S12_VMALL 0x28
#define CMDQ_OP_TLBI_S2_IPA 0x2a
#define CMDQ_OP_TLBI_NSNH_ALL 0x30
struct {
u8 num;
u8 scale;
u16 asid;
u16 vmid;
bool leaf;
u8 ttl;
u8 tg;
u64 addr;
} tlbi;
#define CMDQ_OP_ATC_INV 0x40
#define ATC_INV_SIZE_ALL 52
struct {
u32 sid;
u32 ssid;
u64 addr;
u8 size;
bool global;
} atc;
#define CMDQ_OP_PRI_RESP 0x41
struct {
u32 sid;
u32 ssid;
u16 grpid;
enum pri_resp resp;
} pri;
#define CMDQ_OP_RESUME 0x44
struct {
u32 sid;
u16 stag;
u8 resp;
} resume;
#define CMDQ_OP_CMD_SYNC 0x46
struct {
u64 msiaddr;
} sync;
};
};
struct arm_smmu_ll_queue {
union {
u64 val;
@@ -633,17 +700,17 @@ struct arm_smmu_cmdq {
atomic_long_t *valid_map;
atomic_t owner_prod;
atomic_t lock;
bool (*supports_cmd)(struct arm_smmu_cmdq_ent *ent);
bool (*supports_cmd)(struct arm_smmu_cmd *cmd);
};
static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmdq_ent *ent)
struct arm_smmu_cmd *cmd)
{
return cmdq->supports_cmd ? cmdq->supports_cmd(ent) : true;
return cmdq->supports_cmd ? cmdq->supports_cmd(cmd) : true;
}
struct arm_smmu_cmdq_batch {
u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
struct arm_smmu_cmd cmds[CMDQ_BATCH_ENTRIES];
struct arm_smmu_cmdq *cmdq;
int num;
};
@@ -807,7 +874,7 @@ struct arm_smmu_impl_ops {
void (*device_remove)(struct arm_smmu_device *smmu);
int (*init_structures)(struct arm_smmu_device *smmu);
struct arm_smmu_cmdq *(*get_secondary_cmdq)(
struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent);
struct arm_smmu_device *smmu, struct arm_smmu_cmd *cmd);
/*
* An implementation should define its own type other than the default
* IOMMU_HW_INFO_TYPE_ARM_SMMUV3. And it must validate the input @type
@@ -943,6 +1010,7 @@ struct arm_smmu_master {
bool ats_enabled : 1;
bool ste_ats_enabled : 1;
bool stall_enabled;
bool ats_always_on;
unsigned int ssid_bits;
unsigned int iopf_refcount;
};
@@ -1140,7 +1208,8 @@ void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
const struct arm_smmu_ste *target);
int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
struct arm_smmu_cmdq *cmdq,
struct arm_smmu_cmd *cmds, int n,
bool sync);
#ifdef CONFIG_ARM_SMMU_V3_SVA

16
drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
View File

@@ -367,9 +367,9 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
/* Command Queue Function */
static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent *ent)
static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmd *cmd)
{
switch (ent->opcode) {
switch (FIELD_GET(CMDQ_0_OP, cmd->data[0])) {
case CMDQ_OP_TLBI_NH_ASID:
case CMDQ_OP_TLBI_NH_VA:
case CMDQ_OP_ATC_INV:
@@ -381,7 +381,7 @@ static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent *ent)
static struct arm_smmu_cmdq *
tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_ent *ent)
struct arm_smmu_cmd *cmd)
{
struct tegra241_cmdqv *cmdqv =
container_of(smmu, struct tegra241_cmdqv, smmu);
@@ -409,7 +409,7 @@ tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu,
return NULL;
/* Unsupported CMD goes for smmu->cmdq pathway */
if (!arm_smmu_cmdq_supports_cmd(&vcmdq->cmdq, ent))
if (!arm_smmu_cmdq_supports_cmd(&vcmdq->cmdq, cmd))
return NULL;
return &vcmdq->cmdq;
}
@@ -427,16 +427,16 @@ tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu,
static void tegra241_vcmdq_hw_flush_timeout(struct tegra241_vcmdq *vcmdq)
{
struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu;
u64 cmd_sync[CMDQ_ENT_DWORDS] = {};
struct arm_smmu_cmd cmd_sync = {};
cmd_sync[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) |
FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
cmd_sync.data[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) |
FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
/*
* It does not hurt to insert another CMD_SYNC, taking advantage of the
* arm_smmu_cmdq_issue_cmdlist() that waits for the CMD_SYNC completion.
*/
arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, cmd_sync, 1, true);
arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &cmd_sync, 1, true);
}
/* This function is for LVCMDQ, so @vcmdq must not be unmapped yet */

13
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
View File

@@ -39,8 +39,10 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = {
.data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) },
{ .compatible = "qcom,adreno-smmu",
.data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) },
{ .compatible = "qcom,fastrpc",
{ .compatible = "qcom,fastrpc-compute-cb",
.data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) },
{ .compatible = "qcom,glymur-mdss",
.data = (const void *) (PREFETCH_DEFAULT | CMTLB) },
{ .compatible = "qcom,qcm2290-mdss",
.data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) },
{ .compatible = "qcom,sa8775p-mdss",
@@ -259,6 +261,7 @@ static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
int ret;
/* The domain must have split pagetables already enabled */
if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
@@ -288,8 +291,16 @@ static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
}
ret = pm_runtime_resume_and_get(smmu_domain->smmu->dev);
if (ret < 0) {
dev_err(smmu_domain->smmu->dev, "failed to get runtime PM: %d\n", ret);
return -ENODEV;
}
arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
pm_runtime_put_autosuspend(smmu_domain->smmu->dev);
return 0;
}

2
drivers/iommu/dma-iommu.c
View File

@@ -1465,7 +1465,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
*/
s->dma_address = pci_p2pdma_bus_addr_map(
p2pdma_state.mem, sg_phys(s));
sg_dma_len(s) = sg->length;
sg_dma_len(s) = s->length;
sg_dma_mark_bus_address(s);
continue;
default:

1
drivers/iommu/generic_pt/fmt/amdv1.h
View File

@@ -404,6 +404,7 @@ amdv1pt_iommu_fmt_hw_info(struct pt_iommu_amdv1 *table,
static const struct pt_iommu_amdv1_cfg amdv1_kunit_fmt_cfgs[] = {
/* Matches what io_pgtable does */
[0] = { .starting_level = 2 },
[1] = { .starting_level = 2, .common.hw_max_vasz_lg2 = 32 },
};
#define kunit_fmt_cfgs amdv1_kunit_fmt_cfgs
enum { KUNIT_FMT_FEATURES = 0 };

4
drivers/iommu/generic_pt/fmt/iommu_riscv64.c
View File

@@ -6,6 +6,8 @@
#define PT_FMT_VARIANT 64
#define PT_SUPPORTED_FEATURES \
(BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | \
BIT(PT_FEAT_RISCV_SVNAPOT_64K))
BIT(PT_FEAT_RISCV_SVNAPOT_64K) | \
BIT(PT_FEAT_DETAILED_GATHER))
#define PT_FORCE_ENABLED_FEATURES BIT(PT_FEAT_DETAILED_GATHER)
#include "iommu_template.h"

11
drivers/iommu/generic_pt/fmt/riscv.h
View File

@@ -64,6 +64,8 @@ enum {
RISCVPT_PPN64 = GENMASK_ULL(53, 10),
RISCVPT_PPN64_64K = GENMASK_ULL(53, 14),
RISCVPT_PBMT = GENMASK_ULL(62, 61),
RISCVPT_NC = BIT_ULL(61),
RISCVPT_IO = BIT_ULL(62),
RISCVPT_N = BIT_ULL(63),
/* Svnapot encodings for ppn[0] */
@@ -201,7 +203,8 @@ static inline void riscvpt_attr_from_entry(const struct pt_state *pts,
{
attrs->descriptor_bits =
pts->entry & (RISCVPT_R | RISCVPT_W | RISCVPT_X | RISCVPT_U |
RISCVPT_G | RISCVPT_A | RISCVPT_D);
RISCVPT_G | RISCVPT_A | RISCVPT_D | RISCVPT_NC |
RISCVPT_IO);
}
#define pt_attr_from_entry riscvpt_attr_from_entry
@@ -237,6 +240,12 @@ static inline int riscvpt_iommu_set_prot(struct pt_common *common,
pte |= RISCVPT_R;
if (!(iommu_prot & IOMMU_NOEXEC))
pte |= RISCVPT_X;
if (common->features & BIT(PT_FEAT_RISCV_SVPBMT)) {
if (iommu_prot & IOMMU_MMIO)
pte |= RISCVPT_IO;
else if (!(iommu_prot & IOMMU_CACHE))
pte |= RISCVPT_NC;
}
/* Caller must specify a supported combination of flags */
if (unlikely((pte & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) == 0))

110
drivers/iommu/generic_pt/iommu_pt.h
View File

@@ -40,15 +40,40 @@ static void flush_writes_item(const struct pt_state *pts)
PT_ITEM_WORD_SIZE);
}
static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather,
struct pt_iommu *iommu_table, pt_vaddr_t iova,
pt_vaddr_t len,
struct iommu_pages_list *free_list)
struct iommupt_pending_gather {
struct iommu_iotlb_gather *iotlb_gather;
struct iommu_pages_list free_list;
u8 leaf_levels_bitmap;
u8 table_levels_bitmap;
};
static void gather_add_table(struct iommupt_pending_gather *pending,
const struct pt_state *pts,
struct pt_table_p *table)
{
iommu_pages_list_add(&pending->free_list, table);
if (pts_feature(pts, PT_FEAT_DETAILED_GATHER))
pending->table_levels_bitmap |= BIT(pts->level);
}
static void gather_add_leaf(struct iommupt_pending_gather *pending,
const struct pt_state *pts)
{
if (!pts_feature(pts, PT_FEAT_DETAILED_GATHER))
return;
pending->leaf_levels_bitmap |= BIT(pts->level);
}
static void gather_range_pending(struct iommupt_pending_gather *pending,
struct pt_iommu *iommu_table, pt_vaddr_t iova,
pt_vaddr_t len)
{
struct iommu_iotlb_gather *iotlb_gather = pending->iotlb_gather;
struct pt_common *common = common_from_iommu(iommu_table);
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
iommu_pages_stop_incoherent_list(free_list,
iommu_pages_stop_incoherent_list(&pending->free_list,
iommu_table->iommu_device);
/*
@@ -72,7 +97,17 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather,
iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
}
iommu_pages_list_splice(free_list, &iotlb_gather->freelist);
iommu_pages_list_splice(&pending->free_list, &iotlb_gather->freelist);
INIT_LIST_HEAD(&pending->free_list.pages);
if (pt_feature(common, PT_FEAT_DETAILED_GATHER)) {
iotlb_gather->pt.leaf_levels_bitmap |=
pending->leaf_levels_bitmap;
iotlb_gather->pt.table_levels_bitmap |=
pending->table_levels_bitmap;
pending->leaf_levels_bitmap = 0;
pending->table_levels_bitmap = 0;
}
}
#define DOMAIN_NS(op) CONCATENATE(CONCATENATE(pt_iommu_, PTPFX), op)
@@ -341,7 +376,7 @@ static int __maybe_unused NS(set_dirty)(struct pt_iommu *iommu_table,
}
struct pt_iommu_collect_args {
struct iommu_pages_list free_list;
struct iommupt_pending_gather pending;
/* Fail if any OAs are within the range */
u8 check_mapped : 1;
};
@@ -358,7 +393,8 @@ static int __collect_tables(struct pt_range *range, void *arg,
for_each_pt_level_entry(&pts) {
if (pts.type == PT_ENTRY_TABLE) {
iommu_pages_list_add(&collect->free_list, pts.table_lower);
gather_add_table(&collect->pending, &pts,
pts.table_lower);
ret = pt_descend(&pts, arg, __collect_tables);
if (ret)
return ret;
@@ -493,15 +529,18 @@ static int clear_contig(const struct pt_state *start_pts,
struct pt_range range = *start_pts->range;
struct pt_state pts =
pt_init(&range, start_pts->level, start_pts->table);
struct pt_iommu_collect_args collect = { .check_mapped = true };
struct pt_iommu_collect_args collect = {
.check_mapped = true,
.pending.iotlb_gather = iotlb_gather,
.pending.free_list = IOMMU_PAGES_LIST_INIT(
collect.pending.free_list),
};
int ret;
pts.index = start_pts->index;
pts.end_index = start_pts->index + step;
for (; _pt_iter_load(&pts); pt_next_entry(&pts)) {
if (pts.type == PT_ENTRY_TABLE) {
collect.free_list =
IOMMU_PAGES_LIST_INIT(collect.free_list);
ret = pt_walk_descend_all(&pts, __collect_tables,
&collect);
if (ret)
@@ -514,12 +553,11 @@ static int clear_contig(const struct pt_state *start_pts,
pt_clear_entries(&pts, ilog2(1));
flush_writes_item(&pts);
iommu_pages_list_add(&collect.free_list,
pt_table_ptr(&pts));
gather_range_pages(
iotlb_gather, iommu_table, range.va,
log2_to_int(pt_table_item_lg2sz(&pts)),
&collect.free_list);
gather_add_table(&collect.pending, &pts,
pts.table_lower);
gather_range_pending(
&collect.pending, iommu_table, range.va,
log2_to_int(pt_table_item_lg2sz(&pts)));
} else if (pts.type != PT_ENTRY_EMPTY) {
return -EADDRINUSE;
}
@@ -968,7 +1006,7 @@ static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
}
struct pt_unmap_args {
struct iommu_pages_list free_list;
struct iommupt_pending_gather pending;
pt_vaddr_t unmapped;
};
@@ -1031,8 +1069,8 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg,
* succeed in clearing the lower table levels.
*/
if (fully_covered) {
iommu_pages_list_add(&unmap->free_list,
pts.table_lower);
gather_add_table(&unmap->pending, &pts,
pts.table_lower);
pt_clear_entries(&pts, ilog2(1));
if (pts.index < flush_start_index)
flush_start_index = pts.index;
@@ -1049,6 +1087,7 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg,
*/
num_contig_lg2 = pt_entry_num_contig_lg2(&pts);
pt_clear_entries(&pts, num_contig_lg2);
gather_add_leaf(&unmap->pending, &pts);
num_oas += log2_to_int(num_contig_lg2);
if (pts.index < flush_start_index)
flush_start_index = pts.index;
@@ -1071,8 +1110,11 @@ static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
dma_addr_t len,
struct iommu_iotlb_gather *iotlb_gather)
{
struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT(
unmap.free_list) };
struct pt_unmap_args unmap = {
.pending.iotlb_gather = iotlb_gather,
.pending.free_list = IOMMU_PAGES_LIST_INIT(
unmap.pending.free_list),
};
struct pt_range range;
int ret;
@@ -1082,8 +1124,7 @@ static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
pt_walk_range(&range, __unmap_range, &unmap);
gather_range_pages(iotlb_gather, iommu_table, iova, unmap.unmapped,
&unmap.free_list);
gather_range_pending(&unmap.pending, iommu_table, iova, unmap.unmapped);
return unmap.unmapped;
}
@@ -1108,8 +1149,12 @@ static void NS(get_info)(struct pt_iommu *iommu_table,
pgsize_bitmap |= pt_possible_sizes(&pts);
}
/* Hide page sizes larger than the maximum OA */
info->pgsize_bitmap = oalog2_mod(pgsize_bitmap, common->max_oasz_lg2);
/*
* Hide page sizes larger than the maximum. -1 because a whole table
* pgsize is not allowed
*/
info->pgsize_bitmap = log2_mod(pgsize_bitmap, common->max_vasz_lg2 - 1);
info->pgsize_bitmap = oalog2_mod(info->pgsize_bitmap, common->max_oasz_lg2);
}
static void NS(deinit)(struct pt_iommu *iommu_table)
@@ -1117,10 +1162,11 @@ static void NS(deinit)(struct pt_iommu *iommu_table)
struct pt_common *common = common_from_iommu(iommu_table);
struct pt_range range = pt_all_range(common);
struct pt_iommu_collect_args collect = {
.free_list = IOMMU_PAGES_LIST_INIT(collect.free_list),
.pending.free_list = IOMMU_PAGES_LIST_INIT(
collect.pending.free_list),
};
iommu_pages_list_add(&collect.free_list, range.top_table);
iommu_pages_list_add(&collect.pending.free_list, range.top_table);
pt_walk_range(&range, __collect_tables, &collect);
/*
@@ -1128,9 +1174,9 @@ static void NS(deinit)(struct pt_iommu *iommu_table)
* and invalidated any caching referring to this memory.
*/
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
iommu_pages_stop_incoherent_list(&collect.free_list,
iommu_pages_stop_incoherent_list(&collect.pending.free_list,
iommu_table->iommu_device);
iommu_put_pages_list(&collect.free_list);
iommu_put_pages_list(&collect.pending.free_list);
}
static const struct pt_iommu_ops NS(ops) = {
@@ -1151,10 +1197,6 @@ static int pt_init_common(struct pt_common *common)
if (PT_WARN_ON(top_range.top_level > PT_MAX_TOP_LEVEL))
return -EINVAL;
if (top_range.top_level == PT_MAX_TOP_LEVEL ||
common->max_vasz_lg2 == top_range.max_vasz_lg2)
common->features &= ~BIT(PT_FEAT_DYNAMIC_TOP);
if (top_range.max_vasz_lg2 == PT_VADDR_MAX_LG2)
common->features |= BIT(PT_FEAT_FULL_VA);

10
drivers/iommu/generic_pt/kunit_generic_pt.h
View File

@@ -438,6 +438,9 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts,
{
unsigned int num_items_lg2 = safe_pt_num_items_lg2(pts);
pt_vaddr_t pgsize_bitmap = pt_possible_sizes(pts);
/* Matches get_info() */
pt_vaddr_t limited_pgsize_bitmap =
log2_mod(pgsize_bitmap, pts->range->common->max_vasz_lg2 - 1);
unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
if (!pt_can_have_leaf(pts)) {
@@ -448,7 +451,8 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts,
/* No bits for sizes that would be outside this table */
KUNIT_ASSERT_EQ(test, log2_mod(pgsize_bitmap, isz_lg2), 0);
KUNIT_ASSERT_EQ(
test, fvalog2_div(pgsize_bitmap, num_items_lg2 + isz_lg2), 0);
test,
fvalog2_div(limited_pgsize_bitmap, num_items_lg2 + isz_lg2), 0);
/*
* Non contiguous must be supported. AMDv1 has a HW bug where it does
@@ -463,8 +467,8 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts,
/* A contiguous entry should not span the whole table */
if (num_items_lg2 + isz_lg2 != PT_VADDR_MAX_LG2)
KUNIT_ASSERT_FALSE(
test,
pgsize_bitmap & log2_to_int(num_items_lg2 + isz_lg2));
test, limited_pgsize_bitmap &
log2_to_int(num_items_lg2 + isz_lg2));
}
static void test_entry_possible_sizes(struct kunit *test)

5
drivers/iommu/generic_pt/kunit_iommu_pt.h
View File

@@ -112,8 +112,9 @@ static void test_increase_level(struct kunit *test)
if (IS_32BIT)
kunit_skip(test, "Unable to test on 32bit");
KUNIT_ASSERT_GT(test, common->max_vasz_lg2,
pt_top_range(common).max_vasz_lg2);
if (common->max_vasz_lg2 <= pt_top_range(common).max_vasz_lg2)
kunit_skip(test,
"max_vasz_lg2 fits in starting level, no growth possible");
/* Add every possible level to the max */
while (common->max_vasz_lg2 != pt_top_range(common).max_vasz_lg2) {

9
drivers/iommu/intel/dmar.c
View File

@@ -1894,7 +1894,8 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
reason = dmar_get_fault_reason(fault_reason, &fault_type);
if (fault_type == INTR_REMAP) {
pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n",
pr_err("[INTR-REMAP] Request device [%04x:%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n",
iommu->segment,
source_id >> 8, PCI_SLOT(source_id & 0xFF),
PCI_FUNC(source_id & 0xFF), addr >> 48,
fault_reason, reason);
@@ -1903,14 +1904,16 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
}
if (pasid == IOMMU_PASID_INVALID)
pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
pr_err("[%s NO_PASID] Request device [%04x:%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
type ? "DMA Read" : "DMA Write",
iommu->segment,
source_id >> 8, PCI_SLOT(source_id & 0xFF),
PCI_FUNC(source_id & 0xFF), addr,
fault_reason, reason);
else
pr_err("[%s PASID 0x%x] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
pr_err("[%s PASID 0x%x] Request device [%04x:%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
type ? "DMA Read" : "DMA Write", pasid,
iommu->segment,
source_id >> 8, PCI_SLOT(source_id & 0xFF),
PCI_FUNC(source_id & 0xFF), addr,
fault_reason, reason);

6
drivers/iommu/intel/iommu.c
View File

@@ -157,7 +157,10 @@ static void device_rbtree_remove(struct device_domain_info *info)
unsigned long flags;
spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
rb_erase(&info->node, &iommu->device_rbtree);
if (!RB_EMPTY_NODE(&info->node)) {
rb_erase(&info->node, &iommu->device_rbtree);
RB_CLEAR_NODE(&info->node);
}
spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
}
@@ -3254,6 +3257,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
info->dev = dev;
info->iommu = iommu;
RB_CLEAR_NODE(&info->node);
if (dev_is_pci(dev)) {
if (ecap_dev_iotlb_support(iommu->ecap) &&
pci_ats_supported(pdev) &&

11
drivers/iommu/intel/iommu.h
View File

@@ -1254,18 +1254,29 @@ void intel_iommu_disable_iopf(struct device *dev);
static inline int iopf_for_domain_set(struct iommu_domain *domain,
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
if (!domain || !domain->iopf_handler)
return 0;
/* SVA with non-IOMMU/PRI IOPF handling is allowed. */
if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported)
return 0;
return intel_iommu_enable_iopf(dev);
}
static inline void iopf_for_domain_remove(struct iommu_domain *domain,
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
if (!domain || !domain->iopf_handler)
return;
if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported)
return;
intel_iommu_disable_iopf(dev);
}

16
drivers/iommu/intel/pasid.c
View File

@@ -618,11 +618,11 @@ void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,
intel_pasid_flush_present(iommu, dev, pasid, did, pte);
}
static void pasid_pte_config_nestd(struct intel_iommu *iommu,
struct pasid_entry *pte,
struct iommu_hwpt_vtd_s1 *s1_cfg,
struct dmar_domain *s2_domain,
u16 did)
static void pasid_pte_config_nested(struct intel_iommu *iommu,
struct pasid_entry *pte,
struct iommu_hwpt_vtd_s1 *s1_cfg,
struct dmar_domain *s2_domain,
u16 did)
{
struct pt_iommu_vtdss_hw_info pt_info;
@@ -720,7 +720,7 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
return -EBUSY;
}
pasid_pte_config_nestd(iommu, pte, s1_cfg, s2_domain, did);
pasid_pte_config_nested(iommu, pte, s1_cfg, s2_domain, did);
spin_unlock(&iommu->lock);
pasid_flush_caches(iommu, pte, pasid, did);
@@ -748,10 +748,12 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn)
}
did = context_domain_id(context);
context_clear_entry(context);
context_clear_present(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock(&iommu->lock);
intel_context_flush_no_pasid(info, context, did);
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data)

12
drivers/iommu/intel/svm.c
View File

@@ -164,12 +164,9 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
if (IS_ERR(dev_pasid))
return PTR_ERR(dev_pasid);
/* SVA with non-IOMMU/PRI IOPF handling is allowed. */
if (info->pri_supported) {
ret = iopf_for_domain_replace(domain, old, dev);
if (ret)
goto out_remove_dev_pasid;
}
ret = iopf_for_domain_replace(domain, old, dev);
if (ret)
goto out_remove_dev_pasid;
/* Setup the pasid table: */
sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
@@ -184,8 +181,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
return 0;
out_unwind_iopf:
if (info->pri_supported)
iopf_for_domain_replace(old, domain, dev);
iopf_for_domain_replace(old, domain, dev);
out_remove_dev_pasid:
domain_remove_dev_pasid(domain, dev, pasid);
return ret;

88
drivers/iommu/io-pgtable-arm.c
View File

@@ -143,7 +143,7 @@
#define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
/* IOPTE accessors */
#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
#define iopte_deref(pte, d) phys_to_virt(iopte_to_paddr(pte, d))
#define iopte_type(pte) \
(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
@@ -248,26 +248,15 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages)
return (dma_addr_t)virt_to_phys(pages);
}
static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
struct io_pgtable_cfg *cfg,
void *cookie)
static void *__arm_lpae_cfg_alloc(size_t size, gfp_t gfp,
struct io_pgtable_cfg *cfg,
void *cookie)
{
struct device *dev = cfg->iommu_dev;
size_t alloc_size;
dma_addr_t dma;
void *pages;
/*
* For very small starting-level translation tables the HW requires a
* minimum alignment of at least 64 to cover all cases.
*/
alloc_size = max(size, 64);
if (cfg->alloc)
pages = cfg->alloc(cookie, alloc_size, gfp);
else
pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp,
alloc_size);
pages = cfg->alloc(cookie, size, gfp);
if (!pages)
return NULL;
@@ -291,24 +280,69 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
out_free:
if (cfg->free)
cfg->free(cookie, pages, size);
else
iommu_free_pages(pages);
cfg->free(cookie, pages, size);
return NULL;
}
static void __arm_lpae_cfg_free(void *pages, size_t size,
struct io_pgtable_cfg *cfg,
void *cookie)
{
if (!cfg->coherent_walk)
dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
size, DMA_TO_DEVICE);
cfg->free(cookie, pages, size);
}
static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
struct io_pgtable_cfg *cfg,
void *cookie)
{
struct device *dev = cfg->iommu_dev;
void *pages;
/*
* For very small starting-level translation tables the HW requires a
* minimum alignment of at least 64 to cover all cases.
*/
size = max(size, 64);
if (cfg->alloc)
return __arm_lpae_cfg_alloc(size, gfp, cfg, cookie);
pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, size);
if (!pages)
return NULL;
if (!cfg->coherent_walk) {
int ret = iommu_pages_start_incoherent(pages, dev);
if (ret) {
if (ret == -EOPNOTSUPP)
dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
iommu_free_pages(pages);
return NULL;
}
}
return pages;
}
static void __arm_lpae_free_pages(void *pages, size_t size,
struct io_pgtable_cfg *cfg,
void *cookie)
{
if (!cfg->coherent_walk)
dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
size, DMA_TO_DEVICE);
/* See __arm_lpae_alloc_pages(). */
size = max(size, 64);
if (cfg->free)
cfg->free(cookie, pages, size);
if (cfg->free) {
__arm_lpae_cfg_free(pages, size, cfg, cookie);
return;
}
if (!cfg->coherent_walk)
iommu_pages_free_incoherent(pages, cfg->iommu_dev);
else
iommu_free_pages(pages);
}
@@ -395,7 +429,7 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
arm_lpae_iopte old, new;
struct io_pgtable_cfg *cfg = &data->iop.cfg;
new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
new = paddr_to_iopte(virt_to_phys(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
new |= ARM_LPAE_PTE_NSTABLE;

123
drivers/iommu/riscv/iommu-bits.h
View File

@@ -63,6 +63,8 @@
#define RISCV_IOMMU_CAPABILITIES_PD8 BIT_ULL(38)
#define RISCV_IOMMU_CAPABILITIES_PD17 BIT_ULL(39)
#define RISCV_IOMMU_CAPABILITIES_PD20 BIT_ULL(40)
#define RISCV_IOMMU_CAPABILITIES_NL BIT_ULL(42)
#define RISCV_IOMMU_CAPABILITIES_S BIT_ULL(43)
/**
* enum riscv_iommu_igs_settings - Interrupt Generation Support Settings
@@ -460,31 +462,33 @@ struct riscv_iommu_command {
};
/* Fields on dword0, common for all commands */
#define RISCV_IOMMU_CMD_OPCODE GENMASK_ULL(6, 0)
#define RISCV_IOMMU_CMD_FUNC GENMASK_ULL(9, 7)
#define RISCV_IOMMU_CMD0_OPCODE GENMASK_ULL(6, 0)
#define RISCV_IOMMU_CMD0_FUNC GENMASK_ULL(9, 7)
/* 3.1.1 IOMMU Page-table cache invalidation */
/* Fields on dword0 */
#define RISCV_IOMMU_CMD_IOTINVAL_OPCODE 1
#define RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA 0
#define RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA 1
#define RISCV_IOMMU_CMD_IOTINVAL_AV BIT_ULL(10)
#define RISCV_IOMMU_CMD_IOTINVAL_PSCID GENMASK_ULL(31, 12)
#define RISCV_IOMMU_CMD_IOTINVAL_PSCV BIT_ULL(32)
#define RISCV_IOMMU_CMD_IOTINVAL_GV BIT_ULL(33)
#define RISCV_IOMMU_CMD_IOTINVAL_GSCID GENMASK_ULL(59, 44)
#define RISCV_IOMMU_CMD0_IOTINVAL_AV BIT_ULL(10)
#define RISCV_IOMMU_CMD0_IOTINVAL_PSCID GENMASK_ULL(31, 12)
#define RISCV_IOMMU_CMD0_IOTINVAL_PSCV BIT_ULL(32)
#define RISCV_IOMMU_CMD0_IOTINVAL_GV BIT_ULL(33)
#define RISCV_IOMMU_CMD0_IOTINVAL_GSCID GENMASK_ULL(59, 44)
#define RISCV_IOMMU_CMD0_IOTINVAL_NL BIT_ULL(34)
#define RISCV_IOMMU_CMD1_IOTINVAL_S BIT_ULL(9)
/* dword1[61:10] is the 4K-aligned page address */
#define RISCV_IOMMU_CMD_IOTINVAL_ADDR GENMASK_ULL(61, 10)
#define RISCV_IOMMU_CMD1_IOTINVAL_ADDR GENMASK_ULL(61, 10)
/* 3.1.2 IOMMU Command Queue Fences */
/* Fields on dword0 */
#define RISCV_IOMMU_CMD_IOFENCE_OPCODE 2
#define RISCV_IOMMU_CMD_IOFENCE_FUNC_C 0
#define RISCV_IOMMU_CMD_IOFENCE_AV BIT_ULL(10)
#define RISCV_IOMMU_CMD_IOFENCE_WSI BIT_ULL(11)
#define RISCV_IOMMU_CMD_IOFENCE_PR BIT_ULL(12)
#define RISCV_IOMMU_CMD_IOFENCE_PW BIT_ULL(13)
#define RISCV_IOMMU_CMD_IOFENCE_DATA GENMASK_ULL(63, 32)
#define RISCV_IOMMU_CMD0_IOFENCE_AV BIT_ULL(10)
#define RISCV_IOMMU_CMD0_IOFENCE_WSI BIT_ULL(11)
#define RISCV_IOMMU_CMD0_IOFENCE_PR BIT_ULL(12)
#define RISCV_IOMMU_CMD0_IOFENCE_PW BIT_ULL(13)
#define RISCV_IOMMU_CMD0_IOFENCE_DATA GENMASK_ULL(63, 32)
/* dword1 is the address, word-size aligned and shifted to the right by two bits. */
/* 3.1.3 IOMMU Directory cache invalidation */
@@ -492,9 +496,9 @@ struct riscv_iommu_command {
#define RISCV_IOMMU_CMD_IODIR_OPCODE 3
#define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT 0
#define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT 1
#define RISCV_IOMMU_CMD_IODIR_PID GENMASK_ULL(31, 12)
#define RISCV_IOMMU_CMD_IODIR_DV BIT_ULL(33)
#define RISCV_IOMMU_CMD_IODIR_DID GENMASK_ULL(63, 40)
#define RISCV_IOMMU_CMD0_IODIR_PID GENMASK_ULL(31, 12)
#define RISCV_IOMMU_CMD0_IODIR_DV BIT_ULL(33)
#define RISCV_IOMMU_CMD0_IODIR_DID GENMASK_ULL(63, 40)
/* dword1 is reserved for standard use */
/* 3.1.4 IOMMU PCIe ATS */
@@ -502,25 +506,25 @@ struct riscv_iommu_command {
#define RISCV_IOMMU_CMD_ATS_OPCODE 4
#define RISCV_IOMMU_CMD_ATS_FUNC_INVAL 0
#define RISCV_IOMMU_CMD_ATS_FUNC_PRGR 1
#define RISCV_IOMMU_CMD_ATS_PID GENMASK_ULL(31, 12)
#define RISCV_IOMMU_CMD_ATS_PV BIT_ULL(32)
#define RISCV_IOMMU_CMD_ATS_DSV BIT_ULL(33)
#define RISCV_IOMMU_CMD_ATS_RID GENMASK_ULL(55, 40)
#define RISCV_IOMMU_CMD_ATS_DSEG GENMASK_ULL(63, 56)
#define RISCV_IOMMU_CMD0_ATS_PID GENMASK_ULL(31, 12)
#define RISCV_IOMMU_CMD0_ATS_PV BIT_ULL(32)
#define RISCV_IOMMU_CMD0_ATS_DSV BIT_ULL(33)
#define RISCV_IOMMU_CMD0_ATS_RID GENMASK_ULL(55, 40)
#define RISCV_IOMMU_CMD0_ATS_DSEG GENMASK_ULL(63, 56)
/* dword1 is the ATS payload, two different payload types for INVAL and PRGR */
/* ATS.INVAL payload*/
#define RISCV_IOMMU_CMD_ATS_INVAL_G BIT_ULL(0)
#define RISCV_IOMMU_CMD1_ATS_INVAL_G BIT_ULL(0)
/* Bits 1 - 10 are zeroed */
#define RISCV_IOMMU_CMD_ATS_INVAL_S BIT_ULL(11)
#define RISCV_IOMMU_CMD_ATS_INVAL_UADDR GENMASK_ULL(63, 12)
#define RISCV_IOMMU_CMD1_ATS_INVAL_S BIT_ULL(11)
#define RISCV_IOMMU_CMD1_ATS_INVAL_UADDR GENMASK_ULL(63, 12)
/* ATS.PRGR payload */
/* Bits 0 - 31 are zeroed */
#define RISCV_IOMMU_CMD_ATS_PRGR_PRG_INDEX GENMASK_ULL(40, 32)
#define RISCV_IOMMU_CMD1_ATS_PRGR_PRG_INDEX GENMASK_ULL(40, 32)
/* Bits 41 - 43 are zeroed */
#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44)
#define RISCV_IOMMU_CMD_ATS_PRGR_DST_ID GENMASK_ULL(63, 48)
#define RISCV_IOMMU_CMD1_ATS_PRGR_RESP_CODE GENMASK_ULL(47, 44)
#define RISCV_IOMMU_CMD1_ATS_PRGR_DST_ID GENMASK_ULL(63, 48)
/**
* struct riscv_iommu_fq_record - Fault/Event Queue Record
@@ -711,8 +715,8 @@ struct riscv_iommu_msipte {
static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd)
{
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA);
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA);
cmd->dword1 = 0;
}
@@ -720,67 +724,88 @@ static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cm
u64 addr)
{
cmd->dword1 =
FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, PHYS_PFN(addr));
cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV;
FIELD_PREP(RISCV_IOMMU_CMD1_IOTINVAL_ADDR, PHYS_PFN(addr));
cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_AV;
}
static inline void riscv_iommu_cmd_inval_set_nl(struct riscv_iommu_command *cmd)
{
cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_NL;
}
/*
* Set NAPOT-encoded address for range invalidation (S=1).
* sz_lg2: log2 of total range in bytes, must be >= 13 (8KiB, 2 pages).
* addr must be naturally aligned to 2^sz_lg2.
*/
static inline void riscv_iommu_cmd_inval_set_napot(
struct riscv_iommu_command *cmd, u64 addr, unsigned int sz_lg2)
{
u64 pfn = addr >> 12;
pfn |= BIT_U64(sz_lg2 - 13) - 1;
cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD1_IOTINVAL_ADDR, pfn) |
RISCV_IOMMU_CMD1_IOTINVAL_S;
cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_AV;
}
static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd,
int pscid)
{
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_PSCID, pscid) |
RISCV_IOMMU_CMD_IOTINVAL_PSCV;
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IOTINVAL_PSCID, pscid) |
RISCV_IOMMU_CMD0_IOTINVAL_PSCV;
}
static inline void riscv_iommu_cmd_inval_set_gscid(struct riscv_iommu_command *cmd,
int gscid)
{
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_GSCID, gscid) |
RISCV_IOMMU_CMD_IOTINVAL_GV;
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IOTINVAL_GSCID, gscid) |
RISCV_IOMMU_CMD0_IOTINVAL_GV;
}
static inline void riscv_iommu_cmd_iofence(struct riscv_iommu_command *cmd)
{
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
RISCV_IOMMU_CMD_IOFENCE_PR | RISCV_IOMMU_CMD_IOFENCE_PW;
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
RISCV_IOMMU_CMD0_IOFENCE_PR | RISCV_IOMMU_CMD0_IOFENCE_PW;
cmd->dword1 = 0;
}
static inline void riscv_iommu_cmd_iofence_set_av(struct riscv_iommu_command *cmd,
u64 addr, u32 data)
{
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
FIELD_PREP(RISCV_IOMMU_CMD_IOFENCE_DATA, data) |
RISCV_IOMMU_CMD_IOFENCE_AV;
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
FIELD_PREP(RISCV_IOMMU_CMD0_IOFENCE_DATA, data) |
RISCV_IOMMU_CMD0_IOFENCE_AV;
cmd->dword1 = addr >> 2;
}
static inline void riscv_iommu_cmd_iodir_inval_ddt(struct riscv_iommu_command *cmd)
{
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT);
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT);
cmd->dword1 = 0;
}
static inline void riscv_iommu_cmd_iodir_inval_pdt(struct riscv_iommu_command *cmd)
{
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT);
cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT);
cmd->dword1 = 0;
}
static inline void riscv_iommu_cmd_iodir_set_did(struct riscv_iommu_command *cmd,
unsigned int devid)
{
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_DID, devid) |
RISCV_IOMMU_CMD_IODIR_DV;
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IODIR_DID, devid) |
RISCV_IOMMU_CMD0_IODIR_DV;
}
static inline void riscv_iommu_cmd_iodir_set_pid(struct riscv_iommu_command *cmd,
unsigned int pasid)
{
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_PID, pasid);
cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IODIR_PID, pasid);
}
#endif /* _RISCV_IOMMU_BITS_H_ */

162
drivers/iommu/riscv/iommu.c
View File

@@ -920,20 +920,120 @@ static void riscv_iommu_bond_unlink(struct riscv_iommu_domain *domain,
}
}
/*
* Send IOTLB.INVAL for whole address space for ranges larger than 2MB.
* This limit will be replaced with range invalidations, if supported by
* the hardware, when RISC-V IOMMU architecture specification update for
* range invalidations update will be available.
*/
#define RISCV_IOMMU_IOTLB_INVAL_LIMIT (2 << 20)
struct riscv_iommu_tlbi {
u64 start;
u64 last;
bool non_leaf;
struct {
bool use_global;
u8 stride_lg2;
unsigned int num;
} single;
struct {
u8 sz_lg2;
u64 addr;
} range;
};
static void riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi *tlbi,
struct iommu_iotlb_gather *gather)
{
u8 combined = gather->pt.leaf_levels_bitmap |
gather->pt.table_levels_bitmap;
u64 num;
tlbi->non_leaf = gather->pt.table_levels_bitmap != 0;
tlbi->start = gather->start;
tlbi->last = gather->end;
/* No level information available */
if (!combined) {
tlbi->single.use_global = true;
tlbi->range.sz_lg2 = 0;
return;
}
/*
* Calculate the smallest NAPOT range containing [start, last].
* NAPOT encoding requires a power-of-two sized, naturally aligned
* range. Over-invalidation is always safe.
*/
tlbi->range.sz_lg2 = fls64(tlbi->start ^ tlbi->last);
if (unlikely(tlbi->range.sz_lg2 >= 64)) {
tlbi->single.use_global = true;
tlbi->range.sz_lg2 = 0;
return;
}
tlbi->range.addr = tlbi->start & ~(BIT_U64(tlbi->range.sz_lg2) - 1);
/*
* Calculate stride from the lowest changed level. RISC-V uses 4KiB
* granule with 9 bits per level.
*/
tlbi->single.stride_lg2 = 9 * __ffs(combined) + 12;
num = (tlbi->last - tlbi->start + 1) >> tlbi->single.stride_lg2;
if (!num || num > 512) {
tlbi->single.use_global = true;
} else {
tlbi->single.num = num;
tlbi->single.use_global = false;
}
}
static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu,
int pscid,
struct riscv_iommu_tlbi *tlbi)
{
bool use_nl = tlbi->non_leaf &&
(iommu->caps & RISCV_IOMMU_CAPABILITIES_NL);
struct riscv_iommu_command cmd;
unsigned int i;
riscv_iommu_cmd_inval_vma(&cmd);
riscv_iommu_cmd_inval_set_pscid(&cmd, pscid);
/*
* If non-leaf entries were changed and the IOMMU doesn't
* support NL, we must fall back to global invalidation (AV=0).
*/
if (tlbi->non_leaf && !use_nl)
goto global;
if (iommu->caps & RISCV_IOMMU_CAPABILITIES_S &&
tlbi->range.sz_lg2 >= 13) {
riscv_iommu_cmd_inval_set_napot(&cmd, tlbi->range.addr,
tlbi->range.sz_lg2);
if (use_nl)
riscv_iommu_cmd_inval_set_nl(&cmd);
riscv_iommu_cmd_send(iommu, &cmd);
} else {
unsigned long iova;
if (tlbi->single.use_global)
goto global;
iova = tlbi->start;
for (i = 0; i < tlbi->single.num; i++) {
riscv_iommu_cmd_inval_set_addr(&cmd, iova);
if (use_nl)
riscv_iommu_cmd_inval_set_nl(&cmd);
riscv_iommu_cmd_send(iommu, &cmd);
iova += 1ULL << tlbi->single.stride_lg2;
}
}
return;
global:
riscv_iommu_cmd_send(iommu, &cmd);
}
static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
unsigned long start, unsigned long end)
struct iommu_iotlb_gather *gather)
{
struct riscv_iommu_bond *bond;
struct riscv_iommu_device *iommu, *prev;
struct riscv_iommu_command cmd;
struct riscv_iommu_bond *bond;
struct riscv_iommu_tlbi tlbi;
riscv_iommu_tlbi_calc(&tlbi, gather);
/*
* For each IOMMU linked with this protection domain (via bonds->dev),
@@ -974,19 +1074,7 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
if (iommu == prev)
continue;
riscv_iommu_cmd_inval_vma(&cmd);
riscv_iommu_cmd_inval_set_pscid(&cmd, domain->pscid);
if (end - start < RISCV_IOMMU_IOTLB_INVAL_LIMIT - 1) {
unsigned long iova = start;
do {
riscv_iommu_cmd_inval_set_addr(&cmd, iova);
riscv_iommu_cmd_send(iommu, &cmd);
} while (!check_add_overflow(iova, PAGE_SIZE, &iova) &&
iova < end);
} else {
riscv_iommu_cmd_send(iommu, &cmd);
}
riscv_iommu_iotlb_inval_iommu(iommu, domain->pscid, &tlbi);
prev = iommu;
}
@@ -1145,8 +1233,14 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
static void riscv_iommu_iotlb_flush_all(struct iommu_domain *iommu_domain)
{
struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
struct iommu_iotlb_gather gather = {
.start = 0,
.end = ULONG_MAX,
.pt.leaf_levels_bitmap = 0xFF,
.pt.table_levels_bitmap = 0xFE,
};
riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
riscv_iommu_iotlb_inval(domain, &gather);
}
static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
@@ -1154,19 +1248,8 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
{
struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
if (iommu_pages_list_empty(&gather->freelist)) {
riscv_iommu_iotlb_inval(domain, gather->start, gather->end);
} else {
/*
* In 1.0 spec version, the smallest scope we can use to
* invalidate all levels of page table (i.e. leaf and non-leaf)
* is an invalidate-all-PSCID IOTINVAL.VMA with AV=0.
* This will be updated with hardware support for
* capability.NL (non-leaf) IOTINVAL command.
*/
riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
iommu_put_pages_list(&gather->freelist);
}
riscv_iommu_iotlb_inval(domain, gather);
iommu_put_pages_list(&gather->freelist);
}
static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
@@ -1267,7 +1350,10 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev)
*/
cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) |
BIT(PT_FEAT_FLUSH_RANGE) |
BIT(PT_FEAT_RISCV_SVNAPOT_64K);
BIT(PT_FEAT_RISCV_SVNAPOT_64K) |
BIT(PT_FEAT_DETAILED_GATHER);
if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT)
cfg.common.features |= BIT(PT_FEAT_RISCV_SVPBMT);
domain->riscvpt.iommu.nid = dev_to_node(iommu->dev);
domain->domain.ops = &riscv_iommu_paging_domain_ops;

8
drivers/iommu/rockchip-iommu.c
View File

@@ -76,6 +76,8 @@
#define SPAGE_ORDER 12
#define SPAGE_SIZE (1 << SPAGE_ORDER)
#define DISABLE_FETCH_DTE_TIME_LIMIT BIT(31)
/*
* Support mapping any size that fits in one page table:
* 4 KiB to 4 MiB
@@ -930,6 +932,7 @@ static int rk_iommu_enable(struct rk_iommu *iommu)
struct iommu_domain *domain = iommu->domain;
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
int ret, i;
u32 auto_gate;
ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
if (ret)
@@ -948,6 +951,11 @@ static int rk_iommu_enable(struct rk_iommu *iommu)
rk_ops->mk_dtentries(rk_domain->dt_dma));
rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);
/* Workaround for iommu blocked, BIT(31) default to 1 */
auto_gate = rk_iommu_read(iommu->bases[i], RK_MMU_AUTO_GATING);
auto_gate |= DISABLE_FETCH_DTE_TIME_LIMIT;
rk_iommu_write(iommu->bases[i], RK_MMU_AUTO_GATING, auto_gate);
}
ret = rk_iommu_enable_paging(iommu);

791
drivers/iommu/vsi-iommu.c Normal file
View File

@@ -0,0 +1,791 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2025 Collabora Ltd.
*
* IOMMU API for Verisilicon
*
* Module Authors: Yandong Lin <yandong.lin@rock-chips.com>
* Simon Xue <xxm@rock-chips.com>
* Benjamin Gaignard <benjamin.gaignard@collabora.com>
*
* This hardware block is using a 2 pages tables allocation structure.
* That make very similar to Rockhip iommu hardware blocks but it has
* it own driver because the registers offset and configuration bits
* are completely different. An additional reason is that this hardware
* has been developed by Verisilicon to be used by their hardware video
* decoders and not for a general purpose like Rockchip iommus.
*/
#include <linux/clk.h>
#include <linux/compiler.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_iommu.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "iommu-pages.h"
struct vsi_iommu {
struct device *dev;
void __iomem *regs;
struct clk_bulk_data *clocks;
int num_clocks;
struct iommu_device iommu;
struct list_head node; /* entry in vsi_iommu_domain.iommus */
struct iommu_domain *domain; /* domain to which iommu is attached */
spinlock_t lock; /* lock to protect vsi_iommu fields */
int irq;
bool enable;
};
struct vsi_iommu_domain {
struct list_head iommus;
struct device *dev;
u32 *dt;
dma_addr_t dt_dma;
struct iommu_domain domain;
u64 *pta;
dma_addr_t pta_dma;
spinlock_t lock; /* lock to protect vsi_iommu_domain fields */
};
static struct iommu_domain vsi_identity_domain;
#define NUM_DT_ENTRIES 1024
#define NUM_PT_ENTRIES 1024
#define SPAGE_SIZE BIT(12)
/* vsi iommu regs address */
#define VSI_MMU_CONFIG1_BASE 0x1ac
#define VSI_MMU_AHB_EXCEPTION_BASE 0x380
#define VSI_MMU_AHB_CONTROL_BASE 0x388
#define VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE 0x38C
/* MMU register offsets */
#define VSI_MMU_FLUSH_BASE 0x184
#define VSI_MMU_BIT_FLUSH BIT(4)
#define VSI_MMU_PAGE_FAULT_ADDR 0x380
#define VSI_MMU_STATUS_BASE 0x384 /* IRQ status */
#define VSI_MMU_BIT_ENABLE BIT(0)
#define VSI_MMU_OUT_OF_BOUND BIT(28)
/* Irq mask */
#define VSI_MMU_IRQ_MASK 0x7
#define VSI_DTE_PT_ADDRESS_MASK 0xffffffc0
#define VSI_DTE_PT_VALID BIT(0)
#define VSI_PAGE_DESC_LO_MASK 0xfffff000
#define VSI_PAGE_DESC_HI_MASK GENMASK_ULL(39, 32)
#define VSI_PAGE_DESC_HI_SHIFT (32 - 4)
static inline phys_addr_t vsi_dte_pt_address(u32 dte)
{
return (phys_addr_t)dte & VSI_DTE_PT_ADDRESS_MASK;
}
static inline u32 vsi_mk_dte(u32 dte)
{
return (phys_addr_t)dte | VSI_DTE_PT_VALID;
}
#define VSI_PTE_PAGE_WRITABLE BIT(2)
#define VSI_PTE_PAGE_VALID BIT(0)
static inline phys_addr_t vsi_pte_page_address(u64 pte)
{
return ((pte << VSI_PAGE_DESC_HI_SHIFT) & VSI_PAGE_DESC_HI_MASK) |
(pte & VSI_PAGE_DESC_LO_MASK);
}
static u32 vsi_mk_pte(phys_addr_t page, int prot)
{
u32 flags = 0;
flags |= (prot & IOMMU_WRITE) ? VSI_PTE_PAGE_WRITABLE : 0;
page = (page & VSI_PAGE_DESC_LO_MASK) |
((page & VSI_PAGE_DESC_HI_MASK) >> VSI_PAGE_DESC_HI_SHIFT);
return page | flags | VSI_PTE_PAGE_VALID;
}
#define VSI_DTE_PT_VALID BIT(0)
static inline bool vsi_dte_is_pt_valid(u32 dte)
{
return dte & VSI_DTE_PT_VALID;
}
static inline bool vsi_pte_is_page_valid(u32 pte)
{
return pte & VSI_PTE_PAGE_VALID;
}
static u32 vsi_mk_pte_invalid(u32 pte)
{
return pte & ~VSI_PTE_PAGE_VALID;
}
#define VSI_MASTER_TLB_MASK GENMASK_ULL(31, 10)
/* mode 0 : 4k */
#define VSI_PTA_4K_MODE 0
static u64 vsi_mk_pta(dma_addr_t dt_dma)
{
u64 val = (dt_dma & VSI_MASTER_TLB_MASK) | VSI_PTA_4K_MODE;
return val;
}
static struct vsi_iommu_domain *to_vsi_domain(struct iommu_domain *dom)
{
return container_of(dom, struct vsi_iommu_domain, domain);
}
static inline void vsi_table_flush(struct vsi_iommu_domain *vsi_domain, dma_addr_t dma,
unsigned int count)
{
size_t size = count * sizeof(u32); /* count of u32 entry */
dma_sync_single_for_device(vsi_domain->dev, dma, size, DMA_TO_DEVICE);
}
#define VSI_IOVA_DTE_MASK 0xffc00000
#define VSI_IOVA_DTE_SHIFT 22
#define VSI_IOVA_PTE_MASK 0x003ff000
#define VSI_IOVA_PTE_SHIFT 12
#define VSI_IOVA_PAGE_MASK 0x00000fff
#define VSI_IOVA_PAGE_SHIFT 0
static u32 vsi_iova_dte_index(u32 iova)
{
return (iova & VSI_IOVA_DTE_MASK) >> VSI_IOVA_DTE_SHIFT;
}
static u32 vsi_iova_pte_index(u32 iova)
{
return (iova & VSI_IOVA_PTE_MASK) >> VSI_IOVA_PTE_SHIFT;
}
static u32 vsi_iova_page_offset(u32 iova)
{
return (iova & VSI_IOVA_PAGE_MASK) >> VSI_IOVA_PAGE_SHIFT;
}
static irqreturn_t vsi_iommu_irq(int irq, void *dev_id)
{
struct vsi_iommu *iommu = dev_id;
unsigned long flags;
dma_addr_t iova;
u32 status;
if (pm_runtime_resume_and_get(iommu->dev) < 0)
return IRQ_NONE;
spin_lock_irqsave(&iommu->lock, flags);
status = readl(iommu->regs + VSI_MMU_STATUS_BASE);
if (status & VSI_MMU_IRQ_MASK) {
dev_err(iommu->dev, "unexpected int_status=%08x\n", status);
iova = readl(iommu->regs + VSI_MMU_PAGE_FAULT_ADDR);
report_iommu_fault(iommu->domain, iommu->dev, iova, status);
}
writel(0, iommu->regs + VSI_MMU_STATUS_BASE);
spin_unlock_irqrestore(&iommu->lock, flags);
pm_runtime_put_autosuspend(iommu->dev);
return IRQ_HANDLED;
}
static struct vsi_iommu *vsi_iommu_get_from_dev(struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct device *iommu_dev = bus_find_device_by_fwnode(&platform_bus_type,
fwspec->iommu_fwnode);
put_device(iommu_dev);
return iommu_dev ? dev_get_drvdata(iommu_dev) : NULL;
}
static struct iommu_domain *vsi_iommu_domain_alloc_paging(struct device *dev)
{
struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
struct vsi_iommu_domain *vsi_domain;
vsi_domain = kzalloc(sizeof(*vsi_domain), GFP_KERNEL);
if (!vsi_domain)
return NULL;
vsi_domain->dev = iommu->dev;
spin_lock_init(&vsi_domain->lock);
/*
* iommu use a 2 level pagetable.
* Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
* Allocate one 4 KiB page for each table.
*/
vsi_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
SPAGE_SIZE);
if (!vsi_domain->dt)
goto err_free_domain;
vsi_domain->dt_dma = dma_map_single(vsi_domain->dev, vsi_domain->dt,
SPAGE_SIZE, DMA_TO_DEVICE);
if (dma_mapping_error(vsi_domain->dev, vsi_domain->dt_dma)) {
dev_err(dev, "DMA map error for DT\n");
goto err_free_dt;
}
vsi_domain->pta = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
SPAGE_SIZE);
if (!vsi_domain->pta)
goto err_unmap_dt;
vsi_domain->pta[0] = vsi_mk_pta(vsi_domain->dt_dma);
vsi_domain->pta_dma = dma_map_single(vsi_domain->dev, vsi_domain->pta,
SPAGE_SIZE, DMA_TO_DEVICE);
if (dma_mapping_error(vsi_domain->dev, vsi_domain->pta_dma)) {
dev_err(dev, "DMA map error for PTA\n");
goto err_free_pta;
}
INIT_LIST_HEAD(&vsi_domain->iommus);
vsi_domain->domain.geometry.aperture_start = 0;
vsi_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
vsi_domain->domain.geometry.force_aperture = true;
vsi_domain->domain.pgsize_bitmap = SZ_4K;
return &vsi_domain->domain;
err_free_pta:
iommu_free_pages(vsi_domain->pta);
err_unmap_dt:
dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
SPAGE_SIZE, DMA_TO_DEVICE);
err_free_dt:
iommu_free_pages(vsi_domain->dt);
err_free_domain:
kfree(vsi_domain);
return NULL;
}
static phys_addr_t vsi_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
phys_addr_t pt_phys, phys = 0;
unsigned long flags;
u32 dte, pte;
u32 *page_table;
spin_lock_irqsave(&vsi_domain->lock, flags);
dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
if (!vsi_dte_is_pt_valid(dte))
goto unlock;
pt_phys = vsi_dte_pt_address(dte);
page_table = (u32 *)phys_to_virt(pt_phys);
pte = page_table[vsi_iova_pte_index(iova)];
if (!vsi_pte_is_page_valid(pte))
goto unlock;
phys = vsi_pte_page_address(pte) + vsi_iova_page_offset(iova);
unlock:
spin_unlock_irqrestore(&vsi_domain->lock, flags);
return phys;
}
static size_t vsi_iommu_unmap_iova(struct vsi_iommu_domain *vsi_domain,
u32 *pte_addr, dma_addr_t pte_dma,
size_t size)
{
unsigned int pte_count;
unsigned int pte_total = size / SPAGE_SIZE;
for (pte_count = 0;
pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
u32 pte = pte_addr[pte_count];
if (!vsi_pte_is_page_valid(pte))
break;
pte_addr[pte_count] = vsi_mk_pte_invalid(pte);
}
vsi_table_flush(vsi_domain, pte_dma, pte_total);
return pte_count * SPAGE_SIZE;
}
static int vsi_iommu_map_iova(struct vsi_iommu_domain *vsi_domain, u32 *pte_addr,
dma_addr_t pte_dma, dma_addr_t iova,
phys_addr_t paddr, size_t size, int prot)
{
unsigned int pte_count;
unsigned int pte_total = size / SPAGE_SIZE;
for (pte_count = 0;
pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
u32 pte = pte_addr[pte_count];
if (vsi_pte_is_page_valid(pte))
return (pte_count - 1) * SPAGE_SIZE;
pte_addr[pte_count] = vsi_mk_pte(paddr, prot);
paddr += SPAGE_SIZE;
}
vsi_table_flush(vsi_domain, pte_dma, pte_total);
return 0;
}
static void vsi_iommu_flush_tlb(struct iommu_domain *domain)
{
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
struct vsi_iommu *iommu;
list_for_each_entry(iommu, &vsi_domain->iommus, node) {
if (pm_runtime_get(iommu->dev) < 0)
continue;
spin_lock(&iommu->lock);
if (iommu->enable) {
writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
}
spin_unlock(&iommu->lock);
pm_runtime_put_autosuspend(iommu->dev);
}
}
static size_t vsi_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
size_t size, size_t count, struct iommu_iotlb_gather *gather)
{
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
unsigned long flags;
phys_addr_t pt_phys;
u32 dte;
u32 *pte_addr;
size_t unmap_size = 0;
spin_lock_irqsave(&vsi_domain->lock, flags);
dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
/* Just return 0 if iova is unmapped */
if (!vsi_dte_is_pt_valid(dte))
goto unlock;
pt_phys = vsi_dte_pt_address(dte);
pte_addr = (u32 *)phys_to_virt(pt_phys) + vsi_iova_pte_index(iova);
pte_dma = pt_phys + vsi_iova_pte_index(iova) * sizeof(u32);
unmap_size = vsi_iommu_unmap_iova(vsi_domain, pte_addr, pte_dma, size);
if (!unmap_size)
goto unlock;
vsi_iommu_flush_tlb(domain);
unlock:
spin_unlock_irqrestore(&vsi_domain->lock, flags);
return unmap_size;
}
static u32 *vsi_dte_get_page_table(struct vsi_iommu_domain *vsi_domain,
dma_addr_t iova, gfp_t gfp)
{
u32 *page_table, *dte_addr;
u32 dte_index, dte;
phys_addr_t pt_phys;
dma_addr_t pt_dma;
gfp_t flags;
dte_index = vsi_iova_dte_index(iova);
dte_addr = &vsi_domain->dt[dte_index];
dte = *dte_addr;
if (vsi_dte_is_pt_valid(dte))
goto done;
/* Do not allow to sleep while allocating the buffer */
flags = (gfp & ~GFP_KERNEL) | GFP_ATOMIC | GFP_DMA32;
page_table = iommu_alloc_pages_sz(flags, PAGE_SIZE);
if (!page_table)
return ERR_PTR(-ENOMEM);
pt_dma = dma_map_single(vsi_domain->dev, page_table, PAGE_SIZE, DMA_TO_DEVICE);
if (dma_mapping_error(vsi_domain->dev, pt_dma)) {
dev_err(vsi_domain->dev, "DMA mapping error while allocating page table\n");
iommu_free_pages(page_table);
return ERR_PTR(-ENOMEM);
}
dte = vsi_mk_dte(pt_dma);
*dte_addr = dte;
vsi_table_flush(vsi_domain,
vsi_domain->dt_dma + dte_index * sizeof(u32), 1);
done:
pt_phys = vsi_dte_pt_address(dte);
return (u32 *)phys_to_virt(pt_phys);
}
static int vsi_iommu_map(struct iommu_domain *domain, unsigned long _iova,
phys_addr_t paddr, size_t size, size_t count,
int prot, gfp_t gfp, size_t *mapped)
{
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
u32 *page_table, *pte_addr;
u32 dte, pte_index;
unsigned long flags;
int ret;
spin_lock_irqsave(&vsi_domain->lock, flags);
page_table = vsi_dte_get_page_table(vsi_domain, iova, gfp);
if (IS_ERR(page_table)) {
spin_unlock_irqrestore(&vsi_domain->lock, flags);
return PTR_ERR(page_table);
}
dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
pte_index = vsi_iova_pte_index(iova);
pte_addr = &page_table[pte_index];
pte_dma = vsi_dte_pt_address(dte) + pte_index * sizeof(u32);
ret = vsi_iommu_map_iova(vsi_domain, pte_addr, pte_dma, iova,
paddr, size, prot);
if (!ret)
*mapped = size;
vsi_iommu_flush_tlb(domain);
spin_unlock_irqrestore(&vsi_domain->lock, flags);
return ret;
}
static void vsi_iommu_disable(struct vsi_iommu *iommu)
{
writel(0, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
iommu->enable = false;
}
static int vsi_iommu_identity_attach(struct iommu_domain *domain,
struct device *dev, struct iommu_domain *old)
{
struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
unsigned long flags;
int ret;
ret = pm_runtime_resume_and_get(iommu->dev);
if (ret < 0)
return ret;
spin_lock_irqsave(&vsi_domain->lock, flags);
spin_lock(&iommu->lock);
if (iommu->domain == domain)
goto unlock;
vsi_iommu_disable(iommu);
list_del_init(&iommu->node);
iommu->domain = domain;
unlock:
spin_unlock(&iommu->lock);
spin_unlock_irqrestore(&vsi_domain->lock, flags);
pm_runtime_put_autosuspend(iommu->dev);
return 0;
}
static const struct iommu_domain_ops vsi_identity_ops = {
.attach_dev = vsi_iommu_identity_attach,
};
static struct iommu_domain vsi_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &vsi_identity_ops,
};
static void vsi_iommu_enable(struct vsi_iommu *iommu, struct iommu_domain *domain)
{
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
if (domain == &vsi_identity_domain)
return;
writel(vsi_domain->pta_dma, iommu->regs + VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE);
writel(VSI_MMU_OUT_OF_BOUND, iommu->regs + VSI_MMU_CONFIG1_BASE);
writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_EXCEPTION_BASE);
writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
iommu->enable = true;
}
static int vsi_iommu_attach_device(struct iommu_domain *domain,
struct device *dev, struct iommu_domain *old)
{
struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
unsigned long flags;
int ret = 0;
ret = pm_runtime_resume_and_get(iommu->dev);
if (ret < 0)
return ret;
spin_lock_irqsave(&vsi_domain->lock, flags);
spin_lock(&iommu->lock);
vsi_iommu_enable(iommu, domain);
writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
list_del_init(&iommu->node);
list_add_tail(&iommu->node, &vsi_domain->iommus);
iommu->domain = domain;
spin_unlock(&iommu->lock);
spin_unlock_irqrestore(&vsi_domain->lock, flags);
pm_runtime_put_autosuspend(iommu->dev);
return ret;
}
static void vsi_iommu_domain_free(struct iommu_domain *domain)
{
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
unsigned long flags;
int i;
spin_lock_irqsave(&vsi_domain->lock, flags);
WARN_ON(!list_empty(&vsi_domain->iommus));
for (i = 0; i < NUM_DT_ENTRIES; i++) {
u32 dte = vsi_domain->dt[i];
if (vsi_dte_is_pt_valid(dte)) {
phys_addr_t pt_phys = vsi_dte_pt_address(dte);
u32 *page_table = phys_to_virt(pt_phys);
dma_unmap_single(vsi_domain->dev, pt_phys,
SPAGE_SIZE, DMA_TO_DEVICE);
iommu_free_pages(page_table);
}
}
dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
SPAGE_SIZE, DMA_TO_DEVICE);
iommu_free_pages(vsi_domain->dt);
dma_unmap_single(vsi_domain->dev, vsi_domain->pta_dma,
SPAGE_SIZE, DMA_TO_DEVICE);
iommu_free_pages(vsi_domain->pta);
spin_unlock_irqrestore(&vsi_domain->lock, flags);
kfree(vsi_domain);
}
static struct iommu_device *vsi_iommu_probe_device(struct device *dev)
{
struct vsi_iommu *iommu = vsi_iommu_get_from_dev(dev);
struct device_link *link;
link = device_link_add(dev, iommu->dev,
DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
if (!link)
dev_err(dev, "Unable to link %s\n", dev_name(iommu->dev));
dev_iommu_priv_set(dev, iommu);
return &iommu->iommu;
}
static void vsi_iommu_release_device(struct device *dev)
{
struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
device_link_remove(dev, iommu->dev);
}
static int vsi_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
{
return iommu_fwspec_add_ids(dev, args->args, 1);
}
static const struct iommu_ops vsi_iommu_ops = {
.identity_domain = &vsi_identity_domain,
.release_domain = &vsi_identity_domain,
.domain_alloc_paging = vsi_iommu_domain_alloc_paging,
.of_xlate = vsi_iommu_of_xlate,
.probe_device = vsi_iommu_probe_device,
.release_device = vsi_iommu_release_device,
.device_group = generic_single_device_group,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = vsi_iommu_attach_device,
.map_pages = vsi_iommu_map,
.unmap_pages = vsi_iommu_unmap,
.iova_to_phys = vsi_iommu_iova_to_phys,
.free = vsi_iommu_domain_free,
}
};
static const struct of_device_id vsi_iommu_dt_ids[] = {
{
.compatible = "verisilicon,iommu-1.2",
},
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, vsi_iommu_dt_ids);
static int vsi_iommu_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct vsi_iommu *iommu;
int err;
iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
if (!iommu)
return -ENOMEM;
iommu->dev = dev;
spin_lock_init(&iommu->lock);
INIT_LIST_HEAD(&iommu->node);
iommu->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(iommu->regs))
return -ENOMEM;
iommu->num_clocks = devm_clk_bulk_get_all(dev, &iommu->clocks);
if (iommu->num_clocks < 0)
return iommu->num_clocks;
err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
if (err)
return err;
iommu->irq = platform_get_irq(pdev, 0);
if (iommu->irq < 0)
return iommu->irq;
err = devm_request_irq(iommu->dev, iommu->irq, vsi_iommu_irq,
IRQF_SHARED, dev_name(dev), iommu);
if (err)
goto err_unprepare_clocks;
dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
platform_set_drvdata(pdev, iommu);
pm_runtime_set_autosuspend_delay(dev, 100);
pm_runtime_use_autosuspend(dev);
pm_runtime_enable(dev);
err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, "%s",
dev_name(dev));
if (err)
goto err_runtime_disable;
err = iommu_device_register(&iommu->iommu, &vsi_iommu_ops, dev);
if (err)
goto err_remove_sysfs;
return 0;
err_remove_sysfs:
iommu_device_sysfs_remove(&iommu->iommu);
err_runtime_disable:
pm_runtime_disable(dev);
err_unprepare_clocks:
clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
return err;
}
static void vsi_iommu_shutdown(struct platform_device *pdev)
{
struct vsi_iommu *iommu = platform_get_drvdata(pdev);
disable_irq(iommu->irq);
pm_runtime_force_suspend(&pdev->dev);
}
static int __maybe_unused vsi_iommu_suspend(struct device *dev)
{
struct vsi_iommu *iommu = dev_get_drvdata(dev);
vsi_iommu_disable(iommu);
clk_bulk_disable(iommu->num_clocks, iommu->clocks);
return 0;
}
static int __maybe_unused vsi_iommu_resume(struct device *dev)
{
struct vsi_iommu *iommu = dev_get_drvdata(dev);
unsigned long flags;
int ret;
ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
if (ret)
return ret;
if (iommu->domain) {
struct vsi_iommu_domain *vsi_domain = to_vsi_domain(iommu->domain);
spin_lock_irqsave(&vsi_domain->lock, flags);
spin_lock(&iommu->lock);
vsi_iommu_enable(iommu, iommu->domain);
spin_unlock(&iommu->lock);
spin_unlock_irqrestore(&vsi_domain->lock, flags);
}
return 0;
}
static DEFINE_RUNTIME_DEV_PM_OPS(vsi_iommu_pm_ops,
vsi_iommu_suspend, vsi_iommu_resume,
NULL);
static struct platform_driver rockchip_vsi_iommu_driver = {
.probe = vsi_iommu_probe,
.shutdown = vsi_iommu_shutdown,
.driver = {
.name = "vsi_iommu",
.of_match_table = vsi_iommu_dt_ids,
.pm = pm_sleep_ptr(&vsi_iommu_pm_ops),
.suppress_bind_attrs = true,
},
};
module_platform_driver(rockchip_vsi_iommu_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Benjamin Gaignard <benjamin.gaignard@collabora.com>");
MODULE_DESCRIPTION("Verisilicon IOMMU driver");

47
drivers/pci/ats.c
View File

@@ -205,6 +205,53 @@ int pci_ats_page_aligned(struct pci_dev *pdev)
return 0;
}
/*
* CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on
* CXL.cache, devices need to get the Host Physical Address (HPA) from the Host
* by means of an ATS request on CXL.io.
*
* In other words, CXL.cache devices cannot access host physical memory without
* ATS.
*
* Check Cache_Capable instead of Cache_Enable because CXL.cache may be enabled
* after the caller uses this to make its ATS decision.
*/
static bool pci_cxl_ats_required(struct pci_dev *pdev)
{
int offset;
u16 cap;
offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
PCI_DVSEC_CXL_DEVICE);
if (!offset)
return false;
if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap))
return false;
return cap & PCI_DVSEC_CXL_CACHE_CAPABLE;
}
/**
* pci_ats_required - Whether the PCI device requires ATS
* @pdev: the PCI device
*
* Returns true, if the PCI device requires ATS for basic functional operation.
*/
bool pci_ats_required(struct pci_dev *pdev)
{
if (!pci_ats_supported(pdev))
return false;
/* A VF inherits its PF's requirement for ATS function */
if (pdev->is_virtfn)
pdev = pci_physfn(pdev);
return pci_cxl_ats_required(pdev) ||
pci_dev_specific_ats_required(pdev);
}
EXPORT_SYMBOL_GPL(pci_ats_required);
#ifdef CONFIG_PCI_PRI
void pci_pri_init(struct pci_dev *pdev)
{

9
drivers/pci/pci.h
View File

@@ -1155,6 +1155,15 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, bool probe)
}
#endif
#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_PCI_ATS)
bool pci_dev_specific_ats_required(struct pci_dev *dev);
#else
static inline bool pci_dev_specific_ats_required(struct pci_dev *dev)
{
return false;
}
#endif
#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64)
int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment,
struct resource *res);

42
drivers/pci/quirks.c
View File

@@ -5715,6 +5715,48 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1457, quirk_intel_e2000_no_ats);
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1459, quirk_intel_e2000_no_ats);
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145a, quirk_intel_e2000_no_ats);
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145c, quirk_intel_e2000_no_ats);
static bool quirk_nvidia_gpu_ats_required(struct pci_dev *pdev)
{
switch (pdev->device) {
case 0x2e00 ... 0x2e3f: /* GB20B */
return true;
}
return false;
}
static const struct pci_dev_ats_required {
u16 vendor;
u16 device;
bool (*ats_required)(struct pci_dev *dev);
} pci_dev_ats_required[] = {
/* NVIDIA GPUs */
{ PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, quirk_nvidia_gpu_ats_required },
/* NVIDIA CX10 Family NVlink-C2C */
{ PCI_VENDOR_ID_MELLANOX, 0x2101, NULL },
{ 0 }
};
/*
* Some NVIDIA devices do not implement CXL config space, but present as PCIe
* devices that can issue CXL-like cache operations like CXL.cache. Thus, they
* require ATS to obtain host physical addresses, like pci_cxl_ats_required().
*/
bool pci_dev_specific_ats_required(struct pci_dev *pdev)
{
const struct pci_dev_ats_required *i;
for (i = pci_dev_ats_required; i->vendor; i++) {
if (i->vendor != pdev->vendor)
continue;
if (i->ats_required && i->ats_required(pdev))
return true;
if (!i->ats_required && i->device == pdev->device)
return true;
}
return false;
}
#endif /* CONFIG_PCI_ATS */
/* Freescale PCIe doesn't support MSI in RC mode */

9
include/linux/generic_pt/common.h
View File

@@ -134,6 +134,11 @@ enum pt_features {
* significant amount of page table.
*/
PT_FEAT_FLUSH_RANGE_NO_GAPS,
/**
* @PT_FEAT_DETAILED_GATHER: Fill in the struct iommu_iotlb_gather pt
* sub structure with information about which levels were changed.
*/
PT_FEAT_DETAILED_GATHER,
/* private: */
PT_FEAT_FMT_START,
};
@@ -188,6 +193,10 @@ enum {
* Support the 64k contiguous page size following the Svnapot extension.
*/
PT_FEAT_RISCV_SVNAPOT_64K = PT_FEAT_FMT_START,
/*
* Support Svpbmt extension: encode page-based memory type (PBMT) in PTEs.
*/
PT_FEAT_RISCV_SVPBMT,
};

42
include/linux/iommu.h
View File

@@ -345,12 +345,6 @@ struct iommu_pages_list {
/**
* struct iommu_iotlb_gather - Range information for a pending IOTLB flush
*
* @start: IOVA representing the start of the range to be flushed
* @end: IOVA representing the end of the range to be flushed (inclusive)
* @pgsize: The interval at which to perform the flush
* @freelist: Removed pages to free after sync
* @queued: Indicates that the flush will be queued
*
* This structure is intended to be updated by multiple calls to the
* ->unmap() function in struct iommu_ops before eventually being passed
* into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after
@@ -359,10 +353,44 @@ struct iommu_pages_list {
* later instead of ->iotlb_sync(), so drivers may optimise accordingly.
*/
struct iommu_iotlb_gather {
/** @start: IOVA representing the start of the range to be flushed */
unsigned long start;
/**
* @end: IOVA representing the end of the range to be
* flushed (inclusive)
*/
unsigned long end;
size_t pgsize;
union {
/**
* @pgsize: The interval at which to perform the flush, only
* used by arm-smmu-v3
*/
size_t pgsize;
struct {
/**
* @pt.leaf_levels_bitmap: Bitmap of generic_pt
* levels where leaf entries were unmapped. Bit 0
* means the leaf only level. If 0 no leafs
* were unmapped.
*/
u8 leaf_levels_bitmap;
/**
* @pt.table_levels_bitmap: Bitmap of generic_pt levels
* of table entries that were removed. Bit 0 is never
* set, bit 1 means a table of all leafs was removed.
* When freelist is empty this must be 0.
*/
u8 table_levels_bitmap;
} pt;
};
/**
* @freelist: Removed pages to free after sync, only used by
* iommupt
*/
struct iommu_pages_list freelist;
/** @queued: True if the gather will be completed with a flush all */
bool queued;
};

3
include/linux/pci-ats.h
View File

@@ -12,6 +12,7 @@ int pci_prepare_ats(struct pci_dev *dev, int ps);
void pci_disable_ats(struct pci_dev *dev);
int pci_ats_queue_depth(struct pci_dev *dev);
int pci_ats_page_aligned(struct pci_dev *dev);
bool pci_ats_required(struct pci_dev *dev);
#else /* CONFIG_PCI_ATS */
static inline bool pci_ats_supported(struct pci_dev *d)
{ return false; }
@@ -24,6 +25,8 @@ static inline int pci_ats_queue_depth(struct pci_dev *d)
{ return -ENODEV; }
static inline int pci_ats_page_aligned(struct pci_dev *dev)
{ return 0; }
static inline bool pci_ats_required(struct pci_dev *dev)
{ return false; }
#endif /* CONFIG_PCI_ATS */
#ifdef CONFIG_PCI_PRI

1
include/uapi/linux/pci_regs.h
View File

@@ -1349,6 +1349,7 @@
/* CXL r4.0, 8.1.3: PCIe DVSEC for CXL Device */
#define PCI_DVSEC_CXL_DEVICE 0
#define PCI_DVSEC_CXL_CAP 0xA
#define PCI_DVSEC_CXL_CACHE_CAPABLE _BITUL(0)
#define PCI_DVSEC_CXL_MEM_CAPABLE _BITUL(2)
#define PCI_DVSEC_CXL_HDM_COUNT __GENMASK(5, 4)
#define PCI_DVSEC_CXL_CTRL 0xC