mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-04-30 20:30:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd

Browse Source 
Pull iommufd updates from Jason Gunthorpe:
 "Two significant new items:

   - Allow reporting IOMMU HW events to userspace when the events are
     clearly linked to a device.

     This is linked to the VIOMMU object and is intended to be used by a
     VMM to forward HW events to the virtual machine as part of
     emulating a vIOMMU. ARM SMMUv3 is the first driver to use this
     mechanism. Like the existing fault events the data is delivered
     through a simple FD returning event records on read().

   - PASID support in VFIO.

     The "Process Address Space ID" is a PCI feature that allows the
     device to tag all PCI DMA operations with an ID. The IOMMU will
     then use the ID to select a unique translation for those DMAs. This
     is part of Intel's vIOMMU support as VT-D HW requires the
     hypervisor to manage each PASID entry.

     The support is generic so any VFIO user could attach any
     translation to a PASID, and the support should work on ARM SMMUv3
     as well. AMD requires additional driver work.

  Some minor updates, along with fixes:

   - Prevent using nested parents with fault's, no driver support today

   - Put a single "cookie_type" value in the iommu_domain to indicate
     what owns the various opaque owner fields"

* tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (49 commits)
  iommufd: Test attach before detaching pasid
  iommufd: Fix iommu_vevent_header tables markup
  iommu: Convert unreachable() to BUG()
  iommufd: Balance veventq->num_events inc/dec
  iommufd: Initialize the flags of vevent in iommufd_viommu_report_event()
  iommufd/selftest: Add coverage for reporting max_pasid_log2 via IOMMU_HW_INFO
  iommufd: Extend IOMMU_GET_HW_INFO to report PASID capability
  vfio: VFIO_DEVICE_[AT|DE]TACH_IOMMUFD_PT support pasid
  vfio-iommufd: Support pasid [at|de]tach for physical VFIO devices
  ida: Add ida_find_first_range()
  iommufd/selftest: Add coverage for iommufd pasid attach/detach
  iommufd/selftest: Add test ops to test pasid attach/detach
  iommufd/selftest: Add a helper to get test device
  iommufd/selftest: Add set_dev_pasid in mock iommu
  iommufd: Allow allocating PASID-compatible domain
  iommu/vt-d: Add IOMMU_HWPT_ALLOC_PASID support
  iommufd: Enforce PASID-compatible domain for RID
  iommufd: Support pasid attach/replace
  iommufd: Enforce PASID-compatible domain in PASID path
  iommufd/device: Add pasid_attach array to track per-PASID attach
  ...
This commit is contained in:
Linus Torvalds
2025-04-01 18:03:46 -07:00
parent 792b8307ec 7be11d34f6
commit 48552153cf
39 changed files with 3155 additions and 837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Documentation/userspace-api/iommufd.rst
View File

@@ -63,6 +63,13 @@ Following IOMMUFD objects are exposed to userspace:
space usually has mappings from guest-level I/O virtual addresses to guest-
level physical addresses.
- IOMMUFD_FAULT, representing a software queue for an HWPT reporting IO page
faults using the IOMMU HW's PRI (Page Request Interface). This queue object
provides user space an FD to poll the page fault events and also to respond
to those events. A FAULT object must be created first to get a fault_id that
could be then used to allocate a fault-enabled HWPT via the IOMMU_HWPT_ALLOC
command by setting the IOMMU_HWPT_FAULT_ID_VALID bit in its flags field.
- IOMMUFD_OBJ_VIOMMU, representing a slice of the physical IOMMU instance,
passed to or shared with a VM. It may be some HW-accelerated virtualization
features and some SW resources used by the VM. For examples:
@@ -109,6 +116,14 @@ Following IOMMUFD objects are exposed to userspace:
vIOMMU, which is a separate ioctl call from attaching the same device to an
HWPT_PAGING that the vIOMMU holds.
- IOMMUFD_OBJ_VEVENTQ, representing a software queue for a vIOMMU to report its
events such as translation faults occurred to a nested stage-1 (excluding I/O
page faults that should go through IOMMUFD_OBJ_FAULT) and HW-specific events.
This queue object provides user space an FD to poll/read the vIOMMU events. A
vIOMMU object must be created first to get its viommu_id, which could be then
used to allocate a vEVENTQ. Each vIOMMU can support multiple types of vEVENTS,
but is confined to one vEVENTQ per vEVENTQ type.
All user-visible objects are destroyed via the IOMMU_DESTROY uAPI.
The diagrams below show relationships between user-visible objects and kernel
@@ -251,8 +266,10 @@ User visible objects are backed by following datastructures:
- iommufd_device for IOMMUFD_OBJ_DEVICE.
- iommufd_hwpt_paging for IOMMUFD_OBJ_HWPT_PAGING.
- iommufd_hwpt_nested for IOMMUFD_OBJ_HWPT_NESTED.
- iommufd_fault for IOMMUFD_OBJ_FAULT.
- iommufd_viommu for IOMMUFD_OBJ_VIOMMU.
- iommufd_vdevice for IOMMUFD_OBJ_VDEVICE.
- iommufd_veventq for IOMMUFD_OBJ_VEVENTQ.
Several terminologies when looking at these datastructures:

60
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
View File

@@ -43,6 +43,8 @@ static void arm_smmu_make_nested_cd_table_ste(
target->data[0] |= nested_domain->ste[0] &
~cpu_to_le64(STRTAB_STE_0_CFG);
target->data[1] |= nested_domain->ste[1];
/* Merge events for DoS mitigations on eventq */
target->data[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
}
/*
@@ -85,6 +87,47 @@ static void arm_smmu_make_nested_domain_ste(
}
}
int arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,
struct arm_smmu_nested_domain *nested_domain)
{
struct arm_smmu_vmaster *vmaster;
unsigned long vsid;
int ret;
iommu_group_mutex_assert(state->master->dev);
ret = iommufd_viommu_get_vdev_id(&nested_domain->vsmmu->core,
state->master->dev, &vsid);
if (ret)
return ret;
vmaster = kzalloc(sizeof(*vmaster), GFP_KERNEL);
if (!vmaster)
return -ENOMEM;
vmaster->vsmmu = nested_domain->vsmmu;
vmaster->vsid = vsid;
state->vmaster = vmaster;
return 0;
}
void arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state)
{
struct arm_smmu_master *master = state->master;
mutex_lock(&master->smmu->streams_mutex);
kfree(master->vmaster);
master->vmaster = state->vmaster;
mutex_unlock(&master->smmu->streams_mutex);
}
void arm_smmu_master_clear_vmaster(struct arm_smmu_master *master)
{
struct arm_smmu_attach_state state = { .master = master };
arm_smmu_attach_commit_vmaster(&state);
}
static int arm_smmu_attach_dev_nested(struct iommu_domain *domain,
struct device *dev)
{
@@ -392,4 +435,21 @@ struct iommufd_viommu *arm_vsmmu_alloc(struct device *dev,
return &vsmmu->core;
}
int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster, u64 *evt)
{
struct iommu_vevent_arm_smmuv3 vevt;
int i;
lockdep_assert_held(&vmaster->vsmmu->smmu->streams_mutex);
vevt.evt[0] = cpu_to_le64((evt[0] & ~EVTQ_0_SID) |
FIELD_PREP(EVTQ_0_SID, vmaster->vsid));
for (i = 1; i < EVTQ_ENT_DWORDS; i++)
vevt.evt[i] = cpu_to_le64(evt[i]);
return iommufd_viommu_report_event(&vmaster->vsmmu->core,
IOMMU_VEVENTQ_TYPE_ARM_SMMUV3, &vevt,
sizeof(vevt));
}
MODULE_IMPORT_NS("IOMMUFD");

80
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
View File

@@ -1052,7 +1052,7 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
STRTAB_STE_1_EATS);
STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
/*
@@ -1068,7 +1068,7 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
if (cfg & BIT(1)) {
used_bits[1] |=
cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
STRTAB_STE_1_SHCFG);
STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
used_bits[2] |=
cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
@@ -1813,8 +1813,8 @@ static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw,
mutex_unlock(&smmu->streams_mutex);
}
static int arm_smmu_handle_event(struct arm_smmu_device *smmu,
struct arm_smmu_event *event)
static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,
struct arm_smmu_event *event)
{
int ret = 0;
u32 perm = 0;
@@ -1823,6 +1823,10 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu,
struct iommu_fault *flt = &fault_evt.fault;
switch (event->id) {
case EVT_ID_BAD_STE_CONFIG:
case EVT_ID_STREAM_DISABLED_FAULT:
case EVT_ID_BAD_SUBSTREAMID_CONFIG:
case EVT_ID_BAD_CD_CONFIG:
case EVT_ID_TRANSLATION_FAULT:
case EVT_ID_ADDR_SIZE_FAULT:
case EVT_ID_ACCESS_FAULT:
@@ -1832,31 +1836,30 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu,
return -EOPNOTSUPP;
}
if (!event->stall)
return -EOPNOTSUPP;
if (event->stall) {
if (event->read)
perm |= IOMMU_FAULT_PERM_READ;
else
perm |= IOMMU_FAULT_PERM_WRITE;
if (event->read)
perm |= IOMMU_FAULT_PERM_READ;
else
perm |= IOMMU_FAULT_PERM_WRITE;
if (event->instruction)
perm |= IOMMU_FAULT_PERM_EXEC;
if (event->instruction)
perm |= IOMMU_FAULT_PERM_EXEC;
if (event->privileged)
perm |= IOMMU_FAULT_PERM_PRIV;
if (event->privileged)
perm |= IOMMU_FAULT_PERM_PRIV;
flt->type = IOMMU_FAULT_PAGE_REQ;
flt->prm = (struct iommu_fault_page_request){
.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
.grpid = event->stag,
.perm = perm,
.addr = event->iova,
};
flt->type = IOMMU_FAULT_PAGE_REQ;
flt->prm = (struct iommu_fault_page_request) {
.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
.grpid = event->stag,
.perm = perm,
.addr = event->iova,
};
if (event->ssv) {
flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
flt->prm.pasid = event->ssid;
if (event->ssv) {
flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
flt->prm.pasid = event->ssid;
}
}
mutex_lock(&smmu->streams_mutex);
@@ -1866,7 +1869,12 @@ static int arm_smmu_handle_event(struct arm_smmu_device *smmu,
goto out_unlock;
}
ret = iommu_report_device_fault(master->dev, &fault_evt);
if (event->stall)
ret = iommu_report_device_fault(master->dev, &fault_evt);
else if (master->vmaster && !event->s2)
ret = arm_vmaster_report_event(master->vmaster, evt);
else
ret = -EOPNOTSUPP; /* Unhandled events should be pinned */
out_unlock:
mutex_unlock(&smmu->streams_mutex);
return ret;
@@ -1944,7 +1952,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
do {
while (!queue_remove_raw(q, evt)) {
arm_smmu_decode_event(smmu, evt, &event);
if (arm_smmu_handle_event(smmu, &event))
if (arm_smmu_handle_event(smmu, evt, &event))
arm_smmu_dump_event(smmu, evt, &event, &rs);
put_device(event.dev);
@@ -2803,6 +2811,7 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
struct arm_smmu_domain *smmu_domain =
to_smmu_domain_devices(new_domain);
unsigned long flags;
int ret;
/*
* arm_smmu_share_asid() must not see two domains pointing to the same
@@ -2832,9 +2841,18 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
}
if (smmu_domain) {
if (new_domain->type == IOMMU_DOMAIN_NESTED) {
ret = arm_smmu_attach_prepare_vmaster(
state, to_smmu_nested_domain(new_domain));
if (ret)
return ret;
}
master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
if (!master_domain)
if (!master_domain) {
kfree(state->vmaster);
return -ENOMEM;
}
master_domain->master = master;
master_domain->ssid = state->ssid;
if (new_domain->type == IOMMU_DOMAIN_NESTED)
@@ -2861,6 +2879,7 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
spin_unlock_irqrestore(&smmu_domain->devices_lock,
flags);
kfree(master_domain);
kfree(state->vmaster);
return -EINVAL;
}
@@ -2893,6 +2912,8 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
lockdep_assert_held(&arm_smmu_asid_lock);
arm_smmu_attach_commit_vmaster(state);
if (state->ats_enabled && !master->ats_enabled) {
arm_smmu_enable_ats(master);
} else if (state->ats_enabled && master->ats_enabled) {
@@ -3162,6 +3183,7 @@ static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
struct arm_smmu_ste ste;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
arm_smmu_master_clear_vmaster(master);
arm_smmu_make_bypass_ste(master->smmu, &ste);
arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
return 0;
@@ -3180,7 +3202,9 @@ static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
struct device *dev)
{
struct arm_smmu_ste ste;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
arm_smmu_master_clear_vmaster(master);
arm_smmu_make_abort_ste(&ste);
arm_smmu_attach_dev_ste(domain, dev, &ste,
STRTAB_STE_1_S1DSS_TERMINATE);

36
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
View File

@@ -266,6 +266,7 @@ static inline u32 arm_smmu_strtab_l2_idx(u32 sid)
#define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
#define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
#define STRTAB_STE_1_MEV (1UL << 19)
#define STRTAB_STE_1_S2FWB (1UL << 25)
#define STRTAB_STE_1_S1STALLD (1UL << 27)
@@ -799,6 +800,11 @@ struct arm_smmu_stream {
struct rb_node node;
};
struct arm_smmu_vmaster {
struct arm_vsmmu *vsmmu;
unsigned long vsid;
};
struct arm_smmu_event {
u8 stall : 1,
ssv : 1,
@@ -824,6 +830,7 @@ struct arm_smmu_master {
struct arm_smmu_device *smmu;
struct device *dev;
struct arm_smmu_stream *streams;
struct arm_smmu_vmaster *vmaster; /* use smmu->streams_mutex */
/* Locked by the iommu core using the group mutex */
struct arm_smmu_ctx_desc_cfg cd_table;
unsigned int num_streams;
@@ -972,6 +979,7 @@ struct arm_smmu_attach_state {
bool disable_ats;
ioasid_t ssid;
/* Resulting state */
struct arm_smmu_vmaster *vmaster;
bool ats_enabled;
};
@@ -1055,9 +1063,37 @@ struct iommufd_viommu *arm_vsmmu_alloc(struct device *dev,
struct iommu_domain *parent,
struct iommufd_ctx *ictx,
unsigned int viommu_type);
int arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,
struct arm_smmu_nested_domain *nested_domain);
void arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state);
void arm_smmu_master_clear_vmaster(struct arm_smmu_master *master);
int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster, u64 *evt);
#else
#define arm_smmu_hw_info NULL
#define arm_vsmmu_alloc NULL
static inline int
arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,
struct arm_smmu_nested_domain *nested_domain)
{
return 0;
}
static inline void
arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state)
{
}
static inline void
arm_smmu_master_clear_vmaster(struct arm_smmu_master *master)
{
}
static inline int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster,
u64 *evt)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_ARM_SMMU_V3_IOMMUFD */
#endif /* _ARM_SMMU_V3_H */

206
drivers/iommu/dma-iommu.c
View File

@@ -42,11 +42,6 @@ struct iommu_dma_msi_page {
phys_addr_t phys;
};
enum iommu_dma_cookie_type {
IOMMU_DMA_IOVA_COOKIE,
IOMMU_DMA_MSI_COOKIE,
};
enum iommu_dma_queue_type {
IOMMU_DMA_OPTS_PER_CPU_QUEUE,
IOMMU_DMA_OPTS_SINGLE_QUEUE,
@@ -59,34 +54,30 @@ struct iommu_dma_options {
};
struct iommu_dma_cookie {
enum iommu_dma_cookie_type type;
struct iova_domain iovad;
struct list_head msi_page_list;
/* Flush queue */
union {
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
struct {
struct iova_domain iovad;
/* Flush queue */
union {
struct iova_fq *single_fq;
struct iova_fq __percpu *percpu_fq;
};
/* Number of TLB flushes that have been started */
atomic64_t fq_flush_start_cnt;
/* Number of TLB flushes that have been finished */
atomic64_t fq_flush_finish_cnt;
/* Timer to regularily empty the flush queues */
struct timer_list fq_timer;
/* 1 when timer is active, 0 when not */
atomic_t fq_timer_on;
};
/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
dma_addr_t msi_iova;
struct iova_fq *single_fq;
struct iova_fq __percpu *percpu_fq;
};
struct list_head msi_page_list;
/* Number of TLB flushes that have been started */
atomic64_t fq_flush_start_cnt;
/* Number of TLB flushes that have been finished */
atomic64_t fq_flush_finish_cnt;
/* Timer to regularily empty the flush queues */
struct timer_list fq_timer;
/* 1 when timer is active, 0 when not */
atomic_t fq_timer_on;
/* Domain for flush queue callback; NULL if flush queue not in use */
struct iommu_domain *fq_domain;
struct iommu_domain *fq_domain;
/* Options for dma-iommu use */
struct iommu_dma_options options;
struct iommu_dma_options options;
};
struct iommu_dma_msi_cookie {
dma_addr_t msi_iova;
struct list_head msi_page_list;
};
static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
@@ -102,9 +93,6 @@ static int __init iommu_dma_forcedac_setup(char *str)
}
early_param("iommu.forcedac", iommu_dma_forcedac_setup);
static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr);
/* Number of entries per flush queue */
#define IOVA_DEFAULT_FQ_SIZE 256
#define IOVA_SINGLE_FQ_SIZE 32768
@@ -368,39 +356,24 @@ int iommu_dma_init_fq(struct iommu_domain *domain)
return 0;
}
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
{
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
return cookie->iovad.granule;
return PAGE_SIZE;
}
static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
{
struct iommu_dma_cookie *cookie;
cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
if (cookie) {
INIT_LIST_HEAD(&cookie->msi_page_list);
cookie->type = type;
}
return cookie;
}
/**
* iommu_get_dma_cookie - Acquire DMA-API resources for a domain
* @domain: IOMMU domain to prepare for DMA-API usage
*/
int iommu_get_dma_cookie(struct iommu_domain *domain)
{
if (domain->iova_cookie)
struct iommu_dma_cookie *cookie;
if (domain->cookie_type != IOMMU_COOKIE_NONE)
return -EEXIST;
domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);
if (!domain->iova_cookie)
cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
if (!cookie)
return -ENOMEM;
iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);
INIT_LIST_HEAD(&cookie->msi_page_list);
domain->cookie_type = IOMMU_COOKIE_DMA_IOVA;
domain->iova_cookie = cookie;
return 0;
}
@@ -418,54 +391,56 @@ int iommu_get_dma_cookie(struct iommu_domain *domain)
*/
int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
{
struct iommu_dma_cookie *cookie;
struct iommu_dma_msi_cookie *cookie;
if (domain->type != IOMMU_DOMAIN_UNMANAGED)
return -EINVAL;
if (domain->iova_cookie)
if (domain->cookie_type != IOMMU_COOKIE_NONE)
return -EEXIST;
cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
if (!cookie)
return -ENOMEM;
cookie->msi_iova = base;
domain->iova_cookie = cookie;
iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);
INIT_LIST_HEAD(&cookie->msi_page_list);
domain->cookie_type = IOMMU_COOKIE_DMA_MSI;
domain->msi_cookie = cookie;
return 0;
}
EXPORT_SYMBOL(iommu_get_msi_cookie);
/**
* iommu_put_dma_cookie - Release a domain's DMA mapping resources
* @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or
* iommu_get_msi_cookie()
* @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
*/
void iommu_put_dma_cookie(struct iommu_domain *domain)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iommu_dma_msi_page *msi, *tmp;
#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
if (domain->sw_msi != iommu_dma_sw_msi)
return;
#endif
if (!cookie)
return;
if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
if (cookie->iovad.granule) {
iommu_dma_free_fq(cookie);
put_iova_domain(&cookie->iovad);
}
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
list_del(&msi->list);
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list)
kfree(msi);
kfree(cookie);
}
/**
* iommu_put_msi_cookie - Release a domain's MSI mapping resources
* @domain: IOMMU domain previously prepared by iommu_get_msi_cookie()
*/
void iommu_put_msi_cookie(struct iommu_domain *domain)
{
struct iommu_dma_msi_cookie *cookie = domain->msi_cookie;
struct iommu_dma_msi_page *msi, *tmp;
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list)
kfree(msi);
}
kfree(cookie);
domain->iova_cookie = NULL;
}
/**
@@ -685,7 +660,7 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev
struct iova_domain *iovad;
int ret;
if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
if (!cookie || domain->cookie_type != IOMMU_COOKIE_DMA_IOVA)
return -EINVAL;
iovad = &cookie->iovad;
@@ -768,9 +743,9 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
struct iova_domain *iovad = &cookie->iovad;
unsigned long shift, iova_len, iova;
if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
cookie->msi_iova += size;
return cookie->msi_iova - size;
if (domain->cookie_type == IOMMU_COOKIE_DMA_MSI) {
domain->msi_cookie->msi_iova += size;
return domain->msi_cookie->msi_iova - size;
}
shift = iova_shift(iovad);
@@ -807,16 +782,16 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
return (dma_addr_t)iova << shift;
}
static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
static void iommu_dma_free_iova(struct iommu_domain *domain, dma_addr_t iova,
size_t size, struct iommu_iotlb_gather *gather)
{
struct iova_domain *iovad = &cookie->iovad;
struct iova_domain *iovad = &domain->iova_cookie->iovad;
/* The MSI case is only ever cleaning up its most recent allocation */
if (cookie->type == IOMMU_DMA_MSI_COOKIE)
cookie->msi_iova -= size;
if (domain->cookie_type == IOMMU_COOKIE_DMA_MSI)
domain->msi_cookie->msi_iova -= size;
else if (gather && gather->queued)
queue_iova(cookie, iova_pfn(iovad, iova),
queue_iova(domain->iova_cookie, iova_pfn(iovad, iova),
size >> iova_shift(iovad),
&gather->freelist);
else
@@ -844,7 +819,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
if (!iotlb_gather.queued)
iommu_iotlb_sync(domain, &iotlb_gather);
iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
iommu_dma_free_iova(domain, dma_addr, size, &iotlb_gather);
}
static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -872,7 +847,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
return DMA_MAPPING_ERROR;
if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) {
iommu_dma_free_iova(cookie, iova, size, NULL);
iommu_dma_free_iova(domain, iova, size, NULL);
return DMA_MAPPING_ERROR;
}
return iova + iova_off;
@@ -1009,7 +984,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
out_free_sg:
sg_free_table(sgt);
out_free_iova:
iommu_dma_free_iova(cookie, iova, size, NULL);
iommu_dma_free_iova(domain, iova, size, NULL);
out_free_pages:
__iommu_dma_free_pages(pages, count);
return NULL;
@@ -1486,7 +1461,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
return __finalise_sg(dev, sg, nents, iova);
out_free_iova:
iommu_dma_free_iova(cookie, iova, iova_len, NULL);
iommu_dma_free_iova(domain, iova, iova_len, NULL);
out_restore_sg:
__invalidate_sg(sg, nents);
out:
@@ -1764,17 +1739,47 @@ void iommu_setup_dma_ops(struct device *dev)
dev->dma_iommu = false;
}
static bool has_msi_cookie(const struct iommu_domain *domain)
{
return domain && (domain->cookie_type == IOMMU_COOKIE_DMA_IOVA ||
domain->cookie_type == IOMMU_COOKIE_DMA_MSI);
}
static size_t cookie_msi_granule(const struct iommu_domain *domain)
{
switch (domain->cookie_type) {
case IOMMU_COOKIE_DMA_IOVA:
return domain->iova_cookie->iovad.granule;
case IOMMU_COOKIE_DMA_MSI:
return PAGE_SIZE;
default:
BUG();
};
}
static struct list_head *cookie_msi_pages(const struct iommu_domain *domain)
{
switch (domain->cookie_type) {
case IOMMU_COOKIE_DMA_IOVA:
return &domain->iova_cookie->msi_page_list;
case IOMMU_COOKIE_DMA_MSI:
return &domain->msi_cookie->msi_page_list;
default:
BUG();
};
}
static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
phys_addr_t msi_addr, struct iommu_domain *domain)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct list_head *msi_page_list = cookie_msi_pages(domain);
struct iommu_dma_msi_page *msi_page;
dma_addr_t iova;
int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
size_t size = cookie_msi_granule(cookie);
size_t size = cookie_msi_granule(domain);
msi_addr &= ~(phys_addr_t)(size - 1);
list_for_each_entry(msi_page, &cookie->msi_page_list, list)
list_for_each_entry(msi_page, msi_page_list, list)
if (msi_page->phys == msi_addr)
return msi_page;
@@ -1792,23 +1797,23 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
INIT_LIST_HEAD(&msi_page->list);
msi_page->phys = msi_addr;
msi_page->iova = iova;
list_add(&msi_page->list, &cookie->msi_page_list);
list_add(&msi_page->list, msi_page_list);
return msi_page;
out_free_iova:
iommu_dma_free_iova(cookie, iova, size, NULL);
iommu_dma_free_iova(domain, iova, size, NULL);
out_free_page:
kfree(msi_page);
return NULL;
}
static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr)
int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr)
{
struct device *dev = msi_desc_to_dev(desc);
const struct iommu_dma_msi_page *msi_page;
if (!domain->iova_cookie) {
if (!has_msi_cookie(domain)) {
msi_desc_set_iommu_msi_iova(desc, 0, 0);
return 0;
}
@@ -1818,9 +1823,8 @@ static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
if (!msi_page)
return -ENOMEM;
msi_desc_set_iommu_msi_iova(
desc, msi_page->iova,
ilog2(cookie_msi_granule(domain->iova_cookie)));
msi_desc_set_iommu_msi_iova(desc, msi_page->iova,
ilog2(cookie_msi_granule(domain)));
return 0;
}

14
drivers/iommu/dma-iommu.h
View File

@@ -13,11 +13,15 @@ void iommu_setup_dma_ops(struct device *dev);
int iommu_get_dma_cookie(struct iommu_domain *domain);
void iommu_put_dma_cookie(struct iommu_domain *domain);
void iommu_put_msi_cookie(struct iommu_domain *domain);
int iommu_dma_init_fq(struct iommu_domain *domain);
void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr);
extern bool iommu_dma_forcedac;
#else /* CONFIG_IOMMU_DMA */
@@ -40,9 +44,19 @@ static inline void iommu_put_dma_cookie(struct iommu_domain *domain)
{
}
static inline void iommu_put_msi_cookie(struct iommu_domain *domain)
{
}
static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
{
}
static inline int iommu_dma_sw_msi(struct iommu_domain *domain,
struct msi_desc *desc, phys_addr_t msi_addr)
{
return -ENODEV;
}
#endif /* CONFIG_IOMMU_DMA */
#endif /* __DMA_IOMMU_H */

3
drivers/iommu/intel/iommu.c
View File

@@ -3383,7 +3383,8 @@ intel_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
bool first_stage;
if (flags &
(~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING)))
(~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
IOMMU_HWPT_ALLOC_PASID)))
return ERR_PTR(-EOPNOTSUPP);
if (nested_parent && !nested_supported(iommu))
return ERR_PTR(-EOPNOTSUPP);

2
drivers/iommu/intel/nested.c
View File

@@ -198,7 +198,7 @@ intel_iommu_domain_alloc_nested(struct device *dev, struct iommu_domain *parent,
struct dmar_domain *domain;
int ret;
if (!nested_supported(iommu) || flags)
if (!nested_supported(iommu) || flags & ~IOMMU_HWPT_ALLOC_PASID)
return ERR_PTR(-EOPNOTSUPP);
/* Must be nested domain */

16
drivers/iommu/iommu-priv.h
View File

@@ -5,6 +5,7 @@
#define __LINUX_IOMMU_PRIV_H
#include <linux/iommu.h>
#include <linux/msi.h>
static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
{
@@ -47,4 +48,19 @@ void iommu_detach_group_handle(struct iommu_domain *domain,
int iommu_replace_group_handle(struct iommu_group *group,
struct iommu_domain *new_domain,
struct iommu_attach_handle *handle);
#if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE) && IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr);
#else /* !CONFIG_IOMMUFD_DRIVER_CORE || !CONFIG_IRQ_MSI_IOMMU */
static inline int iommufd_sw_msi(struct iommu_domain *domain,
struct msi_desc *desc, phys_addr_t msi_addr)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_IOMMUFD_DRIVER_CORE && CONFIG_IRQ_MSI_IOMMU */
int iommu_replace_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid,
struct iommu_attach_handle *handle);
#endif /* __LINUX_IOMMU_PRIV_H */

1
drivers/iommu/iommu-sva.c
View File

@@ -310,6 +310,7 @@ static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
}
domain->type = IOMMU_DOMAIN_SVA;
domain->cookie_type = IOMMU_COOKIE_SVA;
mmgrab(mm);
domain->mm = mm;
domain->owner = ops;

160
drivers/iommu/iommu.c
View File

@@ -18,6 +18,7 @@
#include <linux/errno.h>
#include <linux/host1x_context_bus.h>
#include <linux/iommu.h>
#include <linux/iommufd.h>
#include <linux/idr.h>
#include <linux/err.h>
#include <linux/pci.h>
@@ -539,6 +540,13 @@ static void iommu_deinit_device(struct device *dev)
dev_iommu_free(dev);
}
static struct iommu_domain *pasid_array_entry_to_domain(void *entry)
{
if (xa_pointer_tag(entry) == IOMMU_PASID_ARRAY_DOMAIN)
return xa_untag_pointer(entry);
return ((struct iommu_attach_handle *)xa_untag_pointer(entry))->domain;
}
DEFINE_MUTEX(iommu_probe_device_lock);
static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
@@ -1973,8 +1981,10 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
iommu_fault_handler_t handler,
void *token)
{
BUG_ON(!domain);
if (WARN_ON(!domain || domain->cookie_type != IOMMU_COOKIE_NONE))
return;
domain->cookie_type = IOMMU_COOKIE_FAULT_HANDLER;
domain->handler = handler;
domain->handler_token = token;
}
@@ -2044,9 +2054,19 @@ EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc_flags);
void iommu_domain_free(struct iommu_domain *domain)
{
if (domain->type == IOMMU_DOMAIN_SVA)
switch (domain->cookie_type) {
case IOMMU_COOKIE_DMA_IOVA:
iommu_put_dma_cookie(domain);
break;
case IOMMU_COOKIE_DMA_MSI:
iommu_put_msi_cookie(domain);
break;
case IOMMU_COOKIE_SVA:
mmdrop(domain->mm);
iommu_put_dma_cookie(domain);
break;
default:
break;
}
if (domain->ops->free)
domain->ops->free(domain);
}
@@ -3335,14 +3355,15 @@ static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
}
static int __iommu_set_group_pasid(struct iommu_domain *domain,
struct iommu_group *group, ioasid_t pasid)
struct iommu_group *group, ioasid_t pasid,
struct iommu_domain *old)
{
struct group_device *device, *last_gdev;
int ret;
for_each_group_device(group, device) {
ret = domain->ops->set_dev_pasid(domain, device->dev,
pasid, NULL);
pasid, old);
if (ret)
goto err_revert;
}
@@ -3354,7 +3375,15 @@ static int __iommu_set_group_pasid(struct iommu_domain *domain,
for_each_group_device(group, device) {
if (device == last_gdev)
break;
iommu_remove_dev_pasid(device->dev, pasid, domain);
/*
* If no old domain, undo the succeeded devices/pasid.
* Otherwise, rollback the succeeded devices/pasid to the old
* domain. And it is a driver bug to fail attaching with a
* previously good domain.
*/
if (!old || WARN_ON(old->ops->set_dev_pasid(old, device->dev,
pasid, domain)))
iommu_remove_dev_pasid(device->dev, pasid, domain);
}
return ret;
}
@@ -3376,6 +3405,9 @@ static void __iommu_remove_group_pasid(struct iommu_group *group,
* @pasid: the pasid of the device.
* @handle: the attach handle.
*
* Caller should always provide a new handle to avoid race with the paths
* that have lockless reference to handle if it intends to pass a valid handle.
*
* Return: 0 on success, or an error.
*/
int iommu_attach_device_pasid(struct iommu_domain *domain,
@@ -3420,7 +3452,7 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
if (ret)
goto out_unlock;
ret = __iommu_set_group_pasid(domain, group, pasid);
ret = __iommu_set_group_pasid(domain, group, pasid, NULL);
if (ret) {
xa_release(&group->pasid_array, pasid);
goto out_unlock;
@@ -3441,6 +3473,97 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
}
EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
/**
* iommu_replace_device_pasid - Replace the domain that a specific pasid
* of the device is attached to
* @domain: the new iommu domain
* @dev: the attached device.
* @pasid: the pasid of the device.
* @handle: the attach handle.
*
* This API allows the pasid to switch domains. The @pasid should have been
* attached. Otherwise, this fails. The pasid will keep the old configuration
* if replacement failed.
*
* Caller should always provide a new handle to avoid race with the paths
* that have lockless reference to handle if it intends to pass a valid handle.
*
* Return 0 on success, or an error.
*/
int iommu_replace_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid,
struct iommu_attach_handle *handle)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
struct iommu_attach_handle *entry;
struct iommu_domain *curr_domain;
void *curr;
int ret;
if (!group)
return -ENODEV;
if (!domain->ops->set_dev_pasid)
return -EOPNOTSUPP;
if (dev_iommu_ops(dev) != domain->owner ||
pasid == IOMMU_NO_PASID || !handle)
return -EINVAL;
mutex_lock(&group->mutex);
entry = iommu_make_pasid_array_entry(domain, handle);
curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,
XA_ZERO_ENTRY, GFP_KERNEL);
if (xa_is_err(curr)) {
ret = xa_err(curr);
goto out_unlock;
}
/*
* No domain (with or without handle) attached, hence not
* a replace case.
*/
if (!curr) {
xa_release(&group->pasid_array, pasid);
ret = -EINVAL;
goto out_unlock;
}
/*
* Reusing handle is problematic as there are paths that refers
* the handle without lock. To avoid race, reject the callers that
* attempt it.
*/
if (curr == entry) {
WARN_ON(1);
ret = -EINVAL;
goto out_unlock;
}
curr_domain = pasid_array_entry_to_domain(curr);
ret = 0;
if (curr_domain != domain) {
ret = __iommu_set_group_pasid(domain, group,
pasid, curr_domain);
if (ret)
goto out_unlock;
}
/*
* The above xa_cmpxchg() reserved the memory, and the
* group->mutex is held, this cannot fail.
*/
WARN_ON(xa_is_err(xa_store(&group->pasid_array,
pasid, entry, GFP_KERNEL)));
out_unlock:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_NS_GPL(iommu_replace_device_pasid, "IOMMUFD_INTERNAL");
/*
* iommu_detach_device_pasid() - Detach the domain from pasid of device
* @domain: the iommu domain.
@@ -3536,6 +3659,9 @@ EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, "IOMMUFD_INTERNAL");
* This is a variant of iommu_attach_group(). It allows the caller to provide
* an attach handle and use it when the domain is attached. This is currently
* used by IOMMUFD to deliver the I/O page faults.
*
* Caller should always provide a new handle to avoid race with the paths
* that have lockless reference to handle.
*/
int iommu_attach_group_handle(struct iommu_domain *domain,
struct iommu_group *group,
@@ -3605,6 +3731,9 @@ EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, "IOMMUFD_INTERNAL");
*
* If the currently attached domain is a core domain (e.g. a default_domain),
* it will act just like the iommu_attach_group_handle().
*
* Caller should always provide a new handle to avoid race with the paths
* that have lockless reference to handle.
*/
int iommu_replace_group_handle(struct iommu_group *group,
struct iommu_domain *new_domain,
@@ -3662,8 +3791,21 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
return 0;
mutex_lock(&group->mutex);
if (group->domain && group->domain->sw_msi)
ret = group->domain->sw_msi(group->domain, desc, msi_addr);
/* An IDENTITY domain must pass through */
if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
switch (group->domain->cookie_type) {
case IOMMU_COOKIE_DMA_MSI:
case IOMMU_COOKIE_DMA_IOVA:
ret = iommu_dma_sw_msi(group->domain, desc, msi_addr);
break;
case IOMMU_COOKIE_IOMMUFD:
ret = iommufd_sw_msi(group->domain, desc, msi_addr);
break;
default:
ret = -EOPNOTSUPP;
break;
}
}
mutex_unlock(&group->mutex);
return ret;
}

2
drivers/iommu/iommufd/Kconfig
View File

@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
config IOMMUFD_DRIVER_CORE
tristate
bool
default (IOMMUFD_DRIVER || IOMMUFD) if IOMMUFD!=n
config IOMMUFD

2
drivers/iommu/iommufd/Makefile
View File

@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-only
iommufd-y := \
device.o \
fault.o \
eventq.o \
hw_pagetable.o \
io_pagetable.o \
ioas.o \

499
drivers/iommu/iommufd/device.c
View File

@@ -3,9 +3,9 @@
*/
#include <linux/iommu.h>
#include <linux/iommufd.h>
#include <linux/pci-ats.h>
#include <linux/slab.h>
#include <uapi/linux/iommufd.h>
#include <linux/msi.h>
#include "../iommu-priv.h"
#include "io_pagetable.h"
@@ -18,12 +18,17 @@ MODULE_PARM_DESC(
"Allow IOMMUFD to bind to devices even if the platform cannot isolate "
"the MSI interrupt window. Enabling this is a security weakness.");
struct iommufd_attach {
struct iommufd_hw_pagetable *hwpt;
struct xarray device_array;
};
static void iommufd_group_release(struct kref *kref)
{
struct iommufd_group *igroup =
container_of(kref, struct iommufd_group, ref);
WARN_ON(igroup->hwpt || !list_empty(&igroup->device_list));
WARN_ON(!xa_empty(&igroup->pasid_attach));
xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup,
NULL, GFP_KERNEL);
@@ -90,7 +95,7 @@ static struct iommufd_group *iommufd_get_group(struct iommufd_ctx *ictx,
kref_init(&new_igroup->ref);
mutex_init(&new_igroup->lock);
INIT_LIST_HEAD(&new_igroup->device_list);
xa_init(&new_igroup->pasid_attach);
new_igroup->sw_msi_start = PHYS_ADDR_MAX;
/* group reference moves into new_igroup */
new_igroup->group = group;
@@ -294,129 +299,24 @@ u32 iommufd_device_to_id(struct iommufd_device *idev)
}
EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, "IOMMUFD");
/*
* Get a iommufd_sw_msi_map for the msi physical address requested by the irq
* layer. The mapping to IOVA is global to the iommufd file descriptor, every
* domain that is attached to a device using the same MSI parameters will use
* the same IOVA.
*/
static __maybe_unused struct iommufd_sw_msi_map *
iommufd_sw_msi_get_map(struct iommufd_ctx *ictx, phys_addr_t msi_addr,
phys_addr_t sw_msi_start)
static unsigned int iommufd_group_device_num(struct iommufd_group *igroup,
ioasid_t pasid)
{
struct iommufd_sw_msi_map *cur;
unsigned int max_pgoff = 0;
struct iommufd_attach *attach;
struct iommufd_device *idev;
unsigned int count = 0;
unsigned long index;
lockdep_assert_held(&ictx->sw_msi_lock);
lockdep_assert_held(&igroup->lock);
list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) {
if (cur->sw_msi_start != sw_msi_start)
continue;
max_pgoff = max(max_pgoff, cur->pgoff + 1);
if (cur->msi_addr == msi_addr)
return cur;
}
if (ictx->sw_msi_id >=
BITS_PER_BYTE * sizeof_field(struct iommufd_sw_msi_maps, bitmap))
return ERR_PTR(-EOVERFLOW);
cur = kzalloc(sizeof(*cur), GFP_KERNEL);
if (!cur)
return ERR_PTR(-ENOMEM);
cur->sw_msi_start = sw_msi_start;
cur->msi_addr = msi_addr;
cur->pgoff = max_pgoff;
cur->id = ictx->sw_msi_id++;
list_add_tail(&cur->sw_msi_item, &ictx->sw_msi_list);
return cur;
attach = xa_load(&igroup->pasid_attach, pasid);
if (attach)
xa_for_each(&attach->device_array, index, idev)
count++;
return count;
}
static int iommufd_sw_msi_install(struct iommufd_ctx *ictx,
struct iommufd_hwpt_paging *hwpt_paging,
struct iommufd_sw_msi_map *msi_map)
{
unsigned long iova;
lockdep_assert_held(&ictx->sw_msi_lock);
iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
if (!test_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap)) {
int rc;
rc = iommu_map(hwpt_paging->common.domain, iova,
msi_map->msi_addr, PAGE_SIZE,
IOMMU_WRITE | IOMMU_READ | IOMMU_MMIO,
GFP_KERNEL_ACCOUNT);
if (rc)
return rc;
__set_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap);
}
return 0;
}
/*
* Called by the irq code if the platform translates the MSI address through the
* IOMMU. msi_addr is the physical address of the MSI page. iommufd will
* allocate a fd global iova for the physical page that is the same on all
* domains and devices.
*/
#ifdef CONFIG_IRQ_MSI_IOMMU
int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr)
{
struct device *dev = msi_desc_to_dev(desc);
struct iommufd_hwpt_paging *hwpt_paging;
struct iommu_attach_handle *raw_handle;
struct iommufd_attach_handle *handle;
struct iommufd_sw_msi_map *msi_map;
struct iommufd_ctx *ictx;
unsigned long iova;
int rc;
/*
* It is safe to call iommu_attach_handle_get() here because the iommu
* core code invokes this under the group mutex which also prevents any
* change of the attach handle for the duration of this function.
*/
iommu_group_mutex_assert(dev);
raw_handle =
iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
if (IS_ERR(raw_handle))
return 0;
hwpt_paging = find_hwpt_paging(domain->iommufd_hwpt);
handle = to_iommufd_handle(raw_handle);
/* No IOMMU_RESV_SW_MSI means no change to the msi_msg */
if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX)
return 0;
ictx = handle->idev->ictx;
guard(mutex)(&ictx->sw_msi_lock);
/*
* The input msi_addr is the exact byte offset of the MSI doorbell, we
* assume the caller has checked that it is contained with a MMIO region
* that is secure to map at PAGE_SIZE.
*/
msi_map = iommufd_sw_msi_get_map(handle->idev->ictx,
msi_addr & PAGE_MASK,
handle->idev->igroup->sw_msi_start);
if (IS_ERR(msi_map))
return PTR_ERR(msi_map);
rc = iommufd_sw_msi_install(ictx, hwpt_paging, msi_map);
if (rc)
return rc;
__set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap);
iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT);
return 0;
}
#endif
static int iommufd_group_setup_msi(struct iommufd_group *igroup,
struct iommufd_hwpt_paging *hwpt_paging)
{
@@ -443,23 +343,39 @@ static int iommufd_group_setup_msi(struct iommufd_group *igroup,
}
return 0;
}
#else
static inline int
iommufd_group_setup_msi(struct iommufd_group *igroup,
struct iommufd_hwpt_paging *hwpt_paging)
{
return 0;
}
#endif
static bool
iommufd_group_first_attach(struct iommufd_group *igroup, ioasid_t pasid)
{
lockdep_assert_held(&igroup->lock);
return !xa_load(&igroup->pasid_attach, pasid);
}
static int
iommufd_device_attach_reserved_iova(struct iommufd_device *idev,
struct iommufd_hwpt_paging *hwpt_paging)
{
struct iommufd_group *igroup = idev->igroup;
int rc;
lockdep_assert_held(&idev->igroup->lock);
lockdep_assert_held(&igroup->lock);
rc = iopt_table_enforce_dev_resv_regions(&hwpt_paging->ioas->iopt,
idev->dev,
&idev->igroup->sw_msi_start);
&igroup->sw_msi_start);
if (rc)
return rc;
if (list_empty(&idev->igroup->device_list)) {
rc = iommufd_group_setup_msi(idev->igroup, hwpt_paging);
if (iommufd_group_first_attach(igroup, IOMMU_NO_PASID)) {
rc = iommufd_group_setup_msi(igroup, hwpt_paging);
if (rc) {
iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt,
idev->dev);
@@ -471,13 +387,54 @@ iommufd_device_attach_reserved_iova(struct iommufd_device *idev,
/* The device attach/detach/replace helpers for attach_handle */
static bool iommufd_device_is_attached(struct iommufd_device *idev,
ioasid_t pasid)
{
struct iommufd_attach *attach;
attach = xa_load(&idev->igroup->pasid_attach, pasid);
return xa_load(&attach->device_array, idev->obj.id);
}
static int iommufd_hwpt_pasid_compat(struct iommufd_hw_pagetable *hwpt,
struct iommufd_device *idev,
ioasid_t pasid)
{
struct iommufd_group *igroup = idev->igroup;
lockdep_assert_held(&igroup->lock);
if (pasid == IOMMU_NO_PASID) {
unsigned long start = IOMMU_NO_PASID;
if (!hwpt->pasid_compat &&
xa_find_after(&igroup->pasid_attach,
&start, UINT_MAX, XA_PRESENT))
return -EINVAL;
} else {
struct iommufd_attach *attach;
if (!hwpt->pasid_compat)
return -EINVAL;
attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID);
if (attach && attach->hwpt && !attach->hwpt->pasid_compat)
return -EINVAL;
}
return 0;
}
static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
struct iommufd_device *idev)
struct iommufd_device *idev,
ioasid_t pasid)
{
struct iommufd_attach_handle *handle;
int rc;
lockdep_assert_held(&idev->igroup->lock);
rc = iommufd_hwpt_pasid_compat(hwpt, idev, pasid);
if (rc)
return rc;
handle = kzalloc(sizeof(*handle), GFP_KERNEL);
if (!handle)
@@ -490,8 +447,12 @@ static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
}
handle->idev = idev;
rc = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
&handle->handle);
if (pasid == IOMMU_NO_PASID)
rc = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
&handle->handle);
else
rc = iommu_attach_device_pasid(hwpt->domain, idev->dev, pasid,
&handle->handle);
if (rc)
goto out_disable_iopf;
@@ -506,26 +467,31 @@ static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
}
static struct iommufd_attach_handle *
iommufd_device_get_attach_handle(struct iommufd_device *idev)
iommufd_device_get_attach_handle(struct iommufd_device *idev, ioasid_t pasid)
{
struct iommu_attach_handle *handle;
lockdep_assert_held(&idev->igroup->lock);
handle =
iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
iommu_attach_handle_get(idev->igroup->group, pasid, 0);
if (IS_ERR(handle))
return NULL;
return to_iommufd_handle(handle);
}
static void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,
struct iommufd_device *idev)
struct iommufd_device *idev,
ioasid_t pasid)
{
struct iommufd_attach_handle *handle;
handle = iommufd_device_get_attach_handle(idev);
iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
handle = iommufd_device_get_attach_handle(idev, pasid);
if (pasid == IOMMU_NO_PASID)
iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
else
iommu_detach_device_pasid(hwpt->domain, idev->dev, pasid);
if (hwpt->fault) {
iommufd_auto_response_faults(hwpt, handle);
iommufd_fault_iopf_disable(idev);
@@ -534,13 +500,19 @@ static void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,
}
static int iommufd_hwpt_replace_device(struct iommufd_device *idev,
ioasid_t pasid,
struct iommufd_hw_pagetable *hwpt,
struct iommufd_hw_pagetable *old)
{
struct iommufd_attach_handle *handle, *old_handle =
iommufd_device_get_attach_handle(idev);
struct iommufd_attach_handle *handle, *old_handle;
int rc;
rc = iommufd_hwpt_pasid_compat(hwpt, idev, pasid);
if (rc)
return rc;
old_handle = iommufd_device_get_attach_handle(idev, pasid);
handle = kzalloc(sizeof(*handle), GFP_KERNEL);
if (!handle)
return -ENOMEM;
@@ -552,8 +524,12 @@ static int iommufd_hwpt_replace_device(struct iommufd_device *idev,
}
handle->idev = idev;
rc = iommu_replace_group_handle(idev->igroup->group, hwpt->domain,
&handle->handle);
if (pasid == IOMMU_NO_PASID)
rc = iommu_replace_group_handle(idev->igroup->group,
hwpt->domain, &handle->handle);
else
rc = iommu_replace_device_pasid(hwpt->domain, idev->dev,
pasid, &handle->handle);
if (rc)
goto out_disable_iopf;
@@ -575,22 +551,51 @@ static int iommufd_hwpt_replace_device(struct iommufd_device *idev,
}
int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
struct iommufd_device *idev)
struct iommufd_device *idev, ioasid_t pasid)
{
struct iommufd_hwpt_paging *hwpt_paging = find_hwpt_paging(hwpt);
bool attach_resv = hwpt_paging && pasid == IOMMU_NO_PASID;
struct iommufd_group *igroup = idev->igroup;
struct iommufd_hw_pagetable *old_hwpt;
struct iommufd_attach *attach;
int rc;
mutex_lock(&idev->igroup->lock);
mutex_lock(&igroup->lock);
if (idev->igroup->hwpt != NULL && idev->igroup->hwpt != hwpt) {
rc = -EINVAL;
attach = xa_cmpxchg(&igroup->pasid_attach, pasid, NULL,
XA_ZERO_ENTRY, GFP_KERNEL);
if (xa_is_err(attach)) {
rc = xa_err(attach);
goto err_unlock;
}
if (hwpt_paging) {
if (!attach) {
attach = kzalloc(sizeof(*attach), GFP_KERNEL);
if (!attach) {
rc = -ENOMEM;
goto err_release_pasid;
}
xa_init(&attach->device_array);
}
old_hwpt = attach->hwpt;
rc = xa_insert(&attach->device_array, idev->obj.id, XA_ZERO_ENTRY,
GFP_KERNEL);
if (rc) {
WARN_ON(rc == -EBUSY && !old_hwpt);
goto err_free_attach;
}
if (old_hwpt && old_hwpt != hwpt) {
rc = -EINVAL;
goto err_release_devid;
}
if (attach_resv) {
rc = iommufd_device_attach_reserved_iova(idev, hwpt_paging);
if (rc)
goto err_unlock;
goto err_release_devid;
}
/*
@@ -600,51 +605,74 @@ int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
* reserved regions are only updated during individual device
* attachment.
*/
if (list_empty(&idev->igroup->device_list)) {
rc = iommufd_hwpt_attach_device(hwpt, idev);
if (iommufd_group_first_attach(igroup, pasid)) {
rc = iommufd_hwpt_attach_device(hwpt, idev, pasid);
if (rc)
goto err_unresv;
idev->igroup->hwpt = hwpt;
attach->hwpt = hwpt;
WARN_ON(xa_is_err(xa_store(&igroup->pasid_attach, pasid, attach,
GFP_KERNEL)));
}
refcount_inc(&hwpt->obj.users);
list_add_tail(&idev->group_item, &idev->igroup->device_list);
mutex_unlock(&idev->igroup->lock);
WARN_ON(xa_is_err(xa_store(&attach->device_array, idev->obj.id,
idev, GFP_KERNEL)));
mutex_unlock(&igroup->lock);
return 0;
err_unresv:
if (hwpt_paging)
if (attach_resv)
iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, idev->dev);
err_release_devid:
xa_release(&attach->device_array, idev->obj.id);
err_free_attach:
if (iommufd_group_first_attach(igroup, pasid))
kfree(attach);
err_release_pasid:
if (iommufd_group_first_attach(igroup, pasid))
xa_release(&igroup->pasid_attach, pasid);
err_unlock:
mutex_unlock(&idev->igroup->lock);
mutex_unlock(&igroup->lock);
return rc;
}
struct iommufd_hw_pagetable *
iommufd_hw_pagetable_detach(struct iommufd_device *idev)
iommufd_hw_pagetable_detach(struct iommufd_device *idev, ioasid_t pasid)
{
struct iommufd_hw_pagetable *hwpt = idev->igroup->hwpt;
struct iommufd_hwpt_paging *hwpt_paging = find_hwpt_paging(hwpt);
struct iommufd_group *igroup = idev->igroup;
struct iommufd_hwpt_paging *hwpt_paging;
struct iommufd_hw_pagetable *hwpt;
struct iommufd_attach *attach;
mutex_lock(&idev->igroup->lock);
list_del(&idev->group_item);
if (list_empty(&idev->igroup->device_list)) {
iommufd_hwpt_detach_device(hwpt, idev);
idev->igroup->hwpt = NULL;
mutex_lock(&igroup->lock);
attach = xa_load(&igroup->pasid_attach, pasid);
if (!attach) {
mutex_unlock(&igroup->lock);
return NULL;
}
if (hwpt_paging)
hwpt = attach->hwpt;
hwpt_paging = find_hwpt_paging(hwpt);
xa_erase(&attach->device_array, idev->obj.id);
if (xa_empty(&attach->device_array)) {
iommufd_hwpt_detach_device(hwpt, idev, pasid);
xa_erase(&igroup->pasid_attach, pasid);
kfree(attach);
}
if (hwpt_paging && pasid == IOMMU_NO_PASID)
iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, idev->dev);
mutex_unlock(&idev->igroup->lock);
mutex_unlock(&igroup->lock);
/* Caller must destroy hwpt */
return hwpt;
}
static struct iommufd_hw_pagetable *
iommufd_device_do_attach(struct iommufd_device *idev,
iommufd_device_do_attach(struct iommufd_device *idev, ioasid_t pasid,
struct iommufd_hw_pagetable *hwpt)
{
int rc;
rc = iommufd_hw_pagetable_attach(hwpt, idev);
rc = iommufd_hw_pagetable_attach(hwpt, idev, pasid);
if (rc)
return ERR_PTR(rc);
return NULL;
@@ -654,11 +682,14 @@ static void
iommufd_group_remove_reserved_iova(struct iommufd_group *igroup,
struct iommufd_hwpt_paging *hwpt_paging)
{
struct iommufd_attach *attach;
struct iommufd_device *cur;
unsigned long index;
lockdep_assert_held(&igroup->lock);
list_for_each_entry(cur, &igroup->device_list, group_item)
attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID);
xa_for_each(&attach->device_array, index, cur)
iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, cur->dev);
}
@@ -667,14 +698,17 @@ iommufd_group_do_replace_reserved_iova(struct iommufd_group *igroup,
struct iommufd_hwpt_paging *hwpt_paging)
{
struct iommufd_hwpt_paging *old_hwpt_paging;
struct iommufd_attach *attach;
struct iommufd_device *cur;
unsigned long index;
int rc;
lockdep_assert_held(&igroup->lock);
old_hwpt_paging = find_hwpt_paging(igroup->hwpt);
attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID);
old_hwpt_paging = find_hwpt_paging(attach->hwpt);
if (!old_hwpt_paging || hwpt_paging->ioas != old_hwpt_paging->ioas) {
list_for_each_entry(cur, &igroup->device_list, group_item) {
xa_for_each(&attach->device_array, index, cur) {
rc = iopt_table_enforce_dev_resv_regions(
&hwpt_paging->ioas->iopt, cur->dev, NULL);
if (rc)
@@ -693,69 +727,81 @@ iommufd_group_do_replace_reserved_iova(struct iommufd_group *igroup,
}
static struct iommufd_hw_pagetable *
iommufd_device_do_replace(struct iommufd_device *idev,
iommufd_device_do_replace(struct iommufd_device *idev, ioasid_t pasid,
struct iommufd_hw_pagetable *hwpt)
{
struct iommufd_hwpt_paging *hwpt_paging = find_hwpt_paging(hwpt);
bool attach_resv = hwpt_paging && pasid == IOMMU_NO_PASID;
struct iommufd_hwpt_paging *old_hwpt_paging;
struct iommufd_group *igroup = idev->igroup;
struct iommufd_hw_pagetable *old_hwpt;
struct iommufd_attach *attach;
unsigned int num_devices;
int rc;
mutex_lock(&idev->igroup->lock);
mutex_lock(&igroup->lock);
if (igroup->hwpt == NULL) {
attach = xa_load(&igroup->pasid_attach, pasid);
if (!attach) {
rc = -EINVAL;
goto err_unlock;
}
if (hwpt == igroup->hwpt) {
mutex_unlock(&idev->igroup->lock);
old_hwpt = attach->hwpt;
WARN_ON(!old_hwpt || xa_empty(&attach->device_array));
if (!iommufd_device_is_attached(idev, pasid)) {
rc = -EINVAL;
goto err_unlock;
}
if (hwpt == old_hwpt) {
mutex_unlock(&igroup->lock);
return NULL;
}
old_hwpt = igroup->hwpt;
if (hwpt_paging) {
if (attach_resv) {
rc = iommufd_group_do_replace_reserved_iova(igroup, hwpt_paging);
if (rc)
goto err_unlock;
}
rc = iommufd_hwpt_replace_device(idev, hwpt, old_hwpt);
rc = iommufd_hwpt_replace_device(idev, pasid, hwpt, old_hwpt);
if (rc)
goto err_unresv;
old_hwpt_paging = find_hwpt_paging(old_hwpt);
if (old_hwpt_paging &&
if (old_hwpt_paging && pasid == IOMMU_NO_PASID &&
(!hwpt_paging || hwpt_paging->ioas != old_hwpt_paging->ioas))
iommufd_group_remove_reserved_iova(igroup, old_hwpt_paging);
igroup->hwpt = hwpt;
attach->hwpt = hwpt;
num_devices = list_count_nodes(&igroup->device_list);
num_devices = iommufd_group_device_num(igroup, pasid);
/*
* Move the refcounts held by the device_list to the new hwpt. Retain a
* Move the refcounts held by the device_array to the new hwpt. Retain a
* refcount for this thread as the caller will free it.
*/
refcount_add(num_devices, &hwpt->obj.users);
if (num_devices > 1)
WARN_ON(refcount_sub_and_test(num_devices - 1,
&old_hwpt->obj.users));
mutex_unlock(&idev->igroup->lock);
mutex_unlock(&igroup->lock);
/* Caller must destroy old_hwpt */
return old_hwpt;
err_unresv:
if (hwpt_paging)
if (attach_resv)
iommufd_group_remove_reserved_iova(igroup, hwpt_paging);
err_unlock:
mutex_unlock(&idev->igroup->lock);
mutex_unlock(&igroup->lock);
return ERR_PTR(rc);
}
typedef struct iommufd_hw_pagetable *(*attach_fn)(
struct iommufd_device *idev, struct iommufd_hw_pagetable *hwpt);
struct iommufd_device *idev, ioasid_t pasid,
struct iommufd_hw_pagetable *hwpt);
/*
* When automatically managing the domains we search for a compatible domain in
@@ -763,7 +809,7 @@ typedef struct iommufd_hw_pagetable *(*attach_fn)(
* Automatic domain selection will never pick a manually created domain.
*/
static struct iommufd_hw_pagetable *
iommufd_device_auto_get_domain(struct iommufd_device *idev,
iommufd_device_auto_get_domain(struct iommufd_device *idev, ioasid_t pasid,
struct iommufd_ioas *ioas, u32 *pt_id,
attach_fn do_attach)
{
@@ -792,7 +838,7 @@ iommufd_device_auto_get_domain(struct iommufd_device *idev,
hwpt = &hwpt_paging->common;
if (!iommufd_lock_obj(&hwpt->obj))
continue;
destroy_hwpt = (*do_attach)(idev, hwpt);
destroy_hwpt = (*do_attach)(idev, pasid, hwpt);
if (IS_ERR(destroy_hwpt)) {
iommufd_put_object(idev->ictx, &hwpt->obj);
/*
@@ -810,8 +856,8 @@ iommufd_device_auto_get_domain(struct iommufd_device *idev,
goto out_unlock;
}
hwpt_paging = iommufd_hwpt_paging_alloc(idev->ictx, ioas, idev, 0,
immediate_attach, NULL);
hwpt_paging = iommufd_hwpt_paging_alloc(idev->ictx, ioas, idev, pasid,
0, immediate_attach, NULL);
if (IS_ERR(hwpt_paging)) {
destroy_hwpt = ERR_CAST(hwpt_paging);
goto out_unlock;
@@ -819,7 +865,7 @@ iommufd_device_auto_get_domain(struct iommufd_device *idev,
hwpt = &hwpt_paging->common;
if (!immediate_attach) {
destroy_hwpt = (*do_attach)(idev, hwpt);
destroy_hwpt = (*do_attach)(idev, pasid, hwpt);
if (IS_ERR(destroy_hwpt))
goto out_abort;
} else {
@@ -840,8 +886,9 @@ iommufd_device_auto_get_domain(struct iommufd_device *idev,
return destroy_hwpt;
}
static int iommufd_device_change_pt(struct iommufd_device *idev, u32 *pt_id,
attach_fn do_attach)
static int iommufd_device_change_pt(struct iommufd_device *idev,
ioasid_t pasid,
u32 *pt_id, attach_fn do_attach)
{
struct iommufd_hw_pagetable *destroy_hwpt;
struct iommufd_object *pt_obj;
@@ -856,7 +903,7 @@ static int iommufd_device_change_pt(struct iommufd_device *idev, u32 *pt_id,
struct iommufd_hw_pagetable *hwpt =
container_of(pt_obj, struct iommufd_hw_pagetable, obj);
destroy_hwpt = (*do_attach)(idev, hwpt);
destroy_hwpt = (*do_attach)(idev, pasid, hwpt);
if (IS_ERR(destroy_hwpt))
goto out_put_pt_obj;
break;
@@ -865,8 +912,8 @@ static int iommufd_device_change_pt(struct iommufd_device *idev, u32 *pt_id,
struct iommufd_ioas *ioas =
container_of(pt_obj, struct iommufd_ioas, obj);
destroy_hwpt = iommufd_device_auto_get_domain(idev, ioas, pt_id,
do_attach);
destroy_hwpt = iommufd_device_auto_get_domain(idev, pasid, ioas,
pt_id, do_attach);
if (IS_ERR(destroy_hwpt))
goto out_put_pt_obj;
break;
@@ -888,22 +935,26 @@ static int iommufd_device_change_pt(struct iommufd_device *idev, u32 *pt_id,
}
/**
* iommufd_device_attach - Connect a device to an iommu_domain
* iommufd_device_attach - Connect a device/pasid to an iommu_domain
* @idev: device to attach
* @pasid: pasid to attach
* @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HWPT_PAGING
* Output the IOMMUFD_OBJ_HWPT_PAGING ID
*
* This connects the device to an iommu_domain, either automatically or manually
* selected. Once this completes the device could do DMA.
* This connects the device/pasid to an iommu_domain, either automatically
* or manually selected. Once this completes the device could do DMA with
* @pasid. @pasid is IOMMU_NO_PASID if this attach is for no pasid usage.
*
* The caller should return the resulting pt_id back to userspace.
* This function is undone by calling iommufd_device_detach().
*/
int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id)
int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid,
u32 *pt_id)
{
int rc;
rc = iommufd_device_change_pt(idev, pt_id, &iommufd_device_do_attach);
rc = iommufd_device_change_pt(idev, pasid, pt_id,
&iommufd_device_do_attach);
if (rc)
return rc;
@@ -917,8 +968,9 @@ int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id)
EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, "IOMMUFD");
/**
* iommufd_device_replace - Change the device's iommu_domain
* iommufd_device_replace - Change the device/pasid's iommu_domain
* @idev: device to change
* @pasid: pasid to change
* @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HWPT_PAGING
* Output the IOMMUFD_OBJ_HWPT_PAGING ID
*
@@ -929,27 +981,33 @@ EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, "IOMMUFD");
*
* If it fails then no change is made to the attachment. The iommu driver may
* implement this so there is no disruption in translation. This can only be
* called if iommufd_device_attach() has already succeeded.
* called if iommufd_device_attach() has already succeeded. @pasid is
* IOMMU_NO_PASID for no pasid usage.
*/
int iommufd_device_replace(struct iommufd_device *idev, u32 *pt_id)
int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid,
u32 *pt_id)
{
return iommufd_device_change_pt(idev, pt_id,
return iommufd_device_change_pt(idev, pasid, pt_id,
&iommufd_device_do_replace);
}
EXPORT_SYMBOL_NS_GPL(iommufd_device_replace, "IOMMUFD");
/**
* iommufd_device_detach - Disconnect a device to an iommu_domain
* iommufd_device_detach - Disconnect a device/device to an iommu_domain
* @idev: device to detach
* @pasid: pasid to detach
*
* Undo iommufd_device_attach(). This disconnects the idev from the previously
* attached pt_id. The device returns back to a blocked DMA translation.
* @pasid is IOMMU_NO_PASID for no pasid usage.
*/
void iommufd_device_detach(struct iommufd_device *idev)
void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid)
{
struct iommufd_hw_pagetable *hwpt;
hwpt = iommufd_hw_pagetable_detach(idev);
hwpt = iommufd_hw_pagetable_detach(idev, pasid);
if (!hwpt)
return;
iommufd_hw_pagetable_put(idev->ictx, hwpt);
refcount_dec(&idev->obj.users);
}
@@ -1349,7 +1407,7 @@ int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
struct io_pagetable *iopt;
struct iopt_area *area;
unsigned long last_iova;
int rc;
int rc = -EINVAL;
if (!length)
return -EINVAL;
@@ -1405,7 +1463,8 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
void *data;
int rc;
if (cmd->flags || cmd->__reserved)
if (cmd->flags || cmd->__reserved[0] || cmd->__reserved[1] ||
cmd->__reserved[2])
return -EOPNOTSUPP;
idev = iommufd_get_device(ucmd, cmd->dev_id);
@@ -1462,6 +1521,36 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
if (device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING))
cmd->out_capabilities |= IOMMU_HW_CAP_DIRTY_TRACKING;
cmd->out_max_pasid_log2 = 0;
/*
* Currently, all iommu drivers enable PASID in the probe_device()
* op if iommu and device supports it. So the max_pasids stored in
* dev->iommu indicates both PASID support and enable status. A
* non-zero dev->iommu->max_pasids means PASID is supported and
* enabled. The iommufd only reports PASID capability to userspace
* if it's enabled.
*/
if (idev->dev->iommu->max_pasids) {
cmd->out_max_pasid_log2 = ilog2(idev->dev->iommu->max_pasids);
if (dev_is_pci(idev->dev)) {
struct pci_dev *pdev = to_pci_dev(idev->dev);
int ctrl;
ctrl = pci_pasid_status(pdev);
WARN_ON_ONCE(ctrl < 0 ||
!(ctrl & PCI_PASID_CTRL_ENABLE));
if (ctrl & PCI_PASID_CTRL_EXEC)
cmd->out_capabilities |=
IOMMU_HW_CAP_PCI_PASID_EXEC;
if (ctrl & PCI_PASID_CTRL_PRIV)
cmd->out_capabilities |=
IOMMU_HW_CAP_PCI_PASID_PRIV;
}
}
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
out_free:
kfree(data);

198
drivers/iommu/iommufd/driver.c
View File

@@ -49,5 +49,203 @@ struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu,
}
EXPORT_SYMBOL_NS_GPL(iommufd_viommu_find_dev, "IOMMUFD");
/* Return -ENOENT if device is not associated to the vIOMMU */
int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
struct device *dev, unsigned long *vdev_id)
{
struct iommufd_vdevice *vdev;
unsigned long index;
int rc = -ENOENT;
if (WARN_ON_ONCE(!vdev_id))
return -EINVAL;
xa_lock(&viommu->vdevs);
xa_for_each(&viommu->vdevs, index, vdev) {
if (vdev->dev == dev) {
*vdev_id = vdev->id;
rc = 0;
break;
}
}
xa_unlock(&viommu->vdevs);
return rc;
}
EXPORT_SYMBOL_NS_GPL(iommufd_viommu_get_vdev_id, "IOMMUFD");
/*
* Typically called in driver's threaded IRQ handler.
* The @type and @event_data must be defined in include/uapi/linux/iommufd.h
*/
int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
enum iommu_veventq_type type, void *event_data,
size_t data_len)
{
struct iommufd_veventq *veventq;
struct iommufd_vevent *vevent;
int rc = 0;
if (WARN_ON_ONCE(!data_len || !event_data))
return -EINVAL;
down_read(&viommu->veventqs_rwsem);
veventq = iommufd_viommu_find_veventq(viommu, type);
if (!veventq) {
rc = -EOPNOTSUPP;
goto out_unlock_veventqs;
}
spin_lock(&veventq->common.lock);
if (veventq->num_events == veventq->depth) {
vevent = &veventq->lost_events_header;
goto out_set_header;
}
vevent = kzalloc(struct_size(vevent, event_data, data_len), GFP_ATOMIC);
if (!vevent) {
rc = -ENOMEM;
vevent = &veventq->lost_events_header;
goto out_set_header;
}
memcpy(vevent->event_data, event_data, data_len);
vevent->data_len = data_len;
veventq->num_events++;
out_set_header:
iommufd_vevent_handler(veventq, vevent);
spin_unlock(&veventq->common.lock);
out_unlock_veventqs:
up_read(&viommu->veventqs_rwsem);
return rc;
}
EXPORT_SYMBOL_NS_GPL(iommufd_viommu_report_event, "IOMMUFD");
#ifdef CONFIG_IRQ_MSI_IOMMU
/*
* Get a iommufd_sw_msi_map for the msi physical address requested by the irq
* layer. The mapping to IOVA is global to the iommufd file descriptor, every
* domain that is attached to a device using the same MSI parameters will use
* the same IOVA.
*/
static struct iommufd_sw_msi_map *
iommufd_sw_msi_get_map(struct iommufd_ctx *ictx, phys_addr_t msi_addr,
phys_addr_t sw_msi_start)
{
struct iommufd_sw_msi_map *cur;
unsigned int max_pgoff = 0;
lockdep_assert_held(&ictx->sw_msi_lock);
list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) {
if (cur->sw_msi_start != sw_msi_start)
continue;
max_pgoff = max(max_pgoff, cur->pgoff + 1);
if (cur->msi_addr == msi_addr)
return cur;
}
if (ictx->sw_msi_id >=
BITS_PER_BYTE * sizeof_field(struct iommufd_sw_msi_maps, bitmap))
return ERR_PTR(-EOVERFLOW);
cur = kzalloc(sizeof(*cur), GFP_KERNEL);
if (!cur)
return ERR_PTR(-ENOMEM);
cur->sw_msi_start = sw_msi_start;
cur->msi_addr = msi_addr;
cur->pgoff = max_pgoff;
cur->id = ictx->sw_msi_id++;
list_add_tail(&cur->sw_msi_item, &ictx->sw_msi_list);
return cur;
}
int iommufd_sw_msi_install(struct iommufd_ctx *ictx,
struct iommufd_hwpt_paging *hwpt_paging,
struct iommufd_sw_msi_map *msi_map)
{
unsigned long iova;
lockdep_assert_held(&ictx->sw_msi_lock);
iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
if (!test_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap)) {
int rc;
rc = iommu_map(hwpt_paging->common.domain, iova,
msi_map->msi_addr, PAGE_SIZE,
IOMMU_WRITE | IOMMU_READ | IOMMU_MMIO,
GFP_KERNEL_ACCOUNT);
if (rc)
return rc;
__set_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap);
}
return 0;
}
EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi_install, "IOMMUFD_INTERNAL");
/*
* Called by the irq code if the platform translates the MSI address through the
* IOMMU. msi_addr is the physical address of the MSI page. iommufd will
* allocate a fd global iova for the physical page that is the same on all
* domains and devices.
*/
int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr)
{
struct device *dev = msi_desc_to_dev(desc);
struct iommufd_hwpt_paging *hwpt_paging;
struct iommu_attach_handle *raw_handle;
struct iommufd_attach_handle *handle;
struct iommufd_sw_msi_map *msi_map;
struct iommufd_ctx *ictx;
unsigned long iova;
int rc;
/*
* It is safe to call iommu_attach_handle_get() here because the iommu
* core code invokes this under the group mutex which also prevents any
* change of the attach handle for the duration of this function.
*/
iommu_group_mutex_assert(dev);
raw_handle =
iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
if (IS_ERR(raw_handle))
return 0;
hwpt_paging = find_hwpt_paging(domain->iommufd_hwpt);
handle = to_iommufd_handle(raw_handle);
/* No IOMMU_RESV_SW_MSI means no change to the msi_msg */
if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX)
return 0;
ictx = handle->idev->ictx;
guard(mutex)(&ictx->sw_msi_lock);
/*
* The input msi_addr is the exact byte offset of the MSI doorbell, we
* assume the caller has checked that it is contained with a MMIO region
* that is secure to map at PAGE_SIZE.
*/
msi_map = iommufd_sw_msi_get_map(handle->idev->ictx,
msi_addr & PAGE_MASK,
handle->idev->igroup->sw_msi_start);
if (IS_ERR(msi_map))
return PTR_ERR(msi_map);
rc = iommufd_sw_msi_install(ictx, hwpt_paging, msi_map);
if (rc)
return rc;
__set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap);
iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT);
return 0;
}
EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi, "IOMMUFD");
#endif
MODULE_DESCRIPTION("iommufd code shared with builtin modules");
MODULE_IMPORT_NS("IOMMUFD_INTERNAL");
MODULE_LICENSE("GPL");

598
drivers/iommu/iommufd/eventq.c Normal file
View File

@@ -0,0 +1,598 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2024 Intel Corporation
*/
#define pr_fmt(fmt) "iommufd: " fmt
#include <linux/anon_inodes.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/iommufd.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci-ats.h>
#include <linux/poll.h>
#include <uapi/linux/iommufd.h>
#include "../iommu-priv.h"
#include "iommufd_private.h"
/* IOMMUFD_OBJ_FAULT Functions */
int iommufd_fault_iopf_enable(struct iommufd_device *idev)
{
struct device *dev = idev->dev;
int ret;
/*
* Once we turn on PCI/PRI support for VF, the response failure code
* should not be forwarded to the hardware due to PRI being a shared
* resource between PF and VFs. There is no coordination for this
* shared capability. This waits for a vPRI reset to recover.
*/
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
if (pdev->is_virtfn && pci_pri_supported(pdev))
return -EINVAL;
}
mutex_lock(&idev->iopf_lock);
/* Device iopf has already been on. */
if (++idev->iopf_enabled > 1) {
mutex_unlock(&idev->iopf_lock);
return 0;
}
ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
if (ret)
--idev->iopf_enabled;
mutex_unlock(&idev->iopf_lock);
return ret;
}
void iommufd_fault_iopf_disable(struct iommufd_device *idev)
{
mutex_lock(&idev->iopf_lock);
if (!WARN_ON(idev->iopf_enabled == 0)) {
if (--idev->iopf_enabled == 0)
iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
}
mutex_unlock(&idev->iopf_lock);
}
void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
struct iommufd_attach_handle *handle)
{
struct iommufd_fault *fault = hwpt->fault;
struct iopf_group *group, *next;
struct list_head free_list;
unsigned long index;
if (!fault)
return;
INIT_LIST_HEAD(&free_list);
mutex_lock(&fault->mutex);
spin_lock(&fault->common.lock);
list_for_each_entry_safe(group, next, &fault->common.deliver, node) {
if (group->attach_handle != &handle->handle)
continue;
list_move(&group->node, &free_list);
}
spin_unlock(&fault->common.lock);
list_for_each_entry_safe(group, next, &free_list, node) {
list_del(&group->node);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
xa_for_each(&fault->response, index, group) {
if (group->attach_handle != &handle->handle)
continue;
xa_erase(&fault->response, index);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
mutex_unlock(&fault->mutex);
}
void iommufd_fault_destroy(struct iommufd_object *obj)
{
struct iommufd_eventq *eventq =
container_of(obj, struct iommufd_eventq, obj);
struct iommufd_fault *fault = eventq_to_fault(eventq);
struct iopf_group *group, *next;
unsigned long index;
/*
* The iommufd object's reference count is zero at this point.
* We can be confident that no other threads are currently
* accessing this pointer. Therefore, acquiring the mutex here
* is unnecessary.
*/
list_for_each_entry_safe(group, next, &fault->common.deliver, node) {
list_del(&group->node);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
xa_for_each(&fault->response, index, group) {
xa_erase(&fault->response, index);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
xa_destroy(&fault->response);
mutex_destroy(&fault->mutex);
}
static void iommufd_compose_fault_message(struct iommu_fault *fault,
struct iommu_hwpt_pgfault *hwpt_fault,
struct iommufd_device *idev,
u32 cookie)
{
hwpt_fault->flags = fault->prm.flags;
hwpt_fault->dev_id = idev->obj.id;
hwpt_fault->pasid = fault->prm.pasid;
hwpt_fault->grpid = fault->prm.grpid;
hwpt_fault->perm = fault->prm.perm;
hwpt_fault->addr = fault->prm.addr;
hwpt_fault->length = 0;
hwpt_fault->cookie = cookie;
}
/* Fetch the first node out of the fault->deliver list */
static struct iopf_group *
iommufd_fault_deliver_fetch(struct iommufd_fault *fault)
{
struct list_head *list = &fault->common.deliver;
struct iopf_group *group = NULL;
spin_lock(&fault->common.lock);
if (!list_empty(list)) {
group = list_first_entry(list, struct iopf_group, node);
list_del(&group->node);
}
spin_unlock(&fault->common.lock);
return group;
}
/* Restore a node back to the head of the fault->deliver list */
static void iommufd_fault_deliver_restore(struct iommufd_fault *fault,
struct iopf_group *group)
{
spin_lock(&fault->common.lock);
list_add(&group->node, &fault->common.deliver);
spin_unlock(&fault->common.lock);
}
static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
size_t count, loff_t *ppos)
{
size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
struct iommufd_eventq *eventq = filep->private_data;
struct iommufd_fault *fault = eventq_to_fault(eventq);
struct iommu_hwpt_pgfault data = {};
struct iommufd_device *idev;
struct iopf_group *group;
struct iopf_fault *iopf;
size_t done = 0;
int rc = 0;
if (*ppos || count % fault_size)
return -ESPIPE;
mutex_lock(&fault->mutex);
while ((group = iommufd_fault_deliver_fetch(fault))) {
if (done >= count ||
group->fault_count * fault_size > count - done) {
iommufd_fault_deliver_restore(fault, group);
break;
}
rc = xa_alloc(&fault->response, &group->cookie, group,
xa_limit_32b, GFP_KERNEL);
if (rc) {
iommufd_fault_deliver_restore(fault, group);
break;
}
idev = to_iommufd_handle(group->attach_handle)->idev;
list_for_each_entry(iopf, &group->faults, list) {
iommufd_compose_fault_message(&iopf->fault,
&data, idev,
group->cookie);
if (copy_to_user(buf + done, &data, fault_size)) {
xa_erase(&fault->response, group->cookie);
iommufd_fault_deliver_restore(fault, group);
rc = -EFAULT;
break;
}
done += fault_size;
}
}
mutex_unlock(&fault->mutex);
return done == 0 ? rc : done;
}
static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
size_t count, loff_t *ppos)
{
size_t response_size = sizeof(struct iommu_hwpt_page_response);
struct iommufd_eventq *eventq = filep->private_data;
struct iommufd_fault *fault = eventq_to_fault(eventq);
struct iommu_hwpt_page_response response;
struct iopf_group *group;
size_t done = 0;
int rc = 0;
if (*ppos || count % response_size)
return -ESPIPE;
mutex_lock(&fault->mutex);
while (count > done) {
rc = copy_from_user(&response, buf + done, response_size);
if (rc)
break;
static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
(int)IOMMU_PAGE_RESP_SUCCESS);
static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
(int)IOMMU_PAGE_RESP_INVALID);
if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
response.code != IOMMUFD_PAGE_RESP_INVALID) {
rc = -EINVAL;
break;
}
group = xa_erase(&fault->response, response.cookie);
if (!group) {
rc = -EINVAL;
break;
}
iopf_group_response(group, response.code);
iopf_free_group(group);
done += response_size;
}
mutex_unlock(&fault->mutex);
return done == 0 ? rc : done;
}
/* IOMMUFD_OBJ_VEVENTQ Functions */
void iommufd_veventq_abort(struct iommufd_object *obj)
{
struct iommufd_eventq *eventq =
container_of(obj, struct iommufd_eventq, obj);
struct iommufd_veventq *veventq = eventq_to_veventq(eventq);
struct iommufd_viommu *viommu = veventq->viommu;
struct iommufd_vevent *cur, *next;
lockdep_assert_held_write(&viommu->veventqs_rwsem);
list_for_each_entry_safe(cur, next, &eventq->deliver, node) {
list_del(&cur->node);
if (cur != &veventq->lost_events_header)
kfree(cur);
}
refcount_dec(&viommu->obj.users);
list_del(&veventq->node);
}
void iommufd_veventq_destroy(struct iommufd_object *obj)
{
struct iommufd_veventq *veventq = eventq_to_veventq(
container_of(obj, struct iommufd_eventq, obj));
down_write(&veventq->viommu->veventqs_rwsem);
iommufd_veventq_abort(obj);
up_write(&veventq->viommu->veventqs_rwsem);
}
static struct iommufd_vevent *
iommufd_veventq_deliver_fetch(struct iommufd_veventq *veventq)
{
struct iommufd_eventq *eventq = &veventq->common;
struct list_head *list = &eventq->deliver;
struct iommufd_vevent *vevent = NULL;
spin_lock(&eventq->lock);
if (!list_empty(list)) {
struct iommufd_vevent *next;
next = list_first_entry(list, struct iommufd_vevent, node);
/* Make a copy of the lost_events_header for copy_to_user */
if (next == &veventq->lost_events_header) {
vevent = kzalloc(sizeof(*vevent), GFP_ATOMIC);
if (!vevent)
goto out_unlock;
}
list_del(&next->node);
if (vevent)
memcpy(vevent, next, sizeof(*vevent));
else
vevent = next;
}
out_unlock:
spin_unlock(&eventq->lock);
return vevent;
}
static void iommufd_veventq_deliver_restore(struct iommufd_veventq *veventq,
struct iommufd_vevent *vevent)
{
struct iommufd_eventq *eventq = &veventq->common;
struct list_head *list = &eventq->deliver;
spin_lock(&eventq->lock);
if (vevent_for_lost_events_header(vevent)) {
/* Remove the copy of the lost_events_header */
kfree(vevent);
vevent = NULL;
/* An empty list needs the lost_events_header back */
if (list_empty(list))
vevent = &veventq->lost_events_header;
}
if (vevent)
list_add(&vevent->node, list);
spin_unlock(&eventq->lock);
}
static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
size_t count, loff_t *ppos)
{
struct iommufd_eventq *eventq = filep->private_data;
struct iommufd_veventq *veventq = eventq_to_veventq(eventq);
struct iommufd_vevent_header *hdr;
struct iommufd_vevent *cur;
size_t done = 0;
int rc = 0;
if (*ppos)
return -ESPIPE;
while ((cur = iommufd_veventq_deliver_fetch(veventq))) {
/* Validate the remaining bytes against the header size */
if (done >= count || sizeof(*hdr) > count - done) {
iommufd_veventq_deliver_restore(veventq, cur);
break;
}
hdr = &cur->header;
/* If being a normal vEVENT, validate against the full size */
if (!vevent_for_lost_events_header(cur) &&
sizeof(hdr) + cur->data_len > count - done) {
iommufd_veventq_deliver_restore(veventq, cur);
break;
}
if (copy_to_user(buf + done, hdr, sizeof(*hdr))) {
iommufd_veventq_deliver_restore(veventq, cur);
rc = -EFAULT;
break;
}
done += sizeof(*hdr);
if (cur->data_len &&
copy_to_user(buf + done, cur->event_data, cur->data_len)) {
iommufd_veventq_deliver_restore(veventq, cur);
rc = -EFAULT;
break;
}
spin_lock(&eventq->lock);
if (!vevent_for_lost_events_header(cur))
veventq->num_events--;
spin_unlock(&eventq->lock);
done += cur->data_len;
kfree(cur);
}
return done == 0 ? rc : done;
}
/* Common Event Queue Functions */
static __poll_t iommufd_eventq_fops_poll(struct file *filep,
struct poll_table_struct *wait)
{
struct iommufd_eventq *eventq = filep->private_data;
__poll_t pollflags = 0;
if (eventq->obj.type == IOMMUFD_OBJ_FAULT)
pollflags |= EPOLLOUT;
poll_wait(filep, &eventq->wait_queue, wait);
spin_lock(&eventq->lock);
if (!list_empty(&eventq->deliver))
pollflags |= EPOLLIN | EPOLLRDNORM;
spin_unlock(&eventq->lock);
return pollflags;
}
static int iommufd_eventq_fops_release(struct inode *inode, struct file *filep)
{
struct iommufd_eventq *eventq = filep->private_data;
refcount_dec(&eventq->obj.users);
iommufd_ctx_put(eventq->ictx);
return 0;
}
#define INIT_EVENTQ_FOPS(read_op, write_op) \
((const struct file_operations){ \
.owner = THIS_MODULE, \
.open = nonseekable_open, \
.read = read_op, \
.write = write_op, \
.poll = iommufd_eventq_fops_poll, \
.release = iommufd_eventq_fops_release, \
})
static int iommufd_eventq_init(struct iommufd_eventq *eventq, char *name,
struct iommufd_ctx *ictx,
const struct file_operations *fops)
{
struct file *filep;
int fdno;
spin_lock_init(&eventq->lock);
INIT_LIST_HEAD(&eventq->deliver);
init_waitqueue_head(&eventq->wait_queue);
filep = anon_inode_getfile(name, fops, eventq, O_RDWR);
if (IS_ERR(filep))
return PTR_ERR(filep);
eventq->ictx = ictx;
iommufd_ctx_get(eventq->ictx);
eventq->filep = filep;
refcount_inc(&eventq->obj.users);
fdno = get_unused_fd_flags(O_CLOEXEC);
if (fdno < 0)
fput(filep);
return fdno;
}
static const struct file_operations iommufd_fault_fops =
INIT_EVENTQ_FOPS(iommufd_fault_fops_read, iommufd_fault_fops_write);
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
{
struct iommu_fault_alloc *cmd = ucmd->cmd;
struct iommufd_fault *fault;
int fdno;
int rc;
if (cmd->flags)
return -EOPNOTSUPP;
fault = __iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT,
common.obj);
if (IS_ERR(fault))
return PTR_ERR(fault);
xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
mutex_init(&fault->mutex);
fdno = iommufd_eventq_init(&fault->common, "[iommufd-pgfault]",
ucmd->ictx, &iommufd_fault_fops);
if (fdno < 0) {
rc = fdno;
goto out_abort;
}
cmd->out_fault_id = fault->common.obj.id;
cmd->out_fault_fd = fdno;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
if (rc)
goto out_put_fdno;
iommufd_object_finalize(ucmd->ictx, &fault->common.obj);
fd_install(fdno, fault->common.filep);
return 0;
out_put_fdno:
put_unused_fd(fdno);
fput(fault->common.filep);
out_abort:
iommufd_object_abort_and_destroy(ucmd->ictx, &fault->common.obj);
return rc;
}
int iommufd_fault_iopf_handler(struct iopf_group *group)
{
struct iommufd_hw_pagetable *hwpt;
struct iommufd_fault *fault;
hwpt = group->attach_handle->domain->iommufd_hwpt;
fault = hwpt->fault;
spin_lock(&fault->common.lock);
list_add_tail(&group->node, &fault->common.deliver);
spin_unlock(&fault->common.lock);
wake_up_interruptible(&fault->common.wait_queue);
return 0;
}
static const struct file_operations iommufd_veventq_fops =
INIT_EVENTQ_FOPS(iommufd_veventq_fops_read, NULL);
int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd)
{
struct iommu_veventq_alloc *cmd = ucmd->cmd;
struct iommufd_veventq *veventq;
struct iommufd_viommu *viommu;
int fdno;
int rc;
if (cmd->flags || cmd->__reserved ||
cmd->type == IOMMU_VEVENTQ_TYPE_DEFAULT)
return -EOPNOTSUPP;
if (!cmd->veventq_depth)
return -EINVAL;
viommu = iommufd_get_viommu(ucmd, cmd->viommu_id);
if (IS_ERR(viommu))
return PTR_ERR(viommu);
down_write(&viommu->veventqs_rwsem);
if (iommufd_viommu_find_veventq(viommu, cmd->type)) {
rc = -EEXIST;
goto out_unlock_veventqs;
}
veventq = __iommufd_object_alloc(ucmd->ictx, veventq,
IOMMUFD_OBJ_VEVENTQ, common.obj);
if (IS_ERR(veventq)) {
rc = PTR_ERR(veventq);
goto out_unlock_veventqs;
}
veventq->type = cmd->type;
veventq->viommu = viommu;
refcount_inc(&viommu->obj.users);
veventq->depth = cmd->veventq_depth;
list_add_tail(&veventq->node, &viommu->veventqs);
veventq->lost_events_header.header.flags =
IOMMU_VEVENTQ_FLAG_LOST_EVENTS;
fdno = iommufd_eventq_init(&veventq->common, "[iommufd-viommu-event]",
ucmd->ictx, &iommufd_veventq_fops);
if (fdno < 0) {
rc = fdno;
goto out_abort;
}
cmd->out_veventq_id = veventq->common.obj.id;
cmd->out_veventq_fd = fdno;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
if (rc)
goto out_put_fdno;
iommufd_object_finalize(ucmd->ictx, &veventq->common.obj);
fd_install(fdno, veventq->common.filep);
goto out_unlock_veventqs;
out_put_fdno:
put_unused_fd(fdno);
fput(veventq->common.filep);
out_abort:
iommufd_object_abort_and_destroy(ucmd->ictx, &veventq->common.obj);
out_unlock_veventqs:
up_write(&viommu->veventqs_rwsem);
iommufd_put_object(ucmd->ictx, &viommu->obj);
return rc;
}

342
drivers/iommu/iommufd/fault.c
View File

@@ -1,342 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2024 Intel Corporation
*/
#define pr_fmt(fmt) "iommufd: " fmt
#include <linux/anon_inodes.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/iommufd.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci-ats.h>
#include <linux/poll.h>
#include <uapi/linux/iommufd.h>
#include "../iommu-priv.h"
#include "iommufd_private.h"
int iommufd_fault_iopf_enable(struct iommufd_device *idev)
{
struct device *dev = idev->dev;
int ret;
/*
* Once we turn on PCI/PRI support for VF, the response failure code
* should not be forwarded to the hardware due to PRI being a shared
* resource between PF and VFs. There is no coordination for this
* shared capability. This waits for a vPRI reset to recover.
*/
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
if (pdev->is_virtfn && pci_pri_supported(pdev))
return -EINVAL;
}
mutex_lock(&idev->iopf_lock);
/* Device iopf has already been on. */
if (++idev->iopf_enabled > 1) {
mutex_unlock(&idev->iopf_lock);
return 0;
}
ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
if (ret)
--idev->iopf_enabled;
mutex_unlock(&idev->iopf_lock);
return ret;
}
void iommufd_fault_iopf_disable(struct iommufd_device *idev)
{
mutex_lock(&idev->iopf_lock);
if (!WARN_ON(idev->iopf_enabled == 0)) {
if (--idev->iopf_enabled == 0)
iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
}
mutex_unlock(&idev->iopf_lock);
}
void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
struct iommufd_attach_handle *handle)
{
struct iommufd_fault *fault = hwpt->fault;
struct iopf_group *group, *next;
struct list_head free_list;
unsigned long index;
if (!fault)
return;
INIT_LIST_HEAD(&free_list);
mutex_lock(&fault->mutex);
spin_lock(&fault->lock);
list_for_each_entry_safe(group, next, &fault->deliver, node) {
if (group->attach_handle != &handle->handle)
continue;
list_move(&group->node, &free_list);
}
spin_unlock(&fault->lock);
list_for_each_entry_safe(group, next, &free_list, node) {
list_del(&group->node);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
xa_for_each(&fault->response, index, group) {
if (group->attach_handle != &handle->handle)
continue;
xa_erase(&fault->response, index);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
mutex_unlock(&fault->mutex);
}
void iommufd_fault_destroy(struct iommufd_object *obj)
{
struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
struct iopf_group *group, *next;
unsigned long index;
/*
* The iommufd object's reference count is zero at this point.
* We can be confident that no other threads are currently
* accessing this pointer. Therefore, acquiring the mutex here
* is unnecessary.
*/
list_for_each_entry_safe(group, next, &fault->deliver, node) {
list_del(&group->node);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
xa_for_each(&fault->response, index, group) {
xa_erase(&fault->response, index);
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
iopf_free_group(group);
}
xa_destroy(&fault->response);
mutex_destroy(&fault->mutex);
}
static void iommufd_compose_fault_message(struct iommu_fault *fault,
struct iommu_hwpt_pgfault *hwpt_fault,
struct iommufd_device *idev,
u32 cookie)
{
hwpt_fault->flags = fault->prm.flags;
hwpt_fault->dev_id = idev->obj.id;
hwpt_fault->pasid = fault->prm.pasid;
hwpt_fault->grpid = fault->prm.grpid;
hwpt_fault->perm = fault->prm.perm;
hwpt_fault->addr = fault->prm.addr;
hwpt_fault->length = 0;
hwpt_fault->cookie = cookie;
}
static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
size_t count, loff_t *ppos)
{
size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
struct iommufd_fault *fault = filep->private_data;
struct iommu_hwpt_pgfault data = {};
struct iommufd_device *idev;
struct iopf_group *group;
struct iopf_fault *iopf;
size_t done = 0;
int rc = 0;
if (*ppos || count % fault_size)
return -ESPIPE;
mutex_lock(&fault->mutex);
while ((group = iommufd_fault_deliver_fetch(fault))) {
if (done >= count ||
group->fault_count * fault_size > count - done) {
iommufd_fault_deliver_restore(fault, group);
break;
}
rc = xa_alloc(&fault->response, &group->cookie, group,
xa_limit_32b, GFP_KERNEL);
if (rc) {
iommufd_fault_deliver_restore(fault, group);
break;
}
idev = to_iommufd_handle(group->attach_handle)->idev;
list_for_each_entry(iopf, &group->faults, list) {
iommufd_compose_fault_message(&iopf->fault,
&data, idev,
group->cookie);
if (copy_to_user(buf + done, &data, fault_size)) {
xa_erase(&fault->response, group->cookie);
iommufd_fault_deliver_restore(fault, group);
rc = -EFAULT;
break;
}
done += fault_size;
}
}
mutex_unlock(&fault->mutex);
return done == 0 ? rc : done;
}
static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
size_t count, loff_t *ppos)
{
size_t response_size = sizeof(struct iommu_hwpt_page_response);
struct iommufd_fault *fault = filep->private_data;
struct iommu_hwpt_page_response response;
struct iopf_group *group;
size_t done = 0;
int rc = 0;
if (*ppos || count % response_size)
return -ESPIPE;
mutex_lock(&fault->mutex);
while (count > done) {
rc = copy_from_user(&response, buf + done, response_size);
if (rc)
break;
static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
(int)IOMMU_PAGE_RESP_SUCCESS);
static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
(int)IOMMU_PAGE_RESP_INVALID);
if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
response.code != IOMMUFD_PAGE_RESP_INVALID) {
rc = -EINVAL;
break;
}
group = xa_erase(&fault->response, response.cookie);
if (!group) {
rc = -EINVAL;
break;
}
iopf_group_response(group, response.code);
iopf_free_group(group);
done += response_size;
}
mutex_unlock(&fault->mutex);
return done == 0 ? rc : done;
}
static __poll_t iommufd_fault_fops_poll(struct file *filep,
struct poll_table_struct *wait)
{
struct iommufd_fault *fault = filep->private_data;
__poll_t pollflags = EPOLLOUT;
poll_wait(filep, &fault->wait_queue, wait);
spin_lock(&fault->lock);
if (!list_empty(&fault->deliver))
pollflags |= EPOLLIN | EPOLLRDNORM;
spin_unlock(&fault->lock);
return pollflags;
}
static int iommufd_fault_fops_release(struct inode *inode, struct file *filep)
{
struct iommufd_fault *fault = filep->private_data;
refcount_dec(&fault->obj.users);
iommufd_ctx_put(fault->ictx);
return 0;
}
static const struct file_operations iommufd_fault_fops = {
.owner = THIS_MODULE,
.open = nonseekable_open,
.read = iommufd_fault_fops_read,
.write = iommufd_fault_fops_write,
.poll = iommufd_fault_fops_poll,
.release = iommufd_fault_fops_release,
};
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
{
struct iommu_fault_alloc *cmd = ucmd->cmd;
struct iommufd_fault *fault;
struct file *filep;
int fdno;
int rc;
if (cmd->flags)
return -EOPNOTSUPP;
fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
if (IS_ERR(fault))
return PTR_ERR(fault);
fault->ictx = ucmd->ictx;
INIT_LIST_HEAD(&fault->deliver);
xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
mutex_init(&fault->mutex);
spin_lock_init(&fault->lock);
init_waitqueue_head(&fault->wait_queue);
filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
fault, O_RDWR);
if (IS_ERR(filep)) {
rc = PTR_ERR(filep);
goto out_abort;
}
refcount_inc(&fault->obj.users);
iommufd_ctx_get(fault->ictx);
fault->filep = filep;
fdno = get_unused_fd_flags(O_CLOEXEC);
if (fdno < 0) {
rc = fdno;
goto out_fput;
}
cmd->out_fault_id = fault->obj.id;
cmd->out_fault_fd = fdno;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
if (rc)
goto out_put_fdno;
iommufd_object_finalize(ucmd->ictx, &fault->obj);
fd_install(fdno, fault->filep);
return 0;
out_put_fdno:
put_unused_fd(fdno);
out_fput:
fput(filep);
out_abort:
iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
return rc;
}
int iommufd_fault_iopf_handler(struct iopf_group *group)
{
struct iommufd_hw_pagetable *hwpt;
struct iommufd_fault *fault;
hwpt = group->attach_handle->domain->iommufd_hwpt;
fault = hwpt->fault;
spin_lock(&fault->lock);
list_add_tail(&group->node, &fault->deliver);
spin_unlock(&fault->lock);
wake_up_interruptible(&fault->wait_queue);
return 0;
}

42
drivers/iommu/iommufd/hw_pagetable.c
View File

@@ -14,7 +14,7 @@ static void __iommufd_hwpt_destroy(struct iommufd_hw_pagetable *hwpt)
iommu_domain_free(hwpt->domain);
if (hwpt->fault)
refcount_dec(&hwpt->fault->obj.users);
refcount_dec(&hwpt->fault->common.obj.users);
}
void iommufd_hwpt_paging_destroy(struct iommufd_object *obj)
@@ -90,6 +90,7 @@ iommufd_hwpt_paging_enforce_cc(struct iommufd_hwpt_paging *hwpt_paging)
* @ictx: iommufd context
* @ioas: IOAS to associate the domain with
* @idev: Device to get an iommu_domain for
* @pasid: PASID to get an iommu_domain for
* @flags: Flags from userspace
* @immediate_attach: True if idev should be attached to the hwpt
* @user_data: The user provided driver specific data describing the domain to
@@ -105,13 +106,14 @@ iommufd_hwpt_paging_enforce_cc(struct iommufd_hwpt_paging *hwpt_paging)
*/
struct iommufd_hwpt_paging *
iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
struct iommufd_device *idev, u32 flags,
bool immediate_attach,
struct iommufd_device *idev, ioasid_t pasid,
u32 flags, bool immediate_attach,
const struct iommu_user_data *user_data)
{
const u32 valid_flags = IOMMU_HWPT_ALLOC_NEST_PARENT |
IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
IOMMU_HWPT_FAULT_ID_VALID;
IOMMU_HWPT_FAULT_ID_VALID |
IOMMU_HWPT_ALLOC_PASID;
const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
struct iommufd_hwpt_paging *hwpt_paging;
struct iommufd_hw_pagetable *hwpt;
@@ -126,12 +128,16 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
if ((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) &&
!device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING))
return ERR_PTR(-EOPNOTSUPP);
if ((flags & IOMMU_HWPT_FAULT_ID_VALID) &&
(flags & IOMMU_HWPT_ALLOC_NEST_PARENT))
return ERR_PTR(-EOPNOTSUPP);
hwpt_paging = __iommufd_object_alloc(
ictx, hwpt_paging, IOMMUFD_OBJ_HWPT_PAGING, common.obj);
if (IS_ERR(hwpt_paging))
return ERR_CAST(hwpt_paging);
hwpt = &hwpt_paging->common;
hwpt->pasid_compat = flags & IOMMU_HWPT_ALLOC_PASID;
INIT_LIST_HEAD(&hwpt_paging->hwpt_item);
/* Pairs with iommufd_hw_pagetable_destroy() */
@@ -156,7 +162,8 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
goto out_abort;
}
}
iommu_domain_set_sw_msi(hwpt->domain, iommufd_sw_msi);
hwpt->domain->iommufd_hwpt = hwpt;
hwpt->domain->cookie_type = IOMMU_COOKIE_IOMMUFD;
/*
* Set the coherency mode before we do iopt_table_add_domain() as some
@@ -185,7 +192,7 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
* sequence. Once those drivers are fixed this should be removed.
*/
if (immediate_attach) {
rc = iommufd_hw_pagetable_attach(hwpt, idev);
rc = iommufd_hw_pagetable_attach(hwpt, idev, pasid);
if (rc)
goto out_abort;
}
@@ -198,7 +205,7 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
out_detach:
if (immediate_attach)
iommufd_hw_pagetable_detach(idev);
iommufd_hw_pagetable_detach(idev, pasid);
out_abort:
iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
return ERR_PTR(rc);
@@ -227,7 +234,7 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx,
struct iommufd_hw_pagetable *hwpt;
int rc;
if ((flags & ~IOMMU_HWPT_FAULT_ID_VALID) ||
if ((flags & ~(IOMMU_HWPT_FAULT_ID_VALID | IOMMU_HWPT_ALLOC_PASID)) ||
!user_data->len || !ops->domain_alloc_nested)
return ERR_PTR(-EOPNOTSUPP);
if (parent->auto_domain || !parent->nest_parent ||
@@ -239,6 +246,7 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx,
if (IS_ERR(hwpt_nested))
return ERR_CAST(hwpt_nested);
hwpt = &hwpt_nested->common;
hwpt->pasid_compat = flags & IOMMU_HWPT_ALLOC_PASID;
refcount_inc(&parent->common.obj.users);
hwpt_nested->parent = parent;
@@ -252,7 +260,8 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx,
goto out_abort;
}
hwpt->domain->owner = ops;
iommu_domain_set_sw_msi(hwpt->domain, iommufd_sw_msi);
hwpt->domain->iommufd_hwpt = hwpt;
hwpt->domain->cookie_type = IOMMU_COOKIE_IOMMUFD;
if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
rc = -EINVAL;
@@ -282,7 +291,7 @@ iommufd_viommu_alloc_hwpt_nested(struct iommufd_viommu *viommu, u32 flags,
struct iommufd_hw_pagetable *hwpt;
int rc;
if (flags & ~IOMMU_HWPT_FAULT_ID_VALID)
if (flags & ~(IOMMU_HWPT_FAULT_ID_VALID | IOMMU_HWPT_ALLOC_PASID))
return ERR_PTR(-EOPNOTSUPP);
if (!user_data->len)
return ERR_PTR(-EOPNOTSUPP);
@@ -294,6 +303,7 @@ iommufd_viommu_alloc_hwpt_nested(struct iommufd_viommu *viommu, u32 flags,
if (IS_ERR(hwpt_nested))
return ERR_CAST(hwpt_nested);
hwpt = &hwpt_nested->common;
hwpt->pasid_compat = flags & IOMMU_HWPT_ALLOC_PASID;
hwpt_nested->viommu = viommu;
refcount_inc(&viommu->obj.users);
@@ -308,8 +318,9 @@ iommufd_viommu_alloc_hwpt_nested(struct iommufd_viommu *viommu, u32 flags,
hwpt->domain = NULL;
goto out_abort;
}
hwpt->domain->iommufd_hwpt = hwpt;
hwpt->domain->owner = viommu->iommu_dev->ops;
iommu_domain_set_sw_msi(hwpt->domain, iommufd_sw_msi);
hwpt->domain->cookie_type = IOMMU_COOKIE_IOMMUFD;
if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
rc = -EINVAL;
@@ -358,8 +369,8 @@ int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
ioas = container_of(pt_obj, struct iommufd_ioas, obj);
mutex_lock(&ioas->mutex);
hwpt_paging = iommufd_hwpt_paging_alloc(
ucmd->ictx, ioas, idev, cmd->flags, false,
user_data.len ? &user_data : NULL);
ucmd->ictx, ioas, idev, IOMMU_NO_PASID, cmd->flags,
false, user_data.len ? &user_data : NULL);
if (IS_ERR(hwpt_paging)) {
rc = PTR_ERR(hwpt_paging);
goto out_unlock;
@@ -409,10 +420,9 @@ int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
}
hwpt->fault = fault;
hwpt->domain->iopf_handler = iommufd_fault_iopf_handler;
refcount_inc(&fault->obj.users);
iommufd_put_object(ucmd->ictx, &fault->obj);
refcount_inc(&fault->common.obj.users);
iommufd_put_object(ucmd->ictx, &fault->common.obj);
}
hwpt->domain->iommufd_hwpt = hwpt;
cmd->out_hwpt_id = hwpt->obj.id;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));

156
drivers/iommu/iommufd/iommufd_private.h
View File

@@ -32,8 +32,11 @@ struct iommufd_sw_msi_maps {
DECLARE_BITMAP(bitmap, 64);
};
int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr);
#ifdef CONFIG_IRQ_MSI_IOMMU
int iommufd_sw_msi_install(struct iommufd_ctx *ictx,
struct iommufd_hwpt_paging *hwpt_paging,
struct iommufd_sw_msi_map *msi_map);
#endif
struct iommufd_ctx {
struct file *file;
@@ -296,6 +299,7 @@ struct iommufd_hw_pagetable {
struct iommufd_object obj;
struct iommu_domain *domain;
struct iommufd_fault *fault;
bool pasid_compat : 1;
};
struct iommufd_hwpt_paging {
@@ -366,13 +370,13 @@ int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd);
struct iommufd_hwpt_paging *
iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
struct iommufd_device *idev, u32 flags,
bool immediate_attach,
struct iommufd_device *idev, ioasid_t pasid,
u32 flags, bool immediate_attach,
const struct iommu_user_data *user_data);
int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
struct iommufd_device *idev);
struct iommufd_device *idev, ioasid_t pasid);
struct iommufd_hw_pagetable *
iommufd_hw_pagetable_detach(struct iommufd_device *idev);
iommufd_hw_pagetable_detach(struct iommufd_device *idev, ioasid_t pasid);
void iommufd_hwpt_paging_destroy(struct iommufd_object *obj);
void iommufd_hwpt_paging_abort(struct iommufd_object *obj);
void iommufd_hwpt_nested_destroy(struct iommufd_object *obj);
@@ -396,13 +400,14 @@ static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
refcount_dec(&hwpt->obj.users);
}
struct iommufd_attach;
struct iommufd_group {
struct kref ref;
struct mutex lock;
struct iommufd_ctx *ictx;
struct iommu_group *group;
struct iommufd_hw_pagetable *hwpt;
struct list_head device_list;
struct xarray pasid_attach;
struct iommufd_sw_msi_maps required_sw_msi;
phys_addr_t sw_msi_start;
};
@@ -454,49 +459,17 @@ void iopt_remove_access(struct io_pagetable *iopt,
u32 iopt_access_list_id);
void iommufd_access_destroy_object(struct iommufd_object *obj);
/*
* An iommufd_fault object represents an interface to deliver I/O page faults
* to the user space. These objects are created/destroyed by the user space and
* associated with hardware page table objects during page-table allocation.
*/
struct iommufd_fault {
struct iommufd_eventq {
struct iommufd_object obj;
struct iommufd_ctx *ictx;
struct file *filep;
spinlock_t lock; /* protects the deliver list */
struct list_head deliver;
struct mutex mutex; /* serializes response flows */
struct xarray response;
struct wait_queue_head wait_queue;
};
/* Fetch the first node out of the fault->deliver list */
static inline struct iopf_group *
iommufd_fault_deliver_fetch(struct iommufd_fault *fault)
{
struct list_head *list = &fault->deliver;
struct iopf_group *group = NULL;
spin_lock(&fault->lock);
if (!list_empty(list)) {
group = list_first_entry(list, struct iopf_group, node);
list_del(&group->node);
}
spin_unlock(&fault->lock);
return group;
}
/* Restore a node back to the head of the fault->deliver list */
static inline void iommufd_fault_deliver_restore(struct iommufd_fault *fault,
struct iopf_group *group)
{
spin_lock(&fault->lock);
list_add(&group->node, &fault->deliver);
spin_unlock(&fault->lock);
}
struct iommufd_attach_handle {
struct iommu_attach_handle handle;
struct iommufd_device *idev;
@@ -505,12 +478,29 @@ struct iommufd_attach_handle {
/* Convert an iommu attach handle to iommufd handle. */
#define to_iommufd_handle(hdl) container_of(hdl, struct iommufd_attach_handle, handle)
/*
* An iommufd_fault object represents an interface to deliver I/O page faults
* to the user space. These objects are created/destroyed by the user space and
* associated with hardware page table objects during page-table allocation.
*/
struct iommufd_fault {
struct iommufd_eventq common;
struct mutex mutex; /* serializes response flows */
struct xarray response;
};
static inline struct iommufd_fault *
eventq_to_fault(struct iommufd_eventq *eventq)
{
return container_of(eventq, struct iommufd_fault, common);
}
static inline struct iommufd_fault *
iommufd_get_fault(struct iommufd_ucmd *ucmd, u32 id)
{
return container_of(iommufd_get_object(ucmd->ictx, id,
IOMMUFD_OBJ_FAULT),
struct iommufd_fault, obj);
struct iommufd_fault, common.obj);
}
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd);
@@ -522,6 +512,74 @@ void iommufd_fault_iopf_disable(struct iommufd_device *idev);
void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
struct iommufd_attach_handle *handle);
/* An iommufd_vevent represents a vIOMMU event in an iommufd_veventq */
struct iommufd_vevent {
struct iommufd_vevent_header header;
struct list_head node; /* for iommufd_eventq::deliver */
ssize_t data_len;
u64 event_data[] __counted_by(data_len);
};
#define vevent_for_lost_events_header(vevent) \
(vevent->header.flags & IOMMU_VEVENTQ_FLAG_LOST_EVENTS)
/*
* An iommufd_veventq object represents an interface to deliver vIOMMU events to
* the user space. It is created/destroyed by the user space and associated with
* a vIOMMU object during the allocations.
*/
struct iommufd_veventq {
struct iommufd_eventq common;
struct iommufd_viommu *viommu;
struct list_head node; /* for iommufd_viommu::veventqs */
struct iommufd_vevent lost_events_header;
unsigned int type;
unsigned int depth;
/* Use common.lock for protection */
u32 num_events;
u32 sequence;
};
static inline struct iommufd_veventq *
eventq_to_veventq(struct iommufd_eventq *eventq)
{
return container_of(eventq, struct iommufd_veventq, common);
}
static inline struct iommufd_veventq *
iommufd_get_veventq(struct iommufd_ucmd *ucmd, u32 id)
{
return container_of(iommufd_get_object(ucmd->ictx, id,
IOMMUFD_OBJ_VEVENTQ),
struct iommufd_veventq, common.obj);
}
int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd);
void iommufd_veventq_destroy(struct iommufd_object *obj);
void iommufd_veventq_abort(struct iommufd_object *obj);
static inline void iommufd_vevent_handler(struct iommufd_veventq *veventq,
struct iommufd_vevent *vevent)
{
struct iommufd_eventq *eventq = &veventq->common;
lockdep_assert_held(&eventq->lock);
/*
* Remove the lost_events_header and add the new node at the same time.
* Note the new node can be lost_events_header, for a sequence update.
*/
if (list_is_last(&veventq->lost_events_header.node, &eventq->deliver))
list_del(&veventq->lost_events_header.node);
list_add_tail(&vevent->node, &eventq->deliver);
vevent->header.sequence = veventq->sequence;
veventq->sequence = (veventq->sequence + 1) & INT_MAX;
wake_up_interruptible(&eventq->wait_queue);
}
static inline struct iommufd_viommu *
iommufd_get_viommu(struct iommufd_ucmd *ucmd, u32 id)
{
@@ -530,6 +588,20 @@ iommufd_get_viommu(struct iommufd_ucmd *ucmd, u32 id)
struct iommufd_viommu, obj);
}
static inline struct iommufd_veventq *
iommufd_viommu_find_veventq(struct iommufd_viommu *viommu, u32 type)
{
struct iommufd_veventq *veventq, *next;
lockdep_assert_held(&viommu->veventqs_rwsem);
list_for_each_entry_safe(veventq, next, &viommu->veventqs, node) {
if (veventq->type == type)
return veventq;
}
return NULL;
}
int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd);
void iommufd_viommu_destroy(struct iommufd_object *obj);
int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd);

40
drivers/iommu/iommufd/iommufd_test.h
View File

@@ -24,6 +24,11 @@ enum {
IOMMU_TEST_OP_MD_CHECK_IOTLB,
IOMMU_TEST_OP_TRIGGER_IOPF,
IOMMU_TEST_OP_DEV_CHECK_CACHE,
IOMMU_TEST_OP_TRIGGER_VEVENT,
IOMMU_TEST_OP_PASID_ATTACH,
IOMMU_TEST_OP_PASID_REPLACE,
IOMMU_TEST_OP_PASID_DETACH,
IOMMU_TEST_OP_PASID_CHECK_HWPT,
};
enum {
@@ -48,6 +53,7 @@ enum {
enum {
MOCK_FLAGS_DEVICE_NO_DIRTY = 1 << 0,
MOCK_FLAGS_DEVICE_HUGE_IOVA = 1 << 1,
MOCK_FLAGS_DEVICE_PASID = 1 << 2,
};
enum {
@@ -60,6 +66,9 @@ enum {
MOCK_DEV_CACHE_NUM = 4,
};
/* Reserved for special pasid replace test */
#define IOMMU_TEST_PASID_RESERVED 1024
struct iommu_test_cmd {
__u32 size;
__u32 op;
@@ -145,11 +154,36 @@ struct iommu_test_cmd {
__u32 id;
__u32 cache;
} check_dev_cache;
struct {
__u32 dev_id;
} trigger_vevent;
struct {
__u32 pasid;
__u32 pt_id;
/* @id is stdev_id */
} pasid_attach;
struct {
__u32 pasid;
__u32 pt_id;
/* @id is stdev_id */
} pasid_replace;
struct {
__u32 pasid;
/* @id is stdev_id */
} pasid_detach;
struct {
__u32 pasid;
__u32 hwpt_id;
/* @id is stdev_id */
} pasid_check;
};
__u32 last;
};
#define IOMMU_TEST_CMD _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE + 32)
/* Mock device/iommu PASID width */
#define MOCK_PASID_WIDTH 20
/* Mock structs for IOMMU_DEVICE_GET_HW_INFO ioctl */
#define IOMMU_HW_INFO_TYPE_SELFTEST 0xfeedbeef
#define IOMMU_HW_INFO_SELFTEST_REGVAL 0xdeadbeef
@@ -212,4 +246,10 @@ struct iommu_viommu_invalidate_selftest {
__u32 cache_id;
};
#define IOMMU_VEVENTQ_TYPE_SELFTEST 0xbeefbeef
struct iommu_viommu_event_selftest {
__u32 virt_id;
};
#endif

7
drivers/iommu/iommufd/main.c
View File

@@ -317,6 +317,7 @@ union ucmd_buffer {
struct iommu_ioas_unmap unmap;
struct iommu_option option;
struct iommu_vdevice_alloc vdev;
struct iommu_veventq_alloc veventq;
struct iommu_vfio_ioas vfio_ioas;
struct iommu_viommu_alloc viommu;
#ifdef CONFIG_IOMMUFD_TEST
@@ -372,6 +373,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
IOCTL_OP(IOMMU_OPTION, iommufd_option, struct iommu_option, val64),
IOCTL_OP(IOMMU_VDEVICE_ALLOC, iommufd_vdevice_alloc_ioctl,
struct iommu_vdevice_alloc, virt_id),
IOCTL_OP(IOMMU_VEVENTQ_ALLOC, iommufd_veventq_alloc,
struct iommu_veventq_alloc, out_veventq_fd),
IOCTL_OP(IOMMU_VFIO_IOAS, iommufd_vfio_ioas, struct iommu_vfio_ioas,
__reserved),
IOCTL_OP(IOMMU_VIOMMU_ALLOC, iommufd_viommu_alloc_ioctl,
@@ -514,6 +517,10 @@ static const struct iommufd_object_ops iommufd_object_ops[] = {
[IOMMUFD_OBJ_VDEVICE] = {
.destroy = iommufd_vdevice_destroy,
},
[IOMMUFD_OBJ_VEVENTQ] = {
.destroy = iommufd_veventq_destroy,
.abort = iommufd_veventq_abort,
},
[IOMMUFD_OBJ_VIOMMU] = {
.destroy = iommufd_viommu_destroy,
},

297
drivers/iommu/iommufd/selftest.c
View File

@@ -161,9 +161,13 @@ enum selftest_obj_type {
struct mock_dev {
struct device dev;
struct mock_viommu *viommu;
struct rw_semaphore viommu_rwsem;
unsigned long flags;
unsigned long vdev_id;
int id;
u32 cache[MOCK_DEV_CACHE_NUM];
atomic_t pasid_1024_fake_error;
};
static inline struct mock_dev *to_mock_dev(struct device *dev)
@@ -193,15 +197,71 @@ static int mock_domain_nop_attach(struct iommu_domain *domain,
struct device *dev)
{
struct mock_dev *mdev = to_mock_dev(dev);
struct mock_viommu *new_viommu = NULL;
unsigned long vdev_id = 0;
int rc;
if (domain->dirty_ops && (mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY))
return -EINVAL;
iommu_group_mutex_assert(dev);
if (domain->type == IOMMU_DOMAIN_NESTED) {
new_viommu = to_mock_nested(domain)->mock_viommu;
if (new_viommu) {
rc = iommufd_viommu_get_vdev_id(&new_viommu->core, dev,
&vdev_id);
if (rc)
return rc;
}
}
if (new_viommu != mdev->viommu) {
down_write(&mdev->viommu_rwsem);
mdev->viommu = new_viommu;
mdev->vdev_id = vdev_id;
up_write(&mdev->viommu_rwsem);
}
return 0;
}
static int mock_domain_set_dev_pasid_nop(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid,
struct iommu_domain *old)
{
struct mock_dev *mdev = to_mock_dev(dev);
/*
* Per the first attach with pasid 1024, set the
* mdev->pasid_1024_fake_error. Hence the second call of this op
* can fake an error to validate the error path of the core. This
* is helpful to test the case in which the iommu core needs to
* rollback to the old domain due to driver failure. e.g. replace.
* User should be careful about the third call of this op, it shall
* succeed since the mdev->pasid_1024_fake_error is cleared in the
* second call.
*/
if (pasid == 1024) {
if (domain->type == IOMMU_DOMAIN_BLOCKED) {
atomic_set(&mdev->pasid_1024_fake_error, 0);
} else if (atomic_read(&mdev->pasid_1024_fake_error)) {
/*
* Clear the flag, and fake an error to fail the
* replacement.
*/
atomic_set(&mdev->pasid_1024_fake_error, 0);
return -ENOMEM;
} else {
/* Set the flag to fake an error in next call */
atomic_set(&mdev->pasid_1024_fake_error, 1);
}
}
return 0;
}
static const struct iommu_domain_ops mock_blocking_ops = {
.attach_dev = mock_domain_nop_attach,
.set_dev_pasid = mock_domain_set_dev_pasid_nop
};
static struct iommu_domain mock_blocking_domain = {
@@ -343,7 +403,7 @@ mock_domain_alloc_nested(struct device *dev, struct iommu_domain *parent,
struct mock_iommu_domain_nested *mock_nested;
struct mock_iommu_domain *mock_parent;
if (flags)
if (flags & ~IOMMU_HWPT_ALLOC_PASID)
return ERR_PTR(-EOPNOTSUPP);
if (!parent || parent->ops != mock_ops.default_domain_ops)
return ERR_PTR(-EINVAL);
@@ -365,7 +425,8 @@ mock_domain_alloc_paging_flags(struct device *dev, u32 flags,
{
bool has_dirty_flag = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
IOMMU_HWPT_ALLOC_NEST_PARENT;
IOMMU_HWPT_ALLOC_NEST_PARENT |
IOMMU_HWPT_ALLOC_PASID;
struct mock_dev *mdev = to_mock_dev(dev);
bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY;
struct mock_iommu_domain *mock;
@@ -585,7 +646,7 @@ mock_viommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
struct mock_viommu *mock_viommu = to_mock_viommu(viommu);
struct mock_iommu_domain_nested *mock_nested;
if (flags)
if (flags & ~IOMMU_HWPT_ALLOC_PASID)
return ERR_PTR(-EOPNOTSUPP);
mock_nested = __mock_domain_alloc_nested(user_data);
@@ -720,6 +781,7 @@ static const struct iommu_ops mock_ops = {
.map_pages = mock_domain_map_pages,
.unmap_pages = mock_domain_unmap_pages,
.iova_to_phys = mock_domain_iova_to_phys,
.set_dev_pasid = mock_domain_set_dev_pasid_nop,
},
};
@@ -780,6 +842,7 @@ static struct iommu_domain_ops domain_nested_ops = {
.free = mock_domain_free_nested,
.attach_dev = mock_domain_nop_attach,
.cache_invalidate_user = mock_domain_cache_invalidate_user,
.set_dev_pasid = mock_domain_set_dev_pasid_nop,
};
static inline struct iommufd_hw_pagetable *
@@ -839,17 +902,24 @@ static void mock_dev_release(struct device *dev)
static struct mock_dev *mock_dev_create(unsigned long dev_flags)
{
struct property_entry prop[] = {
PROPERTY_ENTRY_U32("pasid-num-bits", 0),
{},
};
const u32 valid_flags = MOCK_FLAGS_DEVICE_NO_DIRTY |
MOCK_FLAGS_DEVICE_HUGE_IOVA |
MOCK_FLAGS_DEVICE_PASID;
struct mock_dev *mdev;
int rc, i;
if (dev_flags &
~(MOCK_FLAGS_DEVICE_NO_DIRTY | MOCK_FLAGS_DEVICE_HUGE_IOVA))
if (dev_flags & ~valid_flags)
return ERR_PTR(-EINVAL);
mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
if (!mdev)
return ERR_PTR(-ENOMEM);
init_rwsem(&mdev->viommu_rwsem);
device_initialize(&mdev->dev);
mdev->flags = dev_flags;
mdev->dev.release = mock_dev_release;
@@ -866,6 +936,15 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags)
if (rc)
goto err_put;
if (dev_flags & MOCK_FLAGS_DEVICE_PASID)
prop[0] = PROPERTY_ENTRY_U32("pasid-num-bits", MOCK_PASID_WIDTH);
rc = device_create_managed_software_node(&mdev->dev, prop, NULL);
if (rc) {
dev_err(&mdev->dev, "add pasid-num-bits property failed, rc: %d", rc);
goto err_put;
}
rc = device_add(&mdev->dev);
if (rc)
goto err_put;
@@ -921,7 +1000,7 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
}
sobj->idev.idev = idev;
rc = iommufd_device_attach(idev, &pt_id);
rc = iommufd_device_attach(idev, IOMMU_NO_PASID, &pt_id);
if (rc)
goto out_unbind;
@@ -936,7 +1015,7 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
return 0;
out_detach:
iommufd_device_detach(idev);
iommufd_device_detach(idev, IOMMU_NO_PASID);
out_unbind:
iommufd_device_unbind(idev);
out_mdev:
@@ -946,39 +1025,49 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
return rc;
}
/* Replace the mock domain with a manually allocated hw_pagetable */
static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd,
unsigned int device_id, u32 pt_id,
struct iommu_test_cmd *cmd)
static struct selftest_obj *
iommufd_test_get_selftest_obj(struct iommufd_ctx *ictx, u32 id)
{
struct iommufd_object *dev_obj;
struct selftest_obj *sobj;
int rc;
/*
* Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure
* it doesn't race with detach, which is not allowed.
*/
dev_obj =
iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST);
dev_obj = iommufd_get_object(ictx, id, IOMMUFD_OBJ_SELFTEST);
if (IS_ERR(dev_obj))
return PTR_ERR(dev_obj);
return ERR_CAST(dev_obj);
sobj = to_selftest_obj(dev_obj);
if (sobj->type != TYPE_IDEV) {
rc = -EINVAL;
goto out_dev_obj;
iommufd_put_object(ictx, dev_obj);
return ERR_PTR(-EINVAL);
}
return sobj;
}
rc = iommufd_device_replace(sobj->idev.idev, &pt_id);
/* Replace the mock domain with a manually allocated hw_pagetable */
static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd,
unsigned int device_id, u32 pt_id,
struct iommu_test_cmd *cmd)
{
struct selftest_obj *sobj;
int rc;
sobj = iommufd_test_get_selftest_obj(ucmd->ictx, device_id);
if (IS_ERR(sobj))
return PTR_ERR(sobj);
rc = iommufd_device_replace(sobj->idev.idev, IOMMU_NO_PASID, &pt_id);
if (rc)
goto out_dev_obj;
goto out_sobj;
cmd->mock_domain_replace.pt_id = pt_id;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
out_dev_obj:
iommufd_put_object(ucmd->ictx, dev_obj);
out_sobj:
iommufd_put_object(ucmd->ictx, &sobj->obj);
return rc;
}
@@ -1597,13 +1686,166 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd,
return 0;
}
static int iommufd_test_trigger_vevent(struct iommufd_ucmd *ucmd,
struct iommu_test_cmd *cmd)
{
struct iommu_viommu_event_selftest test = {};
struct iommufd_device *idev;
struct mock_dev *mdev;
int rc = -ENOENT;
idev = iommufd_get_device(ucmd, cmd->trigger_vevent.dev_id);
if (IS_ERR(idev))
return PTR_ERR(idev);
mdev = to_mock_dev(idev->dev);
down_read(&mdev->viommu_rwsem);
if (!mdev->viommu || !mdev->vdev_id)
goto out_unlock;
test.virt_id = mdev->vdev_id;
rc = iommufd_viommu_report_event(&mdev->viommu->core,
IOMMU_VEVENTQ_TYPE_SELFTEST, &test,
sizeof(test));
out_unlock:
up_read(&mdev->viommu_rwsem);
iommufd_put_object(ucmd->ictx, &idev->obj);
return rc;
}
static inline struct iommufd_hw_pagetable *
iommufd_get_hwpt(struct iommufd_ucmd *ucmd, u32 id)
{
struct iommufd_object *pt_obj;
pt_obj = iommufd_get_object(ucmd->ictx, id, IOMMUFD_OBJ_ANY);
if (IS_ERR(pt_obj))
return ERR_CAST(pt_obj);
if (pt_obj->type != IOMMUFD_OBJ_HWPT_NESTED &&
pt_obj->type != IOMMUFD_OBJ_HWPT_PAGING) {
iommufd_put_object(ucmd->ictx, pt_obj);
return ERR_PTR(-EINVAL);
}
return container_of(pt_obj, struct iommufd_hw_pagetable, obj);
}
static int iommufd_test_pasid_check_hwpt(struct iommufd_ucmd *ucmd,
struct iommu_test_cmd *cmd)
{
u32 hwpt_id = cmd->pasid_check.hwpt_id;
struct iommu_domain *attached_domain;
struct iommu_attach_handle *handle;
struct iommufd_hw_pagetable *hwpt;
struct selftest_obj *sobj;
struct mock_dev *mdev;
int rc = 0;
sobj = iommufd_test_get_selftest_obj(ucmd->ictx, cmd->id);
if (IS_ERR(sobj))
return PTR_ERR(sobj);
mdev = sobj->idev.mock_dev;
handle = iommu_attach_handle_get(mdev->dev.iommu_group,
cmd->pasid_check.pasid, 0);
if (IS_ERR(handle))
attached_domain = NULL;
else
attached_domain = handle->domain;
/* hwpt_id == 0 means to check if pasid is detached */
if (!hwpt_id) {
if (attached_domain)
rc = -EINVAL;
goto out_sobj;
}
hwpt = iommufd_get_hwpt(ucmd, hwpt_id);
if (IS_ERR(hwpt)) {
rc = PTR_ERR(hwpt);
goto out_sobj;
}
if (attached_domain != hwpt->domain)
rc = -EINVAL;
iommufd_put_object(ucmd->ictx, &hwpt->obj);
out_sobj:
iommufd_put_object(ucmd->ictx, &sobj->obj);
return rc;
}
static int iommufd_test_pasid_attach(struct iommufd_ucmd *ucmd,
struct iommu_test_cmd *cmd)
{
struct selftest_obj *sobj;
int rc;
sobj = iommufd_test_get_selftest_obj(ucmd->ictx, cmd->id);
if (IS_ERR(sobj))
return PTR_ERR(sobj);
rc = iommufd_device_attach(sobj->idev.idev, cmd->pasid_attach.pasid,
&cmd->pasid_attach.pt_id);
if (rc)
goto out_sobj;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
if (rc)
iommufd_device_detach(sobj->idev.idev,
cmd->pasid_attach.pasid);
out_sobj:
iommufd_put_object(ucmd->ictx, &sobj->obj);
return rc;
}
static int iommufd_test_pasid_replace(struct iommufd_ucmd *ucmd,
struct iommu_test_cmd *cmd)
{
struct selftest_obj *sobj;
int rc;
sobj = iommufd_test_get_selftest_obj(ucmd->ictx, cmd->id);
if (IS_ERR(sobj))
return PTR_ERR(sobj);
rc = iommufd_device_replace(sobj->idev.idev, cmd->pasid_attach.pasid,
&cmd->pasid_attach.pt_id);
if (rc)
goto out_sobj;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
out_sobj:
iommufd_put_object(ucmd->ictx, &sobj->obj);
return rc;
}
static int iommufd_test_pasid_detach(struct iommufd_ucmd *ucmd,
struct iommu_test_cmd *cmd)
{
struct selftest_obj *sobj;
sobj = iommufd_test_get_selftest_obj(ucmd->ictx, cmd->id);
if (IS_ERR(sobj))
return PTR_ERR(sobj);
iommufd_device_detach(sobj->idev.idev, cmd->pasid_detach.pasid);
iommufd_put_object(ucmd->ictx, &sobj->obj);
return 0;
}
void iommufd_selftest_destroy(struct iommufd_object *obj)
{
struct selftest_obj *sobj = to_selftest_obj(obj);
switch (sobj->type) {
case TYPE_IDEV:
iommufd_device_detach(sobj->idev.idev);
iommufd_device_detach(sobj->idev.idev, IOMMU_NO_PASID);
iommufd_device_unbind(sobj->idev.idev);
mock_dev_destroy(sobj->idev.mock_dev);
break;
@@ -1678,6 +1920,16 @@ int iommufd_test(struct iommufd_ucmd *ucmd)
cmd->dirty.flags);
case IOMMU_TEST_OP_TRIGGER_IOPF:
return iommufd_test_trigger_iopf(ucmd, cmd);
case IOMMU_TEST_OP_TRIGGER_VEVENT:
return iommufd_test_trigger_vevent(ucmd, cmd);
case IOMMU_TEST_OP_PASID_ATTACH:
return iommufd_test_pasid_attach(ucmd, cmd);
case IOMMU_TEST_OP_PASID_REPLACE:
return iommufd_test_pasid_replace(ucmd, cmd);
case IOMMU_TEST_OP_PASID_DETACH:
return iommufd_test_pasid_detach(ucmd, cmd);
case IOMMU_TEST_OP_PASID_CHECK_HWPT:
return iommufd_test_pasid_check_hwpt(ucmd, cmd);
default:
return -EOPNOTSUPP;
}
@@ -1724,6 +1976,7 @@ int __init iommufd_test_init(void)
init_completion(&mock_iommu.complete);
mock_iommu_iopf_queue = iopf_queue_alloc("mock-iopfq");
mock_iommu.iommu_dev.max_pasids = (1 << MOCK_PASID_WIDTH);
return 0;

2
drivers/iommu/iommufd/viommu.c
View File

@@ -59,6 +59,8 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
viommu->ictx = ucmd->ictx;
viommu->hwpt = hwpt_paging;
refcount_inc(&viommu->hwpt->common.obj.users);
INIT_LIST_HEAD(&viommu->veventqs);
init_rwsem(&viommu->veventqs_rwsem);
/*
* It is the most likely case that a physical IOMMU is unpluggable. A
* pluggable IOMMU instance (if exists) is responsible for refcounting

33
drivers/pci/ats.c
View File

@@ -538,4 +538,37 @@ int pci_max_pasids(struct pci_dev *pdev)
return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported));
}
EXPORT_SYMBOL_GPL(pci_max_pasids);
/**
* pci_pasid_status - Check the PASID status
* @pdev: PCI device structure
*
* Returns a negative value when no PASID capability is present.
* Otherwise the value of the control register is returned.
* Status reported are:
*
* PCI_PASID_CTRL_ENABLE - PASID enabled
* PCI_PASID_CTRL_EXEC - Execute permission enabled
* PCI_PASID_CTRL_PRIV - Privileged mode enabled
*/
int pci_pasid_status(struct pci_dev *pdev)
{
int pasid;
u16 ctrl;
if (pdev->is_virtfn)
pdev = pci_physfn(pdev);
pasid = pdev->pasid_cap;
if (!pasid)
return -EINVAL;
pci_read_config_word(pdev, pasid + PCI_PASID_CTRL, &ctrl);
ctrl &= PCI_PASID_CTRL_ENABLE | PCI_PASID_CTRL_EXEC |
PCI_PASID_CTRL_PRIV;
return ctrl;
}
EXPORT_SYMBOL_GPL(pci_pasid_status);
#endif /* CONFIG_PCI_PASID */

60
drivers/vfio/device_cdev.c
View File

@@ -162,9 +162,9 @@ void vfio_df_unbind_iommufd(struct vfio_device_file *df)
int vfio_df_ioctl_attach_pt(struct vfio_device_file *df,
struct vfio_device_attach_iommufd_pt __user *arg)
{
struct vfio_device *device = df->device;
struct vfio_device_attach_iommufd_pt attach;
unsigned long minsz;
struct vfio_device *device = df->device;
unsigned long minsz, xend = 0;
int ret;
minsz = offsetofend(struct vfio_device_attach_iommufd_pt, pt_id);
@@ -172,11 +172,34 @@ int vfio_df_ioctl_attach_pt(struct vfio_device_file *df,
if (copy_from_user(&attach, arg, minsz))
return -EFAULT;
if (attach.argsz < minsz || attach.flags)
if (attach.argsz < minsz)
return -EINVAL;
if (attach.flags & ~VFIO_DEVICE_ATTACH_PASID)
return -EINVAL;
if (attach.flags & VFIO_DEVICE_ATTACH_PASID) {
if (!device->ops->pasid_attach_ioas)
return -EOPNOTSUPP;
xend = offsetofend(struct vfio_device_attach_iommufd_pt, pasid);
}
if (xend) {
if (attach.argsz < xend)
return -EINVAL;
if (copy_from_user((void *)&attach + minsz,
(void __user *)arg + minsz, xend - minsz))
return -EFAULT;
}
mutex_lock(&device->dev_set->lock);
ret = device->ops->attach_ioas(device, &attach.pt_id);
if (attach.flags & VFIO_DEVICE_ATTACH_PASID)
ret = device->ops->pasid_attach_ioas(device,
attach.pasid,
&attach.pt_id);
else
ret = device->ops->attach_ioas(device, &attach.pt_id);
if (ret)
goto out_unlock;
@@ -198,20 +221,41 @@ int vfio_df_ioctl_attach_pt(struct vfio_device_file *df,
int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
struct vfio_device_detach_iommufd_pt __user *arg)
{
struct vfio_device *device = df->device;
struct vfio_device_detach_iommufd_pt detach;
unsigned long minsz;
struct vfio_device *device = df->device;
unsigned long minsz, xend = 0;
minsz = offsetofend(struct vfio_device_detach_iommufd_pt, flags);
if (copy_from_user(&detach, arg, minsz))
return -EFAULT;
if (detach.argsz < minsz || detach.flags)
if (detach.argsz < minsz)
return -EINVAL;
if (detach.flags & ~VFIO_DEVICE_DETACH_PASID)
return -EINVAL;
if (detach.flags & VFIO_DEVICE_DETACH_PASID) {
if (!device->ops->pasid_detach_ioas)
return -EOPNOTSUPP;
xend = offsetofend(struct vfio_device_detach_iommufd_pt, pasid);
}
if (xend) {
if (detach.argsz < xend)
return -EINVAL;
if (copy_from_user((void *)&detach + minsz,
(void __user *)arg + minsz, xend - minsz))
return -EFAULT;
}
mutex_lock(&device->dev_set->lock);
device->ops->detach_ioas(device);
if (detach.flags & VFIO_DEVICE_DETACH_PASID)
device->ops->pasid_detach_ioas(device, detach.pasid);
else
device->ops->detach_ioas(device);
mutex_unlock(&device->dev_set->lock);
return 0;

60
drivers/vfio/iommufd.c
View File

@@ -119,16 +119,24 @@ int vfio_iommufd_physical_bind(struct vfio_device *vdev,
if (IS_ERR(idev))
return PTR_ERR(idev);
vdev->iommufd_device = idev;
ida_init(&vdev->pasids);
return 0;
}
EXPORT_SYMBOL_GPL(vfio_iommufd_physical_bind);
void vfio_iommufd_physical_unbind(struct vfio_device *vdev)
{
int pasid;
lockdep_assert_held(&vdev->dev_set->lock);
while ((pasid = ida_find_first(&vdev->pasids)) >= 0) {
iommufd_device_detach(vdev->iommufd_device, pasid);
ida_free(&vdev->pasids, pasid);
}
if (vdev->iommufd_attached) {
iommufd_device_detach(vdev->iommufd_device);
iommufd_device_detach(vdev->iommufd_device, IOMMU_NO_PASID);
vdev->iommufd_attached = false;
}
iommufd_device_unbind(vdev->iommufd_device);
@@ -146,9 +154,11 @@ int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 *pt_id)
return -EINVAL;
if (vdev->iommufd_attached)
rc = iommufd_device_replace(vdev->iommufd_device, pt_id);
rc = iommufd_device_replace(vdev->iommufd_device,
IOMMU_NO_PASID, pt_id);
else
rc = iommufd_device_attach(vdev->iommufd_device, pt_id);
rc = iommufd_device_attach(vdev->iommufd_device,
IOMMU_NO_PASID, pt_id);
if (rc)
return rc;
vdev->iommufd_attached = true;
@@ -163,11 +173,53 @@ void vfio_iommufd_physical_detach_ioas(struct vfio_device *vdev)
if (WARN_ON(!vdev->iommufd_device) || !vdev->iommufd_attached)
return;
iommufd_device_detach(vdev->iommufd_device);
iommufd_device_detach(vdev->iommufd_device, IOMMU_NO_PASID);
vdev->iommufd_attached = false;
}
EXPORT_SYMBOL_GPL(vfio_iommufd_physical_detach_ioas);
int vfio_iommufd_physical_pasid_attach_ioas(struct vfio_device *vdev,
u32 pasid, u32 *pt_id)
{
int rc;
lockdep_assert_held(&vdev->dev_set->lock);
if (WARN_ON(!vdev->iommufd_device))
return -EINVAL;
if (ida_exists(&vdev->pasids, pasid))
return iommufd_device_replace(vdev->iommufd_device,
pasid, pt_id);
rc = ida_alloc_range(&vdev->pasids, pasid, pasid, GFP_KERNEL);
if (rc < 0)
return rc;
rc = iommufd_device_attach(vdev->iommufd_device, pasid, pt_id);
if (rc)
ida_free(&vdev->pasids, pasid);
return rc;
}
EXPORT_SYMBOL_GPL(vfio_iommufd_physical_pasid_attach_ioas);
void vfio_iommufd_physical_pasid_detach_ioas(struct vfio_device *vdev,
u32 pasid)
{
lockdep_assert_held(&vdev->dev_set->lock);
if (WARN_ON(!vdev->iommufd_device))
return;
if (!ida_exists(&vdev->pasids, pasid))
return;
iommufd_device_detach(vdev->iommufd_device, pasid);
ida_free(&vdev->pasids, pasid);
}
EXPORT_SYMBOL_GPL(vfio_iommufd_physical_pasid_detach_ioas);
/*
* The emulated standard ops mean that vfio_device is going to use the
* "mdev path" and will call vfio_pin_pages()/vfio_dma_rw(). Drivers using this

2
drivers/vfio/pci/vfio_pci.c
View File

@@ -144,6 +144,8 @@ static const struct vfio_device_ops vfio_pci_ops = {
.unbind_iommufd = vfio_iommufd_physical_unbind,
.attach_ioas = vfio_iommufd_physical_attach_ioas,
.detach_ioas = vfio_iommufd_physical_detach_ioas,
.pasid_attach_ioas = vfio_iommufd_physical_pasid_attach_ioas,
.pasid_detach_ioas = vfio_iommufd_physical_pasid_detach_ioas,
};
static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

11
include/linux/idr.h
View File

@@ -274,6 +274,7 @@ struct ida {
int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t);
void ida_free(struct ida *, unsigned int id);
void ida_destroy(struct ida *ida);
int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max);
/**
* ida_alloc() - Allocate an unused ID.
@@ -345,4 +346,14 @@ static inline bool ida_is_empty(const struct ida *ida)
{
return xa_empty(&ida->xa);
}
static inline bool ida_exists(struct ida *ida, unsigned int id)
{
return ida_find_first_range(ida, id, id) == id;
}
static inline int ida_find_first(struct ida *ida)
{
return ida_find_first_range(ida, 0, ~0);
}
#endif /* __IDR_H__ */

35
include/linux/iommu.h
View File

@@ -41,6 +41,7 @@ struct iommu_dirty_ops;
struct notifier_block;
struct iommu_sva;
struct iommu_dma_cookie;
struct iommu_dma_msi_cookie;
struct iommu_fault_param;
struct iommufd_ctx;
struct iommufd_viommu;
@@ -165,6 +166,15 @@ struct iommu_domain_geometry {
bool force_aperture; /* DMA only allowed in mappable range? */
};
enum iommu_domain_cookie_type {
IOMMU_COOKIE_NONE,
IOMMU_COOKIE_DMA_IOVA,
IOMMU_COOKIE_DMA_MSI,
IOMMU_COOKIE_FAULT_HANDLER,
IOMMU_COOKIE_SVA,
IOMMU_COOKIE_IOMMUFD,
};
/* Domain feature flags */
#define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */
#define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API
@@ -211,23 +221,18 @@ struct iommu_domain_geometry {
struct iommu_domain {
unsigned type;
enum iommu_domain_cookie_type cookie_type;
const struct iommu_domain_ops *ops;
const struct iommu_dirty_ops *dirty_ops;
const struct iommu_ops *owner; /* Whose domain_alloc we came from */
unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
struct iommu_domain_geometry geometry;
struct iommu_dma_cookie *iova_cookie;
int (*iopf_handler)(struct iopf_group *group);
#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
int (*sw_msi)(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr);
#endif
union { /* Pointer usable by owner of the domain */
struct iommufd_hw_pagetable *iommufd_hwpt; /* iommufd */
};
union { /* Fault handler */
union { /* cookie */
struct iommu_dma_cookie *iova_cookie;
struct iommu_dma_msi_cookie *msi_cookie;
struct iommufd_hw_pagetable *iommufd_hwpt;
struct {
iommu_fault_handler_t handler;
void *handler_token;
@@ -244,16 +249,6 @@ struct iommu_domain {
};
};
static inline void iommu_domain_set_sw_msi(
struct iommu_domain *domain,
int (*sw_msi)(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr))
{
#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
domain->sw_msi = sw_msi;
#endif
}
static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
{
return domain->type & __IOMMU_DOMAIN_DMA_API;

32
include/linux/iommufd.h
View File

@@ -8,9 +8,11 @@
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/iommu.h>
#include <linux/refcount.h>
#include <linux/types.h>
#include <linux/xarray.h>
#include <uapi/linux/iommufd.h>
struct device;
struct file;
@@ -34,6 +36,7 @@ enum iommufd_object_type {
IOMMUFD_OBJ_FAULT,
IOMMUFD_OBJ_VIOMMU,
IOMMUFD_OBJ_VDEVICE,
IOMMUFD_OBJ_VEVENTQ,
#ifdef CONFIG_IOMMUFD_TEST
IOMMUFD_OBJ_SELFTEST,
#endif
@@ -52,9 +55,11 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
struct device *dev, u32 *id);
void iommufd_device_unbind(struct iommufd_device *idev);
int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id);
int iommufd_device_replace(struct iommufd_device *idev, u32 *pt_id);
void iommufd_device_detach(struct iommufd_device *idev);
int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid,
u32 *pt_id);
int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid,
u32 *pt_id);
void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);
struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);
u32 iommufd_device_to_id(struct iommufd_device *idev);
@@ -93,6 +98,8 @@ struct iommufd_viommu {
const struct iommufd_viommu_ops *ops;
struct xarray vdevs;
struct list_head veventqs;
struct rw_semaphore veventqs_rwsem;
unsigned int type;
};
@@ -187,6 +194,11 @@ struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx,
enum iommufd_object_type type);
struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu,
unsigned long vdev_id);
int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
struct device *dev, unsigned long *vdev_id);
int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
enum iommu_veventq_type type, void *event_data,
size_t data_len);
#else /* !CONFIG_IOMMUFD_DRIVER_CORE */
static inline struct iommufd_object *
_iommufd_object_alloc(struct iommufd_ctx *ictx, size_t size,
@@ -200,6 +212,20 @@ iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id)
{
return NULL;
}
static inline int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
struct device *dev,
unsigned long *vdev_id)
{
return -ENOENT;
}
static inline int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
enum iommu_veventq_type type,
void *event_data, size_t data_len)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_IOMMUFD_DRIVER_CORE */
/*

3
include/linux/pci-ats.h
View File

@@ -42,6 +42,7 @@ int pci_enable_pasid(struct pci_dev *pdev, int features);
void pci_disable_pasid(struct pci_dev *pdev);
int pci_pasid_features(struct pci_dev *pdev);
int pci_max_pasids(struct pci_dev *pdev);
int pci_pasid_status(struct pci_dev *pdev);
#else /* CONFIG_PCI_PASID */
static inline int pci_enable_pasid(struct pci_dev *pdev, int features)
{ return -EINVAL; }
@@ -50,6 +51,8 @@ static inline int pci_pasid_features(struct pci_dev *pdev)
{ return -EINVAL; }
static inline int pci_max_pasids(struct pci_dev *pdev)
{ return -EINVAL; }
static inline int pci_pasid_status(struct pci_dev *pdev)
{ return -EINVAL; }
#endif /* CONFIG_PCI_PASID */
#endif /* LINUX_PCI_ATS_H */

14
include/linux/vfio.h
View File

@@ -67,6 +67,7 @@ struct vfio_device {
struct inode *inode;
#if IS_ENABLED(CONFIG_IOMMUFD)
struct iommufd_device *iommufd_device;
struct ida pasids;
u8 iommufd_attached:1;
#endif
u8 cdev_opened:1;
@@ -91,6 +92,8 @@ struct vfio_device {
* bound iommufd. Undo in unbind_iommufd if @detach_ioas is not
* called.
* @detach_ioas: Opposite of attach_ioas
* @pasid_attach_ioas: The pasid variation of attach_ioas
* @pasid_detach_ioas: Opposite of pasid_attach_ioas
* @open_device: Called when the first file descriptor is opened for this device
* @close_device: Opposite of open_device
* @read: Perform read(2) on device file descriptor
@@ -115,6 +118,9 @@ struct vfio_device_ops {
void (*unbind_iommufd)(struct vfio_device *vdev);
int (*attach_ioas)(struct vfio_device *vdev, u32 *pt_id);
void (*detach_ioas)(struct vfio_device *vdev);
int (*pasid_attach_ioas)(struct vfio_device *vdev, u32 pasid,
u32 *pt_id);
void (*pasid_detach_ioas)(struct vfio_device *vdev, u32 pasid);
int (*open_device)(struct vfio_device *vdev);
void (*close_device)(struct vfio_device *vdev);
ssize_t (*read)(struct vfio_device *vdev, char __user *buf,
@@ -139,6 +145,10 @@ int vfio_iommufd_physical_bind(struct vfio_device *vdev,
void vfio_iommufd_physical_unbind(struct vfio_device *vdev);
int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 *pt_id);
void vfio_iommufd_physical_detach_ioas(struct vfio_device *vdev);
int vfio_iommufd_physical_pasid_attach_ioas(struct vfio_device *vdev,
u32 pasid, u32 *pt_id);
void vfio_iommufd_physical_pasid_detach_ioas(struct vfio_device *vdev,
u32 pasid);
int vfio_iommufd_emulated_bind(struct vfio_device *vdev,
struct iommufd_ctx *ictx, u32 *out_device_id);
void vfio_iommufd_emulated_unbind(struct vfio_device *vdev);
@@ -166,6 +176,10 @@ vfio_iommufd_get_dev_id(struct vfio_device *vdev, struct iommufd_ctx *ictx)
((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL)
#define vfio_iommufd_physical_detach_ioas \
((void (*)(struct vfio_device *vdev)) NULL)
#define vfio_iommufd_physical_pasid_attach_ioas \
((int (*)(struct vfio_device *vdev, u32 pasid, u32 *pt_id)) NULL)
#define vfio_iommufd_physical_pasid_detach_ioas \
((void (*)(struct vfio_device *vdev, u32 pasid)) NULL)
#define vfio_iommufd_emulated_bind \
((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \
u32 *out_device_id)) NULL)

129
include/uapi/linux/iommufd.h
View File

@@ -55,6 +55,7 @@ enum {
IOMMUFD_CMD_VIOMMU_ALLOC = 0x90,
IOMMUFD_CMD_VDEVICE_ALLOC = 0x91,
IOMMUFD_CMD_IOAS_CHANGE_PROCESS = 0x92,
IOMMUFD_CMD_VEVENTQ_ALLOC = 0x93,
};
/**
@@ -392,6 +393,9 @@ struct iommu_vfio_ioas {
* Any domain attached to the non-PASID part of the
* device must also be flagged, otherwise attaching a
* PASID will blocked.
* For the user that wants to attach PASID, ioas is
* not recommended for both the non-PASID part
* and PASID part of the device.
* If IOMMU does not support PASID it will return
* error (-EOPNOTSUPP).
*/
@@ -608,9 +612,17 @@ enum iommu_hw_info_type {
* IOMMU_HWPT_GET_DIRTY_BITMAP
* IOMMU_HWPT_SET_DIRTY_TRACKING
*
* @IOMMU_HW_CAP_PCI_PASID_EXEC: Execute Permission Supported, user ignores it
* when the struct
* iommu_hw_info::out_max_pasid_log2 is zero.
* @IOMMU_HW_CAP_PCI_PASID_PRIV: Privileged Mode Supported, user ignores it
* when the struct
* iommu_hw_info::out_max_pasid_log2 is zero.
*/
enum iommufd_hw_capabilities {
IOMMU_HW_CAP_DIRTY_TRACKING = 1 << 0,
IOMMU_HW_CAP_PCI_PASID_EXEC = 1 << 1,
IOMMU_HW_CAP_PCI_PASID_PRIV = 1 << 2,
};
/**
@@ -626,6 +638,9 @@ enum iommufd_hw_capabilities {
* iommu_hw_info_type.
* @out_capabilities: Output the generic iommu capability info type as defined
* in the enum iommu_hw_capabilities.
* @out_max_pasid_log2: Output the width of PASIDs. 0 means no PASID support.
* PCI devices turn to out_capabilities to check if the
* specific capabilities is supported or not.
* @__reserved: Must be 0
*
* Query an iommu type specific hardware information data from an iommu behind
@@ -649,7 +664,8 @@ struct iommu_hw_info {
__u32 data_len;
__aligned_u64 data_uptr;
__u32 out_data_type;
__u32 __reserved;
__u8 out_max_pasid_log2;
__u8 __reserved[3];
__aligned_u64 out_capabilities;
};
#define IOMMU_GET_HW_INFO _IO(IOMMUFD_TYPE, IOMMUFD_CMD_GET_HW_INFO)
@@ -1014,4 +1030,115 @@ struct iommu_ioas_change_process {
#define IOMMU_IOAS_CHANGE_PROCESS \
_IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_CHANGE_PROCESS)
/**
* enum iommu_veventq_flag - flag for struct iommufd_vevent_header
* @IOMMU_VEVENTQ_FLAG_LOST_EVENTS: vEVENTQ has lost vEVENTs
*/
enum iommu_veventq_flag {
IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0),
};
/**
* struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status
* @flags: Combination of enum iommu_veventq_flag
* @sequence: The sequence index of a vEVENT in the vEVENTQ, with a range of
* [0, INT_MAX] where the following index of INT_MAX is 0
*
* Each iommufd_vevent_header reports a sequence index of the following vEVENT:
*
* +----------------------+-------+----------------------+-------+---+-------+
* | header0 {sequence=0} | data0 | header1 {sequence=1} | data1 |...| dataN |
* +----------------------+-------+----------------------+-------+---+-------+
*
* And this sequence index is expected to be monotonic to the sequence index of
* the previous vEVENT. If two adjacent sequence indexes has a delta larger than
* 1, it means that delta - 1 number of vEVENTs has lost, e.g. two lost vEVENTs:
*
* +-----+----------------------+-------+----------------------+-------+-----+
* | ... | header3 {sequence=3} | data3 | header6 {sequence=6} | data6 | ... |
* +-----+----------------------+-------+----------------------+-------+-----+
*
* If a vEVENT lost at the tail of the vEVENTQ and there is no following vEVENT
* providing the next sequence index, an IOMMU_VEVENTQ_FLAG_LOST_EVENTS header
* would be added to the tail, and no data would follow this header:
*
* +--+----------------------+-------+-----------------------------------------+
* |..| header3 {sequence=3} | data3 | header4 {flags=LOST_EVENTS, sequence=4} |
* +--+----------------------+-------+-----------------------------------------+
*/
struct iommufd_vevent_header {
__u32 flags;
__u32 sequence;
};
/**
* enum iommu_veventq_type - Virtual Event Queue Type
* @IOMMU_VEVENTQ_TYPE_DEFAULT: Reserved for future use
* @IOMMU_VEVENTQ_TYPE_ARM_SMMUV3: ARM SMMUv3 Virtual Event Queue
*/
enum iommu_veventq_type {
IOMMU_VEVENTQ_TYPE_DEFAULT = 0,
IOMMU_VEVENTQ_TYPE_ARM_SMMUV3 = 1,
};
/**
* struct iommu_vevent_arm_smmuv3 - ARM SMMUv3 Virtual Event
* (IOMMU_VEVENTQ_TYPE_ARM_SMMUV3)
* @evt: 256-bit ARM SMMUv3 Event record, little-endian.
* Reported event records: (Refer to "7.3 Event records" in SMMUv3 HW Spec)
* - 0x04 C_BAD_STE
* - 0x06 F_STREAM_DISABLED
* - 0x08 C_BAD_SUBSTREAMID
* - 0x0a C_BAD_CD
* - 0x10 F_TRANSLATION
* - 0x11 F_ADDR_SIZE
* - 0x12 F_ACCESS
* - 0x13 F_PERMISSION
*
* StreamID field reports a virtual device ID. To receive a virtual event for a
* device, a vDEVICE must be allocated via IOMMU_VDEVICE_ALLOC.
*/
struct iommu_vevent_arm_smmuv3 {
__aligned_le64 evt[4];
};
/**
* struct iommu_veventq_alloc - ioctl(IOMMU_VEVENTQ_ALLOC)
* @size: sizeof(struct iommu_veventq_alloc)
* @flags: Must be 0
* @viommu_id: virtual IOMMU ID to associate the vEVENTQ with
* @type: Type of the vEVENTQ. Must be defined in enum iommu_veventq_type
* @veventq_depth: Maximum number of events in the vEVENTQ
* @out_veventq_id: The ID of the new vEVENTQ
* @out_veventq_fd: The fd of the new vEVENTQ. User space must close the
* successfully returned fd after using it
* @__reserved: Must be 0
*
* Explicitly allocate a virtual event queue interface for a vIOMMU. A vIOMMU
* can have multiple FDs for different types, but is confined to one per @type.
* User space should open the @out_veventq_fd to read vEVENTs out of a vEVENTQ,
* if there are vEVENTs available. A vEVENTQ will lose events due to overflow,
* if the number of the vEVENTs hits @veventq_depth.
*
* Each vEVENT in a vEVENTQ encloses a struct iommufd_vevent_header followed by
* a type-specific data structure, in a normal case:
*
* +-+---------+-------+---------+-------+-----+---------+-------+-+
* | | header0 | data0 | header1 | data1 | ... | headerN | dataN | |
* +-+---------+-------+---------+-------+-----+---------+-------+-+
*
* unless a tailing IOMMU_VEVENTQ_FLAG_LOST_EVENTS header is logged (refer to
* struct iommufd_vevent_header).
*/
struct iommu_veventq_alloc {
__u32 size;
__u32 flags;
__u32 viommu_id;
__u32 type;
__u32 veventq_depth;
__u32 out_veventq_id;
__u32 out_veventq_fd;
__u32 __reserved;
};
#define IOMMU_VEVENTQ_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VEVENTQ_ALLOC)
#endif

29
include/uapi/linux/vfio.h
View File

@@ -932,29 +932,34 @@ struct vfio_device_bind_iommufd {
* VFIO_DEVICE_ATTACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 19,
* struct vfio_device_attach_iommufd_pt)
* @argsz: User filled size of this data.
* @flags: Must be 0.
* @flags: Flags for attach.
* @pt_id: Input the target id which can represent an ioas or a hwpt
* allocated via iommufd subsystem.
* Output the input ioas id or the attached hwpt id which could
* be the specified hwpt itself or a hwpt automatically created
* for the specified ioas by kernel during the attachment.
* @pasid: The pasid to be attached, only meaningful when
* VFIO_DEVICE_ATTACH_PASID is set in @flags
*
* Associate the device with an address space within the bound iommufd.
* Undo by VFIO_DEVICE_DETACH_IOMMUFD_PT or device fd close. This is only
* allowed on cdev fds.
*
* If a vfio device is currently attached to a valid hw_pagetable, without doing
* a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl
* passing in another hw_pagetable (hwpt) id is allowed. This action, also known
* as a hw_pagetable replacement, will replace the device's currently attached
* hw_pagetable with a new hw_pagetable corresponding to the given pt_id.
* If a vfio device or a pasid of this device is currently attached to a valid
* hw_pagetable (hwpt), without doing a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second
* VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl passing in another hwpt id is allowed.
* This action, also known as a hw_pagetable replacement, will replace the
* currently attached hwpt of the device or the pasid of this device with a new
* hwpt corresponding to the given pt_id.
*
* Return: 0 on success, -errno on failure.
*/
struct vfio_device_attach_iommufd_pt {
__u32 argsz;
__u32 flags;
#define VFIO_DEVICE_ATTACH_PASID (1 << 0)
__u32 pt_id;
__u32 pasid;
};
#define VFIO_DEVICE_ATTACH_IOMMUFD_PT _IO(VFIO_TYPE, VFIO_BASE + 19)
@@ -963,17 +968,21 @@ struct vfio_device_attach_iommufd_pt {
* VFIO_DEVICE_DETACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 20,
* struct vfio_device_detach_iommufd_pt)
* @argsz: User filled size of this data.
* @flags: Must be 0.
* @flags: Flags for detach.
* @pasid: The pasid to be detached, only meaningful when
* VFIO_DEVICE_DETACH_PASID is set in @flags
*
* Remove the association of the device and its current associated address
* space. After it, the device should be in a blocking DMA state. This is only
* allowed on cdev fds.
* Remove the association of the device or a pasid of the device and its current
* associated address space. After it, the device or the pasid should be in a
* blocking DMA state. This is only allowed on cdev fds.
*
* Return: 0 on success, -errno on failure.
*/
struct vfio_device_detach_iommufd_pt {
__u32 argsz;
__u32 flags;
#define VFIO_DEVICE_DETACH_PASID (1 << 0)
__u32 pasid;
};
#define VFIO_DEVICE_DETACH_IOMMUFD_PT _IO(VFIO_TYPE, VFIO_BASE + 20)

67
lib/idr.c
View File

@@ -476,6 +476,73 @@ int ida_alloc_range(struct ida *ida, unsigned int min, unsigned int max,
}
EXPORT_SYMBOL(ida_alloc_range);
/**
* ida_find_first_range - Get the lowest used ID.
* @ida: IDA handle.
* @min: Lowest ID to get.
* @max: Highest ID to get.
*
* Get the lowest used ID between @min and @max, inclusive. The returned
* ID will not exceed %INT_MAX, even if @max is larger.
*
* Context: Any context. Takes and releases the xa_lock.
* Return: The lowest used ID, or errno if no used ID is found.
*/
int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
{
unsigned long index = min / IDA_BITMAP_BITS;
unsigned int offset = min % IDA_BITMAP_BITS;
unsigned long *addr, size, bit;
unsigned long tmp = 0;
unsigned long flags;
void *entry;
int ret;
if ((int)min < 0)
return -EINVAL;
if ((int)max < 0)
max = INT_MAX;
xa_lock_irqsave(&ida->xa, flags);
entry = xa_find(&ida->xa, &index, max / IDA_BITMAP_BITS, XA_PRESENT);
if (!entry) {
ret = -ENOENT;
goto err_unlock;
}
if (index > min / IDA_BITMAP_BITS)
offset = 0;
if (index * IDA_BITMAP_BITS + offset > max) {
ret = -ENOENT;
goto err_unlock;
}
if (xa_is_value(entry)) {
tmp = xa_to_value(entry);
addr = &tmp;
size = BITS_PER_XA_VALUE;
} else {
addr = ((struct ida_bitmap *)entry)->bitmap;
size = IDA_BITMAP_BITS;
}
bit = find_next_bit(addr, size, offset);
xa_unlock_irqrestore(&ida->xa, flags);
if (bit == size ||
index * IDA_BITMAP_BITS + bit > max)
return -ENOENT;
return index * IDA_BITMAP_BITS + bit;
err_unlock:
xa_unlock_irqrestore(&ida->xa, flags);
return ret;
}
EXPORT_SYMBOL(ida_find_first_range);
/**
* ida_free() - Release an allocated ID.
* @ida: IDA handle.

70
lib/test_ida.c
View File

@@ -189,6 +189,75 @@ static void ida_check_bad_free(struct ida *ida)
IDA_BUG_ON(ida, !ida_is_empty(ida));
}
/*
* Check ida_find_first_range() and varriants.
*/
static void ida_check_find_first(struct ida *ida)
{
/* IDA is empty; all of the below should be not exist */
IDA_BUG_ON(ida, ida_exists(ida, 0));
IDA_BUG_ON(ida, ida_exists(ida, 3));
IDA_BUG_ON(ida, ida_exists(ida, 63));
IDA_BUG_ON(ida, ida_exists(ida, 1023));
IDA_BUG_ON(ida, ida_exists(ida, (1 << 20) - 1));
/* IDA contains a single value entry */
IDA_BUG_ON(ida, ida_alloc_min(ida, 3, GFP_KERNEL) != 3);
IDA_BUG_ON(ida, ida_exists(ida, 0));
IDA_BUG_ON(ida, !ida_exists(ida, 3));
IDA_BUG_ON(ida, ida_exists(ida, 63));
IDA_BUG_ON(ida, ida_exists(ida, 1023));
IDA_BUG_ON(ida, ida_exists(ida, (1 << 20) - 1));
IDA_BUG_ON(ida, ida_alloc_min(ida, 63, GFP_KERNEL) != 63);
IDA_BUG_ON(ida, ida_exists(ida, 0));
IDA_BUG_ON(ida, !ida_exists(ida, 3));
IDA_BUG_ON(ida, !ida_exists(ida, 63));
IDA_BUG_ON(ida, ida_exists(ida, 1023));
IDA_BUG_ON(ida, ida_exists(ida, (1 << 20) - 1));
/* IDA contains a single bitmap */
IDA_BUG_ON(ida, ida_alloc_min(ida, 1023, GFP_KERNEL) != 1023);
IDA_BUG_ON(ida, ida_exists(ida, 0));
IDA_BUG_ON(ida, !ida_exists(ida, 3));
IDA_BUG_ON(ida, !ida_exists(ida, 63));
IDA_BUG_ON(ida, !ida_exists(ida, 1023));
IDA_BUG_ON(ida, ida_exists(ida, (1 << 20) - 1));
/* IDA contains a tree */
IDA_BUG_ON(ida, ida_alloc_min(ida, (1 << 20) - 1, GFP_KERNEL) != (1 << 20) - 1);
IDA_BUG_ON(ida, ida_exists(ida, 0));
IDA_BUG_ON(ida, !ida_exists(ida, 3));
IDA_BUG_ON(ida, !ida_exists(ida, 63));
IDA_BUG_ON(ida, !ida_exists(ida, 1023));
IDA_BUG_ON(ida, !ida_exists(ida, (1 << 20) - 1));
/* Now try to find first */
IDA_BUG_ON(ida, ida_find_first(ida) != 3);
IDA_BUG_ON(ida, ida_find_first_range(ida, -1, 2) != -EINVAL);
IDA_BUG_ON(ida, ida_find_first_range(ida, 0, 2) != -ENOENT); // no used ID
IDA_BUG_ON(ida, ida_find_first_range(ida, 0, 3) != 3);
IDA_BUG_ON(ida, ida_find_first_range(ida, 1, 3) != 3);
IDA_BUG_ON(ida, ida_find_first_range(ida, 3, 3) != 3);
IDA_BUG_ON(ida, ida_find_first_range(ida, 2, 4) != 3);
IDA_BUG_ON(ida, ida_find_first_range(ida, 4, 3) != -ENOENT); // min > max, fail
IDA_BUG_ON(ida, ida_find_first_range(ida, 4, 60) != -ENOENT); // no used ID
IDA_BUG_ON(ida, ida_find_first_range(ida, 4, 64) != 63);
IDA_BUG_ON(ida, ida_find_first_range(ida, 63, 63) != 63);
IDA_BUG_ON(ida, ida_find_first_range(ida, 64, 1026) != 1023);
IDA_BUG_ON(ida, ida_find_first_range(ida, 1023, 1023) != 1023);
IDA_BUG_ON(ida, ida_find_first_range(ida, 1023, (1 << 20) - 1) != 1023);
IDA_BUG_ON(ida, ida_find_first_range(ida, 1024, (1 << 20) - 1) != (1 << 20) - 1);
IDA_BUG_ON(ida, ida_find_first_range(ida, (1 << 20), INT_MAX) != -ENOENT);
ida_free(ida, 3);
ida_free(ida, 63);
ida_free(ida, 1023);
ida_free(ida, (1 << 20) - 1);
IDA_BUG_ON(ida, !ida_is_empty(ida));
}
static DEFINE_IDA(ida);
static int ida_checks(void)
@@ -202,6 +271,7 @@ static int ida_checks(void)
ida_check_max(&ida);
ida_check_conv(&ida);
ida_check_bad_free(&ida);
ida_check_find_first(&ida);
printk("IDA: %u of %u tests passed\n", tests_passed, tests_run);
return (tests_run != tests_passed) ? 0 : -EINVAL;

365
tools/testing/selftests/iommu/iommufd.c
View File

@@ -342,12 +342,14 @@ FIXTURE(iommufd_ioas)
uint32_t hwpt_id;
uint32_t device_id;
uint64_t base_iova;
uint32_t device_pasid_id;
};
FIXTURE_VARIANT(iommufd_ioas)
{
unsigned int mock_domains;
unsigned int memory_limit;
bool pasid_capable;
};
FIXTURE_SETUP(iommufd_ioas)
@@ -372,6 +374,12 @@ FIXTURE_SETUP(iommufd_ioas)
IOMMU_TEST_DEV_CACHE_DEFAULT);
self->base_iova = MOCK_APERTURE_START;
}
if (variant->pasid_capable)
test_cmd_mock_domain_flags(self->ioas_id,
MOCK_FLAGS_DEVICE_PASID,
NULL, NULL,
&self->device_pasid_id);
}
FIXTURE_TEARDOWN(iommufd_ioas)
@@ -387,6 +395,7 @@ FIXTURE_VARIANT_ADD(iommufd_ioas, no_domain)
FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain)
{
.mock_domains = 1,
.pasid_capable = true,
};
FIXTURE_VARIANT_ADD(iommufd_ioas, two_mock_domain)
@@ -439,6 +448,10 @@ TEST_F(iommufd_ioas, alloc_hwpt_nested)
&test_hwpt_id);
test_err_hwpt_alloc(EINVAL, self->device_id, self->device_id, 0,
&test_hwpt_id);
test_err_hwpt_alloc(EOPNOTSUPP, self->device_id, self->ioas_id,
IOMMU_HWPT_ALLOC_NEST_PARENT |
IOMMU_HWPT_FAULT_ID_VALID,
&test_hwpt_id);
test_cmd_hwpt_alloc(self->device_id, self->ioas_id,
IOMMU_HWPT_ALLOC_NEST_PARENT,
@@ -748,6 +761,8 @@ TEST_F(iommufd_ioas, get_hw_info)
} buffer_smaller;
if (self->device_id) {
uint8_t max_pasid = 0;
/* Provide a zero-size user_buffer */
test_cmd_get_hw_info(self->device_id, NULL, 0);
/* Provide a user_buffer with exact size */
@@ -762,6 +777,13 @@ TEST_F(iommufd_ioas, get_hw_info)
* the fields within the size range still gets updated.
*/
test_cmd_get_hw_info(self->device_id, &buffer_smaller, sizeof(buffer_smaller));
test_cmd_get_hw_info_pasid(self->device_id, &max_pasid);
ASSERT_EQ(0, max_pasid);
if (variant->pasid_capable) {
test_cmd_get_hw_info_pasid(self->device_pasid_id,
&max_pasid);
ASSERT_EQ(MOCK_PASID_WIDTH, max_pasid);
}
} else {
test_err_get_hw_info(ENOENT, self->device_id,
&buffer_exact, sizeof(buffer_exact));
@@ -2736,6 +2758,7 @@ TEST_F(iommufd_viommu, viommu_alloc_nested_iopf)
uint32_t iopf_hwpt_id;
uint32_t fault_id;
uint32_t fault_fd;
uint32_t vdev_id;
if (self->device_id) {
test_ioctl_fault_alloc(&fault_id, &fault_fd);
@@ -2752,6 +2775,10 @@ TEST_F(iommufd_viommu, viommu_alloc_nested_iopf)
&iopf_hwpt_id, IOMMU_HWPT_DATA_SELFTEST, &data,
sizeof(data));
/* Must allocate vdevice before attaching to a nested hwpt */
test_err_mock_domain_replace(ENOENT, self->stdev_id,
iopf_hwpt_id);
test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id);
test_cmd_mock_domain_replace(self->stdev_id, iopf_hwpt_id);
EXPECT_ERRNO(EBUSY,
_test_ioctl_destroy(self->fd, iopf_hwpt_id));
@@ -2769,15 +2796,46 @@ TEST_F(iommufd_viommu, vdevice_alloc)
uint32_t viommu_id = self->viommu_id;
uint32_t dev_id = self->device_id;
uint32_t vdev_id = 0;
uint32_t veventq_id;
uint32_t veventq_fd;
int prev_seq = -1;
if (dev_id) {
/* Must allocate vdevice before attaching to a nested hwpt */
test_err_mock_domain_replace(ENOENT, self->stdev_id,
self->nested_hwpt_id);
/* Allocate a vEVENTQ with veventq_depth=2 */
test_cmd_veventq_alloc(viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST,
&veventq_id, &veventq_fd);
test_err_veventq_alloc(EEXIST, viommu_id,
IOMMU_VEVENTQ_TYPE_SELFTEST, NULL, NULL);
/* Set vdev_id to 0x99, unset it, and set to 0x88 */
test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id);
test_cmd_mock_domain_replace(self->stdev_id,
self->nested_hwpt_id);
test_cmd_trigger_vevents(dev_id, 1);
test_cmd_read_vevents(veventq_fd, 1, 0x99, &prev_seq);
test_err_vdevice_alloc(EEXIST, viommu_id, dev_id, 0x99,
&vdev_id);
test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);
test_ioctl_destroy(vdev_id);
/* Try again with 0x88 */
test_cmd_vdevice_alloc(viommu_id, dev_id, 0x88, &vdev_id);
test_cmd_mock_domain_replace(self->stdev_id,
self->nested_hwpt_id);
/* Trigger an overflow with three events */
test_cmd_trigger_vevents(dev_id, 3);
test_err_read_vevents(EOVERFLOW, veventq_fd, 3, 0x88,
&prev_seq);
/* Overflow must be gone after the previous reads */
test_cmd_trigger_vevents(dev_id, 1);
test_cmd_read_vevents(veventq_fd, 1, 0x88, &prev_seq);
close(veventq_fd);
test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);
test_ioctl_destroy(vdev_id);
test_ioctl_destroy(veventq_id);
} else {
test_err_vdevice_alloc(ENOENT, viommu_id, dev_id, 0x99, NULL);
}
@@ -2956,4 +3014,311 @@ TEST_F(iommufd_viommu, vdevice_cache)
}
}
FIXTURE(iommufd_device_pasid)
{
int fd;
uint32_t ioas_id;
uint32_t hwpt_id;
uint32_t stdev_id;
uint32_t device_id;
uint32_t no_pasid_stdev_id;
uint32_t no_pasid_device_id;
};
FIXTURE_VARIANT(iommufd_device_pasid)
{
bool pasid_capable;
};
FIXTURE_SETUP(iommufd_device_pasid)
{
self->fd = open("/dev/iommu", O_RDWR);
ASSERT_NE(-1, self->fd);
test_ioctl_ioas_alloc(&self->ioas_id);
test_cmd_mock_domain_flags(self->ioas_id,
MOCK_FLAGS_DEVICE_PASID,
&self->stdev_id, &self->hwpt_id,
&self->device_id);
if (!variant->pasid_capable)
test_cmd_mock_domain_flags(self->ioas_id, 0,
&self->no_pasid_stdev_id, NULL,
&self->no_pasid_device_id);
}
FIXTURE_TEARDOWN(iommufd_device_pasid)
{
teardown_iommufd(self->fd, _metadata);
}
FIXTURE_VARIANT_ADD(iommufd_device_pasid, no_pasid)
{
.pasid_capable = false,
};
FIXTURE_VARIANT_ADD(iommufd_device_pasid, has_pasid)
{
.pasid_capable = true,
};
TEST_F(iommufd_device_pasid, pasid_attach)
{
struct iommu_hwpt_selftest data = {
.iotlb = IOMMU_TEST_IOTLB_DEFAULT,
};
uint32_t nested_hwpt_id[3] = {};
uint32_t parent_hwpt_id = 0;
uint32_t fault_id, fault_fd;
uint32_t s2_hwpt_id = 0;
uint32_t iopf_hwpt_id;
uint32_t pasid = 100;
uint32_t viommu_id;
/*
* Negative, detach pasid without attaching, this is not expected.
* But it should not result in failure anyway.
*/
test_cmd_pasid_detach(pasid);
/* Allocate two nested hwpts sharing one common parent hwpt */
test_cmd_hwpt_alloc(self->device_id, self->ioas_id,
IOMMU_HWPT_ALLOC_NEST_PARENT,
&parent_hwpt_id);
test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id,
IOMMU_HWPT_ALLOC_PASID,
&nested_hwpt_id[0],
IOMMU_HWPT_DATA_SELFTEST,
&data, sizeof(data));
test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id,
IOMMU_HWPT_ALLOC_PASID,
&nested_hwpt_id[1],
IOMMU_HWPT_DATA_SELFTEST,
&data, sizeof(data));
/* Fault related preparation */
test_ioctl_fault_alloc(&fault_id, &fault_fd);
test_cmd_hwpt_alloc_iopf(self->device_id, parent_hwpt_id, fault_id,
IOMMU_HWPT_FAULT_ID_VALID | IOMMU_HWPT_ALLOC_PASID,
&iopf_hwpt_id,
IOMMU_HWPT_DATA_SELFTEST, &data,
sizeof(data));
/* Allocate a regular nested hwpt based on viommu */
test_cmd_viommu_alloc(self->device_id, parent_hwpt_id,
IOMMU_VIOMMU_TYPE_SELFTEST,
&viommu_id);
test_cmd_hwpt_alloc_nested(self->device_id, viommu_id,
IOMMU_HWPT_ALLOC_PASID,
&nested_hwpt_id[2],
IOMMU_HWPT_DATA_SELFTEST, &data,
sizeof(data));
test_cmd_hwpt_alloc(self->device_id, self->ioas_id,
IOMMU_HWPT_ALLOC_PASID,
&s2_hwpt_id);
/* Attach RID to non-pasid compat domain, */
test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id);
/* then attach to pasid should fail */
test_err_pasid_attach(EINVAL, pasid, s2_hwpt_id);
/* Attach RID to pasid compat domain, */
test_cmd_mock_domain_replace(self->stdev_id, s2_hwpt_id);
/* then attach to pasid should succeed, */
test_cmd_pasid_attach(pasid, nested_hwpt_id[0]);
/* but attach RID to non-pasid compat domain should fail now. */
test_err_mock_domain_replace(EINVAL, self->stdev_id, parent_hwpt_id);
/*
* Detach hwpt from pasid 100, and check if the pasid 100
* has null domain.
*/
test_cmd_pasid_detach(pasid);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, 0));
/* RID is attached to pasid-comapt domain, pasid path is not used */
if (!variant->pasid_capable) {
/*
* PASID-compatible domain can be used by non-PASID-capable
* device.
*/
test_cmd_mock_domain_replace(self->no_pasid_stdev_id, nested_hwpt_id[0]);
test_cmd_mock_domain_replace(self->no_pasid_stdev_id, self->ioas_id);
/*
* Attach hwpt to pasid 100 of non-PASID-capable device,
* should fail, no matter domain is pasid-comapt or not.
*/
EXPECT_ERRNO(EINVAL,
_test_cmd_pasid_attach(self->fd, self->no_pasid_stdev_id,
pasid, parent_hwpt_id));
EXPECT_ERRNO(EINVAL,
_test_cmd_pasid_attach(self->fd, self->no_pasid_stdev_id,
pasid, s2_hwpt_id));
}
/*
* Attach non pasid compat hwpt to pasid-capable device, should
* fail, and have null domain.
*/
test_err_pasid_attach(EINVAL, pasid, parent_hwpt_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, 0));
/*
* Attach ioas to pasid 100, should fail, domain should
* be null.
*/
test_err_pasid_attach(EINVAL, pasid, self->ioas_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, 0));
/*
* Attach the s2_hwpt to pasid 100, should succeed, domain should
* be valid.
*/
test_cmd_pasid_attach(pasid, s2_hwpt_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, s2_hwpt_id));
/*
* Try attach pasid 100 with another hwpt, should FAIL
* as attach does not allow overwrite, use REPLACE instead.
*/
test_err_pasid_attach(EBUSY, pasid, nested_hwpt_id[0]);
/*
* Detach hwpt from pasid 100 for next test, should succeed,
* and have null domain.
*/
test_cmd_pasid_detach(pasid);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, 0));
/*
* Attach nested hwpt to pasid 100, should succeed, domain
* should be valid.
*/
test_cmd_pasid_attach(pasid, nested_hwpt_id[0]);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, nested_hwpt_id[0]));
/* Attach to pasid 100 which has been attached, should fail. */
test_err_pasid_attach(EBUSY, pasid, nested_hwpt_id[0]);
/* cleanup pasid 100 */
test_cmd_pasid_detach(pasid);
/* Replace tests */
pasid = 200;
/*
* Replace pasid 200 without attaching it, should fail
* with -EINVAL.
*/
test_err_pasid_replace(EINVAL, pasid, s2_hwpt_id);
/*
* Attach the s2 hwpt to pasid 200, should succeed, domain should
* be valid.
*/
test_cmd_pasid_attach(pasid, s2_hwpt_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, s2_hwpt_id));
/*
* Replace pasid 200 with self->ioas_id, should fail
* and domain should be the prior s2 hwpt.
*/
test_err_pasid_replace(EINVAL, pasid, self->ioas_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, s2_hwpt_id));
/*
* Replace a nested hwpt for pasid 200, should succeed,
* and have valid domain.
*/
test_cmd_pasid_replace(pasid, nested_hwpt_id[0]);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, nested_hwpt_id[0]));
/*
* Replace with another nested hwpt for pasid 200, should
* succeed, and have valid domain.
*/
test_cmd_pasid_replace(pasid, nested_hwpt_id[1]);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, nested_hwpt_id[1]));
/* cleanup pasid 200 */
test_cmd_pasid_detach(pasid);
/* Negative Tests for pasid replace, use pasid 1024 */
/*
* Attach the s2 hwpt to pasid 1024, should succeed, domain should
* be valid.
*/
pasid = 1024;
test_cmd_pasid_attach(pasid, s2_hwpt_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, s2_hwpt_id));
/*
* Replace pasid 1024 with nested_hwpt_id[0], should fail,
* but have the old valid domain. This is a designed
* negative case. Normally, this shall succeed.
*/
test_err_pasid_replace(ENOMEM, pasid, nested_hwpt_id[0]);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, s2_hwpt_id));
/* cleanup pasid 1024 */
test_cmd_pasid_detach(pasid);
/* Attach to iopf-capable hwpt */
/*
* Attach an iopf hwpt to pasid 2048, should succeed, domain should
* be valid.
*/
pasid = 2048;
test_cmd_pasid_attach(pasid, iopf_hwpt_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, iopf_hwpt_id));
test_cmd_trigger_iopf_pasid(self->device_id, pasid, fault_fd);
/*
* Replace with s2_hwpt_id for pasid 2048, should
* succeed, and have valid domain.
*/
test_cmd_pasid_replace(pasid, s2_hwpt_id);
ASSERT_EQ(0,
test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,
pasid, s2_hwpt_id));
/* cleanup pasid 2048 */
test_cmd_pasid_detach(pasid);
test_ioctl_destroy(iopf_hwpt_id);
close(fault_fd);
test_ioctl_destroy(fault_id);
/* Detach the s2_hwpt_id from RID */
test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);
}
TEST_HARNESS_MAIN

73
tools/testing/selftests/iommu/iommufd_fail_nth.c
View File

@@ -209,12 +209,16 @@ FIXTURE(basic_fail_nth)
{
int fd;
uint32_t access_id;
uint32_t stdev_id;
uint32_t pasid;
};
FIXTURE_SETUP(basic_fail_nth)
{
self->fd = -1;
self->access_id = 0;
self->stdev_id = 0;
self->pasid = 0; //test should use a non-zero value
}
FIXTURE_TEARDOWN(basic_fail_nth)
@@ -226,6 +230,8 @@ FIXTURE_TEARDOWN(basic_fail_nth)
rc = _test_cmd_destroy_access(self->access_id);
assert(rc == 0);
}
if (self->pasid && self->stdev_id)
_test_cmd_pasid_detach(self->fd, self->stdev_id, self->pasid);
teardown_iommufd(self->fd, _metadata);
}
@@ -620,10 +626,11 @@ TEST_FAIL_NTH(basic_fail_nth, device)
};
struct iommu_test_hw_info info;
uint32_t fault_id, fault_fd;
uint32_t veventq_id, veventq_fd;
uint32_t fault_hwpt_id;
uint32_t test_hwpt_id;
uint32_t ioas_id;
uint32_t ioas_id2;
uint32_t stdev_id;
uint32_t idev_id;
uint32_t hwpt_id;
uint32_t viommu_id;
@@ -654,25 +661,30 @@ TEST_FAIL_NTH(basic_fail_nth, device)
fail_nth_enable();
if (_test_cmd_mock_domain(self->fd, ioas_id, &stdev_id, NULL,
&idev_id))
if (_test_cmd_mock_domain_flags(self->fd, ioas_id,
MOCK_FLAGS_DEVICE_PASID,
&self->stdev_id, NULL, &idev_id))
return -1;
if (_test_cmd_get_hw_info(self->fd, idev_id, &info, sizeof(info), NULL))
return -1;
if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0, 0, &hwpt_id,
IOMMU_HWPT_DATA_NONE, 0, 0))
return -1;
if (_test_cmd_mock_domain_replace(self->fd, stdev_id, ioas_id2, NULL))
return -1;
if (_test_cmd_mock_domain_replace(self->fd, stdev_id, hwpt_id, NULL))
if (_test_cmd_get_hw_info(self->fd, idev_id, &info,
sizeof(info), NULL, NULL))
return -1;
if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0,
IOMMU_HWPT_ALLOC_NEST_PARENT, &hwpt_id,
IOMMU_HWPT_ALLOC_PASID, &hwpt_id,
IOMMU_HWPT_DATA_NONE, 0, 0))
return -1;
if (_test_cmd_mock_domain_replace(self->fd, self->stdev_id, ioas_id2, NULL))
return -1;
if (_test_cmd_mock_domain_replace(self->fd, self->stdev_id, hwpt_id, NULL))
return -1;
if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0,
IOMMU_HWPT_ALLOC_NEST_PARENT |
IOMMU_HWPT_ALLOC_PASID,
&hwpt_id,
IOMMU_HWPT_DATA_NONE, 0, 0))
return -1;
@@ -692,6 +704,37 @@ TEST_FAIL_NTH(basic_fail_nth, device)
IOMMU_HWPT_DATA_SELFTEST, &data, sizeof(data)))
return -1;
if (_test_cmd_veventq_alloc(self->fd, viommu_id,
IOMMU_VEVENTQ_TYPE_SELFTEST, &veventq_id,
&veventq_fd))
return -1;
close(veventq_fd);
if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0,
IOMMU_HWPT_ALLOC_PASID,
&test_hwpt_id,
IOMMU_HWPT_DATA_NONE, 0, 0))
return -1;
/* Tests for pasid attach/replace/detach */
self->pasid = 200;
if (_test_cmd_pasid_attach(self->fd, self->stdev_id,
self->pasid, hwpt_id)) {
self->pasid = 0;
return -1;
}
if (_test_cmd_pasid_replace(self->fd, self->stdev_id,
self->pasid, test_hwpt_id))
return -1;
if (_test_cmd_pasid_detach(self->fd, self->stdev_id, self->pasid))
return -1;
self->pasid = 0;
return 0;
}

229
tools/testing/selftests/iommu/iommufd_utils.h
View File

@@ -9,6 +9,7 @@
#include <sys/ioctl.h>
#include <stdint.h>
#include <assert.h>
#include <poll.h>
#include "../kselftest_harness.h"
#include "../../../../drivers/iommu/iommufd/iommufd_test.h"
@@ -757,7 +758,8 @@ static void teardown_iommufd(int fd, struct __test_metadata *_metadata)
/* @data can be NULL */
static int _test_cmd_get_hw_info(int fd, __u32 device_id, void *data,
size_t data_len, uint32_t *capabilities)
size_t data_len, uint32_t *capabilities,
uint8_t *max_pasid)
{
struct iommu_test_hw_info *info = (struct iommu_test_hw_info *)data;
struct iommu_hw_info cmd = {
@@ -802,6 +804,9 @@ static int _test_cmd_get_hw_info(int fd, __u32 device_id, void *data,
assert(!info->flags);
}
if (max_pasid)
*max_pasid = cmd.out_max_pasid_log2;
if (capabilities)
*capabilities = cmd.out_capabilities;
@@ -810,14 +815,19 @@ static int _test_cmd_get_hw_info(int fd, __u32 device_id, void *data,
#define test_cmd_get_hw_info(device_id, data, data_len) \
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, data, \
data_len, NULL))
data_len, NULL, NULL))
#define test_err_get_hw_info(_errno, device_id, data, data_len) \
EXPECT_ERRNO(_errno, _test_cmd_get_hw_info(self->fd, device_id, data, \
data_len, NULL))
data_len, NULL, NULL))
#define test_cmd_get_hw_capabilities(device_id, caps, mask) \
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, 0, &caps))
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, \
0, &caps, NULL))
#define test_cmd_get_hw_info_pasid(device_id, max_pasid) \
ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, \
0, NULL, max_pasid))
static int _test_ioctl_fault_alloc(int fd, __u32 *fault_id, __u32 *fault_fd)
{
@@ -842,14 +852,15 @@ static int _test_ioctl_fault_alloc(int fd, __u32 *fault_id, __u32 *fault_fd)
ASSERT_NE(0, *(fault_fd)); \
})
static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 fault_fd)
static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 pasid,
__u32 fault_fd)
{
struct iommu_test_cmd trigger_iopf_cmd = {
.size = sizeof(trigger_iopf_cmd),
.op = IOMMU_TEST_OP_TRIGGER_IOPF,
.trigger_iopf = {
.dev_id = device_id,
.pasid = 0x1,
.pasid = pasid,
.grpid = 0x2,
.perm = IOMMU_PGFAULT_PERM_READ | IOMMU_PGFAULT_PERM_WRITE,
.addr = 0xdeadbeaf,
@@ -880,7 +891,10 @@ static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 fault_fd)
}
#define test_cmd_trigger_iopf(device_id, fault_fd) \
ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, fault_fd))
ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, 0x1, fault_fd))
#define test_cmd_trigger_iopf_pasid(device_id, pasid, fault_fd) \
ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, \
pasid, fault_fd))
static int _test_cmd_viommu_alloc(int fd, __u32 device_id, __u32 hwpt_id,
__u32 type, __u32 flags, __u32 *viommu_id)
@@ -936,3 +950,204 @@ static int _test_cmd_vdevice_alloc(int fd, __u32 viommu_id, __u32 idev_id,
EXPECT_ERRNO(_errno, \
_test_cmd_vdevice_alloc(self->fd, viommu_id, idev_id, \
virt_id, vdev_id))
static int _test_cmd_veventq_alloc(int fd, __u32 viommu_id, __u32 type,
__u32 *veventq_id, __u32 *veventq_fd)
{
struct iommu_veventq_alloc cmd = {
.size = sizeof(cmd),
.type = type,
.veventq_depth = 2,
.viommu_id = viommu_id,
};
int ret;
ret = ioctl(fd, IOMMU_VEVENTQ_ALLOC, &cmd);
if (ret)
return ret;
if (veventq_id)
*veventq_id = cmd.out_veventq_id;
if (veventq_fd)
*veventq_fd = cmd.out_veventq_fd;
return 0;
}
#define test_cmd_veventq_alloc(viommu_id, type, veventq_id, veventq_fd) \
ASSERT_EQ(0, _test_cmd_veventq_alloc(self->fd, viommu_id, type, \
veventq_id, veventq_fd))
#define test_err_veventq_alloc(_errno, viommu_id, type, veventq_id, \
veventq_fd) \
EXPECT_ERRNO(_errno, \
_test_cmd_veventq_alloc(self->fd, viommu_id, type, \
veventq_id, veventq_fd))
static int _test_cmd_trigger_vevents(int fd, __u32 dev_id, __u32 nvevents)
{
struct iommu_test_cmd trigger_vevent_cmd = {
.size = sizeof(trigger_vevent_cmd),
.op = IOMMU_TEST_OP_TRIGGER_VEVENT,
.trigger_vevent = {
.dev_id = dev_id,
},
};
int ret;
while (nvevents--) {
ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_VEVENT),
&trigger_vevent_cmd);
if (ret < 0)
return -1;
}
return ret;
}
#define test_cmd_trigger_vevents(dev_id, nvevents) \
ASSERT_EQ(0, _test_cmd_trigger_vevents(self->fd, dev_id, nvevents))
static int _test_cmd_read_vevents(int fd, __u32 event_fd, __u32 nvevents,
__u32 virt_id, int *prev_seq)
{
struct pollfd pollfd = { .fd = event_fd, .events = POLLIN };
struct iommu_viommu_event_selftest *event;
struct iommufd_vevent_header *hdr;
ssize_t bytes;
void *data;
int ret, i;
ret = poll(&pollfd, 1, 1000);
if (ret < 0)
return -1;
data = calloc(nvevents, sizeof(*hdr) + sizeof(*event));
if (!data) {
errno = ENOMEM;
return -1;
}
bytes = read(event_fd, data,
nvevents * (sizeof(*hdr) + sizeof(*event)));
if (bytes <= 0) {
errno = EFAULT;
ret = -1;
goto out_free;
}
for (i = 0; i < nvevents; i++) {
hdr = data + i * (sizeof(*hdr) + sizeof(*event));
if (hdr->flags & IOMMU_VEVENTQ_FLAG_LOST_EVENTS ||
hdr->sequence - *prev_seq > 1) {
*prev_seq = hdr->sequence;
errno = EOVERFLOW;
ret = -1;
goto out_free;
}
*prev_seq = hdr->sequence;
event = data + sizeof(*hdr);
if (event->virt_id != virt_id) {
errno = EINVAL;
ret = -1;
goto out_free;
}
}
ret = 0;
out_free:
free(data);
return ret;
}
#define test_cmd_read_vevents(event_fd, nvevents, virt_id, prev_seq) \
ASSERT_EQ(0, _test_cmd_read_vevents(self->fd, event_fd, nvevents, \
virt_id, prev_seq))
#define test_err_read_vevents(_errno, event_fd, nvevents, virt_id, prev_seq) \
EXPECT_ERRNO(_errno, \
_test_cmd_read_vevents(self->fd, event_fd, nvevents, \
virt_id, prev_seq))
static int _test_cmd_pasid_attach(int fd, __u32 stdev_id, __u32 pasid,
__u32 pt_id)
{
struct iommu_test_cmd test_attach = {
.size = sizeof(test_attach),
.op = IOMMU_TEST_OP_PASID_ATTACH,
.id = stdev_id,
.pasid_attach = {
.pasid = pasid,
.pt_id = pt_id,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_ATTACH),
&test_attach);
}
#define test_cmd_pasid_attach(pasid, hwpt_id) \
ASSERT_EQ(0, _test_cmd_pasid_attach(self->fd, self->stdev_id, \
pasid, hwpt_id))
#define test_err_pasid_attach(_errno, pasid, hwpt_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_pasid_attach(self->fd, self->stdev_id, \
pasid, hwpt_id))
static int _test_cmd_pasid_replace(int fd, __u32 stdev_id, __u32 pasid,
__u32 pt_id)
{
struct iommu_test_cmd test_replace = {
.size = sizeof(test_replace),
.op = IOMMU_TEST_OP_PASID_REPLACE,
.id = stdev_id,
.pasid_replace = {
.pasid = pasid,
.pt_id = pt_id,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_REPLACE),
&test_replace);
}
#define test_cmd_pasid_replace(pasid, hwpt_id) \
ASSERT_EQ(0, _test_cmd_pasid_replace(self->fd, self->stdev_id, \
pasid, hwpt_id))
#define test_err_pasid_replace(_errno, pasid, hwpt_id) \
EXPECT_ERRNO(_errno, \
_test_cmd_pasid_replace(self->fd, self->stdev_id, \
pasid, hwpt_id))
static int _test_cmd_pasid_detach(int fd, __u32 stdev_id, __u32 pasid)
{
struct iommu_test_cmd test_detach = {
.size = sizeof(test_detach),
.op = IOMMU_TEST_OP_PASID_DETACH,
.id = stdev_id,
.pasid_detach = {
.pasid = pasid,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_DETACH),
&test_detach);
}
#define test_cmd_pasid_detach(pasid) \
ASSERT_EQ(0, _test_cmd_pasid_detach(self->fd, self->stdev_id, pasid))
static int test_cmd_pasid_check_hwpt(int fd, __u32 stdev_id, __u32 pasid,
__u32 hwpt_id)
{
struct iommu_test_cmd test_pasid_check = {
.size = sizeof(test_pasid_check),
.op = IOMMU_TEST_OP_PASID_CHECK_HWPT,
.id = stdev_id,
.pasid_check = {
.pasid = pasid,
.hwpt_id = hwpt_id,
},
};
return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_CHECK_HWPT),
&test_pasid_check);
}