diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h index 15c05c45ce36..08cdb173b04b 100644 --- a/include/uapi/linux/io_uring/zcrx.h +++ b/include/uapi/linux/io_uring/zcrx.h @@ -116,6 +116,7 @@ enum zcrx_ctrl_op { ZCRX_CTRL_FLUSH_RQ, ZCRX_CTRL_EXPORT, ZCRX_CTRL_ARM_NOTIFICATION, + ZCRX_CTRL_ADD_AREA, __ZCRX_CTRL_LAST, }; @@ -134,6 +135,11 @@ struct zcrx_ctrl_arm_notif { __u32 __resv[11]; }; +struct zcrx_ctrl_add_area { + __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */ + __u64 __resv[5]; +}; + struct zcrx_ctrl { __u32 zcrx_id; __u32 op; /* see enum zcrx_ctrl_op */ @@ -143,6 +149,7 @@ struct zcrx_ctrl { struct zcrx_ctrl_export zc_export; struct zcrx_ctrl_flush_rq zc_flush; struct zcrx_ctrl_arm_notif zc_arm_notif; + struct zcrx_ctrl_add_area zc_area; }; }; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 22f04a9f2410..194418833b9c 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -36,9 +36,15 @@ #define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024) #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) +#define ZCRX_MAX_AREAS 1024 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) +static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx) +{ + return zcrx->nr_areas; +} + static inline u64 zcrx_area_id_to_token(u32 area_id) { return (u64)area_id << IORING_ZCRX_AREA_SHIFT; @@ -334,7 +340,7 @@ static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) { unsigned area_idx; - guard(mutex)(&ifq->pp_lock); + lockdep_assert_held(&ifq->pp_lock); for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]); @@ -469,7 +475,9 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area **areas, **old_areas; unsigned old_nr; - if (WARN_ON_ONCE(ifq->kern_readable != kern_readable)) + if (ifq->kern_readable != kern_readable) + return -EINVAL; + if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS) return -EINVAL; old_areas = ifq->areas; @@ -508,7 +516,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, return -EINVAL; buf_size_shift = ilog2(rx_buf_len); } - if (WARN_ON_ONCE(ifq->niov_shift)) + if (ifq->niov_shift && ifq->niov_shift != buf_size_shift) return -EINVAL; if (!ifq->dev && buf_size_shift != PAGE_SHIFT) return -EOPNOTSUPP; @@ -566,7 +574,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, area->free_count = nr_iovs; /* we're only supporting one area per ifq for now */ - area->area_id = 0; + area->area_id = zcrx_next_area_id(ifq); area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); ret = io_zcrx_append_area(ifq, area); @@ -608,7 +616,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq) { - guard(mutex)(&ifq->pp_lock); + lockdep_assert_held(&ifq->pp_lock); if (!ifq->netdev) return; @@ -636,7 +644,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) if (ifq->if_rxq != -1) netif_mp_close_rxq(netdev, ifq->if_rxq, &p); - io_zcrx_unmap_areas(ifq); + scoped_guard(mutex, &ifq->pp_lock) + io_zcrx_unmap_areas(ifq); netdev_unlock(netdev); netdev_put(netdev, &netdev_tracker); } @@ -994,6 +1003,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area))) return -EFAULT; + if (area.rq_area_token) + return -EINVAL; memset(¬if, 0, sizeof(notif)); if (reg.notif_desc && copy_from_user(¬if, u64_to_user_ptr(reg.notif_desc), @@ -1056,6 +1067,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, goto err; } + WARN_ON_ONCE(!ifq->niov_shift); + reg.zcrx_id = id; scoped_guard(mutex, &ctx->mmap_lock) { @@ -1447,6 +1460,7 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) struct pp_memory_provider_params *p = &rxq->mp_params; struct io_zcrx_ifq *ifq = mp_priv; + guard(mutex)(&ifq->pp_lock); io_zcrx_unmap_areas(ifq); io_zcrx_drop_netdev(ifq); @@ -1549,6 +1563,34 @@ static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, return 0; } +static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, + struct zcrx_ctrl *ctrl) +{ + struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area; + struct io_uring_zcrx_area_reg __user *area_uptr; + struct io_uring_zcrx_area_reg area_reg; + + area_uptr = u64_to_user_ptr(ctrl_add->area_ptr); + if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg))) + return -EFAULT; + if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv))) + return -EINVAL; + if (area_reg.rq_area_token) + return -EINVAL; + + guard(mutex)(&ifq->pp_lock); + if (ifq->dev && !ifq->netdev) + return -EFAULT; + + /* we can't safely roll back area append, copy it out first */ + area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq)); + if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg))) + return -EFAULT; + area_reg.rq_area_token = 0; + + return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift); +} + int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { struct zcrx_ctrl ctrl; @@ -1575,6 +1617,8 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) return zcrx_export(ctx, zcrx, &ctrl, arg); case ZCRX_CTRL_ARM_NOTIFICATION: return zcrx_arm_notif(ctx, zcrx, &ctrl); + case ZCRX_CTRL_ADD_AREA: + return zcrx_ctrl_add_area(ctx, zcrx, &ctrl); } return -EOPNOTSUPP;