From dd2403fe1499bdb19b818e0fc06655e9e4a59dfe Mon Sep 17 00:00:00 2001 From: Surabhi Gogte Date: Thu, 9 Apr 2026 18:15:49 -0600 Subject: [PATCH 001/148] RDMA/addr: Change addr_wq back to unordered workqueue Commit 5fff41e1f89d ("IB/core: Fix race condition in resolving IP to MAC") changed the workqueue "addr_wq" to a single-threaded wq. Commit e19c0d237873 ("RDMA/rdma_cm: Remove process_req and timer sorting") eliminated global work and started using per-req work. Now we no longer have the race, change "addr_wq" back to multi-threaded workqueue to speed up multiple addr resolutions. Signed-off-by: Surabhi Gogte Link: https://patch.msgid.link/20260410001549.3149060-1-sgogte@purestorage.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index a40a765f0307..9f6f03f41204 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -850,7 +850,7 @@ static struct notifier_block nb = { int addr_init(void) { - addr_wq = alloc_ordered_workqueue("ib_addr", 0); + addr_wq = alloc_workqueue("ib_addr", WQ_UNBOUND, 0); if (!addr_wq) return -ENOMEM; From e79389115b9d27287ff6230a9750675106ed7668 Mon Sep 17 00:00:00 2001 From: Prathamesh Deshpande Date: Sun, 12 Apr 2026 02:18:49 +0100 Subject: [PATCH 002/148] IB/mlx5: Fix transport-domain rollback and initialize lb mutex earlier mlx5_ib_alloc_transport_domain() allocates a transport domain and then may fail in mlx5_ib_enable_lb(). In that case, the allocated TD is leaked. Fix this by deallocating the TD when mlx5_ib_enable_lb() returns an error. Also return 0 explicitly in the no-loopback-capability success branch, and move dev->lb.mutex initialization to mlx5_ib_stage_init_init(). Fixes: 146d2f1af324 ("IB/mlx5: Allocate a Transport Domain for each ucontext") Signed-off-by: Prathamesh Deshpande Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/main.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 109661c2ac12..45f5fcd9adf0 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2069,9 +2069,13 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn, if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) || (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) && !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - return err; + return 0; - return mlx5_ib_enable_lb(dev, true, false); + err = mlx5_ib_enable_lb(dev, true, false); + if (err) + mlx5_cmd_dealloc_transport_domain(dev->mdev, *tdn, uid); + + return err; } static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, @@ -4487,6 +4491,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) dev->port[i].roce.last_port_state = IB_PORT_DOWN; } + mutex_init(&dev->lb.mutex); + err = mlx5r_cmd_query_special_mkeys(dev); if (err) return err; @@ -4787,11 +4793,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) if (err) return err; - if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && - (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) || - MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - mutex_init(&dev->lb.mutex); - if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { err = mlx5_ib_init_var_region(dev); From 194762e6e436acde0f8f6aef44200b0058c36791 Mon Sep 17 00:00:00 2001 From: Chenguang Zhao Date: Fri, 10 Apr 2026 15:40:46 +0800 Subject: [PATCH 003/148] RDMA/mlx5: Use QP port when decoding responder CQEs The responder CQE path determines the link layer via rdma_port_get_link_layer(). Use qp->port instead of hardcoding port 1, which can mis-decode completions on multi-port devices. Signed-off-by: Chenguang Zhao Link: https://patch.msgid.link/20260410074046.2044595-1-zhaochenguang@kylinos.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/cq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index a76b7a36087d..ae20d11f04fc 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -169,7 +169,8 @@ enum { static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, struct mlx5_ib_qp *qp) { - enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1); + enum rdma_link_layer ll = + rdma_port_get_link_layer(qp->ibqp.device, qp->port); struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); struct mlx5_ib_srq *srq = NULL; struct mlx5_ib_wq *wq; From a38e4410af9ad8b7ad2217254da06cd4dc21f0ed Mon Sep 17 00:00:00 2001 From: Alexander Chesnokov Date: Mon, 13 Apr 2026 12:14:43 +0300 Subject: [PATCH 004/148] RDMA/hns: Fix arithmetic overflow in calc_hem_config() If bt_num is 3 or 2, then the expressions like l0_idx * chunk_ba_num + l1_idx are computed in 32-bit arithmetic before being assigned to a u64 index field, which can lead to overflow. Cast the first operand to u64 to ensure the arithmetic is performed in 64-bit. Found by Linux Verification Center (linuxtesting.org) with SVACE. Fixes: 2f49de21f3e9 ("RDMA/hns: Optimize mhop get flow for multi-hop addressing") Signed-off-by: Alexander Chesnokov Link: https://patch.msgid.link/20260413091527.39990-1-Alexander.Chesnokov@kaspersky.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hem.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index e7c9e30ad2d8..ccb40f8a48b7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -314,14 +314,14 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev, bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); switch (bt_num) { case 3: - index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l1 = (u64)l0_idx * chunk_ba_num + l1_idx; index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num * chunk_ba_num + - l1_idx * chunk_ba_num + l2_idx; + index->buf = (u64)l0_idx * chunk_ba_num * chunk_ba_num + + (u64)l1_idx * chunk_ba_num + l2_idx; break; case 2: index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num + l1_idx; + index->buf = (u64)l0_idx * chunk_ba_num + l1_idx; break; case 1: index->buf = l0_idx; From dd677609753c847a433cfbc90a6ce9c18281e6a3 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Tue, 14 Apr 2026 14:29:45 +0800 Subject: [PATCH 005/148] RDMA/rxe: remove rxe_ib_device_get_netdev() and RXE_PORT Suggested by Leon, remove the rxe_ib_device_get_netdev() wrapper and the RXE_PORT definition. These additions do not improve readability, and RXE has always had only a single port. Signed-off-by: zhenwei pi Link: https://patch.msgid.link/20260414062948.671658-2-zhenwei.pi@linux.dev Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_mcast.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_net.c | 7 +++---- drivers/infiniband/sw/rxe/rxe_verbs.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ------ 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c index 5cad72073eca..acd03bd87794 100644 --- a/drivers/infiniband/sw/rxe/rxe_mcast.c +++ b/drivers/infiniband/sw/rxe/rxe_mcast.c @@ -34,7 +34,7 @@ static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid) struct net_device *ndev; int ret; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return -ENODEV; @@ -59,7 +59,7 @@ static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid) struct net_device *ndev; int ret; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return -ENODEV; diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 50a2cb5405e2..26e70c4ad749 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -600,7 +600,7 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num) struct net_device *ndev; char *ndev_name; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return NULL; ndev_name = ndev->name; @@ -644,12 +644,11 @@ static void rxe_sock_put(struct sock *sk, void rxe_net_del(struct ib_device *dev) { - struct rxe_dev *rxe = container_of(dev, struct rxe_dev, ib_dev); struct net_device *ndev; struct sock *sk; struct net *net; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(dev, 1); if (!ndev) return; @@ -697,7 +696,7 @@ void rxe_set_port_state(struct rxe_dev *rxe) { struct net_device *ndev; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 4d4891dc2884..eb17b6086d5e 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -50,7 +50,7 @@ static int rxe_query_port(struct ib_device *ibdev, goto err_out; } - ndev = rxe_ib_device_get_netdev(ibdev); + ndev = ib_device_get_netdev(ibdev, 1); if (!ndev) { err = -ENODEV; goto err_out; @@ -1441,7 +1441,7 @@ static int rxe_enable_driver(struct ib_device *ib_dev) struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev); struct net_device *ndev; - ndev = rxe_ib_device_get_netdev(ib_dev); + ndev = ib_device_get_netdev(ib_dev, 1); if (!ndev) return -ENODEV; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index d92f80d16f78..e800545d1046 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -415,7 +415,6 @@ struct rxe_port { u32 qp_gsi_index; }; -#define RXE_PORT 1 struct rxe_dev { struct ib_device ib_dev; struct ib_device_attr attr; @@ -451,11 +450,6 @@ struct rxe_dev { struct rxe_port port; }; -static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev) -{ - return ib_device_get_netdev(dev, RXE_PORT); -} - static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) { atomic64_inc(&rxe->stats_counters[index]); From ed7e5e4a932a05aba0fd8c27859c829882ac0b01 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Tue, 14 Apr 2026 14:29:46 +0800 Subject: [PATCH 006/148] RDMA/rxe: add SENT/RCVD bytes There is a lack of sent/received counter in bytes. Signed-off-by: zhenwei pi Link: https://patch.msgid.link/20260414062948.671658-3-zhenwei.pi@linux.dev Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 ++ drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 ++ drivers/infiniband/sw/rxe/rxe_net.c | 2 ++ drivers/infiniband/sw/rxe/rxe_recv.c | 2 ++ drivers/infiniband/sw/rxe/rxe_verbs.h | 6 ++++++ 5 files changed, 14 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c index 437917a7d8f2..17edaa9a9b9b 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c @@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev, diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h index 051f9e1c3852..01b355103cbc 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h @@ -26,6 +26,8 @@ enum rxe_counters { RXE_CNT_LINK_DOWNED, RXE_CNT_RDMA_SEND, RXE_CNT_RDMA_RECV, + RXE_CNT_SENT_BYTES, + RXE_CNT_RCVD_BYTES, RXE_NUM_OF_COUNTERS }; diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 26e70c4ad749..67415455b118 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -501,6 +501,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, int err; int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); + unsigned int skblen = skb->len; unsigned long flags; spin_lock_irqsave(&qp->state_lock, flags); @@ -524,6 +525,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, } rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); + rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen); goto done; drop: diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index f79214738c2b..1853014362fd 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb) int err; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); struct rxe_dev *rxe = pkt->rxe; + unsigned int skblen = skb->len - skb_network_offset(skb); if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop; @@ -342,6 +343,7 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(err)) goto drop; + rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen); rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS); if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN)) diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index e800545d1046..0f5ffd94643f 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -455,6 +455,12 @@ static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) atomic64_inc(&rxe->stats_counters[index]); } +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, + s64 val) +{ + atomic64_add(val, &rxe->stats_counters[index]); +} + static inline struct rxe_dev *to_rdev(struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL; From 1648bcf1ed5cc32c6d93f9f4fcd3876e789b1f8f Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Tue, 14 Apr 2026 14:29:47 +0800 Subject: [PATCH 007/148] RDMA/rxe: support perf mgmt GET method In RXE, hardware counters are already supported, but not in a standardized manner. For instance, user-space monitoring tools like atop only read from the *counters* directory. Therefore, it is necessary to add perf management support to RXE. Also use rxe_counter_get instead of raw atomic64_read in hw-counters. Signed-off-by: zhenwei pi Link: https://patch.msgid.link/20260414062948.671658-4-zhenwei.pi@linux.dev Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 ++ drivers/infiniband/sw/rxe/rxe_mad.c | 97 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 1 + 4 files changed, 105 insertions(+) create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile index 3977f4f13258..e097c1ca1874 100644 --- a/drivers/infiniband/sw/rxe/Makefile +++ b/drivers/infiniband/sw/rxe/Makefile @@ -23,6 +23,7 @@ rdma_rxe-y := \ rxe_task.o \ rxe_net.o \ rxe_hw_counters.o \ + rxe_mad.o \ rxe_ns.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index e095c12699cb..64d636bf80fd 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -242,4 +242,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd, #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, + const struct ib_wc *in_wc, const struct ib_grh *in_grh, + const struct ib_mad *in, struct ib_mad *out, + size_t *out_mad_size, u16 *out_mad_pkey_index); + #endif /* RXE_LOC_H */ diff --git a/drivers/infiniband/sw/rxe/rxe_mad.c b/drivers/infiniband/sw/rxe/rxe_mad.c new file mode 100644 index 000000000000..cb2d55830f42 --- /dev/null +++ b/drivers/infiniband/sw/rxe/rxe_mad.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2026 zhenwei pi + */ + +#include +#include "rxe.h" +#include "rxe_hw_counters.h" + +static int rxe_get_pma_info(struct ib_mad *out) +{ + struct ib_class_port_info cpi = {}; + + cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH; + memcpy((out->data + 40), &cpi, sizeof(cpi)); + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_get_pma_counters(struct rxe_dev *rxe, struct ib_mad *out) +{ + struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data + 40); + s64 val; + + /* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop + * at all ones. + */ + val = atomic64_read(&rxe->stats_counters[RXE_CNT_LINK_DOWNED]); + pma_cnt->link_downed_counter = clamp(val, 0, U8_MAX); + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_get_pma_counters_ext(struct rxe_dev *rxe, struct ib_mad *out) +{ + struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)(out->data + 40); + s64 val; + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_SENT_BYTES]); + pma_cnt_ext->port_xmit_data = cpu_to_be64(val >> 2); + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_RCVD_BYTES]); + pma_cnt_ext->port_rcv_data = cpu_to_be64(val >> 2); + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_SENT_PKTS]); + pma_cnt_ext->port_xmit_packets = cpu_to_be64(val); + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_RCVD_PKTS]); + pma_cnt_ext->port_rcv_packets = cpu_to_be64(val); + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_get_perf_mgmt(struct rxe_dev *rxe, const struct ib_mad *in, struct ib_mad *out) +{ + switch (in->mad_hdr.attr_id) { + case IB_PMA_CLASS_PORT_INFO: + return rxe_get_pma_info(out); + + case IB_PMA_PORT_COUNTERS: + return rxe_get_pma_counters(rxe, out); + + case IB_PMA_PORT_COUNTERS_EXT: + return rxe_get_pma_counters_ext(rxe, out); + + default: + out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB); + return IB_MAD_RESULT_SUCCESS; + } +} + +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, + const struct ib_wc *in_wc, const struct ib_grh *in_grh, + const struct ib_mad *in, struct ib_mad *out, + size_t *out_mad_size, u16 *out_mad_pkey_index) +{ + struct rxe_dev *rxe = to_rdev(ibdev); + u8 mgmt_class = in->mad_hdr.mgmt_class; + u8 method = in->mad_hdr.method; + + if (port_num != 1) + return IB_MAD_RESULT_FAILURE; + + memset(out, 0, sizeof(*out)); + switch (mgmt_class) { + case IB_MGMT_CLASS_PERF_MGMT: + if (method == IB_MGMT_METHOD_GET) + return rxe_get_perf_mgmt(rxe, in, out); + break; + + default: + out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD); + return IB_MAD_RESULT_SUCCESS; + } + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index eb17b6086d5e..8edd4dd1f031 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -1496,6 +1496,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey, From c67bd973695c33f448a2789de5da765c04a78c71 Mon Sep 17 00:00:00 2001 From: Zhu Yanjun Date: Tue, 14 Apr 2026 14:29:48 +0800 Subject: [PATCH 008/148] selftest/rxe: Add selftests for perf Create a virtual TUN net device with RXE support, then run rping server and client to invoke networking packets, finally compare both *port_xmit_data* and *port_rcv_data* of such device. Signed-off-by: Zhu Yanjun Link: https://patch.msgid.link/20260414062948.671658-5-zhenwei.pi@linux.dev Signed-off-by: Leon Romanovsky --- tools/testing/selftests/rdma/Makefile | 3 +- .../selftests/rdma/rxe_sent_rcvd_bytes.sh | 75 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100755 tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh diff --git a/tools/testing/selftests/rdma/Makefile b/tools/testing/selftests/rdma/Makefile index 7dd7cba7a73c..07af7f15c1bf 100644 --- a/tools/testing/selftests/rdma/Makefile +++ b/tools/testing/selftests/rdma/Makefile @@ -2,6 +2,7 @@ TEST_PROGS := rxe_rping_between_netns.sh \ rxe_ipv6.sh \ rxe_socket_with_netns.sh \ - rxe_test_NETDEV_UNREGISTER.sh + rxe_test_NETDEV_UNREGISTER.sh \ + rxe_sent_rcvd_bytes.sh include ../lib.mk diff --git a/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh new file mode 100755 index 000000000000..0e4fbfeebd22 --- /dev/null +++ b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Configuration +PORT=4791 +MODS=("tun" "rdma_rxe") + +exec > /dev/null + +# --- Helper: Cleanup Routine --- +cleanup() { + echo "Cleaning up resources..." + rdma link del rxe0 2>/dev/null + ip link del tun0 2>/dev/null + for m in "${MODS[@]}"; do modprobe -r "$m" 2>/dev/null; done +} + +# Ensure cleanup runs on script exit or interrupt +trap cleanup EXIT + +# --- Phase 1: Environment Check --- +if [[ $EUID -ne 0 ]]; then + echo "Error: This script must be run as root." + exit 1 +fi + +for m in "${MODS[@]}"; do + modprobe "$m" || { echo "Error: Failed to load $m"; exit 1; } +done + +# --- Phase 2: Create Interfaces & RXE Links --- +echo "Creating tun0 (1.1.1.1) and rxe0..." +ip tuntap add mode tun tun0 +ip addr add 1.1.1.1/24 dev tun0 +ip link set tun0 up +rdma link add rxe0 type rxe netdev tun0 + +# Verify port 4791 is listening +if ! ss -Huln sport = :$PORT | grep -q ":$PORT"; then + echo "Error: UDP port $PORT not found after rxe0 creation" + exit 1 +fi + +orig_s=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_xmit_data` +orig_r=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_rcv_data` + +rping -s -a 1.1.1.1 -C 3 -v & +sleep 1 +rping -c -a 1.1.1.1 -C 3 -d -v + +new_s=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_xmit_data` +new_r=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_rcv_data` + +echo sent $new_s $orig_s +echo rcvd $new_r $orig_r + +result0=$((new_s - orig_s)) +result1=$((new_r - orig_r)) + +if [ $result0 != $result1 ]; then + echo "Error: sent and rcvd bytes different" + echo $result0 + echo $result1 + exit 1 +fi + +echo "Deleting rxe0..." +rdma link del rxe0 + +# Port should now be gone +if ss -Huln sport = :$PORT | grep -q ":$PORT"; then + echo "Error: UDP port $PORT still exists after all links deleted" + exit 1 +fi + +echo "Test passed successfully." From 7de165740ce8d006cbe80bca9d8207ba05a4cfc5 Mon Sep 17 00:00:00 2001 From: Yonatan Nachum Date: Thu, 9 Apr 2026 07:49:05 +0000 Subject: [PATCH 009/148] RDMA/efa: Add checksum support for admin responses EFA devices added support for CRC16 checksum on admin responses and to expose it to the driver the API version increased to 0.2. Add a check for support on device init and if supported validate the checksum on each admin response the driver receives. If the checksum validation failed, drop the CQE. Add the CRC16 module to Kconfig to have the in-tree dependency. Reviewed-by: Firas Jahjah Reviewed-by: Michael Margolin Signed-off-by: Yonatan Nachum Link: https://patch.msgid.link/20260409074905.3126023-1-ynachum@amazon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/Kconfig | 3 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 3 -- drivers/infiniband/hw/efa/efa_admin_defs.h | 15 +++--- drivers/infiniband/hw/efa/efa_com.c | 50 ++++++++++++++++--- drivers/infiniband/hw/efa/efa_com.h | 4 +- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/hw/efa/Kconfig b/drivers/infiniband/hw/efa/Kconfig index 457e18ba1d57..ff7f7c0870b3 100644 --- a/drivers/infiniband/hw/efa/Kconfig +++ b/drivers/infiniband/hw/efa/Kconfig @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. # # Amazon fabric device configuration # @@ -8,6 +8,7 @@ config INFINIBAND_EFA tristate "Amazon Elastic Fabric Adapter (EFA) support" depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN depends on INFINIBAND_USER_ACCESS + select CRC16 help This driver supports Amazon Elastic Fabric Adapter (EFA). diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index ad34ea5da6b0..826790ca9d83 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -6,9 +6,6 @@ #ifndef _EFA_ADMIN_CMDS_H_ #define _EFA_ADMIN_CMDS_H_ -#define EFA_ADMIN_API_VERSION_MAJOR 0 -#define EFA_ADMIN_API_VERSION_MINOR 1 - /* EFA admin queue opcodes */ enum efa_admin_aq_opcode { EFA_ADMIN_CREATE_QP = 1, diff --git a/drivers/infiniband/hw/efa/efa_admin_defs.h b/drivers/infiniband/hw/efa/efa_admin_defs.h index 35700c93e639..02f86edabed8 100644 --- a/drivers/infiniband/hw/efa/efa_admin_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_defs.h @@ -1,18 +1,20 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* - * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_ADMIN_H_ #define _EFA_ADMIN_H_ +#define EFA_ADMIN_API_VERSION_MAJOR 0 +#define EFA_ADMIN_API_VERSION_MINOR 2 + enum efa_admin_aq_completion_status { EFA_ADMIN_SUCCESS = 0, EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1, EFA_ADMIN_BAD_OPCODE = 2, EFA_ADMIN_UNSUPPORTED_OPCODE = 3, EFA_ADMIN_MALFORMED_REQUEST = 4, - /* Additional status is provided in ACQ entry extended_status */ EFA_ADMIN_ILLEGAL_PARAMETER = 5, EFA_ADMIN_UNKNOWN_ERROR = 6, EFA_ADMIN_RESOURCE_BUSY = 7, @@ -78,13 +80,10 @@ struct efa_admin_acq_common_desc { */ u8 flags; - u16 extended_status; + /* Poly 0x8005 CRC16 with initial value 0xFFFF and final XOR of 0xFFFF */ + u16 checksum; - /* - * indicates to the driver which AQ entry has been consumed by the - * device and could be reused - */ - u16 sq_head_indx; + u16 reserved; }; struct efa_admin_acq_entry { diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index e97b5f0d7003..7cc3f4af0bb9 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -3,6 +3,7 @@ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ +#include #include #include "efa_com.h" @@ -22,6 +23,14 @@ #define EFA_CTRL_MINOR 0 #define EFA_CTRL_SUB_MINOR 1 +#define EFA_CRC16_INIT_VAL 0xffff + +#define EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR 0 +#define EFA_CRC_MIN_ADMIN_API_VERSION_MINOR 2 + +#define EFA_MIN_ADMIN_API_VERSION_MAJOR 0 +#define EFA_MIN_ADMIN_API_VERSION_MINOR 1 + enum efa_cmd_status { EFA_CMD_UNUSED, EFA_CMD_ALLOCATED, @@ -167,9 +176,8 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev) struct efa_com_admin_queue *aq = &edev->aq; struct efa_com_admin_cq *cq = &aq->cq; u16 size = aq->depth * sizeof(*cq->entries); - u32 acq_caps = 0; - u32 addr_high; - u32 addr_low; + u32 acq_caps = 0, crc_min_ver = 0; + u32 addr_high, addr_low; cq->entries = dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); @@ -178,6 +186,11 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev) spin_lock_init(&cq->lock); + EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR); + EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MINOR); + if (edev->dev_api_ver >= crc_min_ver) + cq->validate_checksum = true; + cq->cc = 0; cq->phase = 1; @@ -409,12 +422,35 @@ static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, return 0; } +static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq, + struct efa_admin_acq_entry *cqe) +{ + u16 cqe_checksum = cqe->acq_common_descriptor.checksum; + u16 calc_checksum; + + cqe->acq_common_descriptor.checksum = 0; + + calc_checksum = crc16(EFA_CRC16_INIT_VAL, (u8 *)cqe, sizeof(*cqe)) ^ EFA_CRC16_INIT_VAL; + if (calc_checksum != cqe_checksum) { + ibdev_err(aq->efa_dev, + "Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n", + cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc, + aq->cq.cc); + return false; + } + + return true; +} + static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, struct efa_admin_acq_entry *cqe) { struct efa_comp_ctx *comp_ctx; u16 cmd_id; + if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe)) + return -EINVAL; + cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); @@ -954,16 +990,16 @@ int efa_com_validate_version(struct efa_com_dev *edev) EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION), EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION)); - EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION, - EFA_ADMIN_API_VERSION_MAJOR); - EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION, - EFA_ADMIN_API_VERSION_MINOR); + EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MAJOR); + EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MINOR); if (ver < min_ver) { ibdev_err(edev->efa_dev, "EFA version is lower than the minimal version the driver supports\n"); return -EOPNOTSUPP; } + edev->dev_api_ver = ver; + ibdev_dbg( edev->efa_dev, "efa controller version: %d.%d.%d implementation version %d\n", diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h index 4d9ca97e4296..f8c692b0e092 100644 --- a/drivers/infiniband/hw/efa/efa_com.h +++ b/drivers/infiniband/hw/efa/efa_com.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_COM_H_ @@ -25,6 +25,7 @@ struct efa_com_admin_cq { struct efa_admin_acq_entry *entries; dma_addr_t dma_addr; spinlock_t lock; /* Protects ACQ */ + bool validate_checksum; u16 cc; /* consumer counter */ u8 phase; @@ -112,6 +113,7 @@ struct efa_com_dev { u32 supported_features; u32 dma_addr_bits; + u32 dev_api_ver; struct efa_com_mmio_read mmio_read; }; From 1eae35b37923cb71b0cb5136d00671440d488b9f Mon Sep 17 00:00:00 2001 From: Prathamesh Deshpande Date: Sun, 26 Apr 2026 14:23:41 +0100 Subject: [PATCH 010/148] RDMA/mlx5: Fix UMR XLT cleanup on ODP populate failure mlx5r_umr_update_xlt() allocates and DMA maps an XLT buffer with mlx5r_umr_create_xlt(). The buffer is released by the common cleanup path through mlx5r_umr_unmap_free_xlt(). After mlx5_odp_populate_xlt() became fallible, its error path returned directly and skipped that cleanup. This leaks the XLT DMA mapping and buffer. If the emergency XLT page was used, it also leaves xlt_emergency_page_mutex locked. Break out of the loop so execution falls through the existing cleanup path. Fixes: 1efe8c0670d6 ("RDMA/core: Convert UMEM ODP DMA mapping to caching IOVA and page linkage") Signed-off-by: Prathamesh Deshpande Link: https://patch.msgid.link/20260426132356.22264-1-prathameshdeshpande7@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/umr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 29488fba21a0..71f331ce6d89 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -915,7 +915,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, */ err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); if (err) - return err; + break; dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT); From 43f8f7946814c8e5f464518246fdbc69b6e32326 Mon Sep 17 00:00:00 2001 From: Prathamesh Deshpande Date: Tue, 28 Apr 2026 23:42:49 +0100 Subject: [PATCH 011/148] RDMA/mlx5: Fix devx subscribe-event unwind NULL dereference MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT() links event_sub into sub_list before initializing the fields used by the shared error path. If eventfd_ctx_fdget() then fails, the unwind path dereferences event_sub->ev_file in uverbs_uobject_put() and calls subscribe_event_xa_dealloc() with an unset xa_key_level1. subscribe_event_xa_alloc() creates the XA entry exactly once for a given key_level1, on the first occurrence of that key. The unwind path must therefore call subscribe_event_xa_dealloc() exactly once for it as well. Enforce that by adding devx_key_in_sub_list() and calling subscribe_event_xa_dealloc() only when the last matching pending entry is being cleaned up. Fixes: 759738537142 ("IB/mlx5: Enable subscription for device events over DEVX") Signed-off-by: Prathamesh Deshpande Link: https://patch.msgid.link/20260428224319.37682-1-prathameshdeshpande7@gmail.com Reviewed-by: Yishai Hadas Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/devx.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 645ebcc0832d..c2ae5a140471 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -1913,6 +1913,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( return err; } +static bool devx_key_in_sub_list(struct list_head *list, u32 key_level1) +{ + struct devx_event_subscription *s; + + list_for_each_entry(s, list, event_list) + if (s->xa_key_level1 == key_level1) + return true; + + return false; +} + static void subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table, u32 key_level1, @@ -2160,10 +2171,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( event_sub = kzalloc_obj(*event_sub); if (!event_sub) { + if (!devx_key_in_sub_list(&sub_list, key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + key_level1, + obj, + obj_id); err = -ENOMEM; goto err; } + event_sub->ev_file = ev_file; + event_sub->xa_key_level1 = key_level1; list_add_tail(&event_sub->event_list, &sub_list); uverbs_uobject_get(&ev_file->uobj); if (use_eventfd) { @@ -2178,9 +2196,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( } event_sub->cookie = cookie; - event_sub->ev_file = ev_file; - /* May be needed upon cleanup the devx object/subscription */ - event_sub->xa_key_level1 = key_level1; event_sub->xa_key_level2 = obj_id; INIT_LIST_HEAD(&event_sub->obj_list); } @@ -2225,10 +2240,11 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { list_del(&event_sub->event_list); - subscribe_event_xa_dealloc(devx_event_table, - event_sub->xa_key_level1, - obj, - obj_id); + if (!devx_key_in_sub_list(&sub_list, event_sub->xa_key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + event_sub->xa_key_level1, + obj, + obj_id); if (event_sub->eventfd) eventfd_ctx_put(event_sub->eventfd); From d38e69adc27e88111ebf13a51c83756c6249f734 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Sat, 2 May 2026 22:33:49 -0400 Subject: [PATCH 012/148] IB/mlx5: Reduce spinlock contention by moving free operations outside The functions kfree() and kvfree() can occasionally trigger a long chain of calls or face contention in the slab allocator. Executing these inside a spinlock increases the risk of CPU stalls and increases lock contention under heavy event load. Move the memory freeing logic out of the critical sections in devx.c by using temporary lists and local flags. This narrows the lock's scope to only protect the list integrity and state transitions. Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260503023349.1959-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/devx.c | 36 ++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index c2ae5a140471..c5929d023ee3 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -2528,7 +2528,7 @@ static int deliver_event(struct devx_event_subscription *event_sub, const void *data) { struct devx_async_event_file *ev_file; - struct devx_async_event_data *event_data; + struct devx_async_event_data *event_data, *to_free; unsigned long flags; ev_file = event_sub->ev_file; @@ -2559,12 +2559,17 @@ static int deliver_event(struct devx_event_subscription *event_sub, event_data->hdr.cookie = event_sub->cookie; memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe)); + to_free = NULL; + spin_lock_irqsave(&ev_file->lock, flags); if (!ev_file->is_destroyed) list_add_tail(&event_data->list, &ev_file->event_list); else - kfree(event_data); + to_free = event_data; spin_unlock_irqrestore(&ev_file->lock, flags); + + kfree(to_free); + wake_up_interruptible(&ev_file->poll_wait); return 0; @@ -2958,6 +2963,7 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, uobj); struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; struct devx_async_data *entry, *tmp; + LIST_HEAD(tmp_list); spin_lock_irq(&ev_queue->lock); ev_queue->is_destroyed = 1; @@ -2967,12 +2973,15 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); spin_lock_irq(&comp_ev_file->ev_queue.lock); - list_for_each_entry_safe(entry, tmp, - &comp_ev_file->ev_queue.event_list, list) { + /* Move all entries to a temporary list and free them outside lock */ + list_splice_init(&comp_ev_file->ev_queue.event_list, &tmp_list); + spin_unlock_irq(&comp_ev_file->ev_queue.lock); + + /* Free memory outside of critical section */ + list_for_each_entry_safe(entry, tmp, &tmp_list, list) { list_del(&entry->list); kvfree(entry); } - spin_unlock_irq(&comp_ev_file->ev_queue.lock); }; static void devx_async_event_destroy_uobj(struct ib_uobject *uobj, @@ -2982,7 +2991,9 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj, container_of(uobj, struct devx_async_event_file, uobj); struct devx_event_subscription *event_sub, *event_sub_tmp; + struct devx_async_event_data *entry, *tmp; struct mlx5_ib_dev *dev = ev_file->dev; + LIST_HEAD(tmp_list); spin_lock_irq(&ev_file->lock); ev_file->is_destroyed = 1; @@ -2996,18 +3007,19 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj, list_del_init(&event_sub->event_list); } else { - struct devx_async_event_data *entry, *tmp; - - list_for_each_entry_safe(entry, tmp, &ev_file->event_list, - list) { - list_del(&entry->list); - kfree(entry); - } + /* Move all entries to a temporary list */ + list_splice_init(&ev_file->event_list, &tmp_list); } spin_unlock_irq(&ev_file->lock); wake_up_interruptible(&ev_file->poll_wait); + /* Free event data outside of critical section */ + list_for_each_entry_safe(entry, tmp, &tmp_list, list) { + list_del(&entry->list); + kfree(entry); + } + mutex_lock(&dev->devx_event_table.event_xa_lock); /* delete the subscriptions which are related to this FD */ list_for_each_entry_safe(event_sub, event_sub_tmp, From 44ea9d022312097a8a25215f89e9c1a69a77b1d9 Mon Sep 17 00:00:00 2001 From: Rohit Chavan Date: Tue, 5 May 2026 13:23:07 +0530 Subject: [PATCH 013/148] RDMA/mlx4: Use secs_to_jiffies() instead of open-coding The conversion from seconds to jiffies is currently performed by multiplying the value by 1000 and passing it to msecs_to_jiffies(). Use the more direct secs_to_jiffies() helper instead. This simplifies the code, improves readability, and avoids the manual multiplication step by using the dedicated kernel API. Signed-off-by: Rohit Chavan Link: https://patch.msgid.link/20260505075308.1754861-1-roheetchavan@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx4/alias_GUID.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c index c6e1b9e4122a..fc23960cf1fd 100644 --- a/drivers/infiniband/hw/mlx4/alias_GUID.c +++ b/drivers/infiniband/hw/mlx4/alias_GUID.c @@ -437,7 +437,7 @@ static void aliasguid_query_handler(int status, queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq, &dev->sriov.alias_guid.ports_guid[port_index]. alias_guid_work, - msecs_to_jiffies(resched_delay_sec * 1000)); + secs_to_jiffies(resched_delay_sec)); } if (cb_ctx->sa_query) { list_del(&cb_ctx->list); From eb4ecdf631fe00e8020bf461503cb9b7017ed796 Mon Sep 17 00:00:00 2001 From: Sara Venkatesh Date: Mon, 4 May 2026 01:00:36 -0700 Subject: [PATCH 014/148] RDMA/srpt: fix integer overflow in immediate data length check imm_buf->len is a user-controlled uint32_t received from the network. Adding it to imm_data_offset without overflow checking allows a malicious initiator to send len=0xFFFFFFFF, causing req_size to wrap around to a small value, bypassing the bounds check, and subsequently passing a ~4GB length to sg_init_one(). Use check_add_overflow() to detect wrapping before the comparison. Fixes: 5dabcd0456d7 ("RDMA/srpt: Add support for immediate data") Reported-by: Carlos Bilbao (Lambda) Signed-off-by: Sara Venkatesh Link: https://patch.msgid.link/20260504080036.3482415-1-sarajvenkatesh@gmail.com Reviewed-by: Carlos Bilbao (Lambda) Reviewed-by: Bart Van Assche Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/srpt/ib_srpt.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 9aec5d80117f..f66cfd70c263 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1129,9 +1129,10 @@ static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx, struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd); void *data = (void *)srp_cmd + imm_data_offset; uint32_t len = be32_to_cpu(imm_buf->len); - uint32_t req_size = imm_data_offset + len; + uint32_t req_size; - if (req_size > srp_max_req_size) { + if (check_add_overflow((uint32_t)imm_data_offset, len, &req_size) || + req_size > srp_max_req_size) { pr_err("Immediate data (length %d + %d) exceeds request size %d\n", imm_data_offset, len, srp_max_req_size); return -EINVAL; From e456dc2d81a6b43c7ace4036cceea7a822d17a13 Mon Sep 17 00:00:00 2001 From: Rohit Chavan Date: Tue, 5 May 2026 15:35:49 +0530 Subject: [PATCH 015/148] RDMA/mlx5: Use max() macro for bfreg calculation Simplify the calculation of medium blue flame registers by using the max() macro instead of open-coded ternary logic. This improves readability and aligns with the subsystem's preference for using standard kernel helpers. Signed-off-by: Rohit Chavan Link: https://patch.msgid.link/20260505100550.1810139-1-roheetchavan@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8f50e7342a76..3048d2d273d7 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -703,12 +703,8 @@ static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi) static int num_med_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi) { - int n; - - n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs - - NUM_NON_BLUE_FLAME_BFREGS; - - return n >= 0 ? n : 0; + return max(0, max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs - + NUM_NON_BLUE_FLAME_BFREGS); } static int first_med_bfreg(struct mlx5_ib_dev *dev, From b4070770506ff516e21b4923afdaf7eb5da0e150 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Thu, 7 May 2026 09:21:46 +0800 Subject: [PATCH 016/148] RDMA/hns: Initialize seqfile before creating file The debugfs file was created before seq->read and seq->data were set, leaving a small window where userspace could access an uninitialized seqfile. Move debugfs_create_file() after the assignments to avoid this issue. Also, inline the original init_debugfs_seqfile() since it is not a really necessary helper. Fixes: ca7ad04cd5d2 ("RDMA/hns: Add debugfs to hns RoCE") Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260507012148.1079712-2-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_debugfs.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index b869cdc54118..db32c5897640 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -26,17 +26,6 @@ static const struct file_operations hns_debugfs_seqfile_fops = { .llseek = seq_lseek }; -static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq, - const char *name, struct dentry *parent, - int (*read_fn)(struct seq_file *, void *), - void *data) -{ - debugfs_create_file(name, 0400, parent, seq, &hns_debugfs_seqfile_fops); - - seq->read = read_fn; - seq->data = data; -} - static const char * const sw_stat_info[] = { [HNS_ROCE_DFX_AEQE_CNT] = "aeqe", [HNS_ROCE_DFX_CEQE_CNT] = "ceqe", @@ -76,10 +65,12 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev, { struct hns_sw_stat_debugfs *dbgfs = &hr_dev->dbgfs.sw_stat_root; - dbgfs->root = debugfs_create_dir("sw_stat", parent); + dbgfs->sw_stat.read = sw_stat_debugfs_show; + dbgfs->sw_stat.data = hr_dev; - init_debugfs_seqfile(&dbgfs->sw_stat, "sw_stat", dbgfs->root, - sw_stat_debugfs_show, hr_dev); + dbgfs->root = debugfs_create_dir("sw_stat", parent); + debugfs_create_file("sw_stat", 0400, dbgfs->root, &dbgfs->sw_stat, + &hns_debugfs_seqfile_fops); } /* debugfs for device */ From bb50659745fa978784571e1402b89fb5f02ff875 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Thu, 7 May 2026 09:21:47 +0800 Subject: [PATCH 017/148] RDMA/hns: Add write support to debugfs Add write support to debugfs. Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260507012148.1079712-3-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_debugfs.c | 20 ++++++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_debugfs.h | 1 + 2 files changed, 21 insertions(+) diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index db32c5897640..724d5ad90bfe 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -18,11 +18,31 @@ static int hns_debugfs_seqfile_open(struct inode *inode, struct file *f) return single_open(f, seqfile->read, seqfile->data); } +static ssize_t hns_debugfs_seqfile_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct hns_debugfs_seqfile *seqfile = file_inode(file)->i_private; + char buf[16] = {}; + + if (!seqfile->write) + return -EOPNOTSUPP; + + if (count >= sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, buffer, count)) + return -EFAULT; + + return seqfile->write(buf, count, seqfile->data); +} + static const struct file_operations hns_debugfs_seqfile_fops = { .owner = THIS_MODULE, .open = hns_debugfs_seqfile_open, .release = single_release, .read = seq_read, + .write = hns_debugfs_seqfile_write, .llseek = seq_lseek }; diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h index 98e87bd3161e..4e77dea0fbf6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h @@ -9,6 +9,7 @@ /* debugfs seqfile */ struct hns_debugfs_seqfile { int (*read)(struct seq_file *seq, void *data); + ssize_t (*write)(char *buf, size_t count, void *data); void *data; }; From df3835fd4e764db3dccbdafa144b15b49028f91e Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Thu, 7 May 2026 09:21:48 +0800 Subject: [PATCH 018/148] RDMA/hns: Support congestion control algorithm parameter configuration hns RoCE supports 4 congestion control algorithms. Each algorihm involves multiple parameters. Support configuring these parameters by debugfs. Here are some examples of this feature: * The directory structure: $ ls /sys/kernel/debug/hns_roce/0000\:35\:00.0/ dcqcn_cc_param dip_cc_param hc3_cc_param ldcp_cc_param $ ls /sys/kernel/debug/hns_roce/0000\:35\:00.0/dcqcn_cc_param/ ai al alp ashift cnp_time f g lifespan max_speed tkp tmp * Read the value of a param: $ cat /sys/kernel/debug/hns_roce/0000\:35\:00.0/dcqcn_cc_param/ai 1 * Set a new value for a param: $ echo 2 > /sys/kernel/debug/hns_roce/0000\:35\:00.0/dcqcn_cc_param/ai Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260507012148.1079712-4-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_debugfs.c | 248 +++++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_debugfs.h | 25 ++ drivers/infiniband/hw/hns/hns_roce_device.h | 21 ++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 54 ++++ drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 125 ++++++++++ 5 files changed, 473 insertions(+) diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index 724d5ad90bfe..05630f7c9155 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -3,11 +3,13 @@ * Copyright (c) 2023 Hisilicon Limited. */ +#include #include #include #include #include "hns_roce_device.h" +#include "hns_roce_hw_v2.h" static struct dentry *hns_roce_dbgfs_root; @@ -93,6 +95,250 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev, &hns_debugfs_seqfile_fops); } +#define __HNS_SCC_ATTR(_name, _type, _offset, _size, _min, _max) { \ + .name = _name, \ + .algo_type = _type, \ + .offset = _offset, \ + .size = _size, \ + .min = _min, \ + .max = _max, \ +} + +#define HNS_DCQCN_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DCQCN, \ + HNS_ROCE_DCQCN_##NAME##_OFS, \ + HNS_ROCE_DCQCN_##NAME##_SZ, \ + 0, HNS_ROCE_DCQCN_##NAME##_MAX) + +#define HNS_LDCP_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_LDCP, \ + HNS_ROCE_LDCP_##NAME##_OFS, \ + HNS_ROCE_LDCP_##NAME##_SZ, \ + 0, HNS_ROCE_LDCP_##NAME##_MAX) + +#define HNS_HC3_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_HC3, \ + HNS_ROCE_HC3_##NAME##_OFS, \ + HNS_ROCE_HC3_##NAME##_SZ, \ + HNS_ROCE_HC3_##NAME##_MIN, \ + HNS_ROCE_HC3_##NAME##_MAX) + +#define HNS_DIP_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DIP, \ + HNS_ROCE_DIP_##NAME##_OFS, \ + HNS_ROCE_DIP_##NAME##_SZ, \ + 0, HNS_ROCE_DIP_##NAME##_MAX) + +static const struct hns_roce_cong_attr { + enum hns_roce_cong_type cong_type; + const char *name; + struct hns_roce_cc_param_attr params[HNS_ROCE_CC_PARAM_MAX_NUM]; +} cong_attrs[] = { + { CONG_TYPE_DCQCN, "dcqcn_cc_param", + { + HNS_DCQCN_CC_ATTR_RW("ai", AI), + HNS_DCQCN_CC_ATTR_RW("f", F), + HNS_DCQCN_CC_ATTR_RW("tkp", TKP), + HNS_DCQCN_CC_ATTR_RW("tmp", TMP), + HNS_DCQCN_CC_ATTR_RW("alp", ALP), + HNS_DCQCN_CC_ATTR_RW("max_speed", MAX_SPEED), + HNS_DCQCN_CC_ATTR_RW("g", G), + HNS_DCQCN_CC_ATTR_RW("al", AL), + HNS_DCQCN_CC_ATTR_RW("cnp_time", CNP_TIME), + HNS_DCQCN_CC_ATTR_RW("ashift", ASHIFT), + } + }, + { CONG_TYPE_LDCP, "ldcp_cc_param", + { + HNS_LDCP_CC_ATTR_RW("cwd0", CWD0), + HNS_LDCP_CC_ATTR_RW("alpha", ALPHA), + HNS_LDCP_CC_ATTR_RW("gamma", GAMMA), + HNS_LDCP_CC_ATTR_RW("beta", BETA), + HNS_LDCP_CC_ATTR_RW("eta", ETA), + } + }, + { CONG_TYPE_HC3, "hc3_cc_param", + { + HNS_HC3_CC_ATTR_RW("initial_window", INITIAL_WINDOW), + HNS_HC3_CC_ATTR_RW("bandwidth", BANDWIDTH), + HNS_HC3_CC_ATTR_RW("qlen_shift", QLEN_SHIFT), + HNS_HC3_CC_ATTR_RW("port_usage_shift", PORT_USAGE_SHIFT), + HNS_HC3_CC_ATTR_RW("over_period", OVER_PERIOD), + HNS_HC3_CC_ATTR_RW("max_stage", MAX_STAGE), + HNS_HC3_CC_ATTR_RW("gamma_shift", GAMMA_SHIFT), + } + }, + { CONG_TYPE_DIP, "dip_cc_param", + { + HNS_DIP_CC_ATTR_RW("ai", AI), + HNS_DIP_CC_ATTR_RW("f", F), + HNS_DIP_CC_ATTR_RW("tkp", TKP), + HNS_DIP_CC_ATTR_RW("tmp", TMP), + HNS_DIP_CC_ATTR_RW("alp", ALP), + HNS_DIP_CC_ATTR_RW("max_speed", MAX_SPEED), + HNS_DIP_CC_ATTR_RW("g", G), + HNS_DIP_CC_ATTR_RW("al", AL), + HNS_DIP_CC_ATTR_RW("cnp_time", CNP_TIME), + HNS_DIP_CC_ATTR_RW("ashift", ASHIFT), + } + } +}; + +static int cc_param_debugfs_show(struct seq_file *file, void *offset) +{ + struct hns_cc_param_seqfile *param_seqfile = file->private; + const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr; + int algo_type = param_attr->algo_type; + int index = param_seqfile->index; + struct hns_roce_dev *hr_dev = + container_of(param_seqfile, struct hns_roce_dev, + dbgfs.cc_param_root[algo_type].params[index]); + struct hns_roce_scc_param *scc_param; + __le32 val = 0; + + scc_param = &hr_dev->scc_param[algo_type]; + + scoped_guard(mutex, &scc_param->scc_mutex) { + memcpy(&val, + (void *)scc_param->param + param_attr->offset, + param_attr->size); + } + + seq_printf(file, "%u\n", le32_to_cpu(val)); + + return 0; +} + +static ssize_t cc_param_debugfs_store(char *buf, size_t count, void *data) +{ + struct hns_cc_param_seqfile *param_seqfile = data; + const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr; + int algo_type = param_attr->algo_type; + int index = param_seqfile->index; + struct hns_roce_dev *hr_dev = + container_of(param_seqfile, struct hns_roce_dev, + dbgfs.cc_param_root[algo_type].params[index]); + struct hns_roce_scc_param *scc_param; + __le32 attr_val, old_val = 0; + u32 val; + int ret; + + if (kstrtou32(buf, 0, &val)) + return -EINVAL; + + if (val > param_attr->max || val < param_attr->min) + return -EINVAL; + + attr_val = cpu_to_le32(val); + scc_param = &hr_dev->scc_param[algo_type]; + guard(mutex)(&scc_param->scc_mutex); + + memcpy(&old_val, (void *)scc_param->param + param_attr->offset, + param_attr->size); + memcpy((void *)scc_param->param + param_attr->offset, &attr_val, + param_attr->size); + + ret = hr_dev->hw->config_scc_param(hr_dev, algo_type); + if (ret) { + memcpy((void *)scc_param->param + param_attr->offset, &old_val, + param_attr->size); + return ret; + } + + return count; +} + +static void get_default_scc_param(struct hns_roce_dev *hr_dev) +{ + int ret; + int i; + + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) { + ret = hr_dev->hw->query_scc_param(hr_dev, i); + if (ret && ret != -EOPNOTSUPP) + ibdev_warn_ratelimited(&hr_dev->ib_dev, + "failed to get default parameters of scc algo %d, ret = %d.\n", + i, ret); + } +} + +static int hns_roce_alloc_scc_param(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_scc_param *scc_param; + int i; + + scc_param = kvcalloc(HNS_ROCE_SCC_ALGO_TOTAL, sizeof(*scc_param), + GFP_KERNEL); + if (!scc_param) + return -ENOMEM; + + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) { + scc_param[i].algo_type = i; + scc_param[i].hr_dev = hr_dev; + mutex_init(&scc_param[i].scc_mutex); + } + + hr_dev->scc_param = scc_param; + + get_default_scc_param(hr_dev); + + return 0; +} + +static void hns_roce_dealloc_scc_param(struct hns_roce_dev *hr_dev) +{ + int i; + + if (!hr_dev->scc_param) + return; + + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) + mutex_destroy(&hr_dev->scc_param[i].scc_mutex); + + kvfree(hr_dev->scc_param); +} + +static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev, + struct dentry *parent) +{ + const struct hns_roce_cong_attr *cong_attr; + struct hns_cc_param_debugfs *dbgfs; + int i, j; + int ret; + + if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 || + !(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) || + hr_dev->is_vf) + return; + + ret = hns_roce_alloc_scc_param(hr_dev); + if (ret) + return; + + for (i = 0; i < CONG_TYPE_MAX_NUM; i++) { + cong_attr = &cong_attrs[i]; + if (!test_bit(cong_attr->cong_type, + (unsigned long *)&hr_dev->caps.cong_cap)) + continue; + + dbgfs = &hr_dev->dbgfs.cc_param_root[i]; + dbgfs->root = debugfs_create_dir(cong_attr->name, parent); + for (j = 0; j < HNS_ROCE_CC_PARAM_MAX_NUM; j++) { + if (!cong_attr->params[j].name) + break; + dbgfs->params[j].param_attr = &cong_attr->params[j]; + dbgfs->params[j].index = j; + dbgfs->params[j].seqfile.read = cc_param_debugfs_show; + dbgfs->params[j].seqfile.write = cc_param_debugfs_store; + dbgfs->params[j].seqfile.data = &dbgfs->params[j]; + debugfs_create_file(cong_attr->params[j].name, 0600, + dbgfs->root, + &dbgfs->params[j].seqfile, + &hns_debugfs_seqfile_fops); + } + } +} + /* debugfs for device */ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) { @@ -102,11 +348,13 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) hns_roce_dbgfs_root); create_sw_stat_debugfs(hr_dev, dbgfs->root); + create_cc_param_debugfs(hr_dev, dbgfs->root); } void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev) { debugfs_remove_recursive(hr_dev->dbgfs.root); + hns_roce_dealloc_scc_param(hr_dev); } /* debugfs for hns module */ diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h index 4e77dea0fbf6..116b1e8b6677 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h @@ -18,10 +18,35 @@ struct hns_sw_stat_debugfs { struct hns_debugfs_seqfile sw_stat; }; +struct hns_roce_cc_param_attr { + const char *name; + int algo_type; + u32 offset; + u32 size; + u32 max; + u32 min; +}; + +struct hns_cc_param_seqfile { + struct hns_debugfs_seqfile seqfile; + const struct hns_roce_cc_param_attr *param_attr; + int index; +}; + +#define HNS_ROCE_CC_PARAM_MAX_NUM 11 + +struct hns_cc_param_debugfs { + struct dentry *root; + struct hns_cc_param_seqfile params[HNS_ROCE_CC_PARAM_MAX_NUM]; +}; + +#define CONG_TYPE_MAX_NUM 4 + /* Debugfs for device */ struct hns_roce_dev_debugfs { struct dentry *root; struct hns_sw_stat_debugfs sw_stat_root; + struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM]; }; struct hns_roce_dev; diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 3f032b8038af..eb7b1865e4c7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -726,6 +726,14 @@ struct hns_roce_eq_table { struct hns_roce_eq *eq; }; +enum hns_roce_scc_algo { + HNS_ROCE_SCC_ALGO_DCQCN = 0, + HNS_ROCE_SCC_ALGO_LDCP, + HNS_ROCE_SCC_ALGO_HC3, + HNS_ROCE_SCC_ALGO_DIP, + HNS_ROCE_SCC_ALGO_TOTAL, +}; + struct hns_roce_caps { u64 fw_ver; u8 num_ports; @@ -963,6 +971,18 @@ struct hns_roce_hw { u8 *tc_mode, u8 *priority); const struct ib_device_ops *hns_roce_dev_ops; const struct ib_device_ops *hns_roce_dev_srq_ops; + int (*config_scc_param)(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo algo); + int (*query_scc_param)(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo alog); +}; + +#define HNS_ROCE_SCC_PARAM_SIZE 4 +struct hns_roce_scc_param { + __le32 param[HNS_ROCE_SCC_PARAM_SIZE]; + enum hns_roce_scc_algo algo_type; + struct hns_roce_dev *hr_dev; + struct mutex scc_mutex; /* protect @param */ }; struct hns_roce_dev { @@ -1026,6 +1046,7 @@ struct hns_roce_dev { u64 dwqe_page; struct hns_roce_dev_debugfs dbgfs; atomic64_t *dfx_cnt; + struct hns_roce_scc_param *scc_param; }; enum hns_roce_trace_type { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index fa36700d0db2..0988d91619c1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -31,6 +31,7 @@ */ #include +#include #include #include #include @@ -7196,6 +7197,57 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) kfree(eq_table->eq); } +static const enum hns_roce_opcode_type scc_opcode[] = { + HNS_ROCE_OPC_CFG_DCQCN_PARAM, + HNS_ROCE_OPC_CFG_LDCP_PARAM, + HNS_ROCE_OPC_CFG_HC3_PARAM, + HNS_ROCE_OPC_CFG_DIP_PARAM, +}; + +static int hns_roce_v2_config_scc_param(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo algo) +{ + struct hns_roce_scc_param *scc_param = &hr_dev->scc_param[algo]; + struct hns_roce_cmq_desc desc; + int ret; + + lockdep_assert_held(&scc_param->scc_mutex); + + hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], false); + memcpy(&desc.data, scc_param->param, sizeof(scc_param->param)); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to configure scc param, opcode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + + return ret; +} + +static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo algo) +{ + struct hns_roce_scc_param *scc_param; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to query scc param, opcode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + return ret; + } + + scc_param = &hr_dev->scc_param[algo]; + scoped_guard(mutex, &scc_param->scc_mutex) + memcpy(scc_param->param, &desc.data, sizeof(scc_param->param)); + + return 0; +} + static const struct ib_device_ops hns_roce_v2_dev_ops = { .destroy_qp = hns_roce_v2_destroy_qp, .modify_cq = hns_roce_v2_modify_cq, @@ -7246,6 +7298,8 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .get_dscp = hns_roce_hw_v2_get_dscp, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, + .config_scc_param = hns_roce_v2_config_scc_param, + .query_scc_param = hns_roce_v2_query_scc_param, }; static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 285fe0875fac..8c7856aa24dc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -202,6 +202,10 @@ enum { /* CMQ command */ enum hns_roce_opcode_type { HNS_QUERY_FW_VER = 0x0001, + HNS_ROCE_OPC_CFG_DCQCN_PARAM = 0x1A80, + HNS_ROCE_OPC_CFG_LDCP_PARAM = 0x1A81, + HNS_ROCE_OPC_CFG_HC3_PARAM = 0x1A82, + HNS_ROCE_OPC_CFG_DIP_PARAM = 0x1A83, HNS_ROCE_OPC_QUERY_HW_VER = 0x8000, HNS_ROCE_OPC_CFG_GLOBAL_PARAM = 0x8001, HNS_ROCE_OPC_ALLOC_PF_RES = 0x8004, @@ -1459,6 +1463,127 @@ struct hns_roce_wqe_atomic_seg { __le64 cmp_data; }; +#define HNS_ROCE_DCQCN_AI_OFS 0 +#define HNS_ROCE_DCQCN_AI_SZ sizeof(u16) +#define HNS_ROCE_DCQCN_AI_MAX ((u16)(~0U)) +#define HNS_ROCE_DCQCN_F_OFS (HNS_ROCE_DCQCN_AI_OFS + HNS_ROCE_DCQCN_AI_SZ) +#define HNS_ROCE_DCQCN_F_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_F_MAX ((u8)(~0U)) +#define HNS_ROCE_DCQCN_TKP_OFS (HNS_ROCE_DCQCN_F_OFS + HNS_ROCE_DCQCN_F_SZ) +#define HNS_ROCE_DCQCN_TKP_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_TKP_MAX 10 +#define HNS_ROCE_DCQCN_TMP_OFS (HNS_ROCE_DCQCN_TKP_OFS + HNS_ROCE_DCQCN_TKP_SZ) +#define HNS_ROCE_DCQCN_TMP_SZ sizeof(u16) +#define HNS_ROCE_DCQCN_TMP_MAX 15 +#define HNS_ROCE_DCQCN_ALP_OFS (HNS_ROCE_DCQCN_TMP_OFS + HNS_ROCE_DCQCN_TMP_SZ) +#define HNS_ROCE_DCQCN_ALP_SZ sizeof(u16) +#define HNS_ROCE_DCQCN_ALP_MAX ((u16)(~0U)) +#define HNS_ROCE_DCQCN_MAX_SPEED_OFS (HNS_ROCE_DCQCN_ALP_OFS + \ + HNS_ROCE_DCQCN_ALP_SZ) +#define HNS_ROCE_DCQCN_MAX_SPEED_SZ sizeof(u32) +#define HNS_ROCE_DCQCN_MAX_SPEED_MAX ((u32)(~0U)) +#define HNS_ROCE_DCQCN_G_OFS (HNS_ROCE_DCQCN_MAX_SPEED_OFS + \ + HNS_ROCE_DCQCN_MAX_SPEED_SZ) +#define HNS_ROCE_DCQCN_G_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_G_MAX 15 +#define HNS_ROCE_DCQCN_AL_OFS (HNS_ROCE_DCQCN_G_OFS + HNS_ROCE_DCQCN_G_SZ) +#define HNS_ROCE_DCQCN_AL_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_AL_MAX ((u8)(~0U)) +#define HNS_ROCE_DCQCN_CNP_TIME_OFS (HNS_ROCE_DCQCN_AL_OFS + \ + HNS_ROCE_DCQCN_AL_SZ) +#define HNS_ROCE_DCQCN_CNP_TIME_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_CNP_TIME_MAX ((u8)(~0U)) +#define HNS_ROCE_DCQCN_ASHIFT_OFS (HNS_ROCE_DCQCN_CNP_TIME_OFS + \ + HNS_ROCE_DCQCN_CNP_TIME_SZ) +#define HNS_ROCE_DCQCN_ASHIFT_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_ASHIFT_MAX 15 + +#define HNS_ROCE_LDCP_CWD0_OFS 0 +#define HNS_ROCE_LDCP_CWD0_SZ sizeof(u32) +#define HNS_ROCE_LDCP_CWD0_MAX ((u32)(~0U)) +#define HNS_ROCE_LDCP_ALPHA_OFS (HNS_ROCE_LDCP_CWD0_OFS + HNS_ROCE_LDCP_CWD0_SZ) +#define HNS_ROCE_LDCP_ALPHA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_ALPHA_MAX ((u8)(~0U)) +#define HNS_ROCE_LDCP_GAMMA_OFS (HNS_ROCE_LDCP_ALPHA_OFS + \ + HNS_ROCE_LDCP_ALPHA_SZ) +#define HNS_ROCE_LDCP_GAMMA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_GAMMA_MAX 7 +#define HNS_ROCE_LDCP_BETA_OFS (HNS_ROCE_LDCP_GAMMA_OFS + \ + HNS_ROCE_LDCP_GAMMA_SZ) +#define HNS_ROCE_LDCP_BETA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_BETA_MAX 7 +#define HNS_ROCE_LDCP_ETA_OFS (HNS_ROCE_LDCP_BETA_OFS + HNS_ROCE_LDCP_BETA_SZ) +#define HNS_ROCE_LDCP_ETA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_ETA_MAX 7 + +#define HNS_ROCE_HC3_INITIAL_WINDOW_OFS 0 +#define HNS_ROCE_HC3_INITIAL_WINDOW_SZ sizeof(u32) +#define HNS_ROCE_HC3_INITIAL_WINDOW_MIN 0 +#define HNS_ROCE_HC3_INITIAL_WINDOW_MAX ((u32)(~0U)) +#define HNS_ROCE_HC3_BANDWIDTH_OFS (HNS_ROCE_HC3_INITIAL_WINDOW_OFS + \ + HNS_ROCE_HC3_INITIAL_WINDOW_SZ) +#define HNS_ROCE_HC3_BANDWIDTH_SZ sizeof(u32) +#define HNS_ROCE_HC3_BANDWIDTH_MIN 1000 +#define HNS_ROCE_HC3_BANDWIDTH_MAX ((u32)(~0U)) +#define HNS_ROCE_HC3_QLEN_SHIFT_OFS (HNS_ROCE_HC3_BANDWIDTH_OFS + \ + HNS_ROCE_HC3_BANDWIDTH_SZ) +#define HNS_ROCE_HC3_QLEN_SHIFT_SZ sizeof(u8) +#define HNS_ROCE_HC3_QLEN_SHIFT_MIN 0 +#define HNS_ROCE_HC3_QLEN_SHIFT_MAX 31 +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS (HNS_ROCE_HC3_QLEN_SHIFT_OFS + \ + HNS_ROCE_HC3_QLEN_SHIFT_SZ) +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ sizeof(u8) +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MIN 0 +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MAX 100 +#define HNS_ROCE_HC3_OVER_PERIOD_OFS (HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS + \ + HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ) +#define HNS_ROCE_HC3_OVER_PERIOD_SZ sizeof(u8) +#define HNS_ROCE_HC3_OVER_PERIOD_MIN 0 +#define HNS_ROCE_HC3_OVER_PERIOD_MAX ((u8)(~0U)) +#define HNS_ROCE_HC3_MAX_STAGE_OFS (HNS_ROCE_HC3_OVER_PERIOD_OFS + \ + HNS_ROCE_HC3_OVER_PERIOD_SZ) +#define HNS_ROCE_HC3_MAX_STAGE_SZ sizeof(u8) +#define HNS_ROCE_HC3_MAX_STAGE_MIN 0 +#define HNS_ROCE_HC3_MAX_STAGE_MAX ((u8)(~0U)) +#define HNS_ROCE_HC3_GAMMA_SHIFT_OFS (HNS_ROCE_HC3_MAX_STAGE_OFS + \ + HNS_ROCE_HC3_MAX_STAGE_SZ) +#define HNS_ROCE_HC3_GAMMA_SHIFT_SZ sizeof(u8) +#define HNS_ROCE_HC3_GAMMA_SHIFT_MIN 0 +#define HNS_ROCE_HC3_GAMMA_SHIFT_MAX 15 + +#define HNS_ROCE_DIP_AI_OFS 0 +#define HNS_ROCE_DIP_AI_SZ sizeof(u16) +#define HNS_ROCE_DIP_AI_MAX ((u16)(~0U)) +#define HNS_ROCE_DIP_F_OFS (HNS_ROCE_DIP_AI_OFS + HNS_ROCE_DIP_AI_SZ) +#define HNS_ROCE_DIP_F_SZ sizeof(u8) +#define HNS_ROCE_DIP_F_MAX ((u8)(~0U)) +#define HNS_ROCE_DIP_TKP_OFS (HNS_ROCE_DIP_F_OFS + HNS_ROCE_DIP_F_SZ) +#define HNS_ROCE_DIP_TKP_SZ sizeof(u8) +#define HNS_ROCE_DIP_TKP_MAX 10 +#define HNS_ROCE_DIP_TMP_OFS (HNS_ROCE_DIP_TKP_OFS + HNS_ROCE_DIP_TKP_SZ) +#define HNS_ROCE_DIP_TMP_SZ sizeof(u16) +#define HNS_ROCE_DIP_TMP_MAX 15 +#define HNS_ROCE_DIP_ALP_OFS (HNS_ROCE_DIP_TMP_OFS + HNS_ROCE_DIP_TMP_SZ) +#define HNS_ROCE_DIP_ALP_SZ sizeof(u16) +#define HNS_ROCE_DIP_ALP_MAX ((u16)(~0U)) +#define HNS_ROCE_DIP_MAX_SPEED_OFS (HNS_ROCE_DIP_ALP_OFS + HNS_ROCE_DIP_ALP_SZ) +#define HNS_ROCE_DIP_MAX_SPEED_SZ sizeof(u32) +#define HNS_ROCE_DIP_MAX_SPEED_MAX ((u32)(~0U)) +#define HNS_ROCE_DIP_G_OFS (HNS_ROCE_DIP_MAX_SPEED_OFS + \ + HNS_ROCE_DIP_MAX_SPEED_SZ) +#define HNS_ROCE_DIP_G_SZ sizeof(u8) +#define HNS_ROCE_DIP_G_MAX 15 +#define HNS_ROCE_DIP_AL_OFS (HNS_ROCE_DIP_G_OFS + HNS_ROCE_DIP_G_SZ) +#define HNS_ROCE_DIP_AL_SZ sizeof(u8) +#define HNS_ROCE_DIP_AL_MAX ((u8)(~0U)) +#define HNS_ROCE_DIP_CNP_TIME_OFS (HNS_ROCE_DIP_AL_OFS + HNS_ROCE_DIP_AL_SZ) +#define HNS_ROCE_DIP_CNP_TIME_SZ sizeof(u8) +#define HNS_ROCE_DIP_CNP_TIME_MAX ((u8)(~0U)) +#define HNS_ROCE_DIP_ASHIFT_OFS (HNS_ROCE_DIP_CNP_TIME_OFS + \ + HNS_ROCE_DIP_CNP_TIME_SZ) +#define HNS_ROCE_DIP_ASHIFT_SZ sizeof(u8) +#define HNS_ROCE_DIP_ASHIFT_MAX 15 + struct hns_roce_sccc_clr { __le32 qpn; __le32 rsv[5]; From dd7f5374d1117a4a90f415cbe98ac9772d1f9e59 Mon Sep 17 00:00:00 2001 From: Rohit Chavan Date: Tue, 5 May 2026 14:27:08 +0530 Subject: [PATCH 019/148] RDMA/bng_re: Remove unused variable rc The variable 'rc' is initialized to 0 and returned at the end of bng_re_process_qp_event(), but it is never modified in between. Simplify the function by removing the redundant variable and returning 0 directly. This cleans up the code and avoids potential compiler warnings about unused variables. Signed-off-by: Rohit Chavan Link: https://patch.msgid.link/20260505085709.1755534-1-roheetchavan@gmail.com Reviewed-by: Siva Reddy Kallam Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bng_re/bng_fw.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c index 17d7cc3aa11d..50156c300b33 100644 --- a/drivers/infiniband/hw/bng_re/bng_fw.c +++ b/drivers/infiniband/hw/bng_re/bng_fw.c @@ -123,7 +123,6 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw, bool is_waiter_alive; struct pci_dev *pdev; u32 wait_cmds = 0; - int rc = 0; pdev = rcfw->pdev; switch (qp_event->event) { @@ -152,7 +151,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw, "rcfw timedout: cookie = %#x, free_slots = %d", cookie, crsqe->free_slots); spin_unlock(&hwq->lock); - return rc; + return 0; } if (crsqe->is_waiter_alive) { @@ -182,7 +181,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw, spin_unlock(&hwq->lock); } *num_wait += wait_cmds; - return rc; + return 0; } /* function events */ From c6207055617a7f463c4aac92d704d21081103b94 Mon Sep 17 00:00:00 2001 From: Bernard Metzler Date: Mon, 11 May 2026 16:11:49 +0200 Subject: [PATCH 020/148] RDMA/siw: use kzalloc_flex Simplify umem allocation by using flexible array member. Add __counted_by to get extra runtime analysis. Suggested-by: Rosen Penev Signed-off-by: Bernard Metzler Link: https://patch.msgid.link/20260511141149.52362-1-bernard.metzler@linux.dev Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/siw/siw.h | 5 +-- drivers/infiniband/sw/siw/siw_mem.c | 56 ++++++++++++++--------------- drivers/infiniband/sw/siw/siw_mem.h | 5 ++- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h index f5fd71717b80..dbc998248b1e 100644 --- a/drivers/infiniband/sw/siw/siw.h +++ b/drivers/infiniband/sw/siw/siw.h @@ -119,9 +119,10 @@ struct siw_page_chunk { struct siw_umem { struct ib_umem *base_mem; - struct siw_page_chunk *page_chunk; - int num_pages; + unsigned int num_pages; + unsigned int num_chunks; u64 fp_addr; /* First page base address */ + struct siw_page_chunk page_chunk[] __counted_by(num_chunks); }; struct siw_pble { diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c index 98c802b3ed72..4df792ebe02f 100644 --- a/drivers/infiniband/sw/siw/siw_mem.c +++ b/drivers/infiniband/sw/siw/siw_mem.c @@ -41,16 +41,14 @@ struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index) void siw_umem_release(struct siw_umem *umem) { - int i, num_pages = umem->num_pages; + unsigned int i, num_chunks = umem->num_chunks; if (umem->base_mem) ib_umem_release(umem->base_mem); - for (i = 0; num_pages > 0; i++) { + for (i = 0; i < num_chunks; i++) kfree(umem->page_chunk[i].plist); - num_pages -= PAGES_PER_CHUNK; - } - kfree(umem->page_chunk); + kfree(umem); } @@ -188,7 +186,7 @@ int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr, * lookup is being done and mem is not released it check fails. */ int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge, struct siw_mem *mem[], - enum ib_access_flags perms, u32 off, int len) + enum ib_access_flags perms, u32 off, u32 len) { struct siw_device *sdev = to_siw_dev(pd->device); struct siw_mem *new = NULL; @@ -338,25 +336,20 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start, struct sg_page_iter sg_iter; struct sg_table *sgt; u64 first_page_va; - int num_pages, num_chunks, i, rv = 0; + unsigned int num_pages, num_chunks, i; + int rv = 0; if (!len) return ERR_PTR(-EINVAL); first_page_va = start & PAGE_MASK; num_pages = PAGE_ALIGN(start + len - first_page_va) >> PAGE_SHIFT; - num_chunks = (num_pages >> CHUNK_SHIFT) + 1; + num_chunks = ((num_pages - 1) >> CHUNK_SHIFT) + 1; - umem = kzalloc_obj(*umem); + umem = kzalloc_flex(*umem, page_chunk, num_chunks); if (!umem) return ERR_PTR(-ENOMEM); - umem->page_chunk = - kzalloc_objs(struct siw_page_chunk, num_chunks); - if (!umem->page_chunk) { - rv = -ENOMEM; - goto err_out; - } base_mem = ib_umem_get(base_dev, start, len, rights); if (IS_ERR(base_mem)) { rv = PTR_ERR(base_mem); @@ -365,33 +358,40 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start, } umem->fp_addr = first_page_va; umem->base_mem = base_mem; + umem->num_pages = num_pages; + umem->num_chunks = num_chunks; sgt = &base_mem->sgt_append.sgt; __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); - if (!__sg_page_iter_next(&sg_iter)) { - rv = -EINVAL; - goto err_out; - } - for (i = 0; num_pages > 0; i++) { - int nents = min_t(int, num_pages, PAGES_PER_CHUNK); - struct page **plist = - kzalloc_objs(struct page *, nents); + for (i = 0; i < num_chunks; i++) { + struct page **plist; + unsigned int pix, nents = min(num_pages, PAGES_PER_CHUNK); + plist = kzalloc_objs(struct page *, nents); if (!plist) { rv = -ENOMEM; goto err_out; } umem->page_chunk[i].plist = plist; - while (nents--) { - *plist = sg_page_iter_page(&sg_iter); - umem->num_pages++; - num_pages--; - plist++; + + for (pix = 0; pix < nents; pix++) { if (!__sg_page_iter_next(&sg_iter)) break; + plist[pix] = sg_page_iter_page(&sg_iter); + num_pages--; } } + + if (num_pages) { + /* + * Unexpected size of sg list provided by ib_umem_get() + */ + siw_dbg(base_dev, "Short SG list, missing %u pages\n", + num_pages); + rv = -EINVAL; + goto err_out; + } return umem; err_out: siw_umem_release(umem); diff --git a/drivers/infiniband/sw/siw/siw_mem.h b/drivers/infiniband/sw/siw/siw_mem.h index 8e769d30e2ac..e4f3a5a4f81d 100644 --- a/drivers/infiniband/sw/siw/siw_mem.h +++ b/drivers/infiniband/sw/siw/siw_mem.h @@ -17,7 +17,7 @@ int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr, enum ib_access_flags perms, int len); int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge, struct siw_mem *mem[], enum ib_access_flags perms, - u32 off, int len); + u32 off, u32 len); void siw_wqe_put_mem(struct siw_wqe *wqe, enum siw_opcode op); int siw_mr_add_mem(struct siw_mr *mr, struct ib_pd *pd, void *mem_obj, u64 start, u64 len, int rights); @@ -45,7 +45,6 @@ static inline void siw_unref_mem_sgl(struct siw_mem **mem, unsigned int num_sge) #define CHUNK_SHIFT 9 /* sets number of pages per chunk */ #define PAGES_PER_CHUNK (_AC(1, UL) << CHUNK_SHIFT) #define CHUNK_MASK (~(PAGES_PER_CHUNK - 1)) -#define PAGE_CHUNK_SIZE (PAGES_PER_CHUNK * sizeof(struct page *)) /* * siw_get_upage() @@ -61,7 +60,7 @@ static inline struct page *siw_get_upage(struct siw_umem *umem, u64 addr) chunk_idx = page_idx >> CHUNK_SHIFT, page_in_chunk = page_idx & ~CHUNK_MASK; - if (likely(page_idx < umem->num_pages)) + if (page_idx < umem->num_pages) return umem->page_chunk[chunk_idx].plist[page_in_chunk]; return NULL; From cd31340419af6e0ae62b2c27985db209826a9577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig=20=28The=20Capable=20Hub=29?= Date: Thu, 7 May 2026 09:54:37 +0200 Subject: [PATCH 021/148] RDMA/hns: Use named initializer for pci_device_id array MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While being more verbose using a named initializer yields easier to understand code and doesn't rely on the two hidden zeros in the PCI_VDEVICE macro. While at it, also drop the explicit zero in the terminating entry. This doesn't introduce any changes to the compiled result of the array, which was confirmed on x86 and arm64. Signed-off-by: Uwe Kleine-König (The Capable Hub) Link: https://patch.msgid.link/20260507075437.2669363-2-u.kleine-koenig@baylibre.com Reviewed-by: Junxian Huang Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 0988d91619c1..4afd7d6ae3ca 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -7303,16 +7303,30 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { }; static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), - HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, + { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), + .driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS, + }, /* required last entry */ - {0, } + { } }; MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); From 55e6360d8f0e57de7d9158cb59ffc3ba43aa597f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:30 -0300 Subject: [PATCH 022/148] RDMA: Use ib_is_udata_in_empty() for places calling ib_is_udata_cleared() Convert the pattern: if (udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen)) Using Coccinelle: virtual patch virtual context virtual report @@ expression udata; @@ ( - udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen) + !ib_is_udata_in_empty(udata) | - udata->inlen > 0 && !ib_is_udata_cleared(udata, 0, udata->inlen) + !ib_is_udata_in_empty(udata) ) @@ expression udata; @@ - udata && udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen) + !ib_is_udata_in_empty(udata) These cases are already checking for zeroed data that the kernel does not understand. Run another pass with AI to propagate the return code correctly and remove redundant prints. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/efa_verbs.c | 43 +++++++++------------------ drivers/infiniband/hw/mlx4/main.c | 6 ++-- drivers/infiniband/hw/mlx4/qp.c | 7 ++--- drivers/infiniband/hw/mlx5/main.c | 5 ++-- drivers/infiniband/hw/mlx5/qp.c | 7 ++--- 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 7bd0838ebc99..3ad5d6e27b15 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -218,12 +218,9 @@ int efa_query_device(struct ib_device *ibdev, struct efa_dev *dev = to_edev(ibdev); int err; - if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(ibdev, - "Incompatible ABI params, udata not cleared\n"); - return -EINVAL; - } + err = ib_is_udata_in_empty(udata); + if (err) + return err; dev_attr = &dev->dev_attr; @@ -433,13 +430,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct efa_pd *pd = to_epd(ibpd); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, - "Incompatible ABI params, udata not cleared\n"); - err = -EINVAL; + err = ib_is_udata_in_empty(udata); + if (err) goto err_out; - } err = efa_com_alloc_pd(&dev->edev, &result); if (err) @@ -982,12 +975,9 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, - "Incompatible ABI params, udata not cleared\n"); - return -EINVAL; - } + err = ib_is_udata_in_empty(udata); + if (err) + return err; cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state : qp->state; @@ -1612,13 +1602,11 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, struct efa_dev *dev = to_edev(ibpd->device); int supp_access_flags; struct efa_mr *mr; + int ret; - if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, - "Incompatible ABI params, udata not cleared\n"); - return ERR_PTR(-EINVAL); - } + ret = ib_is_udata_in_empty(udata); + if (ret) + return ERR_PTR(ret); supp_access_flags = IB_ACCESS_LOCAL_WRITE | @@ -2082,12 +2070,9 @@ int efa_create_ah(struct ib_ah *ibah, goto err_out; } - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - err = -EINVAL; + err = ib_is_udata_in_empty(udata); + if (err) goto err_out; - } memcpy(params.dest_addr, ah_attr->grh.dgid.raw, sizeof(params.dest_addr)); diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 464c9ab42516..16e9ce8138cb 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1696,9 +1696,9 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, (flow_attr->type != IB_FLOW_ATTR_NORMAL)) return ERR_PTR(-EOPNOTSUPP); - if (udata && - udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen)) - return ERR_PTR(-EOPNOTSUPP); + err = ib_is_udata_in_empty(udata); + if (err) + return ERR_PTR(err); memset(type, 0, sizeof(type)); diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 790be09d985a..aca8a985ce33 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -4297,10 +4297,9 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table, size_t min_resp_len; int i, err = 0; - if (udata->inlen > 0 && - !ib_is_udata_cleared(udata, 0, - udata->inlen)) - return -EOPNOTSUPP; + err = ib_is_udata_in_empty(udata); + if (err) + return err; min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); if (udata->outlen && udata->outlen < min_resp_len) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 45f5fcd9adf0..01b21530e390 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -965,8 +965,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, resp.response_length = resp_len; - if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen)) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; memset(props, 0, sizeof(*props)); err = mlx5_query_system_image_guid(ibdev, diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 3048d2d273d7..1a37209457ef 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -5529,10 +5529,9 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, u32 *in; void *rqtc; - if (udata->inlen > 0 && - !ib_is_udata_cleared(udata, 0, - udata->inlen)) - return -EOPNOTSUPP; + err = ib_is_udata_in_empty(udata); + if (err) + return err; if (init_attr->log_ind_tbl_size > MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) { From 9ebad5c3ebc77cfc4611f7ed60ea5abcb555d3a8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:31 -0300 Subject: [PATCH 023/148] IB/rdmavt: Don't abuse udata and ib_respond_udata() Use copy_to_user() directly since the data is not being placed in the udata response memory. It is unclear why this is trying to do two copies, but leave it alone. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rdmavt/srq.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c index fe125bf85b27..d022aa56c5bf 100644 --- a/drivers/infiniband/sw/rdmavt/srq.c +++ b/drivers/infiniband/sw/rdmavt/srq.c @@ -128,6 +128,7 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); struct rvt_rq tmp_rq = {}; + __u64 offset_addr; int ret = 0; if (attr_mask & IB_SRQ_MAX_WR) { @@ -149,19 +150,17 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, return -ENOMEM; /* Check that we can write the offset to mmap. */ if (udata && udata->inlen >= sizeof(__u64)) { - __u64 offset_addr; __u64 offset = 0; ret = ib_copy_from_udata(&offset_addr, udata, sizeof(offset_addr)); if (ret) goto bail_free; - udata->outbuf = (void __user *) - (unsigned long)offset_addr; - ret = ib_copy_to_udata(udata, &offset, - sizeof(offset)); - if (ret) + if (copy_to_user(u64_to_user_ptr(offset_addr), &offset, + sizeof(offset))) { + ret = -EFAULT; goto bail_free; + } } spin_lock_irq(&srq->rq.kwq->c_lock); @@ -236,10 +235,10 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, * See rvt_mmap() for details. */ if (udata && udata->inlen >= sizeof(__u64)) { - ret = ib_copy_to_udata(udata, &ip->offset, - sizeof(ip->offset)); - if (ret) - return ret; + if (copy_to_user(u64_to_user_ptr(offset_addr), + &ip->offset, + sizeof(ip->offset))) + return -EFAULT; } /* From a5e9f0ae620f58b6f6fa4ee00accaac523cb1a72 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:32 -0300 Subject: [PATCH 024/148] RDMA: Convert drivers using min to ib_respond_udata() Convert the pattern: ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); Using Coccinelle: @@ identifier resp; expression udata; @@ - ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)) + ib_respond_udata(udata, resp) @@ identifier resp; expression udata; @@ - ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))) + ib_respond_udata(udata, resp) Run another pass with AI to propagate the return code correctly and remove redundant prints. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/efa_verbs.c | 44 +++++------------- drivers/infiniband/hw/erdma/erdma_verbs.c | 3 +- drivers/infiniband/hw/hns/hns_roce_ah.c | 4 +- drivers/infiniband/hw/hns/hns_roce_cq.c | 3 +- drivers/infiniband/hw/hns/hns_roce_main.c | 3 +- drivers/infiniband/hw/hns/hns_roce_pd.c | 8 ++-- drivers/infiniband/hw/hns/hns_roce_qp.c | 13 ++---- drivers/infiniband/hw/hns/hns_roce_srq.c | 6 +-- drivers/infiniband/hw/irdma/verbs.c | 48 +++++++------------- drivers/infiniband/hw/mana/cq.c | 6 +-- drivers/infiniband/hw/mana/qp.c | 6 +-- drivers/infiniband/hw/mlx5/srq.c | 7 +-- drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c | 8 ++-- 13 files changed, 49 insertions(+), 110 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 3ad5d6e27b15..395290ab0584 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -270,13 +270,9 @@ int efa_query_device(struct ib_device *ibdev, if (dev->neqs) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(ibdev, - "Failed to copy udata for query_device\n"); + err = ib_respond_udata(udata, resp); + if (err) return err; - } } return 0; @@ -442,13 +438,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) resp.pdn = result.pdn; if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&dev->ibdev, - "Failed to copy udata for alloc_pd\n"); + err = ib_respond_udata(udata, resp); + if (err) goto err_dealloc_pd; - } } ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn); @@ -782,14 +774,9 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, qp->max_inline_data = init_attr->cap.max_inline_data; if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&dev->ibdev, - "Failed to copy udata for qp[%u]\n", - create_qp_resp.qp_num); + err = ib_respond_udata(udata, resp); + if (err) goto err_remove_mmap_entries; - } } ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num); @@ -1226,13 +1213,9 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(ibdev, - "Failed to copy udata for create_cq\n"); + err = ib_respond_udata(udata, resp); + if (err) goto err_xa_erase; - } } ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", @@ -1935,8 +1918,7 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) resp.max_tx_batch = dev->dev_attr.max_tx_batch; resp.min_sq_wr = dev->dev_attr.min_sq_depth; - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); + err = ib_respond_udata(udata, resp); if (err) goto err_dealloc_uar; @@ -2087,13 +2069,9 @@ int efa_create_ah(struct ib_ah *ibah, resp.efa_address_handle = result.ah; if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&dev->ibdev, - "Failed to copy udata for create_ah response\n"); + err = ib_respond_udata(udata, resp); + if (err) goto err_destroy_ah; - } } ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 5523b4e151e1..9bba470c6e32 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1990,8 +1990,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, uresp.cq_id = cq->cqn; uresp.num_cqe = depth; - ret = ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_res; } else { diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 8a605da8a93c..925ddf15b681 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -32,6 +32,7 @@ #include #include +#include #include "hns_roce_device.h" #include "hns_roce_hw_v2.h" @@ -112,8 +113,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, resp.priority = ah->av.sl; resp.tc_mode = tc_mode; memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); + ret = ib_respond_udata(udata, resp); } err_out: diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 621568e11405..24de651f735e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -452,8 +452,7 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, if (udata) { resp.cqn = hr_cq->cqn; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); + ret = ib_respond_udata(udata, resp); if (ret) goto err_cqc; } diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 0dbe99aab6ad..c17ff5347a01 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -477,8 +477,7 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, resp.cqe_size = hr_dev->caps.cqe_sz; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); + ret = ib_respond_udata(udata, resp); if (ret) goto error_fail_copy_to_udata; diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index 225c3e328e0e..73bb000574c5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include #include "hns_roce_device.h" void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev) @@ -61,12 +62,9 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) if (udata) { struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd->pdn}; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); - if (ret) { + ret = ib_respond_udata(udata, resp); + if (ret) ida_free(&pd_ida->ida, id); - ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret); - } } return ret; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index a27ea85bb063..6d63613dcd5a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1235,12 +1235,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (udata) { resp.cap_flags = hr_qp->en_flags; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); - if (ret) { - ibdev_err(ibdev, "copy qp resp failed!\n"); + ret = ib_respond_udata(udata, resp); + if (ret) goto err_flow_ctrl; - } } if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { @@ -1487,11 +1484,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (udata && udata->outlen) { resp.tc_mode = hr_qp->tc_mode; resp.priority = hr_qp->sl; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); - if (ret) - ibdev_err_ratelimited(&hr_dev->ib_dev, - "failed to copy modify qp resp.\n"); + ret = ib_respond_udata(udata, resp); } out: diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index cb848e8e6bbd..8644c3916367 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -473,11 +473,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, if (udata) { resp.cap_flags = srq->cap_flags; resp.srqn = srq->srqn; - if (ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp)))) { - ret = -EFAULT; + ret = ib_respond_udata(udata, resp); + if (ret) goto err_srqc; - } } srq->event = hns_roce_ib_srq_event; diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 17086048d2d7..79e72a457e79 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -325,9 +325,9 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx, uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; uresp.kernel_ver = req.userspace_ver; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) - return -EFAULT; + ret = ib_respond_udata(udata, uresp); + if (ret) + return ret; } else { u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; @@ -354,10 +354,10 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx, uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta; uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) { + ret = ib_respond_udata(udata, uresp); + if (ret) { rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); - return -EFAULT; + return ret; } } @@ -420,11 +420,9 @@ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) ibucontext); irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); uresp.pd_id = pd_id; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) { - err = -EFAULT; + err = ib_respond_udata(udata, uresp); + if (err) goto error; - } } else { irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); } @@ -1124,10 +1122,8 @@ static int irdma_create_qp(struct ib_qp *ibqp, uresp.qp_id = qp_num; uresp.qp_caps = qp->qp_uk.qp_caps; - err_code = ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen)); + err_code = ib_respond_udata(udata, uresp); if (err_code) { - ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); irdma_destroy_qp(&iwqp->ibqp, udata); return err_code; } @@ -1612,12 +1608,9 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, uresp.push_valid = 1; uresp.push_offset = iwqp->sc_qp.push_offset; } - ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), - udata->outlen)); + ret = ib_respond_udata(udata, uresp); if (ret) { irdma_remove_push_mmap_entries(iwqp); - ibdev_dbg(&iwdev->ibdev, - "VERBS: copy_to_udata failed\n"); return ret; } } @@ -1860,12 +1853,9 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, uresp.push_offset = iwqp->sc_qp.push_offset; } - err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), - udata->outlen)); + err = ib_respond_udata(udata, uresp); if (err) { irdma_remove_push_mmap_entries(iwqp); - ibdev_dbg(&iwdev->ibdev, - "VERBS: copy_to_udata failed\n"); return err; } } @@ -2418,11 +2408,9 @@ static int irdma_create_srq(struct ib_srq *ibsrq, resp.srq_id = iwsrq->srq_num; resp.srq_size = ukinfo->srq_size; - if (ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen))) { - err_code = -EPROTO; + err_code = ib_respond_udata(udata, resp); + if (err_code) goto srq_destroy; - } } return 0; @@ -2664,13 +2652,9 @@ static int irdma_create_cq(struct ib_cq *ibcq, resp.cq_id = info.cq_uk_init_info.cq_id; resp.cq_size = info.cq_uk_init_info.cq_size; - if (ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen))) { - ibdev_dbg(&iwdev->ibdev, - "VERBS: copy to user data\n"); - err_code = -EPROTO; + err_code = ib_respond_udata(udata, resp); + if (err_code) goto cq_destroy; - } } init_completion(&iwcq->free_cq); @@ -5330,7 +5314,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah, mutex_unlock(&iwdev->rf->ah_tbl_lock); uresp.ah_id = ah->sc_ah.ah_info.ah_idx; - err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); + err = ib_respond_udata(udata, uresp); if (err) irdma_destroy_ah(ibah, attr->flags); diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index f4cbe21763bf..43b3ef65d3fc 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -79,11 +79,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (udata) { resp.cqid = cq->queue.id; - err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + err = ib_respond_udata(udata, resp); + if (err) goto err_remove_cq_cb; - } } spin_lock_init(&cq->cq_lock); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 645581359cee..1538aec6d85c 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -553,11 +553,9 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, resp.queue_id[j] = qp->rc_qp.queues[i].id; j++; } - err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + err = ib_respond_udata(udata, resp); + if (err) goto destroy_qp; - } } err = mana_table_store_qp(mdev, qp); diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 852f6f502d14..3fb8519a4ce0 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -292,12 +292,9 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, .srqn = srq->msrq.srqn, }; - if (ib_copy_to_udata(udata, &resp, min(udata->outlen, - sizeof(resp)))) { - mlx5_ib_dbg(dev, "copy to user failed\n"); - err = -EFAULT; + err = ib_respond_udata(udata, resp); + if (err) goto err_core; - } } init_attr->attr.max_wr = srq->msrq.max - 1; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 16aab967a203..cefcb243c3a6 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -406,12 +406,10 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, qp_resp.qpn = qp->ibqp.qp_num; qp_resp.qp_handle = qp->qp_handle; - if (ib_copy_to_udata(udata, &qp_resp, - min(udata->outlen, sizeof(qp_resp)))) { - dev_warn(&dev->pdev->dev, - "failed to copy back udata\n"); + ret = ib_respond_udata(udata, qp_resp); + if (ret) { __pvrdma_destroy_qp(dev, qp); - return -EINVAL; + return ret; } } From 5189936049e91f4fd54b47f147232cde6f7ec7cc Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:33 -0300 Subject: [PATCH 025/148] RDMA: Convert drivers using sizeof() to ib_respond_udata() Convert the pattern: ib_copy_to_udata(udata, &resp, sizeof(resp)); Using Coccinelle: @@ identifier resp; expression udata; @@ - ib_copy_to_udata(udata, &resp, sizeof(resp)) + ib_respond_udata(udata, resp) Run another pass with AI to propagate the return code correctly and remove redundant prints. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/cxgb4/provider.c | 9 ++++++--- drivers/infiniband/hw/cxgb4/qp.c | 4 ++-- drivers/infiniband/hw/erdma/erdma_verbs.c | 4 ++-- .../infiniband/hw/ionic/ionic_controlpath.c | 8 ++++---- drivers/infiniband/hw/mana/qp.c | 16 ++++------------ drivers/infiniband/hw/mlx4/main.c | 8 ++++---- drivers/infiniband/hw/mlx5/main.c | 5 +++-- drivers/infiniband/hw/mthca/mthca_provider.c | 5 +++-- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 19 +++++++------------ drivers/infiniband/hw/qedr/verbs.c | 7 +------ drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 9 ++------- drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c | 7 +++---- drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c | 6 +++--- .../infiniband/hw/vmw_pvrdma/pvrdma_verbs.c | 11 +++++------ drivers/infiniband/sw/rdmavt/cq.c | 2 +- drivers/infiniband/sw/rdmavt/qp.c | 3 +-- drivers/infiniband/sw/siw/siw_verbs.c | 10 +++++----- 17 files changed, 56 insertions(+), 77 deletions(-) diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 616019ac1da5..a119e8793aef 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "iw_cxgb4.h" @@ -209,8 +210,9 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) { struct c4iw_pd *php = to_c4iw_pd(pd); struct ib_device *ibdev = pd->device; - u32 pdid; struct c4iw_dev *rhp; + u32 pdid; + int ret; pr_debug("ibdev %p\n", ibdev); rhp = (struct c4iw_dev *) ibdev; @@ -223,9 +225,10 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) if (udata) { struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid}; - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + ret = ib_respond_udata(udata, uresp); + if (ret) { c4iw_deallocate_pd(&php->ibpd, udata); - return -EFAULT; + return ret; } } mutex_lock(&rhp->rdev.stats.lock); diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index d9a86e4c5461..f9c7030ac6bf 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -2280,7 +2280,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, ucontext->key += PAGE_SIZE; } spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_ma_sync_key; sq_key_mm->key = uresp.sq_key; @@ -2777,7 +2777,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, uresp.srq_db_gts_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_srq_db_key_mm; srq_key_mm->key = uresp.srq_key; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 9bba470c6e32..92a65970ab6f 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1055,7 +1055,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, uresp.qp_id = QP_ID(qp); uresp.rq_offset = qp->user_qp.rq_offset; - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_out_cmd; } else { @@ -1571,7 +1571,7 @@ int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) uresp.dev_id = dev->pdev->device; - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_put_mmap_entries; diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 7051a81cca94..2b01345848dd 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -414,7 +414,7 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) if (dev->lif_cfg.rq_expdb) resp.expdb_qtypes |= IONIC_EXPDB_RQ; - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; @@ -752,7 +752,7 @@ int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, if (udata) { resp.ahid = ah->ahid; - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; } @@ -1263,7 +1263,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (udata) { resp.udma_mask = vcq->udma_mask; - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; } @@ -2315,7 +2315,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, resp.rq_cmb = qp->rq_cmb; } - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; } diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 1538aec6d85c..c50a1eafee58 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -210,13 +210,9 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, if (ret) goto fail; - ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); - if (ret) { - ibdev_dbg(&mdev->ib_dev, - "Failed to copy to udata create rss-qp, %d\n", - ret); + ret = ib_respond_udata(udata, resp); + if (ret) goto fail; - } kfree(mana_ind_table); @@ -349,13 +345,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, resp.cqid = send_cq->queue.id; resp.tx_vp_offset = pd->tx_vp_offset; - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); - if (err) { - ibdev_dbg(&mdev->ib_dev, - "Failed copy udata for create qp-raw, %d\n", - err); + err = ib_respond_udata(udata, resp); + if (err) goto err_remove_cq_cb; - } return 0; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 16e9ce8138cb..ce77e893065c 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1121,16 +1121,16 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, mutex_init(&context->wqn_ranges_mutex); if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) - err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); + err = ib_respond_udata(udata, resp_v3); else - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); + err = ib_respond_udata(udata, resp); if (err) { mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar); - return -EFAULT; + return err; } - return err; + return 0; } static void mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 01b21530e390..e70d32001c5b 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2796,9 +2796,10 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) pd->uid = uid; if (udata) { resp.pdn = pd->pdn; - if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { + err = ib_respond_udata(udata, resp); + if (err) { mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid); - return -EFAULT; + return err; } } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index e8d5d865c1f1..07c60797c860 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -311,10 +311,11 @@ static int mthca_alloc_ucontext(struct ib_ucontext *uctx, return err; } - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + err = ib_respond_udata(udata, uresp); + if (err) { mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab); mthca_uar_free(to_mdev(ibdev), &context->uar); - return -EFAULT; + return err; } context->reg_mr_warned = 0; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index c17e2a54dbca..083f23fc687b 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -502,7 +502,7 @@ int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) resp.dpp_wqe_size = dev->attr.wqe_size; memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); - status = ib_copy_to_udata(udata, &resp, sizeof(resp)); + status = ib_respond_udata(udata, resp); if (status) goto cpy_err; return 0; @@ -611,7 +611,7 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, rsp.dpp_page_addr_lo = dpp_page_addr; } - status = ib_copy_to_udata(udata, &rsp, sizeof(rsp)); + status = ib_respond_udata(udata, rsp); if (status) goto ucopy_err; @@ -945,12 +945,9 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); uresp.db_page_size = dev->nic_info.db_page_size; uresp.phase_change = cq->phase_change ? 1 : 0; - status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (status) { - pr_err("%s(%d) copy error cqid=0x%x.\n", - __func__, dev->id, cq->id); + status = ib_respond_udata(udata, uresp); + if (status) goto err; - } status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); if (status) goto err; @@ -1206,11 +1203,9 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, uresp.dpp_credit = dpp_credit_lmt; uresp.dpp_offset = dpp_offset; } - status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (status) { - pr_err("%s(%d) user copy error.\n", __func__, dev->id); + status = ib_respond_udata(udata, uresp); + if (status) goto err; - } status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); if (status) @@ -1754,7 +1749,7 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, uresp.db_shift = 16; } - status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + status = ib_respond_udata(udata, uresp); if (status) return status; status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 679aa6f3a63b..3b86ea1cf888 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -1251,15 +1251,10 @@ static int qedr_copy_srq_uresp(struct qedr_dev *dev, struct qedr_srq *srq, struct ib_udata *udata) { struct qedr_create_srq_uresp uresp = {}; - int rc; uresp.srq_id = srq->srq_id; - rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (rc) - DP_ERR(dev, "create srq: problem copying data to user space\n"); - - return rc; + return ib_respond_udata(udata, uresp); } static void qedr_copy_rq_uresp(struct qedr_dev *dev, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 615de9c4209b..e887f03a84d0 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -82,7 +82,6 @@ static void usnic_ib_fw_string_to_u64(char *fw_ver_str, u64 *fw_ver) static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, struct ib_udata *udata) { - struct usnic_ib_dev *us_ibdev; struct usnic_ib_create_qp_resp resp; struct pci_dev *pdev; struct vnic_dev_bar *bar; @@ -92,7 +91,6 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, memset(&resp, 0, sizeof(resp)); - us_ibdev = qp_grp->vf->pf; pdev = usnic_vnic_get_pdev(qp_grp->vf->vnic); if (!pdev) { usnic_err("Failed to get pdev of qp_grp %d\n", @@ -157,12 +155,9 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, struct usnic_ib_qp_grp_flow, link); resp.transport = default_flow->trans_type; - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); - if (err) { - usnic_err("Failed to copy udata for %s", - dev_name(&us_ibdev->ib_dev.dev)); + err = ib_respond_udata(udata, resp); + if (err) return err; - } return 0; } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index bc3adcc1ae67..d5bfdbfe1376 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -203,11 +203,10 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->uar = &context->uar; /* Copy udata back. */ - if (ib_copy_to_udata(udata, &cq_resp, sizeof(cq_resp))) { - dev_warn(&dev->pdev->dev, - "failed to copy back udata\n"); + ret = ib_respond_udata(udata, cq_resp); + if (ret) { pvrdma_destroy_cq(&cq->ibcq, udata); - return -EINVAL; + return ret; } } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c index d31fb692fcaa..e69eadde6c26 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c @@ -195,10 +195,10 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, spin_unlock_irqrestore(&dev->srq_tbl_lock, flags); /* Copy udata back. */ - if (ib_copy_to_udata(udata, &srq_resp, sizeof(srq_resp))) { - dev_warn(&dev->pdev->dev, "failed to copy back udata\n"); + ret = ib_respond_udata(udata, srq_resp); + if (ret) { pvrdma_destroy_srq(&srq->ibsrq, udata); - return -EINVAL; + return ret; } return 0; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index bcd43dc30e21..69a89f609ada 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -320,11 +320,11 @@ int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) /* copy back to user */ uresp.qp_tab_size = vdev->dsr->caps.max_qp; - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) { pvrdma_uar_free(vdev, &context->uar); pvrdma_dealloc_ucontext(&context->ibucontext); - return -EFAULT; + return ret; } return 0; @@ -430,11 +430,10 @@ int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) pd_resp.pdn = resp->pd_handle; if (udata) { - if (ib_copy_to_udata(udata, &pd_resp, sizeof(pd_resp))) { - dev_warn(&dev->pdev->dev, - "failed to copy back protection domain\n"); + ret = ib_respond_udata(udata, pd_resp); + if (ret) { pvrdma_dealloc_pd(&pd->ibpd, udata); - return -EFAULT; + return ret; } } diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c index 30904c6ae852..45404611c9ce 100644 --- a/drivers/infiniband/sw/rdmavt/cq.c +++ b/drivers/infiniband/sw/rdmavt/cq.c @@ -372,7 +372,7 @@ int rvt_resize_cq(struct ib_cq *ibcq, unsigned int cqe, struct ib_udata *udata) if (udata && udata->outlen >= sizeof(__u64)) { __u64 offset = 0; - ret = ib_copy_to_udata(udata, &offset, sizeof(offset)); + ret = ib_respond_udata(udata, offset); if (ret) goto bail_free; } diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 816624e0991a..70e7d08fdce6 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -1192,8 +1192,7 @@ int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, if (!qp->r_rq.wq) { __u64 offset = 0; - ret = ib_copy_to_udata(udata, &offset, - sizeof(offset)); + ret = ib_respond_udata(udata, offset); if (ret) goto bail_qpn; } else { diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 1e1d262a4ae2..b34f3d6547ff 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -102,7 +102,7 @@ int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata) rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; @@ -472,7 +472,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, rv = -EINVAL; goto err_out_xa; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out_xa; } @@ -1205,7 +1205,7 @@ int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr, rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; } @@ -1386,7 +1386,7 @@ struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; } @@ -1646,7 +1646,7 @@ int siw_create_srq(struct ib_srq *base_srq, rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; } From addf681eede57d96a6d45c7649adbad47d9e9371 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:34 -0300 Subject: [PATCH 026/148] RDMA/cxgb4: Convert to ib_respond_udata() These cases carefully work around 32-bit unpadded structures, but the min integrated into ib_respond_udata() handles this automatically. Zero-initialize data that would not have been copied. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/cxgb4/cq.c | 8 +++----- drivers/infiniband/hw/cxgb4/provider.c | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index e31fb9134aa8..47508df4cec0 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -1115,13 +1115,11 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, /* communicate to the userspace that * kernel driver supports 64B CQE */ - uresp.flags |= C4IW_64B_CQE; + if (!ucontext->is_32b_cqe) + uresp.flags |= C4IW_64B_CQE; spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, - ucontext->is_32b_cqe ? - sizeof(uresp) - sizeof(uresp.flags) : - sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_mm2; diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index a119e8793aef..0e3827022c63 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -80,7 +80,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext, struct ib_device *ibdev = ucontext->device; struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext); struct c4iw_dev *rhp = to_c4iw_dev(ibdev); - struct c4iw_alloc_ucontext_resp uresp; + struct c4iw_alloc_ucontext_resp uresp = {}; int ret = 0; struct c4iw_mm_entry *mm = NULL; @@ -106,8 +106,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext, context->key += PAGE_SIZE; spin_unlock(&context->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, - sizeof(uresp) - sizeof(uresp.reserved)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_mm; From 49e9e655619194cabbac4e9b4032fc26dd5305f2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:35 -0300 Subject: [PATCH 027/148] RDMA/qedr: Replace qedr_ib_copy_to_udata() with ib_respond_udata() This is another instance of the min() pattern. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/qedr/verbs.c | 35 +++++------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 3b86ea1cf888..79190c5b8b50 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -64,14 +64,6 @@ enum { QEDR_USER_MMAP_PHYS_PAGE, }; -static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, - size_t len) -{ - size_t min_len = min_t(size_t, len, udata->outlen); - - return ib_copy_to_udata(udata, src, min_len); -} - int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { if (index >= QEDR_ROCE_PKEY_TABLE_LEN) @@ -340,7 +332,7 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) uresp.sges_per_srq_wr = dev->attr.max_srq_sge; uresp.max_cqes = QEDR_MAX_CQES; - rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rc = ib_respond_udata(udata, uresp); if (rc) goto err; @@ -459,9 +451,8 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct qedr_ucontext *context = rdma_udata_to_drv_context( udata, struct qedr_ucontext, ibucontext); - rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rc = ib_respond_udata(udata, uresp); if (rc) { - DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); return rc; } @@ -696,12 +687,10 @@ static void qedr_db_recovery_del(struct qedr_dev *dev, dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); } -static int qedr_copy_cq_uresp(struct qedr_dev *dev, - struct qedr_cq *cq, struct ib_udata *udata, +static int qedr_copy_cq_uresp(struct qedr_cq *cq, struct ib_udata *udata, u32 db_offset) { struct qedr_create_cq_uresp uresp; - int rc; memset(&uresp, 0, sizeof(uresp)); @@ -711,11 +700,7 @@ static int qedr_copy_cq_uresp(struct qedr_dev *dev, uresp.db_rec_addr = rdma_user_mmap_get_offset(cq->q.db_mmap_entry); - rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (rc) - DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); - - return rc; + return ib_respond_udata(udata, uresp); } static void consume_cqe(struct qedr_cq *cq) @@ -994,7 +979,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, spin_lock_init(&cq->cq_lock); if (udata) { - rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); + rc = qedr_copy_cq_uresp(cq, udata, db_offset); if (rc) goto err2; @@ -1298,8 +1283,6 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_udata *udata, struct qedr_create_qp_uresp *uresp) { - int rc; - memset(uresp, 0, sizeof(*uresp)); if (qedr_qp_has_sq(qp)) @@ -1311,13 +1294,7 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev, uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; uresp->qp_id = qp->qp_id; - rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); - if (rc) - DP_ERR(dev, - "create qp: failed a copy to user space with qp icid=0x%x.\n", - qp->icid); - - return rc; + return ib_respond_udata(udata, *uresp); } static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) From 0d10dd1388f8727402d07269640e44b78f79eb05 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:36 -0300 Subject: [PATCH 028/148] RDMA/mlx: Replace response_len with ib_respond_udata() The Mellanox drivers have a pattern where they compute the response length they think they need based on what the user asked for, then blindly write that ignoring the provided size limit on the response structure. Drop this and just use ib_respond_udata() which caps the response struct to the user's memory, which is fine for what mlx5 is doing. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx4/main.c | 2 +- drivers/infiniband/hw/mlx4/qp.c | 2 +- drivers/infiniband/hw/mlx5/ah.c | 2 +- drivers/infiniband/hw/mlx5/main.c | 4 ++-- drivers/infiniband/hw/mlx5/mr.c | 2 +- drivers/infiniband/hw/mlx5/qp.c | 10 +++++----- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index ce77e893065c..4b187ec9e017 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -626,7 +626,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, } if (uhw->outlen) { - err = ib_copy_to_udata(uhw, &resp, resp.response_length); + err = ib_respond_udata(uhw, resp); if (err) goto out; } diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index aca8a985ce33..8dc4196218bf 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -4331,7 +4331,7 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table, if (udata->outlen) { resp.response_length = offsetof(typeof(resp), response_length) + sizeof(resp.response_length); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); } return err; diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c index 531a57f9ee7e..a3aa700d0835 100644 --- a/drivers/infiniband/hw/mlx5/ah.c +++ b/drivers/infiniband/hw/mlx5/ah.c @@ -121,7 +121,7 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, resp.response_length = min_resp_len; memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) return err; } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index e70d32001c5b..302f0a13051b 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1356,7 +1356,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } if (uhw_outlen) { - err = ib_copy_to_udata(uhw, &resp, resp.response_length); + err = ib_respond_udata(uhw, resp); if (err) return err; @@ -2285,7 +2285,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, goto out_mdev; resp.response_length = min(udata->outlen, sizeof(resp)); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto out_mdev; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 3b6da45061a5..d7d8f3ae8b64 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1809,7 +1809,7 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) resp.response_length = min(offsetofend(typeof(resp), response_length), udata->outlen); if (resp.response_length) { - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto free_mkey; } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 1a37209457ef..6f88f9c52ad0 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3323,7 +3323,7 @@ int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, * including MLX5_IB_QPT_DCT, which doesn't need it. * In that case, resp will be filled with zeros. */ - err = ib_copy_to_udata(udata, ¶ms.resp, params.outlen); + err = ib_respond_udata(udata, params.resp); if (err) goto destroy_qp; @@ -4622,7 +4622,7 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr, resp.dctn = qp->dct.mdct.mqp.qpn; if (MLX5_CAP_GEN(dev->mdev, ece_support)) resp.ece_options = MLX5_GET(create_dct_out, out, ece); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) { mlx5_core_destroy_dct(dev, &qp->dct.mdct); return err; @@ -4781,7 +4781,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (!err && resp.response_length && udata->outlen >= resp.response_length) /* Return -EFAULT to the user and expect him to destroy QP. */ - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); out: mutex_unlock(&qp->mutex); @@ -5481,7 +5481,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, if (udata->outlen) { resp.response_length = offsetofend( struct mlx5_ib_create_wq_resp, response_length); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto err_copy; } @@ -5572,7 +5572,7 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, resp.response_length = offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, response_length); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto err_copy; } From 89a497400924af156bebf90d5f1f08f480ee44b1 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:37 -0300 Subject: [PATCH 029/148] RDMA: Use proper driver data response structs instead of open coding At some point the response structs were added and rdma-core is using them, but the kernel was not changed to use them as well. Replace the open-coded copy with the right struct and ib_respond_udata(). Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx4/cq.c | 7 ++-- drivers/infiniband/hw/mlx4/main.c | 11 ++++-- drivers/infiniband/hw/mlx4/srq.c | 12 ++++--- drivers/infiniband/hw/mlx5/cq.c | 7 ++-- drivers/infiniband/hw/mthca/mthca_provider.c | 35 ++++++++++++++------ 5 files changed, 48 insertions(+), 24 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 7a6eb602d4a6..7e4505f6c78b 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -142,6 +142,7 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, { struct ib_udata *udata = &attrs->driver_udata; struct ib_device *ibdev = ibcq->device; + struct mlx4_ib_create_cq_resp uresp = {}; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx4_ib_dev *dev = to_mdev(ibdev); @@ -219,10 +220,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, cq->mcq.event = mlx4_ib_cq_event; cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS; - if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { - err = -EFAULT; + uresp.cqn = cq->mcq.cqn; + err = ib_respond_udata(udata, uresp); + if (err) goto err_cq_free; - } return 0; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 4b187ec9e017..25f9738bd772 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1199,9 +1199,14 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) if (err) return err; - if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) { - mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); - return -EFAULT; + if (udata) { + struct mlx4_ib_alloc_pd_resp uresp = { .pdn = pd->pdn }; + + err = ib_respond_udata(udata, uresp); + if (err) { + mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); + return err; + } } return 0; } diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index 5b23e5f8b84a..0b4df4f48ca1 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -191,11 +191,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, srq->msrq.event = mlx4_ib_srq_event; srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; - if (udata) - if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { - err = -EFAULT; + if (udata) { + struct mlx4_ib_create_srq_resp uresp = { + .srqn = srq->msrq.srqn + }; + + err = ib_respond_udata(udata, uresp); + if (err) goto err_wrid; - } + } init_attr->attr.max_wr = srq->msrq.max - 1; diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index ae20d11f04fc..64fad78cb256 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -950,6 +950,7 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq, { struct ib_udata *udata = &attrs->driver_udata; struct ib_device *ibdev = ibcq->device; + struct mlx5_ib_create_cq_resp uresp = {}; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx5_ib_dev *dev = to_mdev(ibdev); @@ -1016,10 +1017,10 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq, INIT_LIST_HEAD(&cq->wc_list); - if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { - err = -EFAULT; + uresp.cqn = cq->mcq.cqn; + err = ib_respond_udata(udata, uresp); + if (err) goto err_cmd; - } kvfree(cqb); return 0; diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 07c60797c860..afa97d3801f7 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -357,9 +357,12 @@ static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return err; if (udata) { - if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) { + struct mthca_alloc_pd_resp uresp = { .pdn = pd->pd_num }; + + err = ib_respond_udata(udata, uresp); + if (err) { mthca_pd_free(to_mdev(ibdev), pd); - return -EFAULT; + return err; } } @@ -428,11 +431,17 @@ static int mthca_create_srq(struct ib_srq *ibsrq, if (err) return err; - if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) { - mthca_free_srq(to_mdev(ibsrq->device), srq); - mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, - context->db_tab, ucmd.db_index); - return -EFAULT; + if (context) { + struct mthca_create_srq_resp uresp = { .srqn = srq->srqn }; + + err = ib_respond_udata(udata, uresp); + if (err) { + mthca_free_srq(to_mdev(ibsrq->device), srq); + mthca_unmap_user_db(to_mdev(ibsrq->device), + &context->uar, context->db_tab, + ucmd.db_index); + return err; + } } return 0; @@ -631,10 +640,14 @@ static int mthca_create_cq(struct ib_cq *ibcq, if (err) goto err_unmap_arm; - if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) { - mthca_free_cq(to_mdev(ibdev), cq); - err = -EFAULT; - goto err_unmap_arm; + if (udata) { + struct mthca_create_cq_resp uresp = { .cqn = cq->cqn }; + + err = ib_respond_udata(udata, uresp); + if (err) { + mthca_free_cq(to_mdev(ibdev), cq); + goto err_unmap_arm; + } } cq->resize_buf = NULL; From eff628f3bb2f2bd95c0b9718f80e046d153d8abc Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:38 -0300 Subject: [PATCH 030/148] RDMA: Add missed = {} initialization to uresp structs All of these are fully initialized so no bugs are being fixed. Add the missing initializer as a precaution against future changes. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 2 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 2 +- drivers/infiniband/hw/mlx4/main.c | 4 ++-- drivers/infiniband/hw/mlx5/main.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 7ed294516b7e..ccb362d6d2e6 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1884,7 +1884,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } if (udata) { - struct bnxt_re_qp_resp resp; + struct bnxt_re_qp_resp resp = {}; resp.qpid = qp->qplib_qp.id; resp.rsvd = 0; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 92a65970ab6f..c8a35337ba51 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1977,7 +1977,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (!rdma_is_kernel_res(&ibcq->res)) { struct erdma_ureq_create_cq ureq; - struct erdma_uresp_create_cq uresp; + struct erdma_uresp_create_cq uresp = {}; ret = ib_copy_validate_udata_in(udata, ureq, rsvd0); if (ret) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 25f9738bd772..d50743f090bf 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1090,8 +1090,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, struct ib_device *ibdev = uctx->device; struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_ucontext *context = to_mucontext(uctx); - struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3; - struct mlx4_ib_alloc_ucontext_resp resp; + struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3 = {}; + struct mlx4_ib_alloc_ucontext_resp resp = {}; int err; if (!dev->ib_active) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 302f0a13051b..428811fa805b 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2777,7 +2777,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct mlx5_ib_pd *pd = to_mpd(ibpd); struct ib_device *ibdev = ibpd->device; - struct mlx5_ib_alloc_pd_resp resp; + struct mlx5_ib_alloc_pd_resp resp = {}; int err; u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; From 9492e70dc915b8717f6a2b6cf37ee45cee0b2f54 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 May 2026 21:09:39 -0300 Subject: [PATCH 031/148] RDMA: Replace memset with = {} pattern for ib_respond_udata() Most drivers do this already, but some open-code a memset. Switch all instances found. qedr_copy_qp_uresp() is already called with zeroed memory so that memset is redundant. Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/cxgb4/cq.c | 3 +-- drivers/infiniband/hw/cxgb4/qp.c | 6 ++---- drivers/infiniband/hw/erdma/erdma_verbs.c | 4 +--- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 12 ++++-------- drivers/infiniband/hw/qedr/verbs.c | 6 +----- drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 4 +--- 6 files changed, 10 insertions(+), 25 deletions(-) diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 47508df4cec0..d1517f2560b9 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -1004,7 +1004,7 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device); struct c4iw_cq *chp = to_c4iw_cq(ibcq); struct c4iw_create_cq ucmd; - struct c4iw_create_cq_resp uresp; + struct c4iw_create_cq_resp uresp = {}; int ret, wr_len; size_t memsize, hwentries; struct c4iw_mm_entry *mm, *mm2; @@ -1102,7 +1102,6 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (!mm2) goto err_free_mm; - memset(&uresp, 0, sizeof(uresp)); uresp.qid_mask = rhp->rdev.cqmask; uresp.cqid = chp->cq.cqid; uresp.size = chp->cq.size; diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index f9c7030ac6bf..e295f79e0cd3 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -2120,7 +2120,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, struct c4iw_pd *php; struct c4iw_cq *schp; struct c4iw_cq *rchp; - struct c4iw_create_qp_resp uresp; + struct c4iw_create_qp_resp uresp = {}; unsigned int sqsize, rqsize = 0; struct c4iw_ucontext *ucontext = rdma_udata_to_drv_context( udata, struct c4iw_ucontext, ibucontext); @@ -2242,7 +2242,6 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, goto err_free_sq_db_key; } } - memset(&uresp, 0, sizeof(uresp)); if (t4_sq_onchip(&qhp->wq.sq)) { ma_sync_key_mm = kmalloc_obj(*ma_sync_key_mm); if (!ma_sync_key_mm) { @@ -2686,7 +2685,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, struct c4iw_dev *rhp; struct c4iw_srq *srq = to_c4iw_srq(ib_srq); struct c4iw_pd *php; - struct c4iw_create_srq_resp uresp; + struct c4iw_create_srq_resp uresp = {}; struct c4iw_ucontext *ucontext; struct c4iw_mm_entry *srq_key_mm, *srq_db_key_mm; int rqsize; @@ -2764,7 +2763,6 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, ret = -ENOMEM; goto err_free_srq_key_mm; } - memset(&uresp, 0, sizeof(uresp)); uresp.flags = srq->flags; uresp.qid_mask = rhp->rdev.qpmask; uresp.srqid = srq->wq.qid; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index c8a35337ba51..b59c2e3a5306 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -996,7 +996,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, struct erdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct erdma_ucontext, ibucontext); struct erdma_ureq_create_qp ureq; - struct erdma_uresp_create_qp uresp; + struct erdma_uresp_create_qp uresp = {}; void *old_entry; int ret = 0; @@ -1048,8 +1048,6 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (ret) goto err_out_xa; - memset(&uresp, 0, sizeof(uresp)); - uresp.num_sqe = qp->attrs.sq_size; uresp.num_rqe = qp->attrs.rq_size; uresp.qp_id = QP_ID(qp); diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 083f23fc687b..d5fdbd7c8dea 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -586,11 +586,10 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, u64 db_page_addr; u64 dpp_page_addr = 0; u32 db_page_size; - struct ocrdma_alloc_pd_uresp rsp; + struct ocrdma_alloc_pd_uresp rsp = {}; struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct ocrdma_ucontext, ibucontext); - memset(&rsp, 0, sizeof(rsp)); rsp.id = pd->id; rsp.dpp_enabled = pd->dpp_enabled; db_page_addr = ocrdma_get_db_addr(dev, pd->id); @@ -930,13 +929,12 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, int status; struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct ocrdma_ucontext, ibucontext); - struct ocrdma_create_cq_uresp uresp; + struct ocrdma_create_cq_uresp uresp = {}; /* this must be user flow! */ if (!udata) return -EINVAL; - memset(&uresp, 0, sizeof(uresp)); uresp.cq_id = cq->id; uresp.page_size = PAGE_ALIGN(cq->len); uresp.num_pages = 1; @@ -1173,11 +1171,10 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, { int status; u64 usr_db; - struct ocrdma_create_qp_uresp uresp; + struct ocrdma_create_qp_uresp uresp = {}; struct ocrdma_pd *pd = qp->pd; struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); - memset(&uresp, 0, sizeof(uresp)); usr_db = dev->nic_info.unmapped_db + (pd->id * dev->nic_info.db_page_size); uresp.qp_id = qp->id; @@ -1730,9 +1727,8 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, struct ib_udata *udata) { int status; - struct ocrdma_create_srq_uresp uresp; + struct ocrdma_create_srq_uresp uresp = {}; - memset(&uresp, 0, sizeof(uresp)); uresp.rq_dbid = srq->rq.dbid; uresp.num_rq_pages = 1; uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 79190c5b8b50..1af908275ca7 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -690,9 +690,7 @@ static void qedr_db_recovery_del(struct qedr_dev *dev, static int qedr_copy_cq_uresp(struct qedr_cq *cq, struct ib_udata *udata, u32 db_offset) { - struct qedr_create_cq_uresp uresp; - - memset(&uresp, 0, sizeof(uresp)); + struct qedr_create_cq_uresp uresp = {}; uresp.db_offset = db_offset; uresp.icid = cq->icid; @@ -1283,8 +1281,6 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_udata *udata, struct qedr_create_qp_uresp *uresp) { - memset(uresp, 0, sizeof(*uresp)); - if (qedr_qp_has_sq(qp)) qedr_copy_sq_uresp(dev, uresp, qp); diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index e887f03a84d0..261f18a83685 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -82,15 +82,13 @@ static void usnic_ib_fw_string_to_u64(char *fw_ver_str, u64 *fw_ver) static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, struct ib_udata *udata) { - struct usnic_ib_create_qp_resp resp; + struct usnic_ib_create_qp_resp resp = {}; struct pci_dev *pdev; struct vnic_dev_bar *bar; struct usnic_vnic_res_chunk *chunk; struct usnic_ib_qp_grp_flow *default_flow; int i, err; - memset(&resp, 0, sizeof(resp)); - pdev = usnic_vnic_get_pdev(qp_grp->vf->vnic); if (!pdev) { usnic_err("Failed to get pdev of qp_grp %d\n", From 67464f388d52ec172be62c99fc43697437ffa384 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 16 May 2026 11:17:47 +0200 Subject: [PATCH 032/148] RDMA/cma: Constify struct configfs_item_operations and configfs_group_operations 'struct configfs_item_operations' and 'configfs_group_operations' are not modified in this driver. Constifying these structures moves some data to a read-only section, so increases overall security, especially when the structure holds some function pointers. On a x86_64, with allmodconfig: Before: ====== text data bss dec hex filename 6677 2776 64 9517 252d drivers/infiniband/core/cma_configfs.o After: ===== text data bss dec hex filename 6901 2552 64 9517 252d drivers/infiniband/core/cma_configfs.o Signed-off-by: Christophe JAILLET Link: https://patch.msgid.link/6acd9c8a79b868b5e541a7e080a6b4b145e4fd4f.1778923041.git.christophe.jaillet@wanadoo.fr Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/cma_configfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c index 819927ce4f0e..891e52afb8f4 100644 --- a/drivers/infiniband/core/cma_configfs.c +++ b/drivers/infiniband/core/cma_configfs.c @@ -255,7 +255,7 @@ static void release_cma_ports_group(struct config_item *item) cma_dev_group->ports = NULL; }; -static struct configfs_item_operations cma_ports_item_ops = { +static const struct configfs_item_operations cma_ports_item_ops = { .release = release_cma_ports_group }; @@ -264,7 +264,7 @@ static const struct config_item_type cma_ports_group_type = { .ct_owner = THIS_MODULE }; -static struct configfs_item_operations cma_device_item_ops = { +static const struct configfs_item_operations cma_device_item_ops = { .release = release_cma_dev }; @@ -327,7 +327,7 @@ static void drop_cma_dev(struct config_group *cgroup, struct config_item *item) config_item_put(item); } -static struct configfs_group_operations cma_subsys_group_ops = { +static const struct configfs_group_operations cma_subsys_group_ops = { .make_group = make_cma_dev, .drop_item = drop_cma_dev, }; From 9a79dcbedcc7a0330a227f23fbd220515c61d54c Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:33 +0530 Subject: [PATCH 033/148] RDMA/bnxt_re: Refactor bnxt_re_init_user_qp() The umem changes for CQ added a helper - bnxt_re_setup_sginfo(). Use the same helper for QP creation since we support only 4K pages for QP ring memory too. Add a new helper function bnxt_re_get_psn_bytes() to improve readability as this code will be updated in subsequent patches. Link: https://patch.msgid.link/r/20260519150041.7251-2-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 127 +++++++++++++---------- 1 file changed, 73 insertions(+), 54 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index ccb362d6d2e6..d50ec9751842 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1136,34 +1136,64 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, return 0; } +static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, + struct ib_umem *umem, + struct bnxt_qplib_sg_info *sginfo) +{ + unsigned long page_size; + + if (!umem) + return -EINVAL; + + page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); + if (!page_size || page_size != SZ_4K) + return -EINVAL; + + sginfo->umem = umem; + sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); + sginfo->pgsize = page_size; + sginfo->pgshft = __builtin_ctz(page_size); + return 0; +} + +static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *cntx, + struct bnxt_qplib_qp *qplib_qp, + struct bnxt_re_qp_req *ureq) +{ + int psn_sz, psn_nume; + + psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { + psn_nume = ureq->sq_slots; + } else { + psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / + sizeof(struct bnxt_qplib_sge)); + } + if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) + psn_nume = roundup_pow_of_two(psn_nume); + + return psn_nume * psn_sz; +} + static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, struct bnxt_re_qp_req *ureq) { struct bnxt_qplib_qp *qplib_qp; - int bytes = 0, psn_sz; struct ib_umem *umem; - int psn_nume; + int bytes; + int rc; qplib_qp = &qp->qplib_qp; bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); /* Consider mapping PSN search memory only for RC QPs. */ - if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { - psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? - sizeof(struct sq_psn_search_ext) : - sizeof(struct sq_psn_search); - if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { - psn_nume = ureq->sq_slots; - } else { - psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? - qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / - sizeof(struct bnxt_qplib_sge)); - } - if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) - psn_nume = roundup_pow_of_two(psn_nume); - bytes += (psn_nume * psn_sz); - } + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) + bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq); bytes = PAGE_ALIGN(bytes); umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes, @@ -1172,33 +1202,42 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, return PTR_ERR(umem); qp->sumem = umem; - qplib_qp->sq.sg_info.umem = umem; - qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; - qplib_qp->qp_handle = ureq->qp_handle; + rc = bnxt_re_setup_sginfo(rdev, qp->sumem, &qplib_qp->sq.sg_info); + if (rc) + goto fail; - if (!qp->qplib_qp.srq) { - bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); - bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) - goto rqfail; - qp->rumem = umem; - qplib_qp->rq.sg_info.umem = umem; - qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; + if (qp->qplib_qp.srq) + goto done; + + bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + goto fail; } + qp->rumem = umem; + rc = bnxt_re_setup_sginfo(rdev, qp->rumem, &qplib_qp->rq.sg_info); + if (rc) + goto rqfail; + +done: + qplib_qp->qp_handle = ureq->qp_handle; qplib_qp->dpi = &cntx->dpi; qplib_qp->is_user = true; return 0; + rqfail: + ib_umem_release(qp->rumem); + qp->rumem = NULL; + memset(&qplib_qp->rq.sg_info, 0, sizeof(qplib_qp->rq.sg_info)); +fail: ib_umem_release(qp->sumem); qp->sumem = NULL; memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); - - return PTR_ERR(umem); + return rc; } static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah @@ -3345,26 +3384,6 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) return ib_respond_empty_udata(udata); } -static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, - struct ib_umem *umem, - struct bnxt_qplib_sg_info *sginfo) -{ - unsigned long page_size; - - if (!umem) - return -EINVAL; - - page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); - if (!page_size || page_size != SZ_4K) - return -EINVAL; - - sginfo->umem = umem; - sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); - sginfo->pgsize = page_size; - sginfo->pgshft = __builtin_ctz(page_size); - return 0; -} - int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { From a3d27ad0184c81048a226476834d2ce8225dd1f8 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:34 +0530 Subject: [PATCH 034/148] RDMA/bnxt_re: Update rq depth for app allocated QPs For app allocated QPs, there's no need to add extra slots or to round up the slot count. Use 'max_recv_wr' count provided by the application as is. Link: https://patch.msgid.link/r/20260519150041.7251-3-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index d50ec9751842..c3db53af3ff6 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1475,7 +1475,8 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, - struct bnxt_re_ucontext *uctx) + struct bnxt_re_ucontext *uctx, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1500,12 +1501,16 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, init_attr->cap.max_recv_sge = rq->max_sge; rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge, dev_attr->max_qp_sges); - /* Allocate 1 more than what's provided so posting max doesn't - * mean empty. - */ - rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, - dev_attr->max_qp_wqes + 1, - uctx); + if (!fixed_que_attr) { + /* Allocate 1 more than what's provided so posting max doesn't + * mean empty. + */ + rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, + dev_attr->max_qp_wqes + 1, + uctx); + } else { + rq->max_wqe = init_attr->cap.max_recv_wr; + } rq->max_sw_wqe = rq->max_wqe; rq->q_full_delta = 0; rq->sg_info.pgsize = PAGE_SIZE; @@ -1676,6 +1681,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; + bool fixed_que_attr = false; struct bnxt_re_dev *rdev; struct bnxt_re_cq *cq; int rc = 0, qptype; @@ -1724,7 +1730,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, } /* Setup RQ/SRQ */ - rc = bnxt_re_init_rq_attr(qp, init_attr, uctx); + rc = bnxt_re_init_rq_attr(qp, init_attr, uctx, fixed_que_attr); if (rc) return rc; if (init_attr->qp_type == IB_QPT_GSI) From 6e27c30bfdcd691afb0375e0e3e4b1f4d66e2c23 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:35 +0530 Subject: [PATCH 035/148] RDMA/bnxt_re: Update sq depth for app allocated QPs For app allocated QPs, there's no need to reserve extra slots. The application accounts for this while allocating the SQ. Link: https://patch.msgid.link/r/20260519150041.7251-4-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index c3db53af3ff6..e5a8d3972a42 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1541,7 +1541,8 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1582,13 +1583,18 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, sq->max_sw_wqe = sq->max_wqe; } - sq->q_full_delta = diff + 1; - /* - * Reserving one slot for Phantom WQE. Application can - * post one extra entry in this case. But allowing this to avoid - * unexpected Queue full condition - */ - qplqp->sq.q_full_delta -= 1; + if (!fixed_que_attr) { + sq->q_full_delta = diff + 1; + /* + * Reserving one slot for Phantom WQE. Application can + * post one extra entry in this case. But allowing this to avoid + * unexpected Queue full condition + */ + qplqp->sq.q_full_delta -= 1; + } else { + sq->q_full_delta = 0; + } + qplqp->sq.sg_info.pgsize = PAGE_SIZE; qplqp->sq.sg_info.pgshft = PAGE_SHIFT; @@ -1737,7 +1743,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_rq_attr(qp); /* Setup SQ */ - rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq); + rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq, fixed_que_attr); if (rc) return rc; if (init_attr->qp_type == IB_QPT_GSI) From 9ce2a8c81c3099f68d371d040d77aab8cd1a9ed5 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:36 +0530 Subject: [PATCH 036/148] RDMA/bnxt_re: Update msn table size for app allocated QPs For app allocated QPs, the driver shouldn't use slots/round-up logic to compute the msn table size. The application handles this logic and computes 'sq_npsn' and passes it to the driver using a new uapi parameter. Link: https://patch.msgid.link/r/20260519150041.7251-5-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 61 +++++++++++++++--------- include/uapi/rdma/bnxt_re-abi.h | 1 + 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index e5a8d3972a42..942148c2d9f0 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1159,29 +1159,39 @@ static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, struct bnxt_re_ucontext *cntx, struct bnxt_qplib_qp *qplib_qp, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) { int psn_sz, psn_nume; - psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? - sizeof(struct sq_psn_search_ext) : - sizeof(struct sq_psn_search); - if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { - psn_nume = ureq->sq_slots; + if (rdev->dev_attr && + _is_host_msn_table(rdev->dev_attr->dev_cap_flags2)) + psn_sz = sizeof(struct sq_msn_search); + else + psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (!fixed_que_attr) { + if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { + psn_nume = ureq->sq_slots; + } else { + psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / + sizeof(struct bnxt_qplib_sge)); + } + if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) + psn_nume = roundup_pow_of_two(psn_nume); } else { - psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? - qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / - sizeof(struct bnxt_qplib_sge)); + psn_nume = ureq->sq_npsn; } - if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) - psn_nume = roundup_pow_of_two(psn_nume); return psn_nume * psn_sz; } static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) { struct bnxt_qplib_qp *qplib_qp; struct ib_umem *umem; @@ -1193,7 +1203,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); /* Consider mapping PSN search memory only for RC QPs. */ if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) - bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq); + bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr); bytes = PAGE_ALIGN(bytes); umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes, @@ -1647,7 +1657,9 @@ static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, return qptype; } -static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) +static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp, + bool fixed_que_attr, + struct bnxt_re_qp_req *req) { struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; struct bnxt_qplib_q *sq = &qplib_qp->sq; @@ -1670,12 +1682,17 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) /* Update msn tbl size */ if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) { - if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); - else - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2; + if (!fixed_que_attr) { + if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); + else + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) + / 2; + } else { + qplib_qp->msn_tbl_sz = req->sq_npsn; + } qplib_qp->msn = 0; } } @@ -1750,12 +1767,12 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); if (uctx) { /* This will update DPI and qp_handle */ - rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq); + rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr); if (rc) return rc; } - bnxt_re_qp_calculate_msn_psn_size(qp); + bnxt_re_qp_calculate_msn_psn_size(qp, fixed_que_attr, ureq); rc = bnxt_re_setup_qp_hwqs(qp); if (rc) diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index 40955eaba32e..db8400f2ce3b 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -135,6 +135,7 @@ struct bnxt_re_qp_req { __aligned_u64 qp_handle; __aligned_u64 comp_mask; __u32 sq_slots; + __u32 sq_npsn; }; struct bnxt_re_qp_resp { From c35b5fcc4ce81a1850d9d5bf9d616a7e2d69308e Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:37 +0530 Subject: [PATCH 037/148] RDMA/bnxt_re: Update hwq depth for app allocated QPs The hwq depth shouldn't be computed using slots/round-up logic for app allocated QPs, use the max_wqe value saved earlier. Link: https://patch.msgid.link/r/20260519150041.7251-6-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 942148c2d9f0..e0c98b3ec298 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1346,7 +1346,7 @@ static int bnxt_re_qp_alloc_init_xrrq(struct bnxt_re_qp *qp) return rc; } -static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) +static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp, bool fixed_que_attr) { struct bnxt_qplib_res *res = &qp->rdev->qplib_res; struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; @@ -1360,12 +1360,17 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) hwq_attr.res = res; hwq_attr.sginfo = &sq->sg_info; hwq_attr.stride = bnxt_qplib_get_stride(); - hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true); hwq_attr.aux_stride = qplib_qp->psn_sz; - hwq_attr.aux_depth = (qplib_qp->psn_sz) ? - bnxt_qplib_set_sq_size(sq, wqe_mode) : 0; - if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) + if (!fixed_que_attr) { + hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true); + hwq_attr.aux_depth = (qplib_qp->psn_sz) ? + bnxt_qplib_set_sq_size(sq, wqe_mode) : 0; + if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) + hwq_attr.aux_depth = qplib_qp->msn_tbl_sz; + } else { + hwq_attr.depth = sq->max_wqe; hwq_attr.aux_depth = qplib_qp->msn_tbl_sz; + } hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); if (rc) @@ -1376,6 +1381,9 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) CMDQ_CREATE_QP_SQ_LVL_SFT); sq->hwq.pg_sz_lvl = pg_sz_lvl; + if (qplib_qp->srq) + goto done; + hwq_attr.res = res; hwq_attr.sginfo = &rq->sg_info; hwq_attr.stride = bnxt_qplib_get_stride(); @@ -1392,6 +1400,7 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) CMDQ_CREATE_QP_RQ_LVL_SFT); rq->hwq.pg_sz_lvl = pg_sz_lvl; +done: if (qplib_qp->psn_sz) { rc = bnxt_re_qp_alloc_init_xrrq(qp); if (rc) @@ -1460,7 +1469,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; qp->qplib_qp.dpi = &rdev->dpi_privileged; - rc = bnxt_re_setup_qp_hwqs(qp); + rc = bnxt_re_setup_qp_hwqs(qp, false); if (rc) goto fail; @@ -1774,7 +1783,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_qp_calculate_msn_psn_size(qp, fixed_que_attr, ureq); - rc = bnxt_re_setup_qp_hwqs(qp); + rc = bnxt_re_setup_qp_hwqs(qp, fixed_que_attr); if (rc) goto free_umem; From 73607410f4f1b0d5c3d0af2d70a79c265482a0e3 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:38 +0530 Subject: [PATCH 038/148] RDMA/bnxt_re: Enhance dbr usecnt logic in doorbell uapis The current logic in the doorbell cleanup function is not sufficient for a change in a subsequent patch, that fails doorbell remove operation in some conditions. The cleanup should facilitate freeing of the dbr object when the caller may not retry the teardown operation (implicit teardown: process-exit/driver-removal). Extend this counter to use kref mechanism so that the dbr object gets freed (via kref callback) when there are no more references to it, rather than directly freeing it in the cleanup uapi. Link: https://patch.msgid.link/r/20260519150041.7251-7-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.h | 3 ++- drivers/infiniband/hw/bnxt_re/uapi.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 08f71a94d55d..13dac48ed453 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -167,7 +167,7 @@ struct bnxt_re_dbr_obj { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_re_user_mmap_entry *entry; - atomic_t usecnt; /* QPs using this dbr */ + struct kref usecnt; /* 1 (uobject) + n (QPs using this dbr) */ }; struct bnxt_re_flow { @@ -308,4 +308,5 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); struct bnxt_re_user_mmap_entry* bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset, enum bnxt_re_mmap_flag mmap_flag, u64 *offset); +void bnxt_re_dbr_kref_release(struct kref *ref); #endif /* __BNXT_RE_IB_VERBS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 3eaee7101615..b8fc8bfba2ad 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -369,6 +369,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a } obj->rdev = rdev; + kref_init(&obj->usecnt); uobj->object = obj; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE); @@ -391,15 +392,24 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a return ret; } +void bnxt_re_dbr_kref_release(struct kref *ref) +{ + struct bnxt_re_dbr_obj *obj = + container_of(ref, struct bnxt_re_dbr_obj, usecnt); + struct bnxt_re_dev *rdev = obj->rdev; + + rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); + bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + kfree(obj); +} + static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { struct bnxt_re_dbr_obj *obj = uobject->object; - struct bnxt_re_dev *rdev = obj->rdev; - rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); - bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + kref_put(&obj->usecnt, bnxt_re_dbr_kref_release); return 0; } From 54c01d98f480a5622780cb4d5e893e25dda70e57 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:39 +0530 Subject: [PATCH 039/148] RDMA/bnxt_re: Enhance dpi lifecycle logic in doorbell uapis If the DPI is freed when the dbr object is freed, but if the process has not unmapped the page yet, then the DPI slot could get reallocated to another process while the original process still has it mapped. To prevent this, save the DPI info in the mmap entry during dbr allocation and free the DPI slot from bnxt_re_mmap_free(), which enures that there are no references to it. This change is needed to support doorbell allocation to QPs in the next patch. Link: https://patch.msgid.link/r/20260519150041.7251-8-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 4 ++++ drivers/infiniband/hw/bnxt_re/ib_verbs.h | 2 ++ drivers/infiniband/hw/bnxt_re/uapi.c | 13 +++++++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index e0c98b3ec298..c706ba19e719 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -4943,6 +4943,10 @@ void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry) bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry, rdma_entry); + if (bnxt_entry->dpi_valid) + bnxt_qplib_free_uc_dpi(&bnxt_entry->uctx->rdev->qplib_res, + &bnxt_entry->dpi); + kfree(bnxt_entry); } diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 13dac48ed453..6a5bcc3fb289 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -161,6 +161,8 @@ struct bnxt_re_user_mmap_entry { struct bnxt_re_ucontext *uctx; u64 mem_offset; u8 mmap_flag; + bool dpi_valid; + struct bnxt_qplib_dpi dpi; }; struct bnxt_re_dbr_obj { diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index b8fc8bfba2ad..1d44d6225da0 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -368,6 +368,13 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a goto free_dpi; } + /* Save DPI info to the mmap entry so that bnxt_re_mmap_free() + * can free the DPI slot only after the last reference to the + * mmap entry is released. + */ + obj->entry->dpi = *dpi; + obj->entry->dpi_valid = true; + obj->rdev = rdev; kref_init(&obj->usecnt); uobj->object = obj; @@ -396,10 +403,12 @@ void bnxt_re_dbr_kref_release(struct kref *ref) { struct bnxt_re_dbr_obj *obj = container_of(ref, struct bnxt_re_dbr_obj, usecnt); - struct bnxt_re_dev *rdev = obj->rdev; + /* Drop the driver's reference to the mmap entry (_remove()). + * The DPI slot gets freed from bnxt_re_mmap_free() only + * when there's no VMA mapping reference to it. + */ rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); - bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); kfree(obj); } From 7e812673a948395675e9893e75b0fc60d19bbf27 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:40 +0530 Subject: [PATCH 040/148] RDMA/bnxt_re: Support doorbells for app allocated QPs App allocated QPs can use a separate doorbell for each QP. This doorbell region can be passed through a new driver specific DBR_HANDLE attribute, during QP creation. When this attribute is set, associate the QP with the given doorbell region. While the QP holds a reference to the dbr, the dbr itself cannot be destroyed and is rejected with EBUSY error. Link: https://patch.msgid.link/r/20260519150041.7251-9-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 35 ++++++++++++++++++++---- drivers/infiniband/hw/bnxt_re/ib_verbs.h | 1 + drivers/infiniband/hw/bnxt_re/uapi.c | 24 ++++++++++++++++ include/uapi/rdma/bnxt_re-abi.h | 4 +++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index c706ba19e719..a5583f0125ed 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1024,6 +1024,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (rc) ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); + if (qp->dbr_obj) + kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release); + if (rdma_is_kernel_res(&qp->ib_qp.res)) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); @@ -1191,7 +1194,8 @@ static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, struct bnxt_re_qp_req *ureq, - bool fixed_que_attr) + bool fixed_que_attr, + struct bnxt_re_dbr_obj *dbr_obj) { struct bnxt_qplib_qp *qplib_qp; struct ib_umem *umem; @@ -1234,8 +1238,11 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, goto rqfail; done: + if (dbr_obj) + qplib_qp->dpi = &dbr_obj->dpi; + else + qplib_qp->dpi = &cntx->dpi; qplib_qp->qp_handle = ureq->qp_handle; - qplib_qp->dpi = &cntx->dpi; qplib_qp->is_user = true; return 0; @@ -1709,7 +1716,8 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp, static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + struct bnxt_re_dbr_obj *dbr_obj) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1776,7 +1784,8 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); if (uctx) { /* This will update DPI and qp_handle */ - rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr); + rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr, + dbr_obj); if (rc) return rc; } @@ -1912,7 +1921,9 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev) int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { + struct bnxt_re_dbr_obj *dbr_obj = NULL; struct bnxt_qplib_dev_attr *dev_attr; + struct uverbs_attr_bundle *attrs; struct bnxt_re_ucontext *uctx; struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; @@ -1933,6 +1944,17 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0); if (rc) return rc; + + attrs = rdma_udata_to_uverbs_attr_bundle(udata); + if (uverbs_attr_is_valid(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) { + dbr_obj = uverbs_attr_get_obj(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE); + if (IS_ERR(dbr_obj)) + return PTR_ERR(dbr_obj); + kref_get(&dbr_obj->usecnt); + qp->dbr_obj = dbr_obj; + } } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1942,7 +1964,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } qp->rdev = rdev; - rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq); + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, + dbr_obj); if (rc) goto fail; @@ -2012,6 +2035,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); bnxt_re_qp_free_umem(qp); fail: + if (dbr_obj) + kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release); return rc; } diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 6a5bcc3fb289..ebc393e6da4f 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -96,6 +96,7 @@ struct bnxt_re_qp { struct bnxt_re_cq *scq; struct bnxt_re_cq *rcq; struct dentry *dentry; + struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */ }; struct bnxt_re_cq { diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 1d44d6225da0..9e68b4a7e952 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -418,6 +418,15 @@ static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, { struct bnxt_re_dbr_obj *obj = uobject->object; + /* If it is being destroyed explicitly while QPs still hold a + * reference (> 1), reject it with EBUSY. If no QP references + * or implicit teardown (process exit, driver removal), drop + * the uobject reference unconditionally. The object gets freed + * (bnxt_re_dbr_kref_release) when the usecnt goes to zero. + */ + if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1) + return -EBUSY; + kref_put(&obj->usecnt, bnxt_re_dbr_kref_release); return 0; } @@ -478,11 +487,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR)); +ADD_UVERBS_ATTRIBUTES_SIMPLE( + bnxt_re_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE, + BNXT_RE_OBJECT_DBR, + UVERBS_ACCESS_READ, + UA_OPTIONAL)); + +const struct uapi_definition bnxt_re_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create), + {}, +}; + const struct uapi_definition bnxt_re_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR), + UAPI_DEF_CHAIN(bnxt_re_create_qp_defs), {} }; diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index db8400f2ce3b..4da8cda337dc 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -138,6 +138,10 @@ struct bnxt_re_qp_req { __u32 sq_npsn; }; +enum bnxt_re_create_qp_attrs { + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE = UVERBS_ID_DRIVER_NS_WITH_UHW, +}; + struct bnxt_re_qp_resp { __u32 qpid; __u32 rsvd; From 47b730054f05a05acd497c138bf3255a5de1a973 Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 19 May 2026 20:30:41 +0530 Subject: [PATCH 041/148] RDMA/bnxt_re: Enable app allocated QPs The driver supports a new comp_mask: REQ_MASK_FIXED_QUE_ATTR. The application sets this comp_mask bit in the CREATE_QP ureq to indicate direct control of the QP. The driver goes through the required processing for app allocated QPs (previous patches). Only variable WQE mode is supported for these QPs. This patch removes an unused comp_mask: BNXT_RE_QP_REQ_MASK_VAR_WQE_SQ_SLOTS Link: https://patch.msgid.link/r/20260519150041.7251-10-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 19 +++++++++++++++---- include/uapi/rdma/bnxt_re-abi.h | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index a5583f0125ed..5dbfa49d1c6f 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1717,11 +1717,11 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, struct bnxt_re_qp_req *ureq, - struct bnxt_re_dbr_obj *dbr_obj) + struct bnxt_re_dbr_obj *dbr_obj, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; - bool fixed_que_attr = false; struct bnxt_re_dev *rdev; struct bnxt_re_cq *cq; int rc = 0, qptype; @@ -1741,6 +1741,13 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, return qptype; qplqp->type = (u8)qptype; qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx); + if (fixed_que_attr) { + if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE) + return -EOPNOTSUPP; + if (!ureq->sq_npsn || + ureq->sq_npsn > roundup_pow_of_two(ureq->sq_slots / 2)) + return -EINVAL; + } qplqp->dev_cap_flags = dev_attr->dev_cap_flags; qplqp->cctx = rdev->chip_ctx; if (init_attr->qp_type == IB_QPT_RC) { @@ -1925,6 +1932,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct bnxt_qplib_dev_attr *dev_attr; struct uverbs_attr_bundle *attrs; struct bnxt_re_ucontext *uctx; + bool fixed_que_attr = false; struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; struct bnxt_re_pd *pd; @@ -1941,7 +1949,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); if (udata) { - rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0); + rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR); if (rc) return rc; @@ -1955,6 +1964,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, kref_get(&dbr_obj->usecnt); qp->dbr_obj = dbr_obj; } + if (ureq.comp_mask & BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR) + fixed_que_attr = true; } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1965,7 +1976,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, qp->rdev = rdev; rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, - dbr_obj); + dbr_obj, fixed_que_attr); if (rc) goto fail; diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index 4da8cda337dc..a4599d7b736a 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -126,7 +126,7 @@ struct bnxt_re_resize_cq_req { }; enum bnxt_re_qp_mask { - BNXT_RE_QP_REQ_MASK_VAR_WQE_SQ_SLOTS = 0x1, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR = 0x1, }; struct bnxt_re_qp_req { From 992ad0c012402309dacdb54e4427a226ee9f23d5 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Sun, 10 May 2026 21:18:12 -0700 Subject: [PATCH 042/148] RDMA/rtrs: Use flexible array for client path stats Store the client path statistics in the RTRS client path allocation instead of allocating them separately. This ties the stats lifetime directly to the path and removes a separate allocation failure path. Keep freeing the per-CPU stats data separately, but do not free the embedded stats object from error paths or the stats kobject release handler. Link: https://patch.msgid.link/r/20260511041812.378030-1-rosenp@gmail.com Assisted-by: Codex:GPT-5.5 Signed-off-by: Rosen Penev Acked-by: Jack Wang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c | 2 -- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 13 ++----------- drivers/infiniband/ulp/rtrs/rtrs-clt.h | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c index 287e0ea43287..f8b833bd81ad 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c @@ -37,8 +37,6 @@ static void rtrs_clt_path_stats_release(struct kobject *kobj) stats = container_of(kobj, struct rtrs_clt_stats, kobj_stats); free_percpu(stats->pcpu_stats); - - kfree(stats); } static struct kobj_type ktype_stats = { diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index e351552733df..d34d7e5f34d6 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1536,7 +1536,7 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, int cpu; size_t total_con; - clt_path = kzalloc_obj(*clt_path); + clt_path = kzalloc_flex(*clt_path, stats, 1); if (!clt_path) goto err; @@ -1552,10 +1552,6 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, clt_path->s.con_num = total_con; clt_path->s.irq_con_num = con_num + 1; - clt_path->stats = kzalloc_obj(*clt_path->stats); - if (!clt_path->stats) - goto err_free_con; - mutex_init(&clt_path->init_mutex); uuid_gen(&clt_path->s.uuid); memcpy(&clt_path->s.dst_addr, path->dst, @@ -1583,7 +1579,7 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, clt_path->mp_skip_entry = alloc_percpu(typeof(*clt_path->mp_skip_entry)); if (!clt_path->mp_skip_entry) - goto err_free_stats; + goto err_free_con; for_each_possible_cpu(cpu) INIT_LIST_HEAD(per_cpu_ptr(clt_path->mp_skip_entry, cpu)); @@ -1596,8 +1592,6 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, err_free_percpu: free_percpu(clt_path->mp_skip_entry); -err_free_stats: - kfree(clt_path->stats); err_free_con: kfree(clt_path->s.con); err_free_path: @@ -2863,7 +2857,6 @@ struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, list_del_rcu(&clt_path->s.entry); rtrs_clt_close_conns(clt_path, true); free_percpu(clt_path->stats->pcpu_stats); - kfree(clt_path->stats); free_path(clt_path); goto close_all_path; } @@ -2873,7 +2866,6 @@ struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, list_del_rcu(&clt_path->s.entry); rtrs_clt_close_conns(clt_path, true); free_percpu(clt_path->stats->pcpu_stats); - kfree(clt_path->stats); free_path(clt_path); goto close_all_path; } @@ -3166,7 +3158,6 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt, rtrs_clt_remove_path_from_arr(clt_path); rtrs_clt_close_conns(clt_path, true); free_percpu(clt_path->stats->pcpu_stats); - kfree(clt_path->stats); free_path(clt_path); return err; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h index 986239ed2d3b..1305601a6251 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h @@ -142,12 +142,12 @@ struct rtrs_clt_path { u32 flags; struct kobject kobj; u8 for_new_clt; - struct rtrs_clt_stats *stats; /* cache hca_port and hca_name to display in sysfs */ u8 hca_port; char hca_name[IB_DEVICE_NAME_MAX]; struct list_head __percpu *mp_skip_entry; + struct rtrs_clt_stats stats[]; }; struct rtrs_clt_sess { From d28654518c8db5d06d27bd3211c0e9a70c18f7c2 Mon Sep 17 00:00:00 2001 From: Shiraz Saleem Date: Tue, 12 May 2026 02:40:56 -0700 Subject: [PATCH 043/148] RDMA/mana_ib: Use ib_get_eth_speed for reporting port speed Replace hardcoded IB_WIDTH_4X/IB_SPEED_EDR with ib_get_eth_speed() to report the actual link speed in mana_ib_query_port(). Fixes: 4bda1d5332ec ("RDMA/mana_ib: Implement port parameters") Link: https://patch.msgid.link/r/20260512094056.264827-1-kotaranov@linux.microsoft.com Signed-off-by: Shiraz Saleem Signed-off-by: Konstantin Taranov Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mana/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index ac5e75dd3494..c2f0b96f473d 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -600,8 +600,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } - props->active_width = IB_WIDTH_4X; - props->active_speed = IB_SPEED_EDR; + ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { props->gid_tbl_len = 16; From 5ebb3ed757be3e04cf803026004aa0beaeb13e9b Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Tue, 12 May 2026 18:38:52 +0000 Subject: [PATCH 044/148] RDMA/irdma: Fix out-of-bounds write in irdma_copy_user_pgaddrs The irdma_copy_user_pgaddrs function loops through all of the umem DMA blocks to populate the PBLEs and will stop when either the last DMA block is reached or palloc->total_cnt is reached. The issue is that the logic for checking palloc->total_cnt would only work for non-zero values. When irdma_setup_pbles is called with lvl==0, it calls irdma_copy_user_pgaddrs with palloc->total_cnt==0, which means the only way to break out of the loop is to reach the last umem DMA block, which means it could end up going beyond the fixed size of 4 iwmr->pgaddrmem array that is used in the lvl==0 case. In the case of QP/CQ/SRQ rings, the value of lvl is determined by a separate input (for example, req.cq_pages in the case of a CQ). So, we must perform explicit checking to ensure we don't overflow the pgaddrmem array if the user provides a umem that consists of more blocks than their provided req.cq_pages. Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs") Link: https://patch.msgid.link/r/20260512183852.614045-1-jmoroni@google.com Signed-off-by: Jacob Moroni Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/verbs.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 79e72a457e79..3f4811bb5514 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2765,10 +2765,11 @@ static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, * irdma_copy_user_pgaddrs - copy user page address to pble's os locally * @iwmr: iwmr for IB's user page addresses * @pbl: ple pointer to save 1 level or 0 level pble + * @pbl_len: Max number of PBL entries to populate * @level: indicated level 0, 1 or 2 */ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, - enum irdma_pble_level level) + u32 pbl_len, enum irdma_pble_level level) { struct ib_umem *region = iwmr->region; struct irdma_pbl *iwpbl = &iwmr->iwpbl; @@ -2776,7 +2777,9 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, struct irdma_pble_info *pinfo; struct ib_block_iter biter; u32 idx = 0; - u32 pbl_cnt = 0; + + if (!pbl_len) + return; pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; @@ -2785,7 +2788,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { *pbl = rdma_block_iter_dma_address(&biter); - if (++pbl_cnt == palloc->total_cnt) + if (!--pbl_len) break; pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); } @@ -2861,6 +2864,7 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, u64 *pbl; int status; enum irdma_pble_level level = PBLE_LEVEL_1; + u32 pbl_len; if (lvl) { status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, @@ -2868,16 +2872,18 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, if (status) return status; + pbl_len = palloc->total_cnt; iwpbl->pbl_allocated = true; level = palloc->level; pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf; pbl = pinfo->addr; } else { + pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR; pbl = iwmr->pgaddrmem; } - irdma_copy_user_pgaddrs(iwmr, pbl, level); + irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); if (lvl) iwmr->pgaddrmem[0] = *pbl; From 35744ab3d03c5fca8c1752f53fc8fc674e14c561 Mon Sep 17 00:00:00 2001 From: Zhu Yanjun Date: Fri, 15 May 2026 02:25:37 +0200 Subject: [PATCH 045/148] RDMA/rxe: Fix a use-after-free problem in rxe_mmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rxe_mmap() removes a rxe_mmap_info struct from the pending_mmaps list and releases pending_lock while the struct's kref is still at 1: list_del_init(&ip->pending_mmaps); spin_unlock_bh(&rxe->pending_lock); /* ref == 1, no lock held */ ret = remap_vmalloc_range(vma, ip->obj, 0); /* walks PTEs */ [...] rxe_vma_open(vma); /* kref_get, ref → 2 */ remap_vmalloc_range_partial() walks PTEs without any lock. A concurrent DESTROY_CQ ioctl on another CPU calls: kref_put(&q->ip->ref, rxe_mmap_release) /* ref 1→0 */ vfree(ip->obj) /* clears vmalloc PTEs mid-walk */ kfree(ip) /* frees rxe_mmap_info */ This yields: 1. Kernel crash, vmalloc_to_page() returns NULL when vfree wins the per-PTE race -> vm_insert_page(NULL) → GPF in validate_page_before_insert 2. Page UAF, vmalloc_to_page() reads a stale PTE before vfree clears it. User VMA holds a PTE to a free'd page which might eventually get reallocated later by vmalloc which allows the attacker to get a clean page-level UAF. It is worth noting that even though a page-level UAF is possible given the strong primitive, it is statistically very difficult to achieve given the very short time window (after the last insert_page and before the kref_get). The call trace are as below: Oops: general protection fault, probably for non-canonical address 0xdffffc0000000001: 0000 [#1] SMP KASAN NOPTI KASAN: null-ptr-deref in range [0x0000000000000008-0x000000000000000f] CPU: 0 UID: 1000 PID: 413 Comm: poc Not tainted 7.0.0-rc5-dirty #28 PREEMPT(lazy) Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.15.0-1 04/01/2014 RIP: 0010:validate_page_before_insert+0x32/0x300 Code: e5 41 57 41 56 49 89 fe 41 55 41 54 53 48 89 f3 e8 93 b5 a3 ff 48 8d 7b 08 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 <80> 3c 02 00 0f 85 7b 02 00 00 4c 8b 63 08 31 ff 4d 89 e5 41 83 e5 RSP: 0018:ffff88811b15f2f0 EFLAGS: 00000202 RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 0000000000000000 RDX: 0000000000000001 RSI: 0000000000000000 RDI: 0000000000000008 RBP: ffff88811b15f318 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000000 R12: ffff8881181eee00 R13: 0000000000000000 R14: ffff8881181eee00 R15: ffff8881181eee20 FS: 00007b1e000f76c0(0000) GS:ffff8884268e0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007b1e00a24ac0 CR3: 0000000116eb3000 CR4: 00000000000006f0 Call Trace: insert_page+0x8f/0x190 ? __pfx_insert_page+0x10/0x10 ? kasan_save_alloc_info+0x38/0x60 vm_insert_page+0x2e7/0x400 remap_vmalloc_range_partial+0x212/0x3e0 remap_vmalloc_range+0x6e/0xb0 ? __kasan_check_write+0x14/0x30 rxe_mmap+0x2e9/0x5d0 ib_uverbs_mmap+0x1ad/0x2c0 __mmap_region+0x12c2/0x2ad0 ? __pfx___mmap_region+0x10/0x10 ? __sanitizer_cov_trace_switch+0x58/0xb0 ? mas_prev_slot+0x360/0x39c0 ? __sanitizer_cov_trace_switch+0x58/0xb0 ? mas_next_slot+0x1e5b/0x2f40 ? __sanitizer_cov_trace_cmp8+0x18/0x30 ? unmapped_area_topdown+0x4dd/0x610 ? kfree+0x1b1/0x440 ? free_cpumask_var+0x16/0x30 ? __kasan_slab_free+0x7d/0xa0 ? __sanitizer_cov_trace_cmp8+0x18/0x30 mmap_region+0x2e6/0x3c0 do_mmap+0xa3e/0x12a0 ? __pfx_do_mmap+0x10/0x10 ? __kasan_check_write+0x14/0x30 ? down_write_killable+0xba/0x160 ? __pfx_down_write_killable+0x10/0x10 ? __sanitizer_cov_trace_cmp4+0x16/0x30 vm_mmap_pgoff+0x2d4/0x4a0 ? __pfx_vm_mmap_pgoff+0x10/0x10 ? fget+0x1bf/0x270 ksys_mmap_pgoff+0x40c/0x690 ? __sanitizer_cov_trace_const_cmp4+0x16/0x30 ? __pfx_ksys_mmap_pgoff+0x10/0x10 ? __kasan_check_write+0x14/0x30 ? _raw_spin_trylock+0xbb/0x130 ? __pfx__raw_spin_trylock+0x10/0x10 __x64_sys_mmap+0x135/0x1e0 x64_sys_call+0x1c14/0x2790 do_syscall_64+0xd2/0x1050 ? rcu_core+0x352/0x7d0 ? rcu_core_si+0xe/0x20 ? handle_softirqs+0x1aa/0x650 ? __sanitizer_cov_trace_cmp4+0x16/0x30 ? fpregs_assert_state_consistent+0xe1/0x160 ? irqentry_exit+0xb1/0x670 entry_SYSCALL_64_after_hwframe+0x76/0x7e Link: https://patch.msgid.link/r/20260515002537.6209-1-yanjun.zhu@linux.dev Reported-and-tested-by: nasm Suggested-by: nasm Fixes: 8700e3e7c485 ("Soft RoCE driver") Signed-off-by: Zhu Yanjun Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_mmap.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c index db380302149e..7f723a2f3700 100644 --- a/drivers/infiniband/sw/rxe/rxe_mmap.c +++ b/drivers/infiniband/sw/rxe/rxe_mmap.c @@ -93,18 +93,31 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) goto done; found_it: + /* + * Increment refcount and check whether it is being freed atm while + * holding lock to prevent UAF + */ + if (!kref_get_unless_zero(&ip->ref)) { + spin_unlock_bh(&rxe->pending_lock); + ret = -ENXIO; + goto done; + } + list_del_init(&ip->pending_mmaps); spin_unlock_bh(&rxe->pending_lock); + vma->vm_ops = &rxe_vm_ops; + vma->vm_private_data = ip; + ret = remap_vmalloc_range(vma, ip->obj, 0); if (ret) { + vma->vm_private_data = NULL; + vma->vm_ops = NULL; + kref_put(&ip->ref, rxe_mmap_release); rxe_dbg_dev(rxe, "err %d from remap_vmalloc_range\n", ret); goto done; } - vma->vm_ops = &rxe_vm_ops; - vma->vm_private_data = ip; - rxe_vma_open(vma); done: return ret; } From 9a8826fdfbcd7ed2ccf745f5d54208358d939def Mon Sep 17 00:00:00 2001 From: Guangshuo Li Date: Mon, 18 May 2026 10:19:10 +0800 Subject: [PATCH 046/148] IB/mlx4: Fix refcount leak in add_port() error path After kobject_init_and_add(), the lifetime of the embedded struct kobject is expected to be managed through the kobject core reference counting. In add_port(), failure paths after kobject_init_and_add() must not free struct mlx4_port directly, because the embedded kobject is then managed by the kobject core. Freeing it directly leaves the kobject reference counting unbalanced and can lead to incorrect lifetime handling. Allocate the pkey and gid attribute arrays before kobject_init_and_add(), so failures before kobject initialization can be handled by directly freeing the allocated memory. Once kobject_init_and_add() has been called, unwind later failures by removing any successfully created sysfs groups, calling kobject_del(), and then releasing the embedded kobject with kobject_put(). Fixes: c1e7e466120b ("IB/mlx4: Add iov directory in sysfs under the ib device") Link: https://patch.msgid.link/r/20260518021910.972900-1-lgs201920130244@gmail.com Signed-off-by: Guangshuo Li Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx4/sysfs.c | 45 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c index b8fa4ecfc961..e688ad66a895 100644 --- a/drivers/infiniband/hw/mlx4/sysfs.c +++ b/drivers/infiniband/hw/mlx4/sysfs.c @@ -636,12 +636,6 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) p->port_num = port_num; p->slave = slave; - ret = kobject_init_and_add(&p->kobj, &port_type, - kobject_get(dev->dev_ports_parent[slave]), - "%d", port_num); - if (ret) - goto err_alloc; - p->pkey_group.name = "pkey_idx"; p->pkey_group.attrs = alloc_group_attrs(show_port_pkey, @@ -649,13 +643,9 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) dev->dev->caps.pkey_table_len[port_num]); if (!p->pkey_group.attrs) { ret = -ENOMEM; - goto err_alloc; + goto err_free_port; } - ret = sysfs_create_group(&p->kobj, &p->pkey_group); - if (ret) - goto err_free_pkey; - p->gid_group.name = "gid_idx"; p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); if (!p->gid_group.attrs) { @@ -663,28 +653,47 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) goto err_free_pkey; } + ret = kobject_init_and_add(&p->kobj, &port_type, + kobject_get(dev->dev_ports_parent[slave]), + "%d", port_num); + if (ret) + goto err_put; + + ret = sysfs_create_group(&p->kobj, &p->pkey_group); + if (ret) + goto err_del; + ret = sysfs_create_group(&p->kobj, &p->gid_group); if (ret) - goto err_free_gid; + goto err_remove_pkey; ret = add_vf_smi_entries(p); if (ret) - goto err_free_gid; + goto err_remove_gid; list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); return 0; -err_free_gid: - kfree(p->gid_group.attrs[0]); - kfree(p->gid_group.attrs); +err_remove_gid: + sysfs_remove_group(&p->kobj, &p->gid_group); + +err_remove_pkey: + sysfs_remove_group(&p->kobj, &p->pkey_group); + +err_del: + kobject_del(&p->kobj); + +err_put: + kobject_put(dev->dev_ports_parent[slave]); + kobject_put(&p->kobj); + return ret; err_free_pkey: for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) kfree(p->pkey_group.attrs[i]); kfree(p->pkey_group.attrs); -err_alloc: - kobject_put(dev->dev_ports_parent[slave]); +err_free_port: kfree(p); return ret; } From 3f19c2a3852e6ba75f3e92dd5edc4e07f3d07f4a Mon Sep 17 00:00:00 2001 From: Lianfa Weng Date: Wed, 20 May 2026 13:57:58 +0800 Subject: [PATCH 047/148] RDMA/hns: Fix warning in poll cq direct mode CQs allocated by ib_alloc_cq() always have a comp_handler. Though in direct mode this handler is never expected to be called, it is still called when the driver is reset, triggering the following WARN_ONCE(): Call trace: ib_cq_completion_direct+0x38/0x60 hns_roce_cq_completion+0x54/0x90 (hns_roce_hw_v2] hns_roce_handle_device_err+Ox1c8/0x340 [hns_roce_hw_v2] hns_roce_hw_v2_uninit_instance.constprop.0+0x34/0x70 [hns_roce_hw_v2] hns_roce_hw_v2_reset_notify+0xc4/0xe0 [hns_roce_hw_v2] hclge_notify_roce_client+0x60/0xbc [hclge] hclge_reset_rebuild+0x48/0x34c [hclge] hclge_reset_subtask+0xcc/0xec [hclge] hclge_reset_service_task+0x80/0x160 [hclge] hclge_service_task+0x50/0x80 (hclge] process_one_work+0x1cc/0x4d0 worker_thread+0x154/0x414 kthread+0x104/0x144 ret_from_fork+0x10/0x18 Fixes: f295e4cece5c ("RDMA/hns: Delete unnecessary callback functions for cq") Link: https://patch.msgid.link/r/20260520055759.2354037-3-huangjunxian6@hisilicon.com Signed-off-by: Lianfa Weng Signed-off-by: Junxian Huang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index c17ff5347a01..77bad9f5d482 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -1112,7 +1112,7 @@ static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) unsigned long flags; spin_lock_irqsave(&hr_cq->lock, flags); - if (cq->comp_handler) { + if (cq->comp_handler && hr_cq->ib_cq.poll_ctx != IB_POLL_DIRECT) { if (!hr_cq->is_armed) { hr_cq->is_armed = 1; list_add_tail(&hr_cq->node, cq_list); From bbd97d71e53e551890e4115ad9de46b5f2ac0858 Mon Sep 17 00:00:00 2001 From: Lianfa Weng Date: Wed, 20 May 2026 13:57:59 +0800 Subject: [PATCH 048/148] RDMA/hns: Fix log flood after cmd_mbox failure hns_roce_cmd_mbox() is the command interface between driver and hardware. When hardware is abnormal, the unlimited error printings after hns_roce_cmd_mbox() failure will cause log flood and even system crash. Replace ibdev_err() and ibdev_warn() with their ratelimited versions in the error handling path after hns_roce_cmd_mbox() (and its wrappers hns_roce_create_hw_ctx/hns_roce_destroy_hw_ctx) fails. Fixes: 9a4435375cd1 ("IB/hns: Add driver files for hns RoCE driver") Link: https://patch.msgid.link/r/20260520055759.2354037-4-huangjunxian6@hisilicon.com Signed-off-by: Lianfa Weng Signed-off-by: Junxian Huang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_cq.c | 6 +++--- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 18 +++++++++--------- drivers/infiniband/hw/hns/hns_roce_mr.c | 6 +++--- drivers/infiniband/hw/hns/hns_roce_srq.c | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 24de651f735e..1dd0efb5620d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -174,9 +174,9 @@ static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC, hr_cq->cqn); if (ret) - ibdev_err(ibdev, - "failed to send create cmd for CQ(0x%lx), ret = %d.\n", - hr_cq->cqn, ret); + ibdev_err_ratelimited(ibdev, + "failed to send create cmd for CQ(0x%lx), ret = %d.\n", + hr_cq->cqn, ret); hns_roce_free_cmd_mailbox(hr_dev, mailbox); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 4afd7d6ae3ca..332a4816f2ca 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -6193,9 +6193,9 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn); hns_roce_free_cmd_mailbox(hr_dev, mailbox); if (ret) - ibdev_err(&hr_dev->ib_dev, - "failed to handle cmd of modifying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to handle cmd of modifying SRQ, ret = %d.\n", + ret); } out: @@ -6221,9 +6221,9 @@ static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC, srq->srqn); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd of querying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd of querying SRQ, ret = %d.\n", + ret); goto out; } @@ -6329,9 +6329,9 @@ static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, key_to_hw_index(key)); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd when querying MPT, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd when querying MPT, ret = %d.\n", + ret); goto err_mailbox; } diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 896af1828a38..e8a9e7d8f267 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -173,7 +173,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx & (hr_dev->caps.num_mtpts - 1)); if (ret) { - dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + dev_err_ratelimited(dev, "failed to create mpt, ret = %d.\n", ret); goto err_page; } @@ -315,7 +315,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT, mtpt_idx); if (ret) - ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); + ibdev_warn_ratelimited(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); mr->enabled = 0; mr->iova = virt_addr; @@ -346,7 +346,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx); if (ret) { - ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret); + ibdev_err_ratelimited(ib_dev, "failed to create MPT, ret = %d.\n", ret); goto free_cmd_mbox; } diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 8644c3916367..00552a08f21a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -103,7 +103,7 @@ static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ, srq->srqn); if (ret) - ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, "failed to config SRQC, ret = %d.\n", ret); err_mbox: hns_roce_free_cmd_mailbox(hr_dev, mailbox); From 4fbc8230009f5b1bcd13cc74c5a6a43ddba141fd Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 20 May 2026 18:45:45 +0800 Subject: [PATCH 049/148] RDMA/counter: Fix num_counters leak on bind_qp failure in alloc_and_bind() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When __rdma_counter_bind_qp() fails in alloc_and_bind(), the error path jumps to err_mode which frees the counter without decrementing port_counter->num_counters. The only place that decrements is rdma_counter_free(), which is unreachable since the counter was never successfully bound. This leak accumulates across repeated failures, permanently preventing the port from switching to AUTO mode (-EBUSY in __counter_set_mode()) and blocking the MANUAL→NONE auto-revert in rdma_counter_free(). When the mode was NONE before the call, the MANUAL mode set by __counter_set_mode() also leaks since the revert logic is never reached. Add an err_bind label between the num_counters increment and the existing err_mode label. It decrements num_counters and mirrors the MANUAL→NONE revert from rdma_counter_free(), ensuring the port state is fully restored on bind failure. Link: https://patch.msgid.link/r/20260520104546.1776253-2-cuitao@kylinos.cn Signed-off-by: Tao Cui Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/counters.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index c3aa6d7fc66b..5ddd607d5fbe 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -198,12 +198,20 @@ static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port, ret = __rdma_counter_bind_qp(counter, qp, port); if (ret) - goto err_mode; + goto err_bind; rdma_restrack_parent_name(&counter->res, &qp->res); rdma_restrack_add(&counter->res); return counter; +err_bind: + mutex_lock(&port_counter->lock); + port_counter->num_counters--; + if (!port_counter->num_counters && + port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL) + __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, + false); + mutex_unlock(&port_counter->lock); err_mode: rdma_free_hw_stats_struct(counter->stats); err_stats: From b86fd95805a7bd4c5b9465c9e7f75e45bbe7eb6f Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 20 May 2026 18:45:46 +0800 Subject: [PATCH 050/148] RDMA/counter: Fix incorrect port index in rdma_counter_init() error cleanup The error cleanup loop in rdma_counter_init() iterates with variable 'i' but accesses dev->port_data[port] instead of dev->port_data[i]. This causes the failed port's hstats to be freed multiple times while leaking hstats of previously initialized ports. Fixes: 56594ae1d250 ("RDMA/core: Annotate destroy of mutex to ensure that it is released as unlocked") Link: https://patch.msgid.link/r/20260520104546.1776253-3-cuitao@kylinos.cn Signed-off-by: Tao Cui Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/counters.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index 5ddd607d5fbe..a9e189194c13 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -669,7 +669,7 @@ void rdma_counter_init(struct ib_device *dev) fail: for (i = port; i >= rdma_start_port(dev); i--) { - port_counter = &dev->port_data[port].port_counter; + port_counter = &dev->port_data[i].port_counter; rdma_free_hw_stats_struct(port_counter->hstats); port_counter->hstats = NULL; mutex_destroy(&port_counter->lock); From 33d1117eb506dbea0fc0995711686179aed22ff2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 25 May 2026 22:22:37 -0300 Subject: [PATCH 051/148] RDMA/core: Do not compile ib_core_uverbs without USER_ACCESS Remove the entire ib_core_uverbs.c from the build if CONFIG_INFINIBAND_USER_ACCESS is not set. These functions are only used to support uverbs and are never callable even if they happen to get linked in. Provide inlines for the missing ones to return errors to further push code elimination in drivers. Link: https://patch.msgid.link/r/1-v3-43aba1969751+1988-ib_uverbs_support_ko_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/Kconfig | 4 ++ drivers/infiniband/core/Makefile | 3 +- drivers/infiniband/core/ib_core_uverbs.c | 8 ++- drivers/infiniband/core/rdma_core.h | 3 -- include/rdma/ib_verbs.h | 67 +++++++++++++++++++++--- include/rdma/uverbs_ioctl.h | 13 +++-- 6 files changed, 78 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index a7e3f29dc037..086195758a8a 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -37,6 +37,10 @@ config INFINIBAND_USER_ACCESS libibverbs, libibcm and a hardware driver library from rdma-core . +config INFINIBAND_USER_ACCESS_CORE + bool + default y if INFINIBAND_USER_ACCESS != n + config INFINIBAND_USER_MEM bool depends on INFINIBAND_USER_ACCESS != n diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index dce798d8cfe6..8c2ba2ce0351 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -11,13 +11,14 @@ ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \ device.o cache.o netlink.o \ roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \ multicast.o mad.o smi.o agent.o mad_rmpp.o \ - nldev.o restrack.o counters.o ib_core_uverbs.o \ + nldev.o restrack.o counters.o \ trace.o lag.o iter.o frmr_pools.o ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o +ib_core-$(CONFIG_INFINIBAND_USER_ACCESS_CORE) += ib_core_uverbs.o ib_cm-y := cm.o cm_trace.o diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 8a0e6fa2a528..0acb0d4967cb 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -398,7 +398,7 @@ EXPORT_SYMBOL(rdma_user_mmap_entry_insert); * The struct ib_device that is handling the uverbs call. Must not be called if * udata is NULL. The result can be NULL. */ -struct ib_device *rdma_udata_to_dev(struct ib_udata *udata) +static struct ib_device *rdma_udata_to_dev(struct ib_udata *udata) { struct uverbs_attr_bundle *bundle = rdma_udata_to_uverbs_attr_bundle(udata); @@ -415,10 +415,9 @@ struct ib_device *rdma_udata_to_dev(struct ib_udata *udata) return srcu_dereference(bundle->ufile->device->ib_dev, &bundle->ufile->device->disassociate_srcu); } -EXPORT_SYMBOL(rdma_udata_to_dev); -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) -uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata) +typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs); +static uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata) { struct uverbs_attr_bundle *bundle = rdma_udata_to_uverbs_attr_bundle(udata); @@ -502,4 +501,3 @@ int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len) return -EFAULT; } EXPORT_SYMBOL(_ib_respond_udata); -#endif diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 269b393799ab..55f1e3558856 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -151,9 +151,6 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, unsigned int num_attrs); void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile); -typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs); -uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata); - extern const struct uapi_definition uverbs_def_obj_async_fd[]; extern const struct uapi_definition uverbs_def_obj_counters[]; extern const struct uapi_definition uverbs_def_obj_cq[]; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 9dd76f489a0b..01e0eea0703f 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -3101,6 +3101,7 @@ void ib_set_client_data(struct ib_device *device, struct ib_client *client, void ib_set_device_ops(struct ib_device *device, const struct ib_device_ops *ops); +#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma, unsigned long pfn, unsigned long size, pgprot_t prot, struct rdma_user_mmap_entry *entry); @@ -3112,13 +3113,7 @@ int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext, size_t length, u32 min_pgoff, u32 max_pgoff); -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) void rdma_user_mmap_disassociate(struct ib_device *device); -#else -static inline void rdma_user_mmap_disassociate(struct ib_device *device) -{ -} -#endif static inline int rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext, @@ -3138,6 +3133,66 @@ rdma_user_mmap_entry_get(struct ib_ucontext *ucontext, void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry); void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry); +#else +static inline int rdma_user_mmap_io(struct ib_ucontext *ucontext, + struct vm_area_struct *vma, + unsigned long pfn, unsigned long size, + pgprot_t prot, + struct rdma_user_mmap_entry *entry) +{ + return -EINVAL; +} + +static inline int +rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext, + struct rdma_user_mmap_entry *entry, size_t length) +{ + return -EINVAL; +} + +static inline int +rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext, + struct rdma_user_mmap_entry *entry, + size_t length, u32 min_pgoff, u32 max_pgoff) +{ + return -EINVAL; +} + +static inline void rdma_user_mmap_disassociate(struct ib_device *device) +{ +} + +static inline int +rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext, + struct rdma_user_mmap_entry *entry, + size_t length, u32 pgoff) +{ + return -EINVAL; +} + +static inline struct rdma_user_mmap_entry * +rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext, + unsigned long pgoff) +{ + return NULL; +} + +static inline struct rdma_user_mmap_entry * +rdma_user_mmap_entry_get(struct ib_ucontext *ucontext, + struct vm_area_struct *vma) +{ + return NULL; +} + +static inline void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry) +{ +} + +static inline void +rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) +{ +} +#endif static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len) { diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index c89428030d61..9d7575d999e1 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -668,8 +668,6 @@ rdma_udata_to_uverbs_attr_bundle(struct ib_udata *udata) (udata ? container_of(rdma_udata_to_uverbs_attr_bundle(udata)->context, \ drv_dev_struct, member) : (drv_dev_struct *)NULL) -struct ib_device *rdma_udata_to_dev(struct ib_udata *udata); - #define IS_UVERBS_COPY_ERR(_ret) ((_ret) && (_ret) != -ENOENT) static inline const struct uverbs_attr *uverbs_attr_get(const struct uverbs_attr_bundle *attrs_bundle, @@ -902,6 +900,8 @@ int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req, size_t kernel_size, size_t minimum_size); int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len); +int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm, + u64 valid_cm); #else static inline int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, @@ -971,6 +971,12 @@ static inline int _ib_respond_udata(struct ib_udata *udata, const void *src, { return -EINVAL; } + +static inline int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, + u64 req_cm, u64 valid_cm) +{ + return -EINVAL; +} #endif #define uverbs_get_const_signed(_to, _attrs_bundle, _idx) \ @@ -1051,9 +1057,6 @@ uverbs_get_raw_fd(int *to, const struct uverbs_attr_bundle *attrs_bundle, _ib_copy_validate_udata_in(_udata, &(_req), sizeof(_req), \ offsetofend(typeof(_req), _end_member)) -int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm, - u64 valid_cm); - /** * ib_copy_validate_udata_in_cm - Copy the req structure and check the comp_mask * @_udata: The system calls ib_udata struct From ab2d2b2872aed03cbf0b7c85eeb45fda34d8059b Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 25 May 2026 22:22:38 -0300 Subject: [PATCH 052/148] RDMA/core: Move many of the little EXPORTs from uverbs_ioctl into ib_core_uverbs Not as many drivers need these functions but it does free efa from the ib_uverbs.ko dependency and follows the general design better. Link: https://patch.msgid.link/r/2-v3-43aba1969751+1988-ib_uverbs_support_ko_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/ib_core_uverbs.c | 218 +++++++++++++++++++++ drivers/infiniband/core/uverbs.h | 15 ++ drivers/infiniband/core/uverbs_ioctl.c | 204 ------------------- drivers/infiniband/core/uverbs_main.c | 24 --- drivers/infiniband/core/uverbs_std_types.c | 6 - 5 files changed, 233 insertions(+), 234 deletions(-) diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 0acb0d4967cb..b4fc693a3bd8 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -501,3 +501,221 @@ int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len) return -EFAULT; } EXPORT_SYMBOL(_ib_respond_udata); + +/* + * Must be called with the ufile->device->disassociate_srcu held, and the lock + * must be held until use of the ucontext is finished. + */ +struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile) +{ + /* + * We do not hold the hw_destroy_rwsem lock for this flow, instead + * srcu is used. It does not matter if someone races this with + * get_context, we get NULL or valid ucontext. + */ + struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext); + + if (!srcu_dereference(ufile->device->ib_dev, + &ufile->device->disassociate_srcu)) + return ERR_PTR(-EIO); + + if (!ucontext) + return ERR_PTR(-EINVAL); + + return ucontext; +} +EXPORT_SYMBOL(ib_uverbs_get_ucontext_file); + +int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) +{ + return 0; +} +EXPORT_SYMBOL(uverbs_destroy_def_handler); + +/** + * _uverbs_alloc() - Quickly allocate memory for use with a bundle + * @bundle: The bundle + * @size: Number of bytes to allocate + * @flags: Allocator flags + * + * The bundle allocator is intended for allocations that are connected with + * processing the system call related to the bundle. The allocated memory is + * always freed once the system call completes, and cannot be freed any other + * way. + * + * This tries to use a small pool of pre-allocated memory for performance. + */ +__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, + gfp_t flags) +{ + struct bundle_priv *pbundle = + container_of(&bundle->hdr, struct bundle_priv, bundle); + size_t new_used; + void *res; + + if (check_add_overflow(size, pbundle->internal_used, &new_used)) + return ERR_PTR(-EOVERFLOW); + + if (new_used > pbundle->internal_avail) { + struct bundle_alloc_head *buf; + + buf = kvmalloc_flex(*buf, data, size, flags); + if (!buf) + return ERR_PTR(-ENOMEM); + buf->next = pbundle->allocated_mem; + pbundle->allocated_mem = buf; + return buf->data; + } + + res = (void *)pbundle->internal_buffer + pbundle->internal_used; + pbundle->internal_used = + ALIGN(new_used, sizeof(*pbundle->internal_buffer)); + if (want_init_on_alloc(flags)) + memset(res, 0, size); + return res; +} +EXPORT_SYMBOL(_uverbs_alloc); + +int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, + const void *from, size_t size) +{ + const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); + size_t min_size; + + if (IS_ERR(attr)) + return PTR_ERR(attr); + + min_size = min_t(size_t, attr->ptr_attr.len, size); + if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) + return -EFAULT; + + return uverbs_set_output(bundle, attr); +} +EXPORT_SYMBOL(uverbs_copy_to); + +int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, + size_t idx, const void *from, size_t size) +{ + const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); + + if (IS_ERR(attr)) + return PTR_ERR(attr); + + if (size < attr->ptr_attr.len) { + if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, + attr->ptr_attr.len - size)) + return -EFAULT; + } + return uverbs_copy_to(bundle, idx, from, size); +} +EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); + +int _uverbs_get_const_unsigned(u64 *to, + const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 upper_bound, u64 *def_val) +{ + const struct uverbs_attr *attr; + + attr = uverbs_attr_get(attrs_bundle, idx); + if (IS_ERR(attr)) { + if ((PTR_ERR(attr) != -ENOENT) || !def_val) + return PTR_ERR(attr); + + *to = *def_val; + } else { + *to = attr->ptr_attr.data; + } + + if (*to > upper_bound) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(_uverbs_get_const_unsigned); + +int _uverbs_get_const_signed(s64 *to, + const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, s64 lower_bound, u64 upper_bound, + s64 *def_val) +{ + const struct uverbs_attr *attr; + + attr = uverbs_attr_get(attrs_bundle, idx); + if (IS_ERR(attr)) { + if ((PTR_ERR(attr) != -ENOENT) || !def_val) + return PTR_ERR(attr); + + *to = *def_val; + } else { + *to = attr->ptr_attr.data; + } + + if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(_uverbs_get_const_signed); + +int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits) +{ + const struct uverbs_attr *attr; + u64 flags; + + attr = uverbs_attr_get(attrs_bundle, idx); + /* Missing attribute means 0 flags */ + if (IS_ERR(attr)) { + *to = 0; + return 0; + } + + /* + * New userspace code should use 8 bytes to pass flags, but we + * transparently support old userspaces that were using 4 bytes as + * well. + */ + if (attr->ptr_attr.len == 8) + flags = attr->ptr_attr.data; + else if (attr->ptr_attr.len == 4) + flags = *(u32 *)&attr->ptr_attr.data; + else + return -EINVAL; + + if (flags & ~allowed_bits) + return -EINVAL; + + *to = flags; + return 0; +} +EXPORT_SYMBOL(uverbs_get_flags64); + +int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits) +{ + u64 flags; + int ret; + + ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); + if (ret) + return ret; + + if (flags > U32_MAX) + return -EINVAL; + *to = flags; + + return 0; +} +EXPORT_SYMBOL(uverbs_get_flags32); + +/* Once called an abort will call through to the type's destroy_hw() */ +void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, + u16 idx) +{ + struct bundle_priv *pbundle = + container_of(&bundle->hdr, struct bundle_priv, bundle); + + __set_bit(uapi_bkey_attr(uapi_key_attr(idx)), + pbundle->uobj_hw_obj_valid); +} +EXPORT_SYMBOL(uverbs_finalize_uobj_create); diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index f2e192b51e60..1563169c6500 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -263,6 +263,21 @@ struct bundle_priv { u64 internal_buffer[32]; }; +static inline int uverbs_set_output(const struct uverbs_attr_bundle *bundle, + const struct uverbs_attr *attr) +{ + struct bundle_priv *pbundle = + container_of(&bundle->hdr, struct bundle_priv, bundle); + u16 flags; + + flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | + UVERBS_ATTR_F_VALID_OUTPUT; + if (put_user(flags, + &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) + return -EFAULT; + return 0; +} + long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); struct ib_uverbs_flow_spec { diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c index 2552a7efe2fb..6a78288e27a1 100644 --- a/drivers/infiniband/core/uverbs_ioctl.c +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -58,50 +58,6 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE); } -/** - * _uverbs_alloc() - Quickly allocate memory for use with a bundle - * @bundle: The bundle - * @size: Number of bytes to allocate - * @flags: Allocator flags - * - * The bundle allocator is intended for allocations that are connected with - * processing the system call related to the bundle. The allocated memory is - * always freed once the system call completes, and cannot be freed any other - * way. - * - * This tries to use a small pool of pre-allocated memory for performance. - */ -__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, - gfp_t flags) -{ - struct bundle_priv *pbundle = - container_of(&bundle->hdr, struct bundle_priv, bundle); - size_t new_used; - void *res; - - if (check_add_overflow(size, pbundle->internal_used, &new_used)) - return ERR_PTR(-EOVERFLOW); - - if (new_used > pbundle->internal_avail) { - struct bundle_alloc_head *buf; - - buf = kvmalloc_flex(*buf, data, size, flags); - if (!buf) - return ERR_PTR(-ENOMEM); - buf->next = pbundle->allocated_mem; - pbundle->allocated_mem = buf; - return buf->data; - } - - res = (void *)pbundle->internal_buffer + pbundle->internal_used; - pbundle->internal_used = - ALIGN(new_used, sizeof(*pbundle->internal_buffer)); - if (want_init_on_alloc(flags)) - memset(res, 0, size); - return res; -} -EXPORT_SYMBOL(_uverbs_alloc); - static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, u16 len) { @@ -113,21 +69,6 @@ static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, 0, uattr->len - len); } -static int uverbs_set_output(const struct uverbs_attr_bundle *bundle, - const struct uverbs_attr *attr) -{ - struct bundle_priv *pbundle = - container_of(&bundle->hdr, struct bundle_priv, bundle); - u16 flags; - - flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | - UVERBS_ATTR_F_VALID_OUTPUT; - if (put_user(flags, - &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) - return -EFAULT; - return 0; -} - static int uverbs_process_idrs_array(struct bundle_priv *pbundle, const struct uverbs_api_attr *attr_uapi, struct uverbs_objs_arr_attr *attr, @@ -616,57 +557,6 @@ long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return err; } -int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, u64 allowed_bits) -{ - const struct uverbs_attr *attr; - u64 flags; - - attr = uverbs_attr_get(attrs_bundle, idx); - /* Missing attribute means 0 flags */ - if (IS_ERR(attr)) { - *to = 0; - return 0; - } - - /* - * New userspace code should use 8 bytes to pass flags, but we - * transparently support old userspaces that were using 4 bytes as - * well. - */ - if (attr->ptr_attr.len == 8) - flags = attr->ptr_attr.data; - else if (attr->ptr_attr.len == 4) - flags = *(u32 *)&attr->ptr_attr.data; - else - return -EINVAL; - - if (flags & ~allowed_bits) - return -EINVAL; - - *to = flags; - return 0; -} -EXPORT_SYMBOL(uverbs_get_flags64); - -int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, u64 allowed_bits) -{ - u64 flags; - int ret; - - ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); - if (ret) - return ret; - - if (flags > U32_MAX) - return -EINVAL; - *to = flags; - - return 0; -} -EXPORT_SYMBOL(uverbs_get_flags32); - /* * Fill a ib_udata struct (core or uhw) using the given attribute IDs. * This is primarily used to convert the UVERBS_ATTR_UHW() into the @@ -707,24 +597,6 @@ void uverbs_fill_udata(struct uverbs_attr_bundle *bundle, } } -int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, - const void *from, size_t size) -{ - const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); - size_t min_size; - - if (IS_ERR(attr)) - return PTR_ERR(attr); - - min_size = min_t(size_t, attr->ptr_attr.len, size); - if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) - return -EFAULT; - - return uverbs_set_output(bundle, attr); -} -EXPORT_SYMBOL(uverbs_copy_to); - - /* * This is only used if the caller has directly used copy_to_use to write the * data. It signals to user space that the buffer is filled in. @@ -738,79 +610,3 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx) return uverbs_set_output(bundle, attr); } - -int _uverbs_get_const_signed(s64 *to, - const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, s64 lower_bound, u64 upper_bound, - s64 *def_val) -{ - const struct uverbs_attr *attr; - - attr = uverbs_attr_get(attrs_bundle, idx); - if (IS_ERR(attr)) { - if ((PTR_ERR(attr) != -ENOENT) || !def_val) - return PTR_ERR(attr); - - *to = *def_val; - } else { - *to = attr->ptr_attr.data; - } - - if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) - return -EINVAL; - - return 0; -} -EXPORT_SYMBOL(_uverbs_get_const_signed); - -int _uverbs_get_const_unsigned(u64 *to, - const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, u64 upper_bound, u64 *def_val) -{ - const struct uverbs_attr *attr; - - attr = uverbs_attr_get(attrs_bundle, idx); - if (IS_ERR(attr)) { - if ((PTR_ERR(attr) != -ENOENT) || !def_val) - return PTR_ERR(attr); - - *to = *def_val; - } else { - *to = attr->ptr_attr.data; - } - - if (*to > upper_bound) - return -EINVAL; - - return 0; -} -EXPORT_SYMBOL(_uverbs_get_const_unsigned); - -int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, - size_t idx, const void *from, size_t size) -{ - const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); - - if (IS_ERR(attr)) - return PTR_ERR(attr); - - if (size < attr->ptr_attr.len) { - if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, - attr->ptr_attr.len - size)) - return -EFAULT; - } - return uverbs_copy_to(bundle, idx, from, size); -} -EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); - -/* Once called an abort will call through to the type's destroy_hw() */ -void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, - u16 idx) -{ - struct bundle_priv *pbundle = - container_of(&bundle->hdr, struct bundle_priv, bundle); - - __set_bit(uapi_bkey_attr(uapi_key_attr(idx)), - pbundle->uobj_hw_obj_valid); -} -EXPORT_SYMBOL(uverbs_finalize_uobj_create); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index f5837da47299..15d8387718c0 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -91,30 +91,6 @@ static const struct class uverbs_class = { .devnode = uverbs_devnode, }; -/* - * Must be called with the ufile->device->disassociate_srcu held, and the lock - * must be held until use of the ucontext is finished. - */ -struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile) -{ - /* - * We do not hold the hw_destroy_rwsem lock for this flow, instead - * srcu is used. It does not matter if someone races this with - * get_context, we get NULL or valid ucontext. - */ - struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext); - - if (!srcu_dereference(ufile->device->ib_dev, - &ufile->device->disassociate_srcu)) - return ERR_PTR(-EIO); - - if (!ucontext) - return ERR_PTR(-EINVAL); - - return ucontext; -} -EXPORT_SYMBOL(ib_uverbs_get_ucontext_file); - int uverbs_dealloc_mw(struct ib_mw *mw) { struct ib_pd *pd = mw->pd; diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index 13776a66e2e4..e160786e1df1 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -165,12 +165,6 @@ uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, ib_uverbs_free_event_queue(&file->ev_queue); } -int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) -{ - return 0; -} -EXPORT_SYMBOL(uverbs_destroy_def_handler); - DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_COMP_CHANNEL, UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file), From 2c296df0add86a75356f8bb099ecfcfc42419600 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 25 May 2026 22:22:39 -0300 Subject: [PATCH 053/148] RDMA/core: Remove uverbs_async_event_release() Instead of having an alternative fops release always use the standard uverbs_uobject_fd_release() and route the special async behavior back up through uverbs_obj_fd_type ops pointer. This removes a dependency where the technically lower level rdma_core.c is referring to a symbol from uverbs_std_types_async_fd.c. Link: https://patch.msgid.link/r/3-v3-43aba1969751+1988-ib_uverbs_support_ko_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 32 ++++++++++++++++--- drivers/infiniband/core/uverbs.h | 1 - drivers/infiniband/core/uverbs_main.c | 2 +- .../core/uverbs_std_types_async_fd.c | 22 ++++--------- drivers/infiniband/core/uverbs_uapi.c | 13 ++++++++ include/rdma/uverbs_types.h | 8 ++++- 6 files changed, 56 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 5018ec837056..8934d0d227ab 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -465,8 +465,8 @@ alloc_begin_fd_uobject(const struct uverbs_api_object *obj, fd_type = container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); - if (WARN_ON(fd_type->fops && fd_type->fops->release != &uverbs_uobject_fd_release && - fd_type->fops->release != &uverbs_async_event_release)) { + if (WARN_ON(fd_type->fops && + fd_type->fops->release != &uverbs_uobject_fd_release)) { ret = ERR_PTR(-EINVAL); goto err_fd; } @@ -846,13 +846,37 @@ int uverbs_uobject_release(struct ib_uobject *uobj) */ int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) { + void (*release_cleanup)(struct ib_uobject *uobj) = NULL; + struct ib_uobject *uobj = filp->private_data; + const struct uverbs_obj_type *type_attrs; + int ret; + /* * This can only happen if the fput came from alloc_abort_fd_uobject() */ - if (!filp->private_data) + if (!uobj) return 0; - return uverbs_uobject_release(filp->private_data); + /* + * uverbs_disassociate_api() can NULL type_attrs after disassociate, but + * it won't if release_cleanup is used. + */ + type_attrs = READ_ONCE(uobj->uapi_object->type_attrs); + if (type_attrs) + release_cleanup = container_of(type_attrs, + struct uverbs_obj_fd_type, type) + ->release_cleanup; + if (release_cleanup) + uverbs_uobject_get(uobj); + + ret = uverbs_uobject_release(uobj); + + if (release_cleanup) { + release_cleanup(uobj); + uverbs_uobject_put(uobj); + } + + return ret; } EXPORT_SYMBOL(uverbs_uobject_fd_release); diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 1563169c6500..a1de8fe9c90b 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -203,7 +203,6 @@ void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue); void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file); void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue); void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res); -int uverbs_async_event_release(struct inode *inode, struct file *filp); int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs); int ib_init_ucontext(struct uverbs_attr_bundle *attrs); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 15d8387718c0..a937d276c5c0 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -338,7 +338,7 @@ const struct file_operations uverbs_async_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_async_event_read, .poll = ib_uverbs_async_event_poll, - .release = uverbs_async_event_release, + .release = uverbs_uobject_fd_release, .fasync = ib_uverbs_async_event_fasync, }; diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c index cc24cfdf7aee..671f510bca49 100644 --- a/drivers/infiniband/core/uverbs_std_types_async_fd.c +++ b/drivers/infiniband/core/uverbs_std_types_async_fd.c @@ -32,14 +32,9 @@ static void uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, NULL, NULL); } -int uverbs_async_event_release(struct inode *inode, struct file *filp) +static void uverbs_async_event_free_event_queue(struct ib_uobject *uobj) { struct ib_uverbs_async_event_file *event_file; - struct ib_uobject *uobj = filp->private_data; - int ret; - - if (!uobj) - return uverbs_uobject_fd_release(inode, filp); event_file = container_of(uobj, struct ib_uverbs_async_event_file, uobj); @@ -50,11 +45,7 @@ int uverbs_async_event_release(struct inode *inode, struct file *filp) * release. The user knows it has reached the end of the event stream * when it sees IB_EVENT_DEVICE_FATAL. */ - uverbs_uobject_get(uobj); - ret = uverbs_uobject_fd_release(inode, filp); ib_uverbs_free_event_queue(&event_file->ev_queue); - uverbs_uobject_put(uobj); - return ret; } DECLARE_UVERBS_NAMED_METHOD( @@ -66,11 +57,12 @@ DECLARE_UVERBS_NAMED_METHOD( DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_ASYNC_EVENT, - UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file), - uverbs_async_event_destroy_uobj, - &uverbs_async_event_fops, - "[infinibandevent]", - O_RDONLY), + UVERBS_TYPE_ALLOC_FD_RELEASE(sizeof(struct ib_uverbs_async_event_file), + uverbs_async_event_destroy_uobj, + uverbs_async_event_free_event_queue, + &uverbs_async_event_fops, + "[infinibandevent]", + O_RDONLY), &UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC)); const struct uapi_definition uverbs_def_obj_async_fd[] = { diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 31b248295854..4e2e556c8119 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -718,12 +718,25 @@ void uverbs_disassociate_api(struct uverbs_api *uapi) if (uapi_key_is_object(iter.index)) { struct uverbs_api_object *object_elm = rcu_dereference_protected(*slot, true); + const struct uverbs_obj_type *type_attrs = + object_elm->type_attrs; /* * Some type_attrs are in the driver module. We don't * bother to keep track of which since there should be * no use of this after disassociate. + * + * release_cleanup is the exception because + * uverbs_uobject_fd_release() needs it. In this case + * the module reference held by the fops will guarentee + * the type_class remains valid too. */ + if (type_attrs && + type_attrs->type_class == &uverbs_fd_class && + container_of(type_attrs, struct uverbs_obj_fd_type, + type)->release_cleanup) + continue; + object_elm->type_attrs = NULL; } else if (uapi_key_is_attr(iter.index)) { struct uverbs_api_attr *elm = diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index 6a253b7dc5ea..5a07f9a6dcd1 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -147,6 +147,7 @@ struct uverbs_obj_fd_type { struct uverbs_obj_type type; void (*destroy_object)(struct ib_uobject *uobj, enum rdma_remove_reason why); + void (*release_cleanup)(struct ib_uobject *uobj); const struct file_operations *fops; const char *name; int flags; @@ -190,7 +191,8 @@ int uverbs_uobject_release(struct ib_uobject *uobj); #define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) - \ sizeof(char)) -#define UVERBS_TYPE_ALLOC_FD(_obj_size, _destroy_object, _fops, _name, _flags) \ +#define UVERBS_TYPE_ALLOC_FD_RELEASE(_obj_size, _destroy_object, \ + _release_cleanup, _fops, _name, _flags) \ ((&((const struct uverbs_obj_fd_type) \ {.type = { \ .type_class = &uverbs_fd_class, \ @@ -199,9 +201,13 @@ int uverbs_uobject_release(struct ib_uobject *uobj); sizeof(struct ib_uobject)), \ }, \ .destroy_object = _destroy_object, \ + .release_cleanup = _release_cleanup, \ .fops = _fops, \ .name = _name, \ .flags = _flags}))->type) +#define UVERBS_TYPE_ALLOC_FD(_obj_size, _destroy_object, _fops, _name, _flags) \ + UVERBS_TYPE_ALLOC_FD_RELEASE(_obj_size, _destroy_object, NULL, \ + _fops, _name, _flags) #define UVERBS_TYPE_ALLOC_IDR_SZ(_size, _destroy_object) \ ((&((const struct uverbs_obj_idr_type) \ {.type = { \ From 2274d8cb4912ab32bcff80d6c7cbae453614ae08 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 25 May 2026 22:22:40 -0300 Subject: [PATCH 054/148] RDMA/core: Make a new module for the uverbs components needed by drivers To maintain the split where ib_uverbs.ko should not be depended on by drivers, add a new module ib_uverbs_support.ko which contains the driver called functions that are too large or too rare to be placed in ib_uverbs_core.ko Start by moving most of rdma_core.c into this module, making some adjustments to split it from the actual uverbs FD code. This was not done originally because we lacked EXPORT_SYMBOL_NS and I had a fear that drivers would abuse this interface surface. Link: https://patch.msgid.link/r/4-v3-43aba1969751+1988-ib_uverbs_support_ko_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/Makefile | 8 +- drivers/infiniband/core/rdma_core.c | 120 ++++++++++++-------------- drivers/infiniband/core/rdma_core.h | 1 - drivers/infiniband/core/uverbs.h | 9 ++ drivers/infiniband/core/uverbs_main.c | 100 +++++++++++++-------- 5 files changed, 133 insertions(+), 105 deletions(-) diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index 8c2ba2ce0351..e2ff9c2be9d3 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -5,7 +5,9 @@ user_access-$(CONFIG_INFINIBAND_ADDR_TRANS) := rdma_ucm.o obj-$(CONFIG_INFINIBAND) += ib_core.o ib_cm.o iw_cm.o \ $(infiniband-y) obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o -obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y) +obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o \ + $(user_access-y) \ + ib_uverbs_support.o ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \ device.o cache.o netlink.o \ @@ -34,7 +36,7 @@ rdma_ucm-y := ucma.o ib_umad-y := user_mad.o ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ - rdma_core.o uverbs_std_types.o uverbs_ioctl.o \ + uverbs_std_types.o uverbs_ioctl.o \ uverbs_std_types_cq.o \ uverbs_std_types_dmabuf.o \ uverbs_std_types_dmah.o \ @@ -46,3 +48,5 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types_wq.o \ uverbs_std_types_qp.o \ ucaps.o + +ib_uverbs_support-y := rdma_core.o diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 8934d0d227ab..fd5651c003ae 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -42,6 +42,40 @@ #include "core_priv.h" #include "rdma_core.h" +static void release_ufile_idr_uobject(struct ib_uverbs_file *ufile); + +void ib_uverbs_release_file(struct kref *ref) +{ + struct ib_uverbs_file *file = + container_of(ref, struct ib_uverbs_file, ref); + struct ib_device *ib_dev; + int srcu_key; + + release_ufile_idr_uobject(file); + + srcu_key = srcu_read_lock(&file->device->disassociate_srcu); + ib_dev = srcu_dereference(file->device->ib_dev, + &file->device->disassociate_srcu); + if (ib_dev && !ib_dev->ops.disassociate_ucontext) + module_put(ib_dev->ops.owner); + srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); + + if (refcount_dec_and_test(&file->device->refcount)) + ib_uverbs_comp_dev(file->device); + + if (file->default_async_file) + uverbs_uobject_put(&file->default_async_file->uobj); + put_device(&file->device->dev); + + if (file->disassociate_page) + __free_pages(file->disassociate_page, 0); + mutex_destroy(&file->disassociation_lock); + mutex_destroy(&file->umap_lock); + mutex_destroy(&file->ucontext_lock); + kfree(file); +} +EXPORT_SYMBOL_NS_GPL(ib_uverbs_release_file, "rdma_core"); + static void uverbs_uobject_free(struct kref *ref) { kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu); @@ -214,6 +248,7 @@ int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs) up_read(&ufile->hw_destroy_rwsem); return ret; } +EXPORT_SYMBOL_NS_GPL(uobj_destroy, "rdma_core"); /* * uobj_get_destroy destroys the HW object and returns a handle to the uobj @@ -239,6 +274,7 @@ struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj, return uobj; } +EXPORT_SYMBOL_NS_GPL(__uobj_get_destroy, "rdma_core"); /* * Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success @@ -255,6 +291,7 @@ int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, uobj_put_destroy(uobj); return 0; } +EXPORT_SYMBOL_NS_GPL(__uobj_perform_destroy, "rdma_core"); /* alloc_uobj must be undone by uverbs_destroy_uobject() */ static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs, @@ -420,6 +457,7 @@ struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj, uverbs_uobject_put(uobj); return ERR_PTR(ret); } +EXPORT_SYMBOL_NS_GPL(rdma_lookup_get_uobject, "rdma_core"); static struct ib_uobject * alloc_begin_idr_uobject(const struct uverbs_api_object *obj, @@ -522,6 +560,7 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, } return ret; } +EXPORT_SYMBOL_NS_GPL(rdma_alloc_begin_uobject, "rdma_core"); static void alloc_abort_idr_uobject(struct ib_uobject *uobj) { @@ -668,6 +707,7 @@ void rdma_alloc_commit_uobject(struct ib_uobject *uobj, /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); } +EXPORT_SYMBOL_NS_GPL(rdma_alloc_commit_uobject, "rdma_core"); /* * new_uobj will be assigned to the handle currently used by to_uobj, and @@ -697,6 +737,7 @@ void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj */ uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs); } +EXPORT_SYMBOL_NS_GPL(rdma_assign_uobject, "rdma_core"); /* * This consumes the kref for uobj. It is up to the caller to unwind the HW @@ -727,6 +768,7 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); } +EXPORT_SYMBOL_NS_GPL(rdma_alloc_abort_uobject, "rdma_core"); static void lookup_put_idr_uobject(struct ib_uobject *uobj, enum rdma_lookup_mode mode) @@ -770,13 +812,15 @@ void rdma_lookup_put_uobject(struct ib_uobject *uobj, /* Pairs with the kref obtained by type->lookup_get */ uverbs_uobject_put(uobj); } +EXPORT_SYMBOL_NS_GPL(rdma_lookup_put_uobject, "rdma_core"); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile) { xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC); } +EXPORT_SYMBOL_NS_GPL(setup_ufile_idr_uobject, "rdma_core"); -void release_ufile_idr_uobject(struct ib_uverbs_file *ufile) +static void release_ufile_idr_uobject(struct ib_uverbs_file *ufile) { struct ib_uobject *entry; unsigned long id; @@ -839,6 +883,7 @@ int uverbs_uobject_release(struct ib_uobject *uobj) uverbs_uobject_put(uobj); return 0; } +EXPORT_SYMBOL_NS_GPL(uverbs_uobject_release, "rdma_core"); /* * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct @@ -880,41 +925,8 @@ int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) } EXPORT_SYMBOL(uverbs_uobject_fd_release); -/* - * Drop the ucontext off the ufile and completely disconnect it from the - * ib_device - */ -static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile, - enum rdma_remove_reason reason) -{ - struct ib_ucontext *ucontext = ufile->ucontext; - struct ib_device *ib_dev = ucontext->device; - - /* - * If we are closing the FD then the user mmap VMAs must have - * already been destroyed as they hold on to the filep, otherwise - * they need to be zap'd. - */ - if (reason == RDMA_REMOVE_DRIVER_REMOVE) { - uverbs_user_mmap_disassociate(ufile); - if (ib_dev->ops.disassociate_ucontext) - ib_dev->ops.disassociate_ucontext(ucontext); - } - - ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev, - RDMACG_RESOURCE_HCA_HANDLE); - - rdma_restrack_del(&ucontext->res); - - ib_dev->ops.dealloc_ucontext(ucontext); - WARN_ON(!xa_empty(&ucontext->mmap_xa)); - kfree(ucontext); - - ufile->ucontext = NULL; -} - -static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, - enum rdma_remove_reason reason) +int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason) { struct uverbs_attr_bundle attrs = { .ufile = ufile }; struct ib_ucontext *ucontext = ufile->ucontext; @@ -955,36 +967,7 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, } return ret; } - -/* - * Destroy the ucontext and every uobject associated with it. - * - * This is internally locked and can be called in parallel from multiple - * contexts. - */ -void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, - enum rdma_remove_reason reason) -{ - down_write(&ufile->hw_destroy_rwsem); - - /* - * If a ucontext was never created then we can't have any uobjects to - * cleanup, nothing to do. - */ - if (!ufile->ucontext) - goto done; - - while (!list_empty(&ufile->uobjects) && - !__uverbs_cleanup_ufile(ufile, reason)) { - } - - if (WARN_ON(!list_empty(&ufile->uobjects))) - __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE); - ufile_destroy_ucontext(ufile, reason); - -done: - up_write(&ufile->hw_destroy_rwsem); -} +EXPORT_SYMBOL_NS_GPL(__uverbs_cleanup_ufile, "rdma_core"); const struct uverbs_obj_type_class uverbs_fd_class = { .alloc_begin = alloc_begin_fd_uobject, @@ -1022,6 +1005,7 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, return ERR_PTR(-EOPNOTSUPP); } } +EXPORT_SYMBOL_NS_GPL(uverbs_get_uobject_from_file, "rdma_core"); void uverbs_finalize_object(struct ib_uobject *uobj, enum uverbs_obj_access access, bool hw_obj_valid, @@ -1054,6 +1038,7 @@ void uverbs_finalize_object(struct ib_uobject *uobj, WARN_ON(true); } } +EXPORT_SYMBOL_NS_GPL(uverbs_finalize_object, "rdma_core"); /** * rdma_uattrs_has_raw_cap() - Returns whether a rdma device linked to the @@ -1083,3 +1068,6 @@ bool rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs) return has_cap; } EXPORT_SYMBOL(rdma_uattrs_has_raw_cap); + +MODULE_DESCRIPTION("InfiniBand uverbs objects"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 55f1e3558856..b626d3d24d08 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -70,7 +70,6 @@ void uverbs_finalize_object(struct ib_uobject *uobj, int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile); -void release_ufile_idr_uobject(struct ib_uverbs_file *ufile); struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs); diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index a1de8fe9c90b..c64dd6b94e10 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -359,4 +359,13 @@ static inline void ib_uverbs_dmabuf_done(struct kref *kref) complete(&priv->comp); } +int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason); + +static inline void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) +{ + complete(&dev->comp); +} + + #endif /* UVERBS_H */ diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index a937d276c5c0..ab6f1e3cb47a 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -61,6 +61,7 @@ MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace verbs access"); MODULE_LICENSE("Dual BSD/GPL"); +MODULE_IMPORT_NS("rdma_core"); enum { IB_UVERBS_MAJOR = 231, @@ -165,42 +166,6 @@ void ib_uverbs_detach_umcast(struct ib_qp *qp, } } -static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) -{ - complete(&dev->comp); -} - -void ib_uverbs_release_file(struct kref *ref) -{ - struct ib_uverbs_file *file = - container_of(ref, struct ib_uverbs_file, ref); - struct ib_device *ib_dev; - int srcu_key; - - release_ufile_idr_uobject(file); - - srcu_key = srcu_read_lock(&file->device->disassociate_srcu); - ib_dev = srcu_dereference(file->device->ib_dev, - &file->device->disassociate_srcu); - if (ib_dev && !ib_dev->ops.disassociate_ucontext) - module_put(ib_dev->ops.owner); - srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); - - if (refcount_dec_and_test(&file->device->refcount)) - ib_uverbs_comp_dev(file->device); - - if (file->default_async_file) - uverbs_uobject_put(&file->default_async_file->uobj); - put_device(&file->device->dev); - - if (file->disassociate_page) - __free_pages(file->disassociate_page, 0); - mutex_destroy(&file->disassociation_lock); - mutex_destroy(&file->umap_lock); - mutex_destroy(&file->ucontext_lock); - kfree(file); -} - static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, struct file *filp, char __user *buf, size_t count, loff_t *pos, @@ -985,6 +950,69 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp) return ret; } +/* + * Drop the ucontext off the ufile and completely disconnect it from the + * ib_device + */ +static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason) +{ + struct ib_ucontext *ucontext = ufile->ucontext; + struct ib_device *ib_dev = ucontext->device; + + /* + * If we are closing the FD then the user mmap VMAs must have + * already been destroyed as they hold on to the filep, otherwise + * they need to be zap'd. + */ + if (reason == RDMA_REMOVE_DRIVER_REMOVE) { + uverbs_user_mmap_disassociate(ufile); + if (ib_dev->ops.disassociate_ucontext) + ib_dev->ops.disassociate_ucontext(ucontext); + } + + ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev, + RDMACG_RESOURCE_HCA_HANDLE); + + rdma_restrack_del(&ucontext->res); + + ib_dev->ops.dealloc_ucontext(ucontext); + WARN_ON(!xa_empty(&ucontext->mmap_xa)); + kfree(ucontext); + + ufile->ucontext = NULL; +} + +/* + * Destroy the ucontext and every uobject associated with it. + * + * This is internally locked and can be called in parallel from multiple + * contexts. + */ +void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason) +{ + down_write(&ufile->hw_destroy_rwsem); + + /* + * If a ucontext was never created then we can't have any uobjects to + * cleanup, nothing to do. + */ + if (!ufile->ucontext) + goto done; + + while (!list_empty(&ufile->uobjects) && + !__uverbs_cleanup_ufile(ufile, reason)) { + } + + if (WARN_ON(!list_empty(&ufile->uobjects))) + __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE); + ufile_destroy_ucontext(ufile, reason); + +done: + up_write(&ufile->hw_destroy_rwsem); +} + static int ib_uverbs_close(struct inode *inode, struct file *filp) { struct ib_uverbs_file *file = filp->private_data; From 1a76adc9b38c678a176edc04db6dd057bcb2298c Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 25 May 2026 22:22:41 -0300 Subject: [PATCH 055/148] RDMA/core: Move ucaps into ib_uverbs_support.ko mlx5 uses these move them into the support module from ib_uverbs.ko. Link: https://patch.msgid.link/r/5-v3-43aba1969751+1988-ib_uverbs_support_ko_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/Makefile | 6 +++--- drivers/infiniband/core/ucaps.c | 10 ++++------ drivers/infiniband/core/uverbs_main.c | 1 - include/rdma/ib_ucaps.h | 1 - 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index e2ff9c2be9d3..47f645cb76f6 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -46,7 +46,7 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types_async_fd.o \ uverbs_std_types_srq.o \ uverbs_std_types_wq.o \ - uverbs_std_types_qp.o \ - ucaps.o + uverbs_std_types_qp.o -ib_uverbs_support-y := rdma_core.o +ib_uverbs_support-y := rdma_core.o \ + ucaps.o diff --git a/drivers/infiniband/core/ucaps.c b/drivers/infiniband/core/ucaps.c index 948093260dbd..c5721d3b0d33 100644 --- a/drivers/infiniband/core/ucaps.c +++ b/drivers/infiniband/core/ucaps.c @@ -46,12 +46,7 @@ static const struct file_operations ucaps_cdev_fops = { .open = simple_open, }; -/** - * ib_cleanup_ucaps - cleanup all API resources and class. - * - * This is called once, when removing the ib_uverbs module. - */ -void ib_cleanup_ucaps(void) +static __exit void ib_cleanup_ucaps(void) { mutex_lock(&ucaps_mutex); if (!ucaps_class_is_registered) { @@ -265,3 +260,6 @@ int ib_get_ucaps(int *fds, int fd_count, uint64_t *idx_mask) mutex_unlock(&ucaps_mutex); return ret; } +EXPORT_SYMBOL_NS_GPL(ib_get_ucaps, "rdma_core"); + +module_exit(ib_cleanup_ucaps); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index ab6f1e3cb47a..3ccf58e96aed 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -1350,7 +1350,6 @@ static void __exit ib_uverbs_cleanup(void) IB_UVERBS_NUM_FIXED_MINOR); unregister_chrdev_region(dynamic_uverbs_dev, IB_UVERBS_NUM_DYNAMIC_MINOR); - ib_cleanup_ucaps(); mmu_notifier_synchronize(); } diff --git a/include/rdma/ib_ucaps.h b/include/rdma/ib_ucaps.h index d9f96be3a553..b629c99117d8 100644 --- a/include/rdma/ib_ucaps.h +++ b/include/rdma/ib_ucaps.h @@ -14,7 +14,6 @@ enum rdma_user_cap { RDMA_UCAP_MAX }; -void ib_cleanup_ucaps(void); int ib_get_ucaps(int *fds, int fd_count, uint64_t *idx_mask); #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int ib_create_ucap(enum rdma_user_cap type); From 9733e9f580fdda2e8c1cd349caddd93f026ab6f5 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 25 May 2026 22:22:42 -0300 Subject: [PATCH 056/148] RDMA/core: Move flow related functions to ib_uverbs_support.ko mlx5 uses these as part of the driver implementation, move them to the support module instead. Link: https://patch.msgid.link/r/6-v3-43aba1969751+1988-ib_uverbs_support_ko_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/Makefile | 3 +- drivers/infiniband/core/uverbs_cmd.c | 76 -------------------------- drivers/infiniband/core/uverbs_flow.c | 78 +++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 77 deletions(-) create mode 100644 drivers/infiniband/core/uverbs_flow.c diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index 47f645cb76f6..ab7a2197bc86 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -49,4 +49,5 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types_qp.o ib_uverbs_support-y := rdma_core.o \ - ucaps.o + ucaps.o \ + uverbs_flow.o diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 91a62d2ade4d..32914007bae6 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -2594,82 +2594,6 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) return ret; } -struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) -{ - struct ib_uflow_resources *resources; - - resources = kzalloc_obj(*resources); - - if (!resources) - return NULL; - - if (!num_specs) - goto out; - - resources->counters = - kzalloc_objs(*resources->counters, num_specs); - resources->collection = - kzalloc_objs(*resources->collection, num_specs); - - if (!resources->counters || !resources->collection) - goto err; - -out: - resources->max = num_specs; - return resources; - -err: - kfree(resources->counters); - kfree(resources); - - return NULL; -} -EXPORT_SYMBOL(flow_resources_alloc); - -void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) -{ - unsigned int i; - - if (!uflow_res) - return; - - for (i = 0; i < uflow_res->collection_num; i++) - atomic_dec(&uflow_res->collection[i]->usecnt); - - for (i = 0; i < uflow_res->counters_num; i++) - atomic_dec(&uflow_res->counters[i]->usecnt); - - kfree(uflow_res->collection); - kfree(uflow_res->counters); - kfree(uflow_res); -} -EXPORT_SYMBOL(ib_uverbs_flow_resources_free); - -void flow_resources_add(struct ib_uflow_resources *uflow_res, - enum ib_flow_spec_type type, - void *ibobj) -{ - WARN_ON(uflow_res->num >= uflow_res->max); - - switch (type) { - case IB_FLOW_SPEC_ACTION_HANDLE: - atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); - uflow_res->collection[uflow_res->collection_num++] = - (struct ib_flow_action *)ibobj; - break; - case IB_FLOW_SPEC_ACTION_COUNT: - atomic_inc(&((struct ib_counters *)ibobj)->usecnt); - uflow_res->counters[uflow_res->counters_num++] = - (struct ib_counters *)ibobj; - break; - default: - WARN_ON(1); - } - - uflow_res->num++; -} -EXPORT_SYMBOL(flow_resources_add); - static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs, struct ib_uverbs_flow_spec *kern_spec, union ib_flow_spec *ib_spec, diff --git a/drivers/infiniband/core/uverbs_flow.c b/drivers/infiniband/core/uverbs_flow.c new file mode 100644 index 000000000000..1528a294f7f8 --- /dev/null +++ b/drivers/infiniband/core/uverbs_flow.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +#include "uverbs.h" + +struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) +{ + struct ib_uflow_resources *resources; + + resources = kzalloc_obj(*resources); + + if (!resources) + return NULL; + + if (!num_specs) + goto out; + + resources->counters = + kzalloc_objs(*resources->counters, num_specs); + resources->collection = + kzalloc_objs(*resources->collection, num_specs); + + if (!resources->counters || !resources->collection) + goto err; + +out: + resources->max = num_specs; + return resources; + +err: + kfree(resources->counters); + kfree(resources); + + return NULL; +} +EXPORT_SYMBOL(flow_resources_alloc); + +void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) +{ + unsigned int i; + + if (!uflow_res) + return; + + for (i = 0; i < uflow_res->collection_num; i++) + atomic_dec(&uflow_res->collection[i]->usecnt); + + for (i = 0; i < uflow_res->counters_num; i++) + atomic_dec(&uflow_res->counters[i]->usecnt); + + kfree(uflow_res->collection); + kfree(uflow_res->counters); + kfree(uflow_res); +} +EXPORT_SYMBOL(ib_uverbs_flow_resources_free); + +void flow_resources_add(struct ib_uflow_resources *uflow_res, + enum ib_flow_spec_type type, + void *ibobj) +{ + WARN_ON(uflow_res->num >= uflow_res->max); + + switch (type) { + case IB_FLOW_SPEC_ACTION_HANDLE: + atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); + uflow_res->collection[uflow_res->collection_num++] = + (struct ib_flow_action *)ibobj; + break; + case IB_FLOW_SPEC_ACTION_COUNT: + atomic_inc(&((struct ib_counters *)ibobj)->usecnt); + uflow_res->counters[uflow_res->counters_num++] = + (struct ib_counters *)ibobj; + break; + default: + WARN_ON(1); + } + + uflow_res->num++; +} +EXPORT_SYMBOL(flow_resources_add); From 57d87906209a67dfebe6700cb5532eaf7e0f87f1 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Tue, 26 May 2026 13:51:40 -0700 Subject: [PATCH 057/148] MAINTAINERS: Remove bouncing Intel RDMA ethernet protocol maintainer The email for Krzysztof Czurylo is bouncing. Remove the entry. Link: https://patch.msgid.link/r/20260526205140.32714-1-dave.hansen@linux.intel.com Signed-off-by: Dave Hansen Signed-off-by: Jason Gunthorpe --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index b539be153f6a..aed5a1103de7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12912,7 +12912,6 @@ F: include/linux/avf/virtchnl.h F: include/linux/net/intel/*/ INTEL ETHERNET PROTOCOL DRIVER FOR RDMA -M: Krzysztof Czurylo M: Tatyana Nikolova L: linux-rdma@vger.kernel.org S: Supported From ded0abacdc162cf16fdd75d3b04f2be8f67f4654 Mon Sep 17 00:00:00 2001 From: Purushothaman Ramalingam Date: Wed, 27 May 2026 10:45:26 +0000 Subject: [PATCH 058/148] RDMA/rxe: Fix typos in comments Fix typos found by codespell in driver comments: rxe.c: s/mangagement/management/ rxe_param.h: s/interations/iterations/ rxe_resp.c: s/recive/receive/ No functional change. Link: https://patch.msgid.link/r/20260527104527.3222-1-purush.ramalingam@gmail.com Signed-off-by: Purushothaman Ramalingam Reviewed-by: Zhu Yanjun Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe.c | 2 +- drivers/infiniband/sw/rxe/rxe_param.h | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c index b0714f9abe3d..af39209d0fcf 100644 --- a/drivers/infiniband/sw/rxe/rxe.c +++ b/drivers/infiniband/sw/rxe/rxe.c @@ -81,7 +81,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe, struct net_device *ndev) } else { /* * This device does not have a HW address, but - * connection mangagement requires a unique gid. + * connection management requires a unique gid. */ eth_random_addr(rxe->raw_gid); } diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h index 767870568372..1cc77c46bbcb 100644 --- a/drivers/infiniband/sw/rxe/rxe_param.h +++ b/drivers/infiniband/sw/rxe/rxe_param.h @@ -109,7 +109,7 @@ enum rxe_device_param { RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, - /* Max number of interations of each work item + /* Max number of iterations of each work item * before yielding the cpu to let other * work make progress */ diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 9cb2f6fbf2dd..d8d1b7f2f341 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -1505,7 +1505,7 @@ static int flush_recv_wqe(struct rxe_qp *qp, struct rxe_recv_wqe *wqe) return err; } -/* drain and optionally complete the recive queue +/* drain and optionally complete the receive queue * if unable to complete a wqe stop completing and * just flush the remaining wqes */ From af841056860d2dc2754a122fb79abbe92f3752f3 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:42:57 +0200 Subject: [PATCH 059/148] RDMA/umem: Rename ib_umem_get() to ib_umem_get_va() The new umem getter family being introduced in follow-up patches need a fitting name for the central all-source helper that resolves attributes, legacy fillers and a UHW VA fallback. Rename the existing VA-pinning helper ib_umem_get() to ib_umem_get_va() so the name is freed up. The new name is consistent with names of rest of the helpers that are about to be introduced. Link: https://patch.msgid.link/r/20260529134312.2836341-2-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 10 +++---- drivers/infiniband/core/uverbs_std_types_cq.c | 2 +- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 28 +++++++++---------- drivers/infiniband/hw/cxgb4/mem.c | 2 +- drivers/infiniband/hw/efa/efa_verbs.c | 2 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 6 ++-- drivers/infiniband/hw/hns/hns_roce_db.c | 4 +-- drivers/infiniband/hw/hns/hns_roce_mr.c | 4 +-- .../infiniband/hw/ionic/ionic_controlpath.c | 10 +++---- drivers/infiniband/hw/irdma/verbs.c | 4 +-- drivers/infiniband/hw/mana/main.c | 2 +- drivers/infiniband/hw/mana/mr.c | 2 +- drivers/infiniband/hw/mlx4/cq.c | 12 ++++---- drivers/infiniband/hw/mlx4/doorbell.c | 4 +-- drivers/infiniband/hw/mlx4/mr.c | 4 +-- drivers/infiniband/hw/mlx4/qp.c | 4 +-- drivers/infiniband/hw/mlx4/srq.c | 2 +- drivers/infiniband/hw/mlx5/cq.c | 12 ++++---- drivers/infiniband/hw/mlx5/devx.c | 2 +- drivers/infiniband/hw/mlx5/doorbell.c | 4 +-- drivers/infiniband/hw/mlx5/mr.c | 6 ++-- drivers/infiniband/hw/mlx5/qp.c | 10 +++---- drivers/infiniband/hw/mlx5/srq.c | 2 +- drivers/infiniband/hw/mthca/mthca_provider.c | 2 +- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 2 +- drivers/infiniband/hw/qedr/verbs.c | 13 +++++---- drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c | 4 +-- drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c | 2 +- drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c | 10 +++---- drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- drivers/infiniband/sw/siw/siw_mem.c | 4 +-- include/rdma/ib_umem.h | 10 +++---- 34 files changed, 96 insertions(+), 95 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 786fa1aa8e55..b253090bb1ab 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -154,15 +154,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, EXPORT_SYMBOL(ib_umem_find_best_pgsz); /** - * ib_umem_get - Pin and DMA map userspace memory. + * ib_umem_get_va - Pin and DMA map userspace memory. * * @device: IB device to connect UMEM * @addr: userspace virtual address to start at * @size: length of region to pin * @access: IB_ACCESS_xxx flags for memory being pinned */ -struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, - size_t size, int access) +struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, + size_t size, int access) { struct ib_umem *umem; struct page **page_list; @@ -275,10 +275,10 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, } return ret ? ERR_PTR(ret) : umem; } -EXPORT_SYMBOL(ib_umem_get); +EXPORT_SYMBOL(ib_umem_get_va); /** - * ib_umem_release - release memory pinned with ib_umem_get + * ib_umem_release - release pinned memory * @umem: umem struct to release */ void ib_umem_release(struct ib_umem *umem) diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index 79b51f60ce2a..1a6bc8baa52b 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -139,7 +139,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( goto err_event_file; } - umem = ib_umem_get(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { ret = PTR_ERR(umem); goto err_event_file; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 94aa06e3b828..5e8fa7bf99cb 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1210,8 +1210,8 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr); bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&rdev->ibdev, ureq->qpsva, bytes, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) return PTR_ERR(umem); @@ -1225,8 +1225,8 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&rdev->ibdev, ureq->qprva, bytes, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { rc = PTR_ERR(umem); goto fail; @@ -2171,8 +2171,8 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&rdev->ibdev, ureq.srqva, bytes, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) return PTR_ERR(umem); @@ -3496,9 +3496,9 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att dev_attr->max_cq_wqes + 1, uctx); if (!ibcq->umem) { - ibcq->umem = ib_umem_get(&rdev->ibdev, req.cq_va, - entries * sizeof(struct cq_base), - IB_ACCESS_LOCAL_WRITE); + ibcq->umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(ibcq->umem)) return PTR_ERR(ibcq->umem); } @@ -3655,12 +3655,12 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; - cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va, - entries * sizeof(struct cq_base), - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { rc = PTR_ERR(cq->resize_umem); - ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %pe\n", + ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n", __func__, cq->resize_umem); cq->resize_umem = NULL; goto fail; @@ -4670,7 +4670,7 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, if (dmah) return ERR_PTR(-EOPNOTSUPP); - umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); + umem = ib_umem_get_va(&rdev->ibdev, start, length, mr_access_flags); if (IS_ERR(umem)) return ERR_CAST(umem); diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 9fde78b74690..cd1b01014198 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -530,7 +530,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mhp->rhp = rhp; - mhp->umem = ib_umem_get(pd->device, start, length, acc); + mhp->umem = ib_umem_get_va(pd->device, start, length, acc); if (IS_ERR(mhp->umem)) goto err_free_skb; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 395290ab0584..8d97a837fa6a 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1745,7 +1745,7 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, goto err_out; } - mr->umem = ib_umem_get(ibpd->device, start, length, access_flags); + mr->umem = ib_umem_get_va(ibpd->device, start, length, access_flags); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); ibdev_dbg(&dev->ibdev, diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index b59c2e3a5306..9a9dc78c7ab6 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -829,7 +829,7 @@ static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, { int ret = 0; - mem->umem = ib_umem_get(&dev->ibdev, start, len, access); + mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access); if (IS_ERR(mem->umem)) { ret = PTR_ERR(mem->umem); mem->umem = NULL; @@ -896,8 +896,8 @@ static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, page->va = (dbrecords_va & PAGE_MASK); page->refcnt = 0; - page->umem = ib_umem_get(ctx->ibucontext.device, - dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); + page->umem = ib_umem_get_va(ctx->ibucontext.device, + dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); if (IS_ERR(page->umem)) { rv = PTR_ERR(page->umem); kfree(page); diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index f64023f5cf0a..5e008a1700ef 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -29,8 +29,8 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, refcount_set(&page->refcount, 1); page->user_virt = page_addr; - page->umem = ib_umem_get(context->ibucontext.device, page_addr, - PAGE_SIZE, 0); + page->umem = ib_umem_get_va(context->ibucontext.device, page_addr, + PAGE_SIZE, 0); if (IS_ERR(page->umem)) { ret = PTR_ERR(page->umem); kfree(page); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index e8a9e7d8f267..01818a679d0a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -591,8 +591,8 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, if (udata) { mtr->kmem = NULL; - mtr->umem = ib_umem_get(ibdev, user_addr, total_size, - buf_attr->user_access); + mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, + buf_attr->user_access); if (IS_ERR(mtr->umem)) { ibdev_err(ibdev, "failed to get umem, ret = %pe.\n", mtr->umem); diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 2b01345848dd..db18c315cc21 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -110,8 +110,8 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (rc) goto err_qdesc; - cq->umem = ib_umem_get(&dev->ibdev, req_cq->addr, req_cq->size, - IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr, + req_cq->size, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { rc = PTR_ERR(cq->umem); goto err_qdesc; @@ -895,7 +895,7 @@ struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); - mr->umem = ib_umem_get(&dev->ibdev, start, length, access); + mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access); if (IS_ERR(mr->umem)) { rc = PTR_ERR(mr->umem); goto err_umem; @@ -1837,7 +1837,7 @@ static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, qp->sq_meta = NULL; qp->sq_msn_idx = NULL; - qp->sq_umem = ib_umem_get(&dev->ibdev, sq->addr, sq->size, 0); + qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0); if (IS_ERR(qp->sq_umem)) return PTR_ERR(qp->sq_umem); } else { @@ -2050,7 +2050,7 @@ static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, qp->rq_meta = NULL; - qp->rq_umem = ib_umem_get(&dev->ibdev, rq->addr, rq->size, 0); + qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0); if (IS_ERR(qp->rq_umem)) return PTR_ERR(qp->rq_umem); } else { diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 3f4811bb5514..baee48df5fdf 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -3527,7 +3527,7 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) return ERR_PTR(-EINVAL); - region = ib_umem_get(pd->device, start, len, access); + region = ib_umem_get_va(pd->device, start, len, access); if (IS_ERR(region)) { ibdev_dbg(&iwdev->ibdev, @@ -3729,7 +3729,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, struct ib_umem *region; int err; - region = ib_umem_get(pd->device, start, len, iwmr->access); + region = ib_umem_get_va(pd->device, start, len, iwmr->access); if (IS_ERR(region)) return PTR_ERR(region); diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 307ae01bf26f..1e93df6455f5 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -238,7 +238,7 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, queue->id = INVALID_QUEUE_ID; queue->gdma_region = GDMA_INVALID_DMA_REGION; - umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); return PTR_ERR(umem); diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c index 8092a7bb785b..030bfdcfff3c 100644 --- a/drivers/infiniband/hw/mana/mr.c +++ b/drivers/infiniband/hw/mana/mr.c @@ -127,7 +127,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, if (!mr) return ERR_PTR(-ENOMEM); - mr->umem = ib_umem_get(ibdev, start, length, access_flags); + mr->umem = ib_umem_get_va(ibdev, start, length, access_flags); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); ibdev_dbg(ibdev, diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 7e4505f6c78b..f9ec6917d9c9 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -180,9 +180,9 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, buf_addr = (void *)(unsigned long)ucmd.buf_addr; if (!ibcq->umem) - ibcq->umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + ibcq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(ibcq->umem)) { err = PTR_ERR(ibcq->umem); goto err_cq; @@ -344,9 +344,9 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq if (!cq->resize_buf) return -ENOMEM; - cq->resize_umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { err = PTR_ERR(cq->resize_umem); goto err_buf; diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c index 8ba86b1e4e46..22ae728834fc 100644 --- a/drivers/infiniband/hw/mlx4/doorbell.c +++ b/drivers/infiniband/hw/mlx4/doorbell.c @@ -64,8 +64,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt, page->user_virt = (virt & PAGE_MASK); page->refcnt = 0; - page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, - PAGE_SIZE, 0); + page->umem = ib_umem_get_va(context->ibucontext.device, + virt & PAGE_MASK, PAGE_SIZE, 0); if (IS_ERR(page->umem)) { err = PTR_ERR(page->umem); kfree(page); diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 650b4a9121ff..027ffb6f79d3 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -110,7 +110,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start, /* * Force registering the memory as writable if the underlying pages * are writable. This is so rereg can change the access permissions - * from readable to writable without having to run through ib_umem_get + * from readable to writable without having to run through ib_umem_get_va * again */ if (!ib_access_writable(access_flags)) { @@ -135,7 +135,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start, mmap_read_unlock(current->mm); } - return ib_umem_get(device, start, length, access_flags); + return ib_umem_get_va(device, start, length, access_flags); } struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 8dc4196218bf..effc6bcceb76 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -897,7 +897,7 @@ static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + (qp->sq.wqe_cnt << qp->sq.wqe_shift); - qp->umem = ib_umem_get(pd->device, wq.buf_addr, qp->buf_size, 0); + qp->umem = ib_umem_get_va(pd->device, wq.buf_addr, qp->buf_size, 0); if (IS_ERR(qp->umem)) { err = PTR_ERR(qp->umem); goto err; @@ -1080,7 +1080,7 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, goto err; qp->umem = - ib_umem_get(pd->device, ucmd.buf_addr, qp->buf_size, 0); + ib_umem_get_va(pd->device, ucmd.buf_addr, qp->buf_size, 0); if (IS_ERR(qp->umem)) { err = PTR_ERR(qp->umem); goto err; diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index dd868f9b893d..7bcb7e225662 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -116,7 +116,7 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, return err; srq->umem = - ib_umem_get(ib_srq->device, ucmd.buf_addr, buf_size, 0); + ib_umem_get_va(ib_srq->device, ucmd.buf_addr, buf_size, 0); if (IS_ERR(srq->umem)) return PTR_ERR(srq->umem); diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 64fad78cb256..5a89676ebeb2 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -747,9 +747,9 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, *cqe_size = ucmd.cqe_size; if (!cq->ibcq.umem) - cq->ibcq.umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - entries * ucmd.cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->ibcq.umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + entries * ucmd.cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->ibcq.umem)) return PTR_ERR(cq->ibcq.umem); @@ -1242,9 +1242,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1) return -EINVAL; - umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - (size_t)ucmd.cqe_size * entries, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + (size_t)ucmd.cqe_size * entries, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { err = PTR_ERR(umem); return err; diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index c5929d023ee3..fc6f793dcc65 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -2287,7 +2287,7 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, return PTR_ERR(umem_dmabuf); obj->umem = &umem_dmabuf->umem; } else { - obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags); + obj->umem = ib_umem_get_va(&dev->ib_dev, addr, size, access_flags); if (IS_ERR(obj->umem)) return PTR_ERR(obj->umem); } diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c index bd68fcf011f4..020c70328663 100644 --- a/drivers/infiniband/hw/mlx5/doorbell.c +++ b/drivers/infiniband/hw/mlx5/doorbell.c @@ -66,8 +66,8 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, page->user_virt = (virt & PAGE_MASK); page->refcnt = 0; - page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, - PAGE_SIZE, 0); + page->umem = ib_umem_get_va(context->ibucontext.device, + virt & PAGE_MASK, PAGE_SIZE, 0); if (IS_ERR(page->umem)) { err = PTR_ERR(page->umem); kfree(page); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index d7d8f3ae8b64..254e6aa4ccaf 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -874,7 +874,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (access_flags & IB_ACCESS_ON_DEMAND) return create_user_odp_mr(pd, start, length, iova, access_flags, udata); - umem = ib_umem_get(&dev->ib_dev, start, length, access_flags); + umem = ib_umem_get_va(&dev->ib_dev, start, length, access_flags); if (IS_ERR(umem)) return ERR_CAST(umem); return create_real_mr(pd, umem, iova, access_flags, dmah); @@ -1227,8 +1227,8 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, struct ib_umem *new_umem; unsigned long page_size; - new_umem = ib_umem_get(&dev->ib_dev, start, length, - new_access_flags); + new_umem = ib_umem_get_va(&dev->ib_dev, start, length, + new_access_flags); if (IS_ERR(new_umem)) return ERR_CAST(new_umem); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 043b12cf7c1f..dc5e6e3e5a64 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -886,7 +886,7 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (!ucmd->buf_addr) return -EINVAL; - rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0); + rwq->umem = ib_umem_get_va(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0); if (IS_ERR(rwq->umem)) { mlx5_ib_dbg(dev, "umem_get failed\n"); err = PTR_ERR(rwq->umem); @@ -996,8 +996,8 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (ucmd->buf_addr && ubuffer->buf_size) { ubuffer->buf_addr = ucmd->buf_addr; - ubuffer->umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr, - ubuffer->buf_size, 0); + ubuffer->umem = ib_umem_get_va(&dev->ib_dev, ubuffer->buf_addr, + ubuffer->buf_size, 0); if (IS_ERR(ubuffer->umem)) { err = PTR_ERR(ubuffer->umem); goto err_bfreg; @@ -1348,8 +1348,8 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, if (ts_format < 0) return ts_format; - sq->ubuffer.umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr, - ubuffer->buf_size, 0); + sq->ubuffer.umem = ib_umem_get_va(&dev->ib_dev, ubuffer->buf_addr, + ubuffer->buf_size, 0); if (IS_ERR(sq->ubuffer.umem)) return PTR_ERR(sq->ubuffer.umem); page_size = mlx5_umem_find_best_quantized_pgoff( diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 3fb8519a4ce0..44903015c6c9 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -66,7 +66,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); - srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0); + srq->umem = ib_umem_get_va(pd->device, ucmd.buf_addr, buf_size, 0); if (IS_ERR(srq->umem)) { mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); err = PTR_ERR(srq->umem); diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index afa97d3801f7..23e387dba79d 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -878,7 +878,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (!mr) return ERR_PTR(-ENOMEM); - mr->umem = ib_umem_get(pd->device, start, length, acc); + mr->umem = ib_umem_get_va(pd->device, start, length, acc); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); goto err; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 383f1d9c15d1..69caadcff810 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -864,7 +864,7 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr = kzalloc_obj(*mr); if (!mr) return ERR_PTR(status); - mr->umem = ib_umem_get(ibpd->device, start, len, acc); + mr->umem = ib_umem_get_va(ibpd->device, start, len, acc); if (IS_ERR(mr->umem)) { status = -EFAULT; goto umem_err; diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 1af908275ca7..7195eeede530 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -779,9 +779,9 @@ static inline int qedr_init_user_queue(struct ib_udata *udata, q->buf_addr = buf_addr; q->buf_len = buf_len; - q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); + q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access); if (IS_ERR(q->umem)) { - DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", + DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n", PTR_ERR(q->umem)); return PTR_ERR(q->umem); } @@ -1439,13 +1439,14 @@ static int qedr_init_srq_user_params(struct ib_udata *udata, if (rc) return rc; - srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, - sizeof(struct rdma_srq_producers), access); + srq->prod_umem = ib_umem_get_va(srq->ibsrq.device, ureq->prod_pair_addr, + sizeof(struct rdma_srq_producers), + access); if (IS_ERR(srq->prod_umem)) { qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); ib_umem_release(srq->usrq.umem); DP_ERR(srq->dev, - "create srq: failed ib_umem_get for producer, got %ld\n", + "create srq: failed ib_umem_get_va for producer, got %ld\n", PTR_ERR(srq->prod_umem)); return PTR_ERR(srq->prod_umem); } @@ -2932,7 +2933,7 @@ struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->type = QEDR_MR_USER; - mr->umem = ib_umem_get(ibpd->device, start, len, acc); + mr->umem = ib_umem_get_va(ibpd->device, start, len, acc); if (IS_ERR(mr->umem)) { rc = -EFAULT; goto err0; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index d5bfdbfe1376..0bdb4452d6f6 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -138,8 +138,8 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (ret) goto err_cq; - cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, ucmd.buf_size, - IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_va(ibdev, ucmd.buf_addr, ucmd.buf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { ret = PTR_ERR(cq->umem); goto err_cq; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c index 05a6bd991502..942381ab0367 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c @@ -131,7 +131,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, return ERR_PTR(-EINVAL); } - umem = ib_umem_get(pd->device, start, length, access_flags); + umem = ib_umem_get_va(pd->device, start, length, access_flags); if (IS_ERR(umem)) { dev_warn(&dev->pdev->dev, "could not get umem for mem region\n"); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index cefcb243c3a6..e939cd5ce40b 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -268,9 +268,9 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, if (!is_srq) { /* set qp->sq.wqe_cnt, shift, buf_size.. */ - qp->rumem = ib_umem_get(ibqp->device, - ucmd.rbuf_addr, - ucmd.rbuf_size, 0); + qp->rumem = ib_umem_get_va(ibqp->device, + ucmd.rbuf_addr, + ucmd.rbuf_size, 0); if (IS_ERR(qp->rumem)) { ret = PTR_ERR(qp->rumem); goto err_qp; @@ -281,8 +281,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, qp->srq = to_vsrq(init_attr->srq); } - qp->sumem = ib_umem_get(ibqp->device, ucmd.sbuf_addr, - ucmd.sbuf_size, 0); + qp->sumem = ib_umem_get_va(ibqp->device, ucmd.sbuf_addr, + ucmd.sbuf_size, 0); if (IS_ERR(qp->sumem)) { if (!is_srq) ib_umem_release(qp->rumem); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c index e69eadde6c26..345ec486a223 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c @@ -146,7 +146,7 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, if (ret) goto err_srq; - srq->umem = ib_umem_get(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); + srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); if (IS_ERR(srq->umem)) { ret = PTR_ERR(srq->umem); goto err_srq; diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c index 15f1ff917d6c..3266129e63e7 100644 --- a/drivers/infiniband/sw/rdmavt/mr.c +++ b/drivers/infiniband/sw/rdmavt/mr.c @@ -351,7 +351,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (length == 0) return ERR_PTR(-EINVAL); - umem = ib_umem_get(pd->device, start, length, mr_access_flags); + umem = ib_umem_get_va(pd->device, start, length, mr_access_flags); if (IS_ERR(umem)) return ERR_CAST(umem); diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index c696ff874980..875eceb55fdf 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -197,7 +197,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access, mr); - umem = ib_umem_get(&rxe->ib_dev, start, length, access); + umem = ib_umem_get_va(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", (int)PTR_ERR(umem)); diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c index 4df792ebe02f..2a08817d3cce 100644 --- a/drivers/infiniband/sw/siw/siw_mem.c +++ b/drivers/infiniband/sw/siw/siw_mem.c @@ -350,7 +350,7 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start, if (!umem) return ERR_PTR(-ENOMEM); - base_mem = ib_umem_get(base_dev, start, len, rights); + base_mem = ib_umem_get_va(base_dev, start, len, rights); if (IS_ERR(base_mem)) { rv = PTR_ERR(base_mem); siw_dbg(base_dev, "Cannot pin user memory: %d\n", rv); @@ -385,7 +385,7 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start, if (num_pages) { /* - * Unexpected size of sg list provided by ib_umem_get() + * Unexpected size of sg list provided by ib_umem_get_va() */ siw_dbg(base_dev, "Short SG list, missing %u pages\n", num_pages); diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 2ad52cc1d52b..25e90766892e 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -75,8 +75,8 @@ static inline size_t ib_umem_num_pages(struct ib_umem *umem) } #ifdef CONFIG_INFINIBAND_USER_MEM -struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, - size_t size, int access); +struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, + size_t size, int access); void ib_umem_release(struct ib_umem *umem); int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length); @@ -160,9 +160,9 @@ void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); #include -static inline struct ib_umem *ib_umem_get(struct ib_device *device, - unsigned long addr, size_t size, - int access) +static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, + unsigned long addr, size_t size, + int access) { return ERR_PTR(-EOPNOTSUPP); } From 28fb701c642b8ff73e08fabef9265b508509d820 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:42:58 +0200 Subject: [PATCH 060/148] RDMA/umem: Split ib_umem_get_va() into a thin wrapper around __ib_umem_get_va() The follow-up patch is going to introduce ib_umem_get_desc(), the canonical desc-to-umem helper, which needs to pin a userspace VA without going through the exported ib_umem_get_va() helper so later on ib_umem_get_va() would use the ib_umem_get_desc() flow too. Move the existing ib_umem_get_va() to a static __ib_umem_get_va() and have ib_umem_get_va() as a thin wrapper that calls it. Link: https://patch.msgid.link/r/20260529134312.2836341-3-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index b253090bb1ab..0056f23af57b 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -153,16 +153,9 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, } EXPORT_SYMBOL(ib_umem_find_best_pgsz); -/** - * ib_umem_get_va - Pin and DMA map userspace memory. - * - * @device: IB device to connect UMEM - * @addr: userspace virtual address to start at - * @size: length of region to pin - * @access: IB_ACCESS_xxx flags for memory being pinned - */ -struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, - size_t size, int access) +static struct ib_umem *__ib_umem_get_va(struct ib_device *device, + unsigned long addr, size_t size, + int access) { struct ib_umem *umem; struct page **page_list; @@ -275,6 +268,20 @@ struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, } return ret ? ERR_PTR(ret) : umem; } + +/** + * ib_umem_get_va - Pin and DMA map userspace memory. + * + * @device: IB device to connect UMEM + * @addr: userspace virtual address to start at + * @size: length of region to pin + * @access: IB_ACCESS_xxx flags for memory being pinned + */ +struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, + size_t size, int access) +{ + return __ib_umem_get_va(device, addr, size, access); +} EXPORT_SYMBOL(ib_umem_get_va); /** From 3cfdff484d84f04716ae56063a8dac3773bd3f29 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:42:59 +0200 Subject: [PATCH 061/148] RDMA/core: Introduce generic buffer descriptor infrastructure for umem Introduce a per-attribute UVERBS_ATTR_UMEM model so each uverbs command's umem set is explicit in its UAPI definition. Add driver-facing wrapper helpers that pin a umem on demand from an attribute or a VA addr; the driver owns the returned umem and releases it from its destroy/error paths. Link: https://patch.msgid.link/r/20260529134312.2836341-4-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/ib_core_uverbs.c | 25 +++ drivers/infiniband/core/umem.c | 228 +++++++++++++++++++++++ include/rdma/ib_umem.h | 36 ++++ include/rdma/uverbs_ioctl.h | 30 +++ include/uapi/rdma/ib_user_ioctl_verbs.h | 27 +++ 5 files changed, 346 insertions(+) diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index b4fc693a3bd8..6e063e05f796 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -708,6 +708,31 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, } EXPORT_SYMBOL(uverbs_get_flags32); +/** + * uverbs_get_buffer_desc - Read a buffer descriptor from a uverbs attr. + * @attrs_bundle: uverbs attribute bundle. + * @attr_id: id of an UVERBS_ATTR_UMEM-typed attribute. + * @desc: descriptor to fill. + * + * Return: 0 on success, -ENOENT if @attr_id is not set, -EINVAL on a + * malformed descriptor, or any other negative errno propagated from + * uverbs_copy_from() (notably -EFAULT on copy_from_user() failure). + */ +int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle, + u16 attr_id, struct ib_uverbs_buffer_desc *desc) +{ + int ret; + + ret = uverbs_copy_from(desc, attrs_bundle, attr_id); + if (ret) + return ret; + if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK) + return -EINVAL; + desc->optional_flags &= IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK; + return 0; +} +EXPORT_SYMBOL(uverbs_get_buffer_desc); + /* Once called an abort will call through to the type's destroy_hw() */ void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, u16 idx) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 0056f23af57b..7d2256583bc7 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -269,6 +269,175 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, return ret ? ERR_PTR(ret) : umem; } +/** + * ib_umem_get_desc - Pin a umem from a buffer descriptor. + * @device: IB device. + * @desc: buffer descriptor (VA or DMABUF). + * @access: IB access flags. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access) +{ + struct ib_umem_dmabuf *umem_dmabuf; + + if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK) + return ERR_PTR(-EINVAL); + + if (overflows_type(desc->addr, unsigned long) || + overflows_type(desc->length, size_t)) + return ERR_PTR(-EOVERFLOW); + + switch (desc->type) { + case IB_UVERBS_BUFFER_TYPE_DMABUF: + umem_dmabuf = ib_umem_dmabuf_get_pinned(device, desc->addr, + desc->length, desc->fd, + access); + if (IS_ERR(umem_dmabuf)) + return ERR_CAST(umem_dmabuf); + return &umem_dmabuf->umem; + case IB_UVERBS_BUFFER_TYPE_VA: + return __ib_umem_get_va(device, desc->addr, desc->length, + access); + default: + return ERR_PTR(-EINVAL); + } +} +EXPORT_SYMBOL(ib_umem_get_desc); + +/* + * Per-command legacy buffer-desc filler. + * Returns 0 on success (desc filled), -ENODATA if no legacy attrs apply, + * negative errno on validation failure. + */ +typedef int (*ib_umem_buf_desc_filler_t)(const struct uverbs_attr_bundle *attrs, + struct ib_uverbs_buffer_desc *desc); + +/* + * ib_umem_resolve_desc - Resolve a buffer descriptor from a per-command UMEM + * attribute and/or a legacy attr filler. + * + * Return: + * 0 @desc filled. + * -ENOENT no source produced a buffer. + * -EINVAL both the UMEM attribute and the legacy filler produced a buffer. + * -errno propagated from attr read / filler validation. + */ +static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs, + u16 attr_id, + ib_umem_buf_desc_filler_t legacy_filler, + struct ib_uverbs_buffer_desc *desc) +{ + bool have_desc = false; + int ret; + + if (!attrs) + return -ENOENT; + + ret = uverbs_get_buffer_desc(attrs, attr_id, desc); + if (!ret) + have_desc = true; + else if (ret != -ENOENT) + return ret; + + if (legacy_filler) { + struct ib_uverbs_buffer_desc legacy_desc = {}; + + ret = legacy_filler(attrs, &legacy_desc); + if (!ret) { + if (have_desc) + return -EINVAL; + *desc = legacy_desc; + have_desc = true; + } else if (ret != -ENODATA) { + return ret; + } + } + + return have_desc ? 0 : -ENOENT; +} + +/* + * ib_umem_get_desc_check - Pin a umem from @desc and verify it meets + * @min_size. + */ +static struct ib_umem * +ib_umem_get_desc_check(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + size_t min_size, int access) +{ + struct ib_umem *umem; + + umem = ib_umem_get_desc(device, desc, access); + if (IS_ERR(umem)) + return umem; + if (umem->length < min_size) { + ib_umem_release(umem); + return ERR_PTR(-EINVAL); + } + return umem; +} + +/* + * ib_umem_get_from_attrs - Pin a umem from a per-command UMEM attribute + * and/or a legacy attr filler. + * + * Return: caller-owned umem on success; NULL when no source supplied a + * buffer; ERR_PTR(...) on error. + */ +static struct ib_umem * +ib_umem_get_from_attrs(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler, + size_t size, int access) +{ + struct ib_uverbs_buffer_desc desc = {}; + int ret; + + ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc); + if (ret == -ENOENT) + return NULL; + if (ret) + return ERR_PTR(ret); + return ib_umem_get_desc_check(device, &desc, size, access); +} + +/* + * ib_umem_get_from_attrs_or_va - Pin a umem from a per-command UMEM + * attribute and/or a legacy attr filler, + * falling back to a UHW VA when no source + * matched. + * + * @size is always consumed: it is the length to pin on the VA fallback + * path AND the post-pin minimum-length check on the attr / legacy paths. + * Callers must always pass a meaningful, validated value. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +static struct ib_umem * +ib_umem_get_from_attrs_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, + ib_umem_buf_desc_filler_t legacy_filler, + u64 addr, size_t size, int access) +{ + struct ib_uverbs_buffer_desc desc = {}; + int ret; + + ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc); + if (ret == -ENOENT) + desc = (struct ib_uverbs_buffer_desc){ + .type = IB_UVERBS_BUFFER_TYPE_VA, + .addr = addr, + .length = size, + }; + else if (ret) + return ERR_PTR(ret); + return ib_umem_get_desc_check(device, &desc, size, access); +} + /** * ib_umem_get_va - Pin and DMA map userspace memory. * @@ -284,6 +453,65 @@ struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, } EXPORT_SYMBOL(ib_umem_get_va); +/** + * ib_umem_get_attr - Pin a umem from a per-command UMEM attribute. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @attr_id: per-command UMEM attribute id. + * @size: minimum required umem length. + * @access: IB access flags. + * + * Return: caller-owned umem on success; NULL when no source supplied + * a buffer; ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_attr(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, size_t size, int access) +{ + return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size, + access); +} +EXPORT_SYMBOL(ib_umem_get_attr); + +/** + * ib_umem_get_attr_or_va - Pin a umem from a per-command UMEM attribute, + * falling back to a UHW VA. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @attr_id: per-command UMEM attribute id. + * @addr: UHW user VA used when no per-command attribute matched. + * @size: on the attr / legacy paths, the minimum required umem length + * validated post-pin; on the VA fallback path, the length to pin. + * @access: IB access flags. + * + * Like ib_umem_get_attr(), but pins @addr/@size when no per-command + * UMEM attribute is supplied. + * + * IMPORTANT: @size is always consumed. On the attr / legacy paths it is + * used as the post-pin minimum-length check; on the VA fallback path it + * is the length to pin. Callers MUST pass a meaningful, validated value + * even when they expect an attribute-supplied buffer to be used. + * + * Every in-tree caller passes the same value for the two roles of @size + * because no driver today distinguishes a user-passed buffer length from + * a driver-computed minimum. Drivers that currently accept a user-supplied + * length without cross-checking it against a driver minimum (vmw_pvrdma + * CQ/QP/SRQ, qedr CQ/QP/SRQ, mana WQ/QP, ionic CQ/QP), once tightened to + * compute and check a real minimum, will want to introduce a separate + * helper that passes these as distinct values. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, u64 addr, size_t size, + int access) +{ + return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr, + size, access); +} +EXPORT_SYMBOL(ib_umem_get_attr_or_va); + /** * ib_umem_release - release pinned memory * @umem: umem struct to release diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 25e90766892e..0f373679ea81 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -73,10 +73,26 @@ static inline size_t ib_umem_num_pages(struct ib_umem *umem) { return ib_umem_num_dma_blocks(umem, PAGE_SIZE); } + +struct ib_udata; +struct ib_uverbs_buffer_desc; +struct uverbs_attr_bundle; + #ifdef CONFIG_INFINIBAND_USER_MEM +struct ib_umem *ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access); struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, int access); +struct ib_umem *ib_umem_get_attr(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, size_t size, int access); +struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, u64 addr, size_t size, + int access); + void ib_umem_release(struct ib_umem *umem); int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length); @@ -160,12 +176,32 @@ void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); #include +static inline struct ib_umem * +ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, int access) { return ERR_PTR(-EOPNOTSUPP); } +static inline struct ib_umem * +ib_umem_get_attr(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + size_t size, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * +ib_umem_get_attr_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + u64 addr, size_t size, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline void ib_umem_release(struct ib_umem *umem) { } static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length) { diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 9d7575d999e1..24fd36213023 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -590,6 +590,28 @@ struct uapi_definition { UA_OPTIONAL, \ .is_udata = 1) +/* + * Per-attribute UMEM descriptor. The payload is a single + * struct ib_uverbs_buffer_desc identifying a memory region backed by + * dma-buf or user virtual address. _access selects UA_OPTIONAL or + * UA_MANDATORY. Drivers obtain a umem from the attribute via the + * ib_umem_get_*() wrapper helpers. + */ +#define UVERBS_ATTR_UMEM(_attr_id, _access) \ + UVERBS_ATTR_PTR_IN(_attr_id, \ + UVERBS_ATTR_TYPE(struct ib_uverbs_buffer_desc), \ + _access) + +/* + * Bit masks of the @flags / @optional_flags fields of struct + * ib_uverbs_buffer_desc that the kernel understands. @flags is strict: + * any bit outside the known mask makes the call fail with -EINVAL. + * @optional_flags is advisory: bits outside the known mask are silently + * dropped. Both masks are extended as new bits are introduced. + */ +#define IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK 0U +#define IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK 0U + /* ================================================= * Parsing infrastructure * ================================================= @@ -862,6 +884,8 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, size_t idx, u64 allowed_bits); int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx, const void *from, size_t size); +int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle, + u16 attr_id, struct ib_uverbs_buffer_desc *desc); __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, gfp_t flags); @@ -920,6 +944,12 @@ static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, { return -EINVAL; } +static inline int +uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle, + u16 attr_id, struct ib_uverbs_buffer_desc *desc) +{ + return -EINVAL; +} static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size) { diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h index 90c5cd8e7753..51030c27d479 100644 --- a/include/uapi/rdma/ib_user_ioctl_verbs.h +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h @@ -273,4 +273,31 @@ struct ib_uverbs_gid_entry { __u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */ }; +enum ib_uverbs_buffer_type { + IB_UVERBS_BUFFER_TYPE_DMABUF, + IB_UVERBS_BUFFER_TYPE_VA, +}; + +/* + * Describes a single buffer backed by dma-buf or user virtual address. + * Used as the payload of a per-attribute UVERBS_ATTR_UMEM-typed attribute. + * + * @type: buffer type from enum ib_uverbs_buffer_type + * @fd: dma-buf file descriptor (valid for IB_UVERBS_BUFFER_TYPE_DMABUF) + * @flags: required flags; the kernel rejects the call with -EINVAL if any + * bit is not understood. No bits are defined yet. + * @optional_flags: advisory flags; bits the kernel does not understand are + * silently ignored. No bits are defined yet. + * @addr: offset within dma-buf, or user virtual address for VA + * @length: buffer length in bytes + */ +struct ib_uverbs_buffer_desc { + __u32 type; + __s32 fd; + __u32 flags; + __u32 optional_flags; + __aligned_u64 addr; + __aligned_u64 length; +}; + #endif From df67f15c447475b81f77de2eaf22732bc11d6bae Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:00 +0200 Subject: [PATCH 062/148] RDMA/umem: Route ib_umem_get_va() through ib_umem_get_attr_or_va() ib_umem_get_va() is now redundant: ib_umem_get_attr_or_va() with attrs=NULL and attr_id=0 covers the exact same path. Make it a static inline wrapper instead of a separately exported symbol. Link: https://patch.msgid.link/r/20260529134312.2836341-5-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 15 --------------- include/rdma/ib_umem.h | 9 +++++++-- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 7d2256583bc7..680bdbbc5984 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -438,21 +438,6 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device, return ib_umem_get_desc_check(device, &desc, size, access); } -/** - * ib_umem_get_va - Pin and DMA map userspace memory. - * - * @device: IB device to connect UMEM - * @addr: userspace virtual address to start at - * @size: length of region to pin - * @access: IB_ACCESS_xxx flags for memory being pinned - */ -struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, - size_t size, int access) -{ - return __ib_umem_get_va(device, addr, size, access); -} -EXPORT_SYMBOL(ib_umem_get_va); - /** * ib_umem_get_attr - Pin a umem from a per-command UMEM attribute. * @device: IB device. diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 0f373679ea81..908eafa52840 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -83,8 +83,6 @@ struct uverbs_attr_bundle; struct ib_umem *ib_umem_get_desc(struct ib_device *device, const struct ib_uverbs_buffer_desc *desc, int access); -struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, - size_t size, int access); struct ib_umem *ib_umem_get_attr(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, size_t size, int access); @@ -93,6 +91,13 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, u16 attr_id, u64 addr, size_t size, int access); +static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, + unsigned long addr, size_t size, + int access) +{ + return ib_umem_get_attr_or_va(device, NULL, 0, addr, size, access); +} + void ib_umem_release(struct ib_umem *umem); int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length); From 057bb70f531c3e33c18c067716f39f8413b252aa Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:01 +0200 Subject: [PATCH 063/148] RDMA/uverbs: Push out CQ buffer umem processing into a helper Extract the UVERBS_ATTR_CREATE_CQ_BUFFER_* parser from the CQ create handler into uverbs_create_cq_get_buffer_desc(), and wrap it in ib_umem_get_cq_tmp(), the umem-producing helper the cq_create handler now calls. ib_umem_get_cq_tmp() is temporary; subsequent patches replace it with driver-owned ib_umem_get_cq_buf*() wrappers built on the same parser, and remove it once all CQ drivers have switched. Link: https://patch.msgid.link/r/20260529134312.2836341-6-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 79 +++++++++++++++++++ drivers/infiniband/core/uverbs_std_types_cq.c | 60 +------------- include/rdma/ib_umem.h | 7 ++ 3 files changed, 89 insertions(+), 57 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 680bdbbc5984..d364a5ab0558 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -497,6 +497,85 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, } EXPORT_SYMBOL(ib_umem_get_attr_or_va); +static int uverbs_create_cq_get_buffer_desc(struct uverbs_attr_bundle *attrs, + struct ib_uverbs_buffer_desc *desc) +{ + struct ib_device *ib_dev = attrs->context->device; + int ret; + + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) { + ret = uverbs_copy_from(&desc->addr, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_VA); + if (ret) + return ret; + ret = uverbs_copy_from(&desc->length, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); + if (ret) + return ret; + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) || + uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || + !ib_dev->ops.create_user_cq) + return -EINVAL; + desc->type = IB_UVERBS_BUFFER_TYPE_VA; + return 0; + } + + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) { + ret = uverbs_get_raw_fd(&desc->fd, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_FD); + if (ret) + return ret; + + ret = uverbs_copy_from(&desc->addr, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET); + if (ret) + return ret; + ret = uverbs_copy_from(&desc->length, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); + if (ret) + return ret; + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) || + !ib_dev->ops.create_user_cq) + return -EINVAL; + desc->type = IB_UVERBS_BUFFER_TYPE_DMABUF; + return 0; + } + + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || + uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH)) + return -EINVAL; + return -ENODATA; +} + +/** + * ib_umem_get_cq_tmp - Temporary CQ buffer umem getter. + * @device: IB device. + * @attrs: uverbs attribute bundle. + * + * Pins a CQ buffer described by the legacy CQ buffer attributes. + * Returns NULL when none are supplied. + * + * Will be removed once all CQ drivers have switched to get + * their buffer directly. + * + * Return: caller-owned umem on success; NULL when no legacy attribute + * is supplied; ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_cq_tmp(struct ib_device *device, + struct uverbs_attr_bundle *attrs) +{ + struct ib_uverbs_buffer_desc desc = {}; + int ret; + + ret = uverbs_create_cq_get_buffer_desc(attrs, &desc); + if (ret == -ENODATA) + return NULL; + if (ret) + return ERR_PTR(ret); + return ib_umem_get_desc(device, &desc, IB_ACCESS_LOCAL_WRITE); +} +EXPORT_SYMBOL(ib_umem_get_cq_tmp); + /** * ib_umem_release - release pinned memory * @umem: umem struct to release diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index 1a6bc8baa52b..711bad0aa8a3 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -66,16 +66,11 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( typeof(*obj), uevent.uobject); struct ib_uverbs_completion_event_file *ev_file = NULL; struct ib_device *ib_dev = attrs->context->device; - struct ib_umem_dmabuf *umem_dmabuf; struct ib_cq_init_attr attr = {}; struct ib_uobject *ev_file_uobj; struct ib_umem *umem = NULL; - u64 buffer_length; - u64 buffer_offset; struct ib_cq *cq; u64 user_handle; - u64 buffer_va; - int buffer_fd; int ret; if ((!ib_dev->ops.create_cq && !ib_dev->ops.create_user_cq) || @@ -122,58 +117,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( INIT_LIST_HEAD(&obj->comp_list); INIT_LIST_HEAD(&obj->uevent.event_list); - if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) { - - ret = uverbs_copy_from(&buffer_va, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA); - if (ret) - goto err_event_file; - - ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); - if (ret) - goto err_event_file; - - if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) || - uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || - !ib_dev->ops.create_user_cq) { - ret = -EINVAL; - goto err_event_file; - } - - umem = ib_umem_get_va(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - ret = PTR_ERR(umem); - goto err_event_file; - } - } else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) { - - ret = uverbs_get_raw_fd(&buffer_fd, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD); - if (ret) - goto err_event_file; - - ret = uverbs_copy_from(&buffer_offset, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET); - if (ret) - goto err_event_file; - - ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); - if (ret) - goto err_event_file; - - if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) || - !ib_dev->ops.create_user_cq) { - ret = -EINVAL; - goto err_event_file; - } - - umem_dmabuf = ib_umem_dmabuf_get_pinned(ib_dev, buffer_offset, buffer_length, - buffer_fd, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem_dmabuf)) { - ret = PTR_ERR(umem_dmabuf); - goto err_event_file; - } - umem = &umem_dmabuf->umem; - } else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || - uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH)) { - ret = -EINVAL; + umem = ib_umem_get_cq_tmp(ib_dev, attrs); + if (IS_ERR(umem)) { + ret = PTR_ERR(umem); goto err_event_file; } diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 908eafa52840..0d6e90a35f3a 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -90,6 +90,8 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, u64 addr, size_t size, int access); +struct ib_umem *ib_umem_get_cq_tmp(struct ib_device *device, + struct uverbs_attr_bundle *attrs); static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, @@ -207,6 +209,11 @@ ib_umem_get_attr_or_va(struct ib_device *device, { return ERR_PTR(-EOPNOTSUPP); } +static inline struct ib_umem * +ib_umem_get_cq_tmp(struct ib_device *device, struct uverbs_attr_bundle *attrs) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline void ib_umem_release(struct ib_umem *umem) { } static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length) { From f6d2e53ca53ad2e847e5eb64c56c426ee2fd8bdd Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:02 +0200 Subject: [PATCH 064/148] RDMA/uverbs: Add CQ buffer UMEM attribute and driver helpers Add UVERBS_ATTR_CREATE_CQ_BUF_UMEM and two driver-facing wrappers, ib_umem_get_cq_buf() and ib_umem_get_cq_buf_or_va(), that pin a CQ buffer umem from it. The wrappers reuse the existing legacy CQ buffer-attr filler. Link: https://patch.msgid.link/r/20260529134312.2836341-7-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 57 ++++++++++++++++++- drivers/infiniband/core/uverbs_std_types_cq.c | 2 + include/rdma/ib_umem.h | 20 +++++++ include/uapi/rdma/ib_user_ioctl_cmds.h | 1 + 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index d364a5ab0558..b3261b924a71 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -497,7 +497,7 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, } EXPORT_SYMBOL(ib_umem_get_attr_or_va); -static int uverbs_create_cq_get_buffer_desc(struct uverbs_attr_bundle *attrs, +static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *attrs, struct ib_uverbs_buffer_desc *desc) { struct ib_device *ib_dev = attrs->context->device; @@ -547,6 +547,61 @@ static int uverbs_create_cq_get_buffer_desc(struct uverbs_attr_bundle *attrs, return -ENODATA; } +/** + * ib_umem_get_cq_buf - Pin a CQ buffer umem from per-command attributes. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @size: minimum required CQ buffer length. + * @access: IB access flags. + * + * Resolves the CQ buffer from the new UMEM attribute or the legacy + * CQ buffer attributes. There is no UHW VA fallback, so the caller + * must arrange its own backing (typically an in-kernel allocation) + * when no source is available. + * + * Return: caller-owned umem on success; NULL when no source supplied + * a buffer; ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + size_t size, int access) +{ + return ib_umem_get_from_attrs(device, attrs, + UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + uverbs_create_cq_get_buffer_desc, + size, access); +} +EXPORT_SYMBOL(ib_umem_get_cq_buf); + +/** + * ib_umem_get_cq_buf_or_va - Pin a CQ buffer umem with UHW VA fallback. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @addr: UHW user VA used when no per-command attribute matched. + * @size: on the attr / legacy paths, the minimum required umem length + * validated post-pin; on the VA fallback path, the length to pin. + * @access: IB access flags. + * + * Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the + * UMEM attribute nor the legacy CQ buffer attributes are supplied. + * + * See ib_umem_get_attr_or_va() for the note on @size's dual role and + * the migration path for drivers that would distinguish a user-supplied + * length from a driver-computed minimum. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u64 addr, size_t size, int access) +{ + return ib_umem_get_from_attrs_or_va(device, attrs, + UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + uverbs_create_cq_get_buffer_desc, + addr, size, access); +} +EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va); + /** * ib_umem_get_cq_tmp - Temporary CQ buffer umem getter. * @device: IB device. diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index 711bad0aa8a3..05d1294762c0 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -215,6 +215,8 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET, UVERBS_ATTR_TYPE(u64), UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + UA_OPTIONAL), UVERBS_ATTR_UHW()); static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)( diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 0d6e90a35f3a..492c8d365730 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -90,6 +90,12 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, u64 addr, size_t size, int access); +struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + size_t size, int access); +struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u64 addr, size_t size, int access); struct ib_umem *ib_umem_get_cq_tmp(struct ib_device *device, struct uverbs_attr_bundle *attrs); @@ -210,6 +216,20 @@ ib_umem_get_attr_or_va(struct ib_device *device, return ERR_PTR(-EOPNOTSUPP); } static inline struct ib_umem * +ib_umem_get_cq_buf(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, size_t size, + int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * +ib_umem_get_cq_buf_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, u64 addr, + size_t size, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * ib_umem_get_cq_tmp(struct ib_device *device, struct uverbs_attr_bundle *attrs) { return ERR_PTR(-EOPNOTSUPP); diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 72041c1b0ea5..02835b7fd76d 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -117,6 +117,7 @@ enum uverbs_attrs_create_cq_cmd_attr_ids { UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH, UVERBS_ATTR_CREATE_CQ_BUFFER_FD, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET, + UVERBS_ATTR_CREATE_CQ_BUF_UMEM, }; enum uverbs_attrs_destroy_cq_cmd_attr_ids { From e3433474db923168babdb5c83abb9e1af11a9e91 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:03 +0200 Subject: [PATCH 065/148] RDMA/efa: Use ib_umem_get_cq_buf() for user CQ buffer Pin the user CQ buffer with ib_umem_get_cq_buf() and take ownership of the umem in the driver. Fall back to the existing kernel-DMA path on NULL. Link: https://patch.msgid.link/r/20260529134312.2836341-8-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_verbs.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 8d97a837fa6a..434d60235945 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1049,6 +1049,7 @@ int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (cq->cpu_addr) efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); + ib_umem_release(cq->umem); return 0; } @@ -1101,6 +1102,7 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct efa_ibv_create_cq cmd; struct efa_cq *cq = to_ecq(ibcq); int entries = attr->cqe; + struct ib_umem *umem; bool set_src_addr; int err; @@ -1149,26 +1151,29 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->ucontext = ucontext; cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs); - if (ibcq->umem) { - if (ibcq->umem->length < cq->size) { - ibdev_dbg(&dev->ibdev, "External memory too small\n"); - err = -EINVAL; - goto err_out; - } + umem = ib_umem_get_cq_buf(ibcq->device, attrs, cq->size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + err = PTR_ERR(umem); + goto err_out; + } - if (!ib_umem_is_contiguous(ibcq->umem)) { + cq->umem = umem; + + if (umem) { + if (!ib_umem_is_contiguous(umem)) { ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n"); err = -EINVAL; - goto err_out; + goto err_release_umem; } - cq->dma_addr = ib_umem_start_dma_addr(ibcq->umem); + cq->dma_addr = ib_umem_start_dma_addr(umem); } else { cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size, DMA_FROM_DEVICE); if (!cq->cpu_addr) { err = -ENOMEM; - goto err_out; + goto err_release_umem; } } @@ -1235,6 +1240,8 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (cq->cpu_addr) efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); +err_release_umem: + ib_umem_release(cq->umem); err_out: atomic64_inc(&dev->stats.create_cq_err); return err; From ffdc91c993e4d56b110757fa06532c35a2d5c798 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:04 +0200 Subject: [PATCH 066/148] RDMA/mlx5: Use ib_umem_get_cq_buf_or_va() for user CQ buffer Pin the user CQ buffer with ib_umem_get_cq_buf_or_va() and take ownership of the umem in the driver. Apply the same ownership pattern to the resize path. Link: https://patch.msgid.link/r/20260529134312.2836341-9-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/cq.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 5a89676ebeb2..e8f8fcc106c8 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -746,15 +746,15 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, *cqe_size = ucmd.cqe_size; - if (!cq->ibcq.umem) - cq->ibcq.umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - entries * ucmd.cqe_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(cq->ibcq.umem)) - return PTR_ERR(cq->ibcq.umem); + cq->buf.umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, attrs, + ucmd.buf_addr, + entries * ucmd.cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->buf.umem)) + return PTR_ERR(cq->buf.umem); page_size = mlx5_umem_find_best_cq_quantized_pgoff( - cq->ibcq.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT, + cq->buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64, &page_offset_quantized); if (!page_size) { err = -EINVAL; @@ -765,12 +765,12 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, if (err) goto err_umem; - ncont = ib_umem_num_dma_blocks(cq->ibcq.umem, page_size); + ncont = ib_umem_num_dma_blocks(cq->buf.umem, page_size); mlx5_ib_dbg( dev, "addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n", ucmd.buf_addr, entries * ucmd.cqe_size, - ib_umem_num_pages(cq->ibcq.umem), page_size, ncont); + ib_umem_num_pages(cq->buf.umem), page_size, ncont); *inlen = MLX5_ST_SZ_BYTES(create_cq_in) + MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont; @@ -781,7 +781,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, } pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas); - mlx5_ib_populate_pas(cq->ibcq.umem, page_size, pas, 0); + mlx5_ib_populate_pas(cq->buf.umem, page_size, pas, 0); cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context); MLX5_SET(cqc, cqc, log_page_size, @@ -854,7 +854,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, mlx5_ib_db_unmap_user(context, &cq->db); err_umem: - /* UMEM is released by ib_core */ + ib_umem_release(cq->buf.umem); return err; } @@ -864,6 +864,7 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata) udata, struct mlx5_ib_ucontext, ibucontext); mlx5_ib_db_unmap_user(context, &cq->db); + ib_umem_release(cq->buf.umem); } static void init_cq_frag_buf(struct mlx5_ib_cq_buf *buf) @@ -1436,8 +1437,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, unsigned int entries, if (udata) { cq->ibcq.cqe = entries - 1; - ib_umem_release(cq->ibcq.umem); - cq->ibcq.umem = cq->resize_umem; + ib_umem_release(cq->buf.umem); + cq->buf.umem = cq->resize_umem; cq->resize_umem = NULL; } else { struct mlx5_ib_cq_buf tbuf; From c1837879e4443c89805569778d82a3693d278696 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:05 +0200 Subject: [PATCH 067/148] RDMA/bnxt_re: Use ib_umem_get_cq_buf_or_va() for user CQ buffer Pin the user CQ buffer with ib_umem_get_cq_buf_or_va() and take ownership of the umem in the driver. Apply the same ownership pattern to the resize path. Link: https://patch.msgid.link/r/20260529134312.2836341-10-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 35 +++++++++++++----------- drivers/infiniband/hw/bnxt_re/ib_verbs.h | 1 + 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 5e8fa7bf99cb..c1c4ddc615e2 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3455,6 +3455,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) atomic_dec(&rdev->stats.res.cq_count); kfree(cq->cql); + ib_umem_release(cq->umem); return ib_respond_empty_udata(udata); } @@ -3495,17 +3496,15 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att entries = bnxt_re_init_depth(attr->cqe + 1, dev_attr->max_cq_wqes + 1, uctx); - if (!ibcq->umem) { - ibcq->umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, + cq->umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, attrs, req.cq_va, entries * sizeof(struct cq_base), IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(ibcq->umem)) - return PTR_ERR(ibcq->umem); - } + if (IS_ERR(cq->umem)) + return PTR_ERR(cq->umem); - rc = bnxt_re_setup_sginfo(rdev, ibcq->umem, &cq->qplib_cq.sg_info); + rc = bnxt_re_setup_sginfo(rdev, cq->umem, &cq->qplib_cq.sg_info); if (rc) - return rc; + goto free_umem; cq->qplib_cq.dpi = &uctx->dpi; cq->qplib_cq.max_wqe = entries; @@ -3515,7 +3514,7 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); if (rc) - return rc; + goto free_umem; cq->ib_cq.cqe = entries; cq->cq_period = cq->qplib_cq.period; @@ -3528,8 +3527,10 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id); /* Allocate a page */ cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL); - if (!cq->uctx_cq_page) - return -ENOMEM; + if (!cq->uctx_cq_page) { + rc = -ENOMEM; + goto destroy_cq; + } resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT; } @@ -3537,15 +3538,17 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att resp.tail = cq->qplib_cq.hwq.cons; resp.phase = cq->qplib_cq.period; rc = ib_respond_udata(udata, resp); - if (rc) { - bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) goto free_mem; - } return 0; free_mem: free_page((unsigned long)cq->uctx_cq_page); +destroy_cq: + bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); +free_umem: + ib_umem_release(cq->umem); return rc; } @@ -3609,8 +3612,8 @@ static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) cq->qplib_cq.max_wqe = cq->resize_cqe; if (cq->resize_umem) { - ib_umem_release(cq->ib_cq.umem); - cq->ib_cq.umem = cq->resize_umem; + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; cq->resize_umem = NULL; cq->resize_cqe = 0; } @@ -4206,7 +4209,7 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) /* User CQ; the only processing we do is to * complete any pending CQ resize operation. */ - if (cq->ib_cq.umem) { + if (cq->umem) { if (cq->resize_umem) bnxt_re_resize_cq_complete(cq); return 0; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index ebc393e6da4f..4d6d1259a795 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -109,6 +109,7 @@ struct bnxt_re_cq { struct bnxt_qplib_cqe *cql; #define MAX_CQL_PER_POLL 1024 u32 max_cql; + struct ib_umem *umem; struct ib_umem *resize_umem; int resize_cqe; void *uctx_cq_page; From c0a94fecec37765fb7e5c31d4e7986a1a23ce697 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:06 +0200 Subject: [PATCH 068/148] RDMA/mlx4: Use ib_umem_get_cq_buf() for user CQ buffer Pin the user CQ buffer with ib_umem_get_cq_buf() and take ownership of the umem in the driver; fall back to ib_umem_get_va() for the legacy UHW VA path. Apply the same ownership pattern to the resize path. Link: https://patch.msgid.link/r/20260529134312.2836341-11-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx4/cq.c | 50 +++++++++++++++++----------- drivers/infiniband/hw/mlx4/mlx4_ib.h | 1 + 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index f9ec6917d9c9..887912469742 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -173,32 +173,40 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, if (err) goto err_cq; - if (ibcq->umem && - (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT)) - return -EOPNOTSUPP; + cq->umem = ib_umem_get_cq_buf(&dev->ib_dev, attrs, entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->umem)) { + err = PTR_ERR(cq->umem); + goto err_cq; + } + if (cq->umem) { + if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT) { + err = -EOPNOTSUPP; + goto err_umem; + } + } else { + cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->umem)) { + err = PTR_ERR(cq->umem); + goto err_cq; + } + } buf_addr = (void *)(unsigned long)ucmd.buf_addr; - if (!ibcq->umem) - ibcq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(ibcq->umem)) { - err = PTR_ERR(ibcq->umem); - goto err_cq; - } - - shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->ibcq.umem, 0, &n); + shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->umem, 0, &n); if (shift < 0) { err = shift; - goto err_cq; + goto err_umem; } err = mlx4_mtt_init(dev->dev, n, shift, &cq->buf.mtt); if (err) - goto err_cq; + goto err_umem; - err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->ibcq.umem); + err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem); if (err) goto err_mtt; @@ -235,7 +243,9 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, err_mtt: mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); - /* UMEM is released by ib_core */ + +err_umem: + ib_umem_release(cq->umem); err_cq: return err; @@ -472,8 +482,8 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, unsigned int entries, if (ibcq->uobject) { cq->buf = cq->resize_buf->buf; cq->ibcq.cqe = cq->resize_buf->cqe; - ib_umem_release(cq->ibcq.umem); - cq->ibcq.umem = cq->resize_umem; + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; kfree(cq->resize_buf); cq->resize_buf = NULL; @@ -533,7 +543,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) struct mlx4_ib_ucontext, ibucontext), &mcq->db); - /* UMEM is released by ib_core */ + ib_umem_release(mcq->umem); } else { mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); mlx4_db_free(dev->dev, &mcq->db); diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 5a799d6df93e..598954dd0613 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -121,6 +121,7 @@ struct mlx4_ib_cq { struct mlx4_db db; spinlock_t lock; struct mutex resize_mutex; + struct ib_umem *umem; struct ib_umem *resize_umem; /* List of qps that it serves.*/ struct list_head send_qp_list; From 7c5bbaaf4441558a5b7ab0e4db3f153b18a4ec19 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:07 +0200 Subject: [PATCH 069/148] RDMA/uverbs: Remove legacy umem field from struct ib_cq Now that all drivers use helper to get umem and manage the lifetime, legacy umem field in struct ib_cq is no longer needed. Remove it along with ib_umem_get_cq_tmp() helper that populated it and both error and destroy paths. Link: https://patch.msgid.link/r/20260529134312.2836341-12-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 29 ------------------- drivers/infiniband/core/uverbs_cmd.c | 1 - drivers/infiniband/core/uverbs_std_types_cq.c | 17 ----------- drivers/infiniband/core/verbs.c | 7 ----- include/rdma/ib_umem.h | 7 ----- include/rdma/ib_verbs.h | 1 - 6 files changed, 62 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index b3261b924a71..89311fe65619 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -602,35 +602,6 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, } EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va); -/** - * ib_umem_get_cq_tmp - Temporary CQ buffer umem getter. - * @device: IB device. - * @attrs: uverbs attribute bundle. - * - * Pins a CQ buffer described by the legacy CQ buffer attributes. - * Returns NULL when none are supplied. - * - * Will be removed once all CQ drivers have switched to get - * their buffer directly. - * - * Return: caller-owned umem on success; NULL when no legacy attribute - * is supplied; ERR_PTR(...) on error. - */ -struct ib_umem *ib_umem_get_cq_tmp(struct ib_device *device, - struct uverbs_attr_bundle *attrs) -{ - struct ib_uverbs_buffer_desc desc = {}; - int ret; - - ret = uverbs_create_cq_get_buffer_desc(attrs, &desc); - if (ret == -ENODATA) - return NULL; - if (ret) - return ERR_PTR(ret); - return ib_umem_get_desc(device, &desc, IB_ACCESS_LOCAL_WRITE); -} -EXPORT_SYMBOL(ib_umem_get_cq_tmp); - /** * ib_umem_release - release pinned memory * @umem: umem struct to release diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 32914007bae6..86df7ec83b3a 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1084,7 +1084,6 @@ static int create_cq(struct uverbs_attr_bundle *attrs, return uverbs_response(attrs, &resp, sizeof(resp)); err_free: - ib_umem_release(cq->umem); rdma_restrack_put(&cq->res); kfree(cq); err_file: diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index 05d1294762c0..148cdd180dab 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -68,7 +68,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( struct ib_device *ib_dev = attrs->context->device; struct ib_cq_init_attr attr = {}; struct ib_uobject *ev_file_uobj; - struct ib_umem *umem = NULL; struct ib_cq *cq; u64 user_handle; int ret; @@ -117,16 +116,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( INIT_LIST_HEAD(&obj->comp_list); INIT_LIST_HEAD(&obj->uevent.event_list); - umem = ib_umem_get_cq_tmp(ib_dev, attrs); - if (IS_ERR(umem)) { - ret = PTR_ERR(umem); - goto err_event_file; - } - cq = rdma_zalloc_drv_obj(ib_dev, ib_cq); if (!cq) { ret = -ENOMEM; - ib_umem_release(umem); goto err_event_file; } @@ -135,11 +127,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; - /* - * If UMEM is not provided here, legacy drivers will set it during - * CQ creation based on their internal udata. - */ - cq->umem = umem; atomic_set(&cq->usecnt, 0); rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ); @@ -152,9 +139,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( if (ret) goto err_free; - /* Check that driver didn't overrun existing umem */ - WARN_ON(umem && cq->umem != umem); - obj->uevent.uobject.object = cq; obj->uevent.uobject.user_handle = user_handle; rdma_restrack_add(&cq->res); @@ -165,7 +149,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( return ret; err_free: - ib_umem_release(cq->umem); rdma_restrack_put(&cq->res); kfree(cq); err_event_file: diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index bac87de9cc67..de7d19fabd75 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -2221,12 +2221,6 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, kfree(cq); return ERR_PTR(ret); } - /* - * We are in kernel verbs flow and drivers are not allowed - * to set umem pointer, it needs to stay NULL. - */ - WARN_ON_ONCE(cq->umem); - rdma_restrack_add(&cq->res); return cq; } @@ -2257,7 +2251,6 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (ret) return ret; - ib_umem_release(cq->umem); rdma_restrack_del(&cq->res); kfree(cq); return ret; diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 492c8d365730..370d802f0e63 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -96,8 +96,6 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u64 addr, size_t size, int access); -struct ib_umem *ib_umem_get_cq_tmp(struct ib_device *device, - struct uverbs_attr_bundle *attrs); static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, @@ -229,11 +227,6 @@ ib_umem_get_cq_buf_or_va(struct ib_device *device, { return ERR_PTR(-EOPNOTSUPP); } -static inline struct ib_umem * -ib_umem_get_cq_tmp(struct ib_device *device, struct uverbs_attr_bundle *attrs) -{ - return ERR_PTR(-EOPNOTSUPP); -} static inline void ib_umem_release(struct ib_umem *umem) { } static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length) { diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 01e0eea0703f..893cb5b73951 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1738,7 +1738,6 @@ struct ib_cq { u8 interrupt:1; u8 shared:1; unsigned int comp_vector; - struct ib_umem *umem; /* * Implementation details of the RDMA core, don't use in drivers: From cd767d980cb9d808a6811fa043f06480e30e0cd8 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:08 +0200 Subject: [PATCH 070/148] RDMA/uverbs: Use UMEM attributes for QP creation Apply the per-attribute UMEM model to the QP create method. Add three optional UMEM attributes that drivers pick from based on how their user ABI lays out the QP rings: - CREATE_QP_BUF_UMEM is a single user buffer that backs both the SQ and RQ of one QP. This is the common case where userspace pins one contiguous WQE region for the QP. - CREATE_QP_SQ_BUF_UMEM and CREATE_QP_RQ_BUF_UMEM are a pair of user buffers backing the SQ and RQ independently, used when the two rings live in physically distinct user allocations and must be pinned and addressed separately. Existing drivers would map their current umems as follows: - mlx5: BUF for normal QPs (one ucmd->buf_addr covers SQ+RQ); for IB_QPT_RAW_PACKET and IB_QP_CREATE_SOURCE_QPN, the RQ side comes from ucmd->buf_addr (RQ-sized) via RQ_BUF and the SQ from ucmd->sq_buf_addr via SQ_BUF. - mlx4: BUF, single ucmd.buf_addr covering SQ+RQ. - hns: BUF, single ucmd.buf_addr covering SQ + ext-SGE + RQ. - erdma: BUF, single ureq.qbuf_va sliced by the kernel into SQ at offset 0 and RQ at rq_offset. - bnxt_re: SQ_BUF (ureq->qpsva) + RQ_BUF (ureq->qprva, the RQ side is skipped when the QP uses an SRQ). - vmw_pvrdma: SQ_BUF (sbuf_addr) + RQ_BUF (rbuf_addr, the RQ side is skipped when the QP uses an SRQ). - qedr: SQ_BUF (sq_addr) + RQ_BUF (rq_addr) for whichever side the QP type actually has (no SQ for XRC_TGT/GSI; no RQ for XRC_INI/XRC_TGT/SRQ). - ionic: SQ_BUF (req.sq.addr) + RQ_BUF (req.rq.addr); both are skipped when the rings are placed in CMB instead of host memory. - mana: raw-packet QP uses SQ_BUF (sq_buf_addr) only; the RC path uses multiple per-queue user buffers (ucmd.queue_buf[]) that do not fit the SQ/RQ pair semantics of these attrs and stays on the legacy UHW path. - efa, irdma, hfi1, ocrdma, mthca, cxgb4 and usnic do not pin a QP WQE buffer via umem; none of these attributes apply. Link: https://patch.msgid.link/r/20260529134312.2836341-13-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_std_types_qp.c | 6 ++++++ include/uapi/rdma/ib_user_ioctl_cmds.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index be0730e8509e..e44974abc6b5 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -340,6 +340,12 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_QP_NUM, UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_BUF_UMEM, + UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + UA_OPTIONAL), UVERBS_ATTR_UHW()); static int UVERBS_HANDLER(UVERBS_METHOD_QP_DESTROY)( diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 02835b7fd76d..839835bd4b23 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -159,6 +159,9 @@ enum uverbs_attrs_create_qp_cmd_attr_ids { UVERBS_ATTR_CREATE_QP_EVENT_FD, UVERBS_ATTR_CREATE_QP_RESP_CAP, UVERBS_ATTR_CREATE_QP_RESP_QP_NUM, + UVERBS_ATTR_CREATE_QP_BUF_UMEM, + UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, }; enum uverbs_attrs_destroy_qp_cmd_attr_ids { From 38fc5bab6c62dd39f4d0b0f270657024f5f07b1e Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:09 +0200 Subject: [PATCH 071/148] RDMA/mlx5: Use UMEM attributes for QP buffers in create_qp Use the per-attribute UMEM helpers to pin QP buffer umems on demand. The QP-type predicate selects between the BUF and RQ_BUF attrs; raw-packet SQ uses its own dedicated SQ_BUF attr. Link: https://patch.msgid.link/r/20260529134312.2836341-14-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 46 +++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index dc5e6e3e5a64..dd5611800531 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -934,8 +934,17 @@ static int adjust_bfregn(struct mlx5_ib_dev *dev, bfregn % MLX5_NON_FP_BFREGS_PER_UAR; } +static u16 mlx5_qp_buf_attr(struct mlx5_ib_qp *qp) +{ + if (qp->type == IB_QPT_RAW_PACKET || + qp->flags & IB_QP_CREATE_SOURCE_QPN) + return UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM; + return UVERBS_ATTR_CREATE_QP_BUF_UMEM; +} + static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, struct mlx5_ib_qp *qp, struct ib_udata *udata, + struct uverbs_attr_bundle *attrs, struct ib_qp_init_attr *attr, u32 **in, struct mlx5_ib_create_qp_resp *resp, int *inlen, struct mlx5_ib_qp_base *base, @@ -994,14 +1003,20 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (err) goto err_bfreg; - if (ucmd->buf_addr && ubuffer->buf_size) { + ubuffer->umem = NULL; + if (ubuffer->buf_size) { ubuffer->buf_addr = ucmd->buf_addr; - ubuffer->umem = ib_umem_get_va(&dev->ib_dev, ubuffer->buf_addr, - ubuffer->buf_size, 0); + ubuffer->umem = ib_umem_get_attr_or_va(&dev->ib_dev, attrs, + mlx5_qp_buf_attr(qp), + ubuffer->buf_addr, + ubuffer->buf_size, 0); if (IS_ERR(ubuffer->umem)) { err = PTR_ERR(ubuffer->umem); + ubuffer->umem = NULL; goto err_bfreg; } + } + if (ubuffer->umem) { page_size = mlx5_umem_find_best_quantized_pgoff( ubuffer->umem, qpc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64, @@ -1011,8 +1026,6 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, goto err_umem; } ncont = ib_umem_num_dma_blocks(ubuffer->umem, page_size); - } else { - ubuffer->umem = NULL; } *inlen = MLX5_ST_SZ_BYTES(create_qp_in) + @@ -1329,6 +1342,7 @@ static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq, static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, struct ib_udata *udata, + struct uverbs_attr_bundle *attrs, struct mlx5_ib_sq *sq, void *qpin, struct ib_pd *pd, struct mlx5_ib_cq *cq) { @@ -1348,8 +1362,10 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, if (ts_format < 0) return ts_format; - sq->ubuffer.umem = ib_umem_get_va(&dev->ib_dev, ubuffer->buf_addr, - ubuffer->buf_size, 0); + sq->ubuffer.umem = ib_umem_get_attr_or_va(&dev->ib_dev, attrs, + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + ubuffer->buf_addr, + ubuffer->buf_size, 0); if (IS_ERR(sq->ubuffer.umem)) return PTR_ERR(sq->ubuffer.umem); page_size = mlx5_umem_find_best_quantized_pgoff( @@ -1562,6 +1578,7 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev, static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, u32 *in, size_t inlen, struct ib_pd *pd, struct ib_udata *udata, + struct uverbs_attr_bundle *attrs, struct mlx5_ib_create_qp_resp *resp, struct ib_qp_init_attr *init_attr) { @@ -1582,7 +1599,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, if (err) return err; - err = create_raw_packet_qp_sq(dev, udata, sq, in, pd, + err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd, to_mcq(init_attr->send_cq)); if (err) goto err_destroy_tis; @@ -1700,6 +1717,7 @@ static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *q struct mlx5_create_qp_params { struct ib_udata *udata; + struct uverbs_attr_bundle *attrs; size_t inlen; size_t outlen; size_t ucmd_size; @@ -2121,8 +2139,8 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (ts_format < 0) return ts_format; - err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, ¶ms->resp, - &inlen, base, ucmd); + err = _create_user_qp(dev, pd, qp, udata, params->attrs, init_attr, + &in, ¶ms->resp, &inlen, base, ucmd); if (err) return err; @@ -2289,8 +2307,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, return ts_format; } - err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, ¶ms->resp, - &inlen, base, ucmd); + err = _create_user_qp(dev, pd, qp, udata, params->attrs, init_attr, + &in, ¶ms->resp, &inlen, base, ucmd); if (err) return err; @@ -2394,7 +2412,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr; raw_packet_qp_copy_info(qp, &qp->raw_packet_qp); err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata, - ¶ms->resp, init_attr); + params->attrs, ¶ms->resp, + init_attr); } else err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); @@ -3274,6 +3293,7 @@ int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, return err; params.udata = udata; + params.attrs = udata ? rdma_udata_to_uverbs_attr_bundle(udata) : NULL; params.uidx = MLX5_IB_DEFAULT_UIDX; params.attr = attr; params.is_rss_raw = !!attr->rwq_ind_tbl; From 2cc10972f5f4f123e5a7658824db4f7b5abfc410 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:10 +0200 Subject: [PATCH 072/148] RDMA/umem: Add ib_umem_is_contiguous() stub for !CONFIG_INFINIBAND_USER_MEM ib_umem_is_contiguous() is defined under #ifdef CONFIG_INFINIBAND_USER_MEM, but the #else branch lacks a stub. Add the missing inline to fix potential broken build. Fixes: c897c2c8b8e8 ("RDMA/core: Add umem "is_contiguous" and "start_dma_addr" helpers") Link: https://patch.msgid.link/r/20260529134312.2836341-15-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- include/rdma/ib_umem.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 370d802f0e63..bc1e6ed73b3f 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -244,6 +244,10 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, { return 0; } +static inline bool ib_umem_is_contiguous(struct ib_umem *umem) +{ + return false; +} static inline struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, unsigned long offset, From d6ab1d243973acb046aab8f6daea90005041070a Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:11 +0200 Subject: [PATCH 073/148] RDMA/mlx5: Use UMEM attribute for CQ doorbell record Add an optional mlx5 driver-namespace UMEM attribute on CQ create so userspace can supply the doorbell record buffer explicitly. mlx5_ib_db_map_user() resolves the attribute (or falls back to the legacy UHW VA) into a struct ib_uverbs_buffer_desc and runs a unified lookup-then-pin: VA-typed descriptors share a per-page umem across CQ/QP/SRQ in the same process, FD-typed descriptors are pinned per call. Link: https://patch.msgid.link/r/20260529134312.2836341-16-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/cq.c | 8 +- drivers/infiniband/hw/mlx5/doorbell.c | 103 +++++++++++++++++++---- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +- drivers/infiniband/hw/mlx5/qp.c | 4 +- drivers/infiniband/hw/mlx5/srq.c | 2 +- include/uapi/rdma/mlx5_user_ioctl_cmds.h | 1 + 6 files changed, 98 insertions(+), 25 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index e8f8fcc106c8..49b4bf148a4a 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -761,7 +761,9 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, goto err_umem; } - err = mlx5_ib_db_map_user(context, ucmd.db_addr, &cq->db); + err = mlx5_ib_db_map_user(context, attrs, + MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, + ucmd.db_addr, &cq->db); if (err) goto err_umem; @@ -1521,7 +1523,9 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE( UVERBS_ATTR_PTR_IN( MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX, UVERBS_ATTR_TYPE(u32), - UA_OPTIONAL)); + UA_OPTIONAL), + UVERBS_ATTR_UMEM(MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, + UA_OPTIONAL)); const struct uapi_definition mlx5_ib_create_cq_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_CQ, &mlx5_ib_cq_create), diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c index 020c70328663..3108894534a8 100644 --- a/drivers/infiniband/hw/mlx5/doorbell.c +++ b/drivers/infiniband/hw/mlx5/doorbell.c @@ -37,50 +37,95 @@ #include "mlx5_ib.h" +#define MLX5_IB_DBR_SIZE (sizeof(__be32) * 2) + struct mlx5_ib_user_db_page { struct list_head list; struct ib_umem *umem; - unsigned long user_virt; + struct ib_uverbs_buffer_desc desc; int refcnt; struct mm_struct *mm; }; -int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, - struct mlx5_db *db) +static int mlx5_ib_db_map_user_desc(struct mlx5_ib_ucontext *context, + const struct ib_uverbs_buffer_desc *desc, + struct mlx5_db *db) { struct mlx5_ib_user_db_page *page; + struct ib_umem *umem; int err = 0; + if (desc->length < MLX5_IB_DBR_SIZE) + return -EINVAL; + /* + * For VA descriptors the umem is normalized to a single PAGE_SIZE + * region, so reject offsets that would place the 8-byte DBR + * straddling the page boundary. + */ + if (desc->type == IB_UVERBS_BUFFER_TYPE_VA && + (desc->addr & ~PAGE_MASK) > PAGE_SIZE - MLX5_IB_DBR_SIZE) + return -EINVAL; + mutex_lock(&context->db_page_mutex); - list_for_each_entry(page, &context->db_page_list, list) - if ((current->mm == page->mm) && - (page->user_virt == (virt & PAGE_MASK))) - goto found; + /* + * Only VA-typed descriptors are eligible to share a per-page + * doorbell umem; FD-typed descriptors are pinned individually. + */ + if (desc->type == IB_UVERBS_BUFFER_TYPE_VA) { + list_for_each_entry(page, &context->db_page_list, list) { + if (current->mm != page->mm) + continue; + if (page->desc.addr == (desc->addr & PAGE_MASK)) + goto found; + } + } - page = kmalloc_obj(*page); + page = kzalloc_obj(*page); if (!page) { err = -ENOMEM; goto out; } - page->user_virt = (virt & PAGE_MASK); - page->refcnt = 0; - page->umem = ib_umem_get_va(context->ibucontext.device, - virt & PAGE_MASK, PAGE_SIZE, 0); - if (IS_ERR(page->umem)) { - err = PTR_ERR(page->umem); + page->desc = *desc; + + /* + * Normalize VA descriptors to a page-aligned PAGE_SIZE region so + * multiple DBRs that fall in the same user page share one umem. + */ + if (page->desc.type == IB_UVERBS_BUFFER_TYPE_VA) { + page->desc.addr &= PAGE_MASK; + page->desc.length = PAGE_SIZE; + } + + umem = ib_umem_get_desc(context->ibucontext.device, &page->desc, 0); + if (IS_ERR(umem)) { + err = PTR_ERR(umem); kfree(page); goto out; } - mmgrab(current->mm); - page->mm = current->mm; + /* + * The 8-byte DBR is programmed to the device as one DMA address, + * so it must live in a single contiguous DMA segment. + */ + if (!ib_umem_is_contiguous(umem)) { + ib_umem_release(umem); + kfree(page); + err = -EINVAL; + goto out; + } + + page->umem = umem; + if (page->desc.type == IB_UVERBS_BUFFER_TYPE_VA) { + mmgrab(current->mm); + page->mm = current->mm; + } list_add(&page->list, &context->db_page_list); found: db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + - (virt & ~PAGE_MASK); + (desc->addr & ~PAGE_MASK); db->u.user_page = page; ++page->refcnt; @@ -90,13 +135,35 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, return err; } +int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + unsigned long virt, struct mlx5_db *db) +{ + struct ib_uverbs_buffer_desc desc = { + .type = IB_UVERBS_BUFFER_TYPE_VA, + .addr = virt, + .length = MLX5_IB_DBR_SIZE, + }; + + if (attrs) { + int err; + + err = uverbs_get_buffer_desc(attrs, attr_id, &desc); + if (err && err != -ENOENT) + return err; + } + + return mlx5_ib_db_map_user_desc(context, &desc, db); +} + void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db) { mutex_lock(&context->db_page_mutex); if (!--db->u.user_page->refcnt) { list_del(&db->u.user_page->list); - mmdrop(db->u.user_page->mm); + if (db->u.user_page->mm) + mmdrop(db->u.user_page->mm); ib_umem_release(db->u.user_page->umem); kfree(db->u.user_page); } diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e156dc4d7529..078f281bcdac 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1259,8 +1259,9 @@ to_mmmap(struct rdma_user_mmap_entry *rdma_entry) int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev); int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev); -int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, - struct mlx5_db *db); +int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + unsigned long virt, struct mlx5_db *db); void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db); void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq); void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index dd5611800531..58997714df70 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -914,7 +914,7 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas, offset); - err = mlx5_ib_db_map_user(ucontext, ucmd->db_addr, &rwq->db); + err = mlx5_ib_db_map_user(ucontext, NULL, 0, ucmd->db_addr, &rwq->db); if (err) { mlx5_ib_dbg(dev, "map failed\n"); goto err_umem; @@ -1053,7 +1053,7 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, resp->bfreg_index = MLX5_IB_INVALID_BFREG; qp->bfregn = bfregn; - err = mlx5_ib_db_map_user(context, ucmd->db_addr, &qp->db); + err = mlx5_ib_db_map_user(context, NULL, 0, ucmd->db_addr, &qp->db); if (err) { mlx5_ib_dbg(dev, "map failed\n"); goto err_free; diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 44903015c6c9..5bc48fef3744 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -74,7 +74,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, } in->umem = srq->umem; - err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db); + err = mlx5_ib_db_map_user(ucontext, NULL, 0, ucmd.db_addr, &srq->db); if (err) { mlx5_ib_dbg(dev, "map doorbell failed\n"); goto err_umem; diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h index 01a2a050e468..b63e75034cda 100644 --- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h +++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h @@ -274,6 +274,7 @@ enum mlx5_ib_device_query_context_attrs { enum mlx5_ib_create_cq_attrs { MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX = UVERBS_ID_DRIVER_NS_WITH_UHW, + MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, }; enum mlx5_ib_reg_dmabuf_mr_attrs { From 93ce776a5b0453314a5d1d6068762c9d646f5754 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Fri, 29 May 2026 15:43:12 +0200 Subject: [PATCH 074/148] RDMA/mlx5: Use UMEM attribute for QP doorbell record Add an optional mlx5 driver-namespace UMEM attribute on QP create so userspace can supply the doorbell record umem explicitly, symmetric to the CQ side. Resolve it inside mlx5_ib_db_map_user() and use it as a private DBR page when present; otherwise take the existing UHW share-or-pin path that preserves per-page DBR sharing across CQ/QP/SRQ in the same process. Add mlx5's first UVERBS_OBJECT_QP UAPI definition chain to attach the new attr. Link: https://patch.msgid.link/r/20260529134312.2836341-17-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 1 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + drivers/infiniband/hw/mlx5/qp.c | 19 ++++++++++++++++++- include/uapi/rdma/mlx5_user_ioctl_cmds.h | 4 ++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 419489bd2199..12bfefc91b02 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -4456,6 +4456,7 @@ static const struct uapi_definition mlx5_ib_defs[] = { UAPI_DEF_CHAIN(mlx5_ib_std_types_defs), UAPI_DEF_CHAIN(mlx5_ib_dm_defs), UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs), + UAPI_DEF_CHAIN(mlx5_ib_create_qp_defs), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 078f281bcdac..1f03c05b0cbd 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1511,6 +1511,7 @@ extern const struct uapi_definition mlx5_ib_flow_defs[]; extern const struct uapi_definition mlx5_ib_qos_defs[]; extern const struct uapi_definition mlx5_ib_std_types_defs[]; extern const struct uapi_definition mlx5_ib_create_cq_defs[]; +extern const struct uapi_definition mlx5_ib_create_qp_defs[]; static inline int is_qp1(enum ib_qp_type qp_type) { diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 58997714df70..e8d34d54b435 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -44,6 +44,9 @@ #include "qp.h" #include "wr.h" +#define UVERBS_MODULE_NAME mlx5_ib +#include + enum { MLX5_IB_ACK_REQ_FREQ = 8, }; @@ -1053,7 +1056,9 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, resp->bfreg_index = MLX5_IB_INVALID_BFREG; qp->bfregn = bfregn; - err = mlx5_ib_db_map_user(context, NULL, 0, ucmd->db_addr, &qp->db); + err = mlx5_ib_db_map_user(context, attrs, + MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM, + ucmd->db_addr, &qp->db); if (err) { mlx5_ib_dbg(dev, "map failed\n"); goto err_free; @@ -5877,3 +5882,15 @@ void mlx5_ib_qp_event_cleanup(void) { destroy_workqueue(mlx5_ib_qp_event_wq); } + +ADD_UVERBS_ATTRIBUTES_SIMPLE( + mlx5_ib_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_UMEM(MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM, + UA_OPTIONAL)); + +const struct uapi_definition mlx5_ib_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &mlx5_ib_qp_create), + {}, +}; diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h index b63e75034cda..ddb898afd813 100644 --- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h +++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h @@ -277,6 +277,10 @@ enum mlx5_ib_create_cq_attrs { MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, }; +enum mlx5_ib_create_qp_attrs { + MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM = UVERBS_ID_DRIVER_NS_WITH_UHW, +}; + enum mlx5_ib_reg_dmabuf_mr_attrs { MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS = (1U << UVERBS_ID_NS_SHIFT), }; From d7a40b519497fbee5644be4bd823b8d710bb8d55 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Sun, 17 May 2026 16:13:10 +0200 Subject: [PATCH 075/148] RDMA/uverbs: Expose CoCo DMA bounce requirement to userspace In CoCo guests, guest memory is encrypted and untrusted (T=0) devices cannot DMA to it directly; such transfers must go through unencrypted bounce buffers. RDMA registers user pages for direct device access, bypassing the DMA layer and thus any bouncing, so registered memory does not work in this configuration. Until trusted (T=1) device detection is available, conservatively flag every device attached to a CoCo guest. Expose the condition to userspace as IB_UVERBS_DEVICE_CC_DMA_BOUNCE in device_cap_flags_ex so applications can avoid memory registration and fall back to copying buffers through send/recv. Link: https://patch.msgid.link/r/20260517141311.2409230-2-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/device.c | 9 +++++++++ drivers/infiniband/core/uverbs_cmd.c | 2 ++ include/rdma/ib_verbs.h | 3 +++ include/uapi/rdma/ib_user_verbs.h | 2 ++ 4 files changed, 16 insertions(+) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index b89efaaa81ec..21ada0fe9059 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -1419,6 +1420,14 @@ int ib_register_device(struct ib_device *device, const char *name, */ WARN_ON(dma_device && !dma_device->dma_parms); device->dma_device = dma_device; + /* + * In a CoCo guest every device is currently assumed to be untrusted + * (T=0) and therefore subject to DMA bouncing. Once trusted (T=1) + * device detection is wired up, narrow this check to exclude such + * devices. + */ + if (dma_device && cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) + device->cc_dma_bounce = 1; ret = setup_device(device); if (ret) diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 86df7ec83b3a..10bd7cafd976 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -3584,6 +3584,8 @@ static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs) resp.timestamp_mask = attr.timestamp_mask; resp.hca_core_clock = attr.hca_core_clock; resp.device_cap_flags_ex = attr.device_cap_flags; + if (ib_dev->cc_dma_bounce) + resp.device_cap_flags_ex |= IB_UVERBS_DEVICE_CC_DMA_BOUNCE; resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; resp.rss_caps.max_rwq_indirection_tables = attr.rss_caps.max_rwq_indirection_tables; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 893cb5b73951..0daa5089d539 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -275,6 +275,7 @@ enum ib_device_cap_flags { IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL, IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT, IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE, + IB_DEVICE_CC_DMA_BOUNCE = IB_UVERBS_DEVICE_CC_DMA_BOUNCE, }; enum ib_kernel_cap_flags { @@ -2950,6 +2951,8 @@ struct ib_device { u16 kverbs_provider:1; /* CQ adaptive moderation (RDMA DIM) */ u16 use_cq_dim:1; + /* CoCo guest with DMA bounce buffering required */ + u16 cc_dma_bounce:1; u8 node_type; u32 phys_port_cnt; struct ib_device_attr attrs; diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h index 3b7bd99813e9..d2aeadb6d2f9 100644 --- a/include/uapi/rdma/ib_user_verbs.h +++ b/include/uapi/rdma/ib_user_verbs.h @@ -1368,6 +1368,8 @@ enum ib_uverbs_device_cap_flags { IB_UVERBS_DEVICE_FLUSH_PERSISTENT = 1ULL << 39, /* Atomic write attributes */ IB_UVERBS_DEVICE_ATOMIC_WRITE = 1ULL << 40, + /* CoCo guest with DMA bounce buffering required */ + IB_UVERBS_DEVICE_CC_DMA_BOUNCE = 1ULL << 41, }; enum ib_uverbs_raw_packet_caps { From 3b6384dac14a306fd3d8e4357b5191a6f2b7f52e Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Sun, 17 May 2026 16:13:11 +0200 Subject: [PATCH 076/148] RDMA/umem: Block plain userspace memory registration under CoCo bounce When a device requires DMA bounce buffering inside a Confidential Computing guest, __ib_umem_get_va() cannot work. The DMA mapping layer redirects all mappings through swiotlb bounce buffers, so the device receives DMA addresses pointing to bounce buffer memory rather than the user's pages. Since RDMA devices access registered memory directly without CPU involvement, there is no opportunity for swiotlb to synchronize between the bounce buffer and the original pages. The registration would already fail later on, since the umem mapping is requested with DMA_ATTR_REQUIRE_COHERENT and gets rejected under is_swiotlb_force_bounce() with -EIO. Fail early with -EOPNOTSUPP instead, so the user gets a specific error code to react to. Link: https://patch.msgid.link/r/20260517141311.2409230-3-jiri@resnulli.us Signed-off-by: Jiri Pirko Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 89311fe65619..e424a9de66c1 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -167,6 +167,9 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, int pinned, ret; unsigned int gup_flags = FOLL_LONGTERM; + if (device->cc_dma_bounce) + return ERR_PTR(-EOPNOTSUPP); + /* * If the combination of the addr and size requested for this memory * region causes an integer overflow, return error. From 22b8fbded65b8c441b634a185f8da67657df6c50 Mon Sep 17 00:00:00 2001 From: Tristan Madani Date: Mon, 18 May 2026 21:50:39 +0000 Subject: [PATCH 077/148] RDMA/rxe: Fix TOCTOU heap overflow in get_srq_wqe get_srq_wqe() reads wqe->dma.num_sge from the shared receive queue buffer, which is mapped into userspace. It validates num_sge against max_sge, but then re-reads the same field to calculate the memcpy size. A concurrent userspace thread can modify num_sge between validation and use, causing a heap buffer overflow when copying the WQE into qp->resp.srq_wqe. Read num_sge into a local variable and use it for both the bounds check and the size calculation. Fixes: 8700e3e7c485 ("Soft RoCE driver") Link: https://patch.msgid.link/r/20260518215040.1598586-2-tristan@talencesecurity.com Signed-off-by: Tristan Madani Reviewed-by: Zhu Yanjun Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_resp.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index d8d1b7f2f341..29438bcde46f 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -264,6 +264,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) struct rxe_recv_wqe *wqe; struct ib_event ev; unsigned int count; + unsigned int num_sge; size_t size; unsigned long flags; @@ -279,12 +280,13 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) } /* don't trust user space data */ - if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) { + num_sge = wqe->dma.num_sge; + if (unlikely(num_sge > srq->rq.max_sge)) { spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n"); return RESPST_ERR_MALFORMED_WQE; } - size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge); + size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); qp->resp.wqe = &qp->resp.srq_wqe.wqe; From d6ab440240a04b8737ee4c7bb21af9182e451733 Mon Sep 17 00:00:00 2001 From: Tristan Madani Date: Mon, 18 May 2026 21:50:40 +0000 Subject: [PATCH 078/148] RDMA/rxe: Copy WQE to local buffer in non-SRQ receive path For non-SRQ QPs, the responder reads WQE fields directly from the shared queue buffer mapped into userspace. This allows a malicious user to modify fields like num_sge or sge entries while the kernel is processing the WQE, leading to out-of-bounds reads in rxe_resp_check_length() and copy_data(). Introduce get_recv_wqe() that validates num_sge and copies the WQE to a kernel-local buffer before processing, matching the approach already used for SRQ WQEs in get_srq_wqe(). The srq_wqe buffer is reused since SRQ and non-SRQ paths are mutually exclusive per QP. Fixes: 8700e3e7c485 ("Soft RoCE driver") Link: https://patch.msgid.link/r/20260518215040.1598586-3-tristan@talencesecurity.com Signed-off-by: Tristan Madani Reviewed-by: Zhu Yanjun Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_resp.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 29438bcde46f..d8cbdfa70cdb 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -310,6 +310,29 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) return RESPST_CHK_LENGTH; } +static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp) +{ + struct rxe_queue *q = qp->rq.queue; + struct rxe_recv_wqe *wqe; + unsigned int num_sge; + size_t size; + + wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT); + if (!wqe) + return RESPST_ERR_RNR; + + num_sge = wqe->dma.num_sge; + if (unlikely(num_sge > qp->rq.max_sge)) { + rxe_dbg_qp(qp, "invalid num_sge in recv WQE\n"); + return RESPST_ERR_MALFORMED_WQE; + } + size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); + memcpy(&qp->resp.srq_wqe, wqe, size); + + qp->resp.wqe = &qp->resp.srq_wqe.wqe; + return RESPST_CHK_LENGTH; +} + static enum resp_states check_resource(struct rxe_qp *qp, struct rxe_pkt_info *pkt) { @@ -330,9 +353,7 @@ static enum resp_states check_resource(struct rxe_qp *qp, if (srq) return get_srq_wqe(qp); - qp->resp.wqe = queue_head(qp->rq.queue, - QUEUE_TYPE_FROM_CLIENT); - return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR; + return rxe_get_recv_wqe(qp); } return RESPST_CHK_LENGTH; From ba7c4912f7abb0e30b920faf86b32ffb9e5e8b14 Mon Sep 17 00:00:00 2001 From: Maoyi Xie Date: Tue, 26 May 2026 13:46:53 +0800 Subject: [PATCH 079/148] RDMA/hns: drop dead empty check in setup_root_hem() setup_root_hem() reads the first entry of head->root and checks the returned pointer against NULL: root_hem = list_first_entry(&head->root, struct hns_roce_hem_item, list); if (!root_hem) return -ENOMEM; list_first_entry() never returns NULL. On an empty list it returns container_of(head, ..., list), a non-NULL garbage pointer that aliases the head. So the check is dead. The only caller adds an entry to head.root right before invoking setup_root_hem(): list_add(&root_hem->list, &head.root); ret = setup_root_hem(..., &head, ...); So head.root is guaranteed non-empty on entry. Drop the check. Link: https://patch.msgid.link/r/20260526054653.2054800-1-maoyixie.tju@gmail.com Suggested-by: Jason Gunthorpe Signed-off-by: Maoyi Xie Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hem.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index ccb40f8a48b7..7041a8e9134b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -1269,8 +1269,6 @@ setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, root_hem = list_first_entry(&head->root, struct hns_roce_hem_item, list); - if (!root_hem) - return -ENOMEM; total = 0; for (i = 0; i < region_cnt && total <= max_ba_num; i++) { From b548a6c4eee5c428663f3944e173e6c92e2e8d6f Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Fri, 29 May 2026 01:30:11 +0300 Subject: [PATCH 080/148] RDMA/irdma: Fix typo in SQ completions generation When we generate completion for SQ the opcode while being properly read from ring buffer is ignored when written back to completion. Seems to be a simple typo. Link: https://patch.msgid.link/r/ahjB87k54bYdFbft@grain Signed-off-by: Cyrill Gorcunov Reviewed-by: Jacob Moroni Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 495e5daff4b4..7cc7b107db3c 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -2442,7 +2442,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp) cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; sw_wqe = qp->sq_base[wqe_idx].elem; get_64bit_val(sw_wqe, 24, &wqe_qword); - cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); + cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; /* remove the SQ WR by moving SQ tail*/ IRDMA_RING_SET_TAIL(*sq_ring, From 43b57d73ebbe5e7964a8b3ee3ffde0948fae6e2e Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 26 May 2026 13:15:05 -0300 Subject: [PATCH 081/148] RDMA/core: Don't make a dummy ib_udata on the stack in create_qp Sashiko points out the udata for destruction has to be created using uverbs_get_cleared_udata(). Move it to ib_core_uverbs.c so that the core qp code can call it. Rework the call chain to pass the struct uverbs_attr_bundle right up to the driver op callback. Fixes a possible wild stack reference in drivers during error unwinding, mlx5 can call rdma_udata_to_drv_context() from destroy_qp() when destroying a QP. Fixes: 00a79d6b996d ("RDMA/core: Configure selinux QP during creation") Link: https://patch.msgid.link/r/1-v1-922fa8e828ba+f7-ib_udata_stack_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 2 +- drivers/infiniband/core/ib_core_uverbs.c | 12 +++++++++++ drivers/infiniband/core/rdma_core.h | 7 +++++++ drivers/infiniband/core/uverbs_cmd.c | 14 +------------ drivers/infiniband/core/uverbs_std_types_qp.c | 3 +-- drivers/infiniband/core/verbs.c | 20 ++++++++++--------- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index a2c36666e6fc..19104c542b27 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -321,7 +321,7 @@ void nldev_exit(void); struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata, + struct uverbs_attr_bundle *uattrs, struct ib_uqp_object *uobj, const char *caller); void ib_qp_usecnt_inc(struct ib_qp *qp); diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 6e063e05f796..dbbc0875132a 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -532,6 +532,18 @@ int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) } EXPORT_SYMBOL(uverbs_destroy_def_handler); +/* + * When calling a destroy function during an error unwind we need to pass in + * the udata that is sanitized of all user arguments. Ie from the driver + * perspective it looks like no udata was passed. + */ +struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) +{ + attrs->driver_udata = (struct ib_udata){}; + return &attrs->driver_udata; +} +EXPORT_SYMBOL_NS_GPL(uverbs_get_cleared_udata, "rdma_core"); + /** * _uverbs_alloc() - Quickly allocate memory for use with a bundle * @bundle: The bundle diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index b626d3d24d08..56121103e9f4 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -71,7 +71,14 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile); +#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs); +#else +static inline struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) +{ + return NULL; +} +#endif /* * This is the runtime description of the uverbs API, used by the syscall diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 10bd7cafd976..e16eacf40d17 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -163,17 +163,6 @@ static int uverbs_request_finish(struct uverbs_req_iter *iter) return 0; } -/* - * When calling a destroy function during an error unwind we need to pass in - * the udata that is sanitized of all user arguments. Ie from the driver - * perspective it looks like no udata was passed. - */ -struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) -{ - attrs->driver_udata = (struct ib_udata){}; - return &attrs->driver_udata; -} - static struct ib_uverbs_completion_event_file * _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs) { @@ -1461,8 +1450,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, attr.source_qpn = cmd->source_qpn; } - qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj, - KBUILD_MODNAME); + qp = ib_create_qp_user(device, pd, &attr, attrs, obj, KBUILD_MODNAME); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index e44974abc6b5..5767607dd420 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -248,8 +248,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( set_caps(&attr, &cap, true); mutex_init(&obj->mcast_lock); - qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj, - KBUILD_MODNAME); + qp = ib_create_qp_user(device, pd, &attr, attrs, obj, KBUILD_MODNAME); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index de7d19fabd75..c0ba517297b8 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -53,6 +53,7 @@ #include #include +#include "rdma_core.h" #include "core_priv.h" #include @@ -1265,10 +1266,9 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata, + struct uverbs_attr_bundle *uattrs, struct ib_uqp_object *uobj, const char *caller) { - struct ib_udata dummy = {}; struct ib_qp *qp; int ret; @@ -1301,9 +1301,10 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, qp->recv_cq = attr->recv_cq; rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP); - WARN_ONCE(!udata && !caller, "Missing kernel QP owner"); - rdma_restrack_set_name(&qp->res, udata ? NULL : caller); - ret = dev->ops.create_qp(qp, attr, udata); + WARN_ONCE(!uattrs && !caller, "Missing kernel QP owner"); + rdma_restrack_set_name(&qp->res, uattrs ? NULL : caller); + ret = dev->ops.create_qp(qp, attr, + uattrs ? &uattrs->driver_udata : NULL); if (ret) goto err_create; @@ -1322,7 +1323,8 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, return qp; err_security: - qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL); + qp->device->ops.destroy_qp( + qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL); err_create: rdma_restrack_put(&qp->res); kfree(qp); @@ -1338,13 +1340,13 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, * @attr: A list of initial attributes required to create the * QP. If QP creation succeeds, then the attributes are updated to * the actual capabilities of the created QP. - * @udata: User data + * @uattrs: User ioctl attributes and udata * @uobj: uverbs obect * @caller: caller's build-time module name */ struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata, + struct uverbs_attr_bundle *uattrs, struct ib_uqp_object *uobj, const char *caller) { struct ib_qp *qp, *xrc_qp; @@ -1352,7 +1354,7 @@ struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, if (attr->qp_type == IB_QPT_XRC_TGT) qp = create_qp(dev, pd, attr, NULL, NULL, caller); else - qp = create_qp(dev, pd, attr, udata, uobj, NULL); + qp = create_qp(dev, pd, attr, uattrs, uobj, NULL); if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(qp)) return qp; From bad4e98893afdfe0b1a03433d3af53972bafc67c Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 26 May 2026 13:15:06 -0300 Subject: [PATCH 082/148] RDMA: Update the query_device() op This op hasn't followed the normal pattern of passing NULL for udata when invoked by the kernel. Instead the kernel caller creates a dummy ib_udata on the stack and passes that in. It does not seem to currently be a bug, but this flow should be modernized to use the new API flow and in the process accept NULL as well. Only mlx4 uses an input request structure, have every other driver call ib_is_udata_in_empty() to enforce the lack of request structs. Use ib_respond_empty_udata() in every driver that does not use a response struct. Ensure a check for NULL udata before calling ib_respond_udata() in bnxt_re, efa, and mlx5. Make mlx4 safe to be called with NULL. Link: https://patch.msgid.link/r/2-v1-922fa8e828ba+f7-ib_udata_stack_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/device.c | 3 +-- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 5 ++++- drivers/infiniband/hw/cxgb4/provider.c | 8 +++++--- drivers/infiniband/hw/erdma/erdma_verbs.c | 9 +++++++-- drivers/infiniband/hw/hns/hns_roce_main.c | 7 ++++++- drivers/infiniband/hw/ionic/ionic_ibdev.c | 7 ++++++- drivers/infiniband/hw/irdma/verbs.c | 8 +++++--- drivers/infiniband/hw/mana/main.c | 7 ++++++- drivers/infiniband/hw/mlx4/main.c | 13 +++++++------ drivers/infiniband/hw/mthca/mthca_provider.c | 13 ++++++++----- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 8 +++++--- drivers/infiniband/hw/qedr/verbs.c | 7 ++++++- drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 8 +++++--- drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c | 8 +++++--- drivers/infiniband/sw/rdmavt/vt.c | 9 ++++++--- drivers/infiniband/sw/rxe/rxe_verbs.c | 14 ++++---------- drivers/infiniband/sw/siw/siw_verbs.c | 8 +++++--- 17 files changed, 91 insertions(+), 51 deletions(-) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 21ada0fe9059..b8193e077a74 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -1246,7 +1246,6 @@ static int assign_name(struct ib_device *device, const char *name) */ static int setup_device(struct ib_device *device) { - struct ib_udata uhw = {.outlen = 0, .inlen = 0}; int ret; ib_device_check_mandatory(device); @@ -1258,7 +1257,7 @@ static int setup_device(struct ib_device *device) } memset(&device->attrs, 0, sizeof(device->attrs)); - ret = device->ops.query_device(device, &device->attrs, &uhw); + ret = device->ops.query_device(device, &device->attrs, NULL); if (ret) { dev_warn(&device->dev, "Couldn't query the device attributes\n"); diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index c1c4ddc615e2..0ca549c2f5cb 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -265,7 +265,10 @@ int bnxt_re_query_device(struct ib_device *ibdev, resp.packet_pacing_caps.supported_qpts = 1 << IB_QPT_RC; } - return ib_respond_udata(udata, resp); + + if (udata) + return ib_respond_udata(udata, resp); + return 0; } int bnxt_re_modify_device(struct ib_device *ibdev, diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 0e3827022c63..e1eec37ee822 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -259,11 +259,13 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro { struct c4iw_dev *dev; + int err; pr_debug("ibdev %p\n", ibdev); - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; dev = to_c4iw_dev(ibdev); addrconf_addr_eui48((u8 *)&props->sys_image_guid, @@ -298,7 +300,7 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro props->max_fast_reg_page_list_len = t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl); - return 0; + return ib_respond_empty_udata(uhw); } static int c4iw_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 9a9dc78c7ab6..74afe6eb18b0 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -315,9 +315,14 @@ erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, } int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, - struct ib_udata *unused) + struct ib_udata *udata) { struct erdma_dev *dev = to_edev(ibdev); + int err; + + err = ib_is_udata_in_empty(udata); + if (err) + return err; memset(attr, 0, sizeof(*attr)); @@ -358,7 +363,7 @@ int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, addrconf_addr_eui48((u8 *)&attr->sys_image_guid, dev->netdev->dev_addr); - return 0; + return ib_respond_empty_udata(udata); } int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 77bad9f5d482..c6f633bd5a34 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -221,6 +221,11 @@ static int hns_roce_query_device(struct ib_device *ib_dev, struct ib_udata *uhw) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + int ret; + + ret = ib_is_udata_in_empty(uhw); + if (ret) + return ret; memset(props, 0, sizeof(*props)); @@ -274,7 +279,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) props->device_cap_flags |= IB_DEVICE_XRC; - return 0; + return ib_respond_empty_udata(uhw); } static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num, diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index 73a616ae3502..b0449c75f893 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -25,6 +25,11 @@ static int ionic_query_device(struct ib_device *ibdev, { struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); struct net_device *ndev; + int err; + + err = ib_is_udata_in_empty(udata); + if (err) + return err; ndev = ib_device_get_netdev(ibdev, 1); addrconf_ifid_eui48((u8 *)&attr->sys_image_guid, ndev); @@ -69,7 +74,7 @@ static int ionic_query_device(struct ib_device *ibdev, attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; attr->max_pkeys = IONIC_PKEY_TBL_LEN; - return 0; + return ib_respond_empty_udata(udata); } static int ionic_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index baee48df5fdf..b19edf31c800 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -16,9 +16,11 @@ static int irdma_query_device(struct ib_device *ibdev, struct irdma_pci_f *rf = iwdev->rf; struct pci_dev *pcidev = iwdev->rf->pcidev; struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; + int err; - if (udata->inlen || udata->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(udata); + if (err) + return err; memset(props, 0, sizeof(*props)); addrconf_addr_eui48((u8 *)&props->sys_image_guid, @@ -74,7 +76,7 @@ static int irdma_query_device(struct ib_device *ibdev, if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; - return 0; + return ib_respond_empty_udata(udata); } /** diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 1e93df6455f5..a86f80b911ef 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -549,6 +549,11 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, { struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)->dev); + int err; + + err = ib_is_udata_in_empty(uhw); + if (err) + return err; memset(props, 0, sizeof(*props)); props->vendor_id = pdev->vendor; @@ -576,7 +581,7 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, if (!mana_ib_is_rnic(dev)) props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM; - return 0; + return ib_respond_empty_udata(uhw); } int mana_ib_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index d50743f090bf..17073e8f105a 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -444,8 +444,9 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, struct mlx4_uverbs_ex_query_device cmd; struct mlx4_uverbs_ex_query_device_resp resp = {}; struct mlx4_clock_params clock_params; + size_t uhw_outlen = uhw ? uhw->outlen : 0; - if (uhw->inlen) { + if (uhw && uhw->inlen) { err = ib_copy_validate_udata_in_cm(uhw, cmd, reserved, 0); if (err) return err; @@ -572,7 +573,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, props->cq_caps.max_cq_moderation_count = MLX4_MAX_CQ_COUNT; props->cq_caps.max_cq_moderation_period = MLX4_MAX_CQ_PERIOD; - if (uhw->outlen >= resp.response_length + sizeof(resp.hca_core_clock_offset)) { + if (uhw_outlen >= resp.response_length + sizeof(resp.hca_core_clock_offset)) { resp.response_length += sizeof(resp.hca_core_clock_offset); if (!mlx4_get_internal_clock_params(dev->dev, &clock_params)) { resp.comp_mask |= MLX4_IB_QUERY_DEV_RESP_MASK_CORE_CLOCK_OFFSET; @@ -580,14 +581,14 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, } } - if (uhw->outlen >= resp.response_length + + if (uhw_outlen >= resp.response_length + sizeof(resp.max_inl_recv_sz)) { resp.response_length += sizeof(resp.max_inl_recv_sz); resp.max_inl_recv_sz = dev->dev->caps.max_rq_sg * sizeof(struct mlx4_wqe_data_seg); } - if (offsetofend(typeof(resp), rss_caps) <= uhw->outlen) { + if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) { if (props->rss_caps.supported_qpts) { resp.rss_caps.rx_hash_function = MLX4_IB_RX_HASH_FUNC_TOEPLITZ; @@ -611,7 +612,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, sizeof(resp.rss_caps); } - if (offsetofend(typeof(resp), tso_caps) <= uhw->outlen) { + if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) { if (dev->dev->caps.max_gso_sz && ((mlx4_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) || @@ -625,7 +626,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, sizeof(resp.tso_caps); } - if (uhw->outlen) { + if (uhw_outlen) { err = ib_respond_udata(uhw, resp); if (err) goto out; diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 23e387dba79d..f90f67afc8fa 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -55,16 +55,19 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr { struct ib_smp *in_mad; struct ib_smp *out_mad; - int err = -ENOMEM; + int err; struct mthca_dev *mdev = to_mdev(ibdev); - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; in_mad = kzalloc_obj(*in_mad); out_mad = kmalloc_obj(*out_mad); - if (!in_mad || !out_mad) + if (!in_mad || !out_mad) { + err = -ENOMEM; goto out; + } memset(props, 0, sizeof *props); @@ -111,7 +114,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * props->max_mcast_grp; - err = 0; + err = ib_respond_empty_udata(uhw); out: kfree(in_mad); kfree(out_mad); diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 69caadcff810..53ba32d168a1 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -68,9 +68,11 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *uhw) { struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); + int err; - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; memset(attr, 0, sizeof *attr); memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], @@ -110,7 +112,7 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; attr->max_pkeys = 1; - return 0; + return ib_respond_empty_udata(uhw); } static inline void get_link_speed_and_width(struct ocrdma_dev *dev, diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 7195eeede530..c90a1b5c8ee7 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -105,6 +105,7 @@ int qedr_query_device(struct ib_device *ibdev, { struct qedr_dev *dev = get_qedr_dev(ibdev); struct qedr_device_attr *qattr = &dev->attr; + int rc; if (!dev->rdma_ctx) { DP_ERR(dev, @@ -113,6 +114,10 @@ int qedr_query_device(struct ib_device *ibdev, return -EINVAL; } + rc = ib_is_udata_in_empty(udata); + if (rc) + return rc; + memset(attr, 0, sizeof(*attr)); attr->fw_ver = qattr->fw_ver; @@ -155,7 +160,7 @@ int qedr_query_device(struct ib_device *ibdev, attr->max_pkeys = qattr->max_pkey; attr->max_ah = qattr->max_ah; - return 0; + return ib_respond_empty_udata(udata); } static inline void get_link_speed_and_width(int speed, u16 *ib_speed, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 261f18a83685..dc355b00f61c 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -275,10 +275,12 @@ int usnic_ib_query_device(struct ib_device *ibdev, union ib_gid gid; struct ethtool_drvinfo info; int qp_per_vf; + int err; usnic_dbg("\n"); - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; mutex_lock(&us_ibdev->usdev_lock); us_ibdev->netdev->ethtool_ops->get_drvinfo(us_ibdev->netdev, &info); @@ -322,7 +324,7 @@ int usnic_ib_query_device(struct ib_device *ibdev, * max_qp_wr, max_sge, max_sge_rd, max_cqe */ mutex_unlock(&us_ibdev->usdev_lock); - return 0; + return ib_respond_empty_udata(uhw); } int usnic_ib_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index b9c3202b9545..1d29a535f76a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -67,9 +67,11 @@ int pvrdma_query_device(struct ib_device *ibdev, struct ib_udata *uhw) { struct pvrdma_dev *dev = to_vdev(ibdev); + int err; - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; props->fw_ver = dev->dsr->caps.fw_ver; props->sys_image_guid = dev->dsr->caps.sys_image_guid; @@ -114,7 +116,7 @@ int pvrdma_query_device(struct ib_device *ibdev, props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN; - return 0; + return ib_respond_empty_udata(uhw); } /** diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index 40aa64208364..5fa3a1f33326 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "vt.h" #include "cq.h" #include "trace.h" @@ -79,14 +80,16 @@ static int rvt_query_device(struct ib_device *ibdev, struct ib_udata *uhw) { struct rvt_dev_info *rdi = ib_to_rvt(ibdev); + int err; - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; /* * Return rvt_dev_info.dparms.props contents */ *props = rdi->dparms.props; - return 0; + return ib_respond_empty_udata(uhw); } static int rvt_get_numa_node(struct ib_device *ibdev) diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 8edd4dd1f031..5815ce34d970 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -22,19 +22,13 @@ static int rxe_query_device(struct ib_device *ibdev, struct rxe_dev *rxe = to_rdev(ibdev); int err; - if (udata->inlen || udata->outlen) { - rxe_dbg_dev(rxe, "malformed udata\n"); - err = -EINVAL; - goto err_out; - } + err = ib_is_udata_in_empty(udata); + if (err) + return err; memcpy(attr, &rxe->attr, sizeof(*attr)); - return 0; - -err_out: - rxe_err_dev(rxe, "returned err = %d\n", err); - return err; + return ib_respond_empty_udata(udata); } static int rxe_query_port(struct ib_device *ibdev, diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index b34f3d6547ff..b74ac85c1b8b 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -130,9 +130,11 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, struct ib_udata *udata) { struct siw_device *sdev = to_siw_dev(base_dev); + int rv; - if (udata->inlen || udata->outlen) - return -EINVAL; + rv = ib_is_udata_in_empty(udata); + if (rv) + return rv; memset(attr, 0, sizeof(*attr)); @@ -165,7 +167,7 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, addrconf_addr_eui48((u8 *)&attr->sys_image_guid, sdev->raw_gid); - return 0; + return ib_respond_empty_udata(udata); } int siw_query_port(struct ib_device *base_dev, u32 port, From 09ea6837a0434fb4db99528a5055b6d822135dcf Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 1 Jun 2026 13:52:32 -0300 Subject: [PATCH 083/148] RDMA/umem: Be careful about boundary conditions in ib_umem_find_best_pgsz() Several corner cases, especially important on 32 bits: - umem->iova is u64, the function argument should pass in u64 or iova will be truncated - Check that the length is not too large for the iova - Check that lengths > 4G don't overflow the GENMASK Link: https://patch.msgid.link/r/2-v1-88303e9e509f+f7-ib_umem_types_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 18 ++++++++++++------ include/rdma/ib_umem.h | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index e424a9de66c1..fd8f3888da52 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -84,14 +84,17 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d */ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt) + u64 virt) { unsigned long curr_len = 0; dma_addr_t curr_base = ~0; - unsigned long va, pgoff; + unsigned long pgoff; struct scatterlist *sg; - dma_addr_t mask; + unsigned long mask = 0; + unsigned int bits; dma_addr_t end; + u64 last_va; + u64 va; int i; umem->iova = va = virt; @@ -109,9 +112,12 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, * number of required pages. Compute the largest page size that could * work based on VA address bits that don't change. */ - mask = pgsz_bitmap & - GENMASK(BITS_PER_LONG - 1, - bits_per((umem->length - 1 + virt) ^ virt)); + if (check_add_overflow(umem->length - 1, virt, &last_va)) + return 0; + bits = bits_per(virt ^ last_va); + if (bits < BITS_PER_LONG) + mask = pgsz_bitmap & GENMASK(BITS_PER_LONG - 1, bits); + /* offset into first SGL */ pgoff = umem->address & ~PAGE_MASK; diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index bc1e6ed73b3f..4c8f433ba246 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -109,7 +109,7 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length); unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt); + u64 virt); /** * ib_umem_find_best_pgoff - Find best HW page size @@ -234,7 +234,7 @@ static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offs } static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt) + u64 virt) { return 0; } From 55d984dae65262fbb0a673cff7cc9824a5e19bdb Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 1 Jun 2026 13:52:33 -0300 Subject: [PATCH 084/148] RDMA/umem: Make ib_umem_is_contiguous() safe on 32 bit Sashiko points out the roundup_pow_of_two() only uses unsigned long but dma_addr_t can be u64. Change this algorithm to be simpler, compute the page size, if any page size is found and it results in a single block then it is contiguous. Link: https://patch.msgid.link/r/3-v1-88303e9e509f+f7-ib_umem_types_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- include/rdma/ib_umem.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 4c8f433ba246..bb4005a9c690 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -145,16 +145,11 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, static inline bool ib_umem_is_contiguous(struct ib_umem *umem) { - dma_addr_t dma_addr; unsigned long pgsz; - /* - * Select the smallest aligned page that can contain the whole umem if - * it was contiguous. - */ - dma_addr = ib_umem_start_dma_addr(umem); - pgsz = roundup_pow_of_two((dma_addr ^ (umem->length - 1 + dma_addr)) + 1); - return !!ib_umem_find_best_pgoff(umem, pgsz, U64_MAX); + pgsz = ib_umem_find_best_pgsz(umem, ULONG_MAX, + ib_umem_start_dma_addr(umem)); + return pgsz && ib_umem_num_dma_blocks(umem, pgsz) == 1; } struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, From 0ee8ac903e5aa100a70bef8d0afc19a336ffa775 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 2 Jun 2026 16:04:34 +0200 Subject: [PATCH 085/148] RDMA/hfi1: Open-code rvt_set_ibdev_name() clang warns about a function missing a printf attribute: include/rdma/rdma_vt.h:457:47: error: diagnostic behavior may be improved by adding the 'format(printf, 2, 3)' attribute to the declaration of 'rvt_set_ibdev_name' [-Werror,-Wmissing-format-attribute] 447 | static inline void rvt_set_ibdev_name(struct rvt_dev_info *rdi, | __attribute__((format(printf, 2, 3))) 448 | const char *fmt, const char *name, 449 | const int unit) The helper was originally added as an abstraction for the hfi1 and qib drivers needing the same thing, but now qib is gone, and hfi1 is the only remaining user of rdma_vt. Avoid the warning and allow the compiler to check the format string by open-coding the helper and directly assigning the device name. Fixes: 5084c8ff21f2 ("IB/{rdmavt, hfi1, qib}: Self determine driver name") Link: https://patch.msgid.link/r/20260602140453.3542427-1-arnd@kernel.org Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/init.c | 13 ++++++++++++- include/rdma/rdma_vt.h | 20 -------------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 8b5a5b32b0fa..b7fd8b1fbbbd 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1206,6 +1206,7 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra) { struct hfi1_devdata *dd; + struct ib_device *ibdev; int ret, nports; /* extra is * number of ports */ @@ -1227,7 +1228,17 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, "Could not allocate unit ID: error %d\n", -ret); goto bail; } - rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit); + + /* + * FIXME: rvt and its users want to touch the ibdev before + * registration and have things like the name work. We don't have the + * infrastructure in the core to support this directly today, hack it + * to work by setting the name manually here. + */ + ibdev = &dd->verbs_dev.rdi.ibdev; + dev_set_name(&ibdev->dev, "%s_%d", class_name(), dd->unit); + strscpy(ibdev->name, dev_name(&ibdev->dev), IB_DEVICE_NAME_MAX); + /* * If the BIOS does not have the NUMA node information set, select * NUMA 0 so we get consistent performance. diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h index 7d8de561f71b..7ffc83262a01 100644 --- a/include/rdma/rdma_vt.h +++ b/include/rdma/rdma_vt.h @@ -438,26 +438,6 @@ struct rvt_dev_info { struct rvt_wss *wss; }; -/** - * rvt_set_ibdev_name - Craft an IB device name from client info - * @rdi: pointer to the client rvt_dev_info structure - * @name: client specific name - * @unit: client specific unit number. - */ -static inline void rvt_set_ibdev_name(struct rvt_dev_info *rdi, - const char *fmt, const char *name, - const int unit) -{ - /* - * FIXME: rvt and its users want to touch the ibdev before - * registration and have things like the name work. We don't have the - * infrastructure in the core to support this directly today, hack it - * to work by setting the name manually here. - */ - dev_set_name(&rdi->ibdev.dev, fmt, name, unit); - strscpy(rdi->ibdev.name, dev_name(&rdi->ibdev.dev), IB_DEVICE_NAME_MAX); -} - /** * rvt_get_ibdev_name - return the IB name * @rdi: rdmavt device From 6219a17c3741461cee83cf82252b15c7f99a9f2a Mon Sep 17 00:00:00 2001 From: Sriharsha Basavapatna Date: Tue, 2 Jun 2026 20:26:18 +0530 Subject: [PATCH 086/148] RDMA/bnxt_re: Update create_qp to use QP buffer umem attrs Use ib_umem_get_attr_or_va() helper to pin QP buffer umems. Pass attribute ids SQ_BUF_UMEM and RQ_BUF_UMEM for respective buffers. Link: https://patch.msgid.link/r/20260602145618.21643-1-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 0ca549c2f5cb..b1c489867fc7 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1198,7 +1198,8 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, struct bnxt_re_qp_req *ureq, bool fixed_que_attr, - struct bnxt_re_dbr_obj *dbr_obj) + struct bnxt_re_dbr_obj *dbr_obj, + struct uverbs_attr_bundle *attrs) { struct bnxt_qplib_qp *qplib_qp; struct ib_umem *umem; @@ -1213,8 +1214,9 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr); bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get_va(&rdev->ibdev, ureq->qpsva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_attr_or_va(&rdev->ibdev, attrs, + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + ureq->qpsva, bytes, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) return PTR_ERR(umem); @@ -1228,8 +1230,9 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get_va(&rdev->ibdev, ureq->qprva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_attr_or_va(&rdev->ibdev, attrs, + UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + ureq->qprva, bytes, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { rc = PTR_ERR(umem); goto fail; @@ -1721,7 +1724,8 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct bnxt_re_ucontext *uctx, struct bnxt_re_qp_req *ureq, struct bnxt_re_dbr_obj *dbr_obj, - bool fixed_que_attr) + bool fixed_que_attr, + struct uverbs_attr_bundle *attrs) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1795,7 +1799,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, if (uctx) { /* This will update DPI and qp_handle */ rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr, - dbr_obj); + dbr_obj, attrs); if (rc) return rc; } @@ -1931,9 +1935,9 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev) int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { + struct uverbs_attr_bundle *attrs = NULL; struct bnxt_re_dbr_obj *dbr_obj = NULL; struct bnxt_qplib_dev_attr *dev_attr; - struct uverbs_attr_bundle *attrs; struct bnxt_re_ucontext *uctx; bool fixed_que_attr = false; struct bnxt_re_qp_req ureq; @@ -1979,7 +1983,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, qp->rdev = rdev; rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, - dbr_obj, fixed_que_attr); + dbr_obj, fixed_que_attr, attrs); if (rc) goto fail; From 9b2207bc5cdb955bdae34b3eec80f04979e17081 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 2 Jun 2026 16:37:28 -0300 Subject: [PATCH 087/148] IB/cm: Fix av cm device leak on an error path in cm_init_av_by_path() Codex pointed out that cm_init_av_by_path() can call cm_set_av_port() which takes a reference on the cm device, but then can immediately return error if ib_init_ah_attr_from_path() fails. Since callers like ib_send_cm_req() put the av on the stack this leaks that cm device reference. Re-order cm_init_av_by_path() so it doesn't touch the av until it has done all its failable work, and then update the av in one shot so it is either left alone or fully init'd. Sashiko also pointed out that the cm_destroy_av() prior to cm_init_av_by_path() is harmful as it leaves the AV broken in the error case and thus the REJ won't send. Since cm_init_av_by_path() is now atomic it is safe to delete the cm_destroy_av(). On succees the av from cm_init_av_for_response() is cleaned up by cm_init_av_by_path(), on failure the 'goto rejected' guarentees the av is destroyed during ib_destroy_cm_id(). Fixes: 76039ac9095f ("IB/cm: Protect cm_dev, cm_ports and mad_agent with kref and lock") Link: https://patch.msgid.link/r/0-v1-38292501f539+14f-ib_cm_av_leak_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 6ab9a0aee1ec..1a2c2775b14d 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -530,6 +530,7 @@ static int cm_init_av_by_path(struct sa_path_rec *path, struct rdma_ah_attr new_ah_attr; struct cm_device *cm_dev; struct cm_port *port; + u16 pkey_index; int ret; port = get_cm_port_from_path(path, sgid_attr); @@ -538,12 +539,10 @@ static int cm_init_av_by_path(struct sa_path_rec *path, cm_dev = port->cm_dev; ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num, - be16_to_cpu(path->pkey), &av->pkey_index); + be16_to_cpu(path->pkey), &pkey_index); if (ret) return ret; - cm_set_av_port(av, port); - /* * av->ah_attr might be initialized based on wc or during * request processing time which might have reference to sgid_attr. @@ -558,6 +557,8 @@ static int cm_init_av_by_path(struct sa_path_rec *path, if (ret) return ret; + av->pkey_index = pkey_index; + cm_set_av_port(av, port); av->timeout = path->packet_life_time + 1; rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); return 0; @@ -2184,8 +2185,10 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->av.ah_attr.roce.dmac); work->path[0].hop_limit = grh->hop_limit; - /* This destroy call is needed to pair with cm_init_av_for_response */ - cm_destroy_av(&cm_id_priv->av); + /* + * cm_init_av_by_path() will internally pair with the above + * cm_init_av_for_response() if it succeeds. + */ ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av); if (ret) { int err; From 309cacf953aa7e289ce50b3e7e4424c63103c333 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Mon, 1 Jun 2026 05:56:54 -0400 Subject: [PATCH 088/148] RDMA/mlx5: Fix error propagation in __mlx5_ib_add __mlx5_ib_add() currently returns -ENOMEM on any stage initialization failure, losing the actual error code returned by the init function. This makes it impossible for callers to distinguish between different failure reasons (e.g. -EINVAL, -EIO, -EOPNOTSUPP) and leads to misleading error handling. Fix it by returning the actual error code stored in 'err'. Link: https://patch.msgid.link/r/20260601095654.2178-1-lirongqing@baidu.com Signed-off-by: Li RongQing Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 12bfefc91b02..e2e8feb1b60d 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -5116,7 +5116,7 @@ int __mlx5_ib_add(struct mlx5_ib_dev *dev, if (profile->stage[i].cleanup) profile->stage[i].cleanup(dev); } - return -ENOMEM; + return err; } static const struct mlx5_ib_profile pf_profile = { From 0d32eabccbe4b2f8d45be3192c5f3c76c8af703d Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Mon, 1 Jun 2026 05:58:18 -0400 Subject: [PATCH 089/148] RDMA/mlx5: Fix state and counter desync on loopback enable failure In mlx5_ib_enable_lb(), dev->lb.enabled was unconditionally set to true even if mlx5_nic_vport_update_local_lb() failed. Fix this by only setting dev->lb.enabled on success. On failure, roll back the reference counters and return the error. Link: https://patch.msgid.link/r/20260601095818.2227-1-lirongqing@baidu.com Signed-off-by: Li RongQing Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index e2e8feb1b60d..a558ac5bb219 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2024,6 +2024,9 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) dev->lb.qps == 1) { if (!dev->lb.enabled) { err = mlx5_nic_vport_update_local_lb(dev->mdev, true); + if (err) + goto err_rollback; + dev->lb.enabled = true; } } @@ -2031,6 +2034,14 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) mutex_unlock(&dev->lb.mutex); return err; + +err_rollback: + if (td) + dev->lb.user_td--; + if (qp) + dev->lb.qps--; + mutex_unlock(&dev->lb.mutex); + return err; } void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) From 7d29f7e9dbd844cae4d3e559cf78324b9642fd6b Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Tue, 2 Jun 2026 15:47:00 -0400 Subject: [PATCH 090/148] RDMA/siw: bound Read Response placement to the RREAD length In drivers/infiniband/sw/siw/siw_qp_rx.c, siw_proc_rresp() places each inbound Read Response DDP segment at sge->laddr + wqe->processed and then accumulates wqe->processed, but it never checks the running total against the sink buffer length on continuation segments. siw_check_sge() resolves and validates the sink memory only on the first fragment (the if (!*mem) branch), and siw_rresp_check_ntoh() compares the cumulative length against wqe->bytes only on the final segment (the !frx->more_ddp_segs guard). A connected siw peer that answers an outstanding RREAD with Read Response segments that keep the DDP Last flag clear, carrying more total payload than the RREAD requested, drives wqe->processed past the validated sink buffer; the next siw_rx_data() call writes out of bounds at sge->laddr + wqe->processed. siw runs iWARP over ordinary routable TCP, so the peer is the remote end of an established RDMA connection and needs no local privilege. Bound every segment before placement, exactly as siw_proc_send() and siw_proc_write() already do for their tagged and untagged paths, and terminate the connection with a base-or-bounds DDP error when the Read Response would overrun the sink buffer. This is the second receive-path length fix for this file. A separate change rejects an MPA FPDU length that underflows the per-fragment remainder in the header decode; that guard does not cover this case, because here each individual segment length is self-consistent and only the accumulated placement offset overruns the buffer. Fixes: 8b6a361b8c48 ("rdma/siw: receive path") Link: https://patch.msgid.link/r/20260602194700.2273758-1-michael.bommarito@gmail.com Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Michael Bommarito Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/siw/siw_qp_rx.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index 34d03584160c..b566d163c5aa 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -844,6 +844,15 @@ int siw_proc_rresp(struct siw_qp *qp) } mem_p = *mem; + if (unlikely(wqe->processed + srx->fpdu_part_rem > wqe->bytes)) { + siw_dbg_qp(qp, "rresp len: %d + %d > %d\n", + wqe->processed, srx->fpdu_part_rem, wqe->bytes); + wqe->wc_status = SIW_WC_LOC_LEN_ERR; + siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, + DDP_ETYPE_TAGGED_BUF, + DDP_ECODE_T_BASE_BOUNDS, 0); + return -EINVAL; + } bytes = min(srx->fpdu_part_rem, srx->skb_new); rv = siw_rx_data(mem_p, srx, &frx->pbl_idx, sge->laddr + wqe->processed, bytes); From ed8621be482bf18dcd217aa024e95758bf9d28f2 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Tue, 2 Jun 2026 21:44:22 +0000 Subject: [PATCH 091/148] RDMA/irdma: Remove redundant legacy_mode checks The driver has the following invariants: 1. legacy_mode is only allowed on GEN_1 hardware (enforced in irdma_alloc_ucontext). 2. GEN_1 hardware does not set IRDMA_FEATURE_CQ_RESIZE or IRDMA_FEATURE_RTS_AE. These feature flags are only set for GEN_2 and GEN_3 hardware. Therefore, legacy_mode is always false if IRDMA_FEATURE_CQ_RESIZE or IRDMA_FEATURE_RTS_AE is set, so remove the redundant checks. Link: https://patch.msgid.link/r/20260602214423.1315105-1-jmoroni@google.com Signed-off-by: Jacob Moroni Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/uk.c | 9 +++------ drivers/infiniband/hw/irdma/user.h | 1 - drivers/infiniband/hw/irdma/verbs.c | 7 +------ 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c index 4718acf6c6fd..a34883fe9983 100644 --- a/drivers/infiniband/hw/irdma/uk.c +++ b/drivers/infiniband/hw/irdma/uk.c @@ -1568,15 +1568,12 @@ static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = { * irdma_setup_connection_wqes - setup WQEs necessary to complete * connection. * @qp: hw qp (user and kernel) - * @info: qp initialization info */ -static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, - struct irdma_qp_uk_init_info *info) +static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp) { u16 move_cnt = 1; - if (!info->legacy_mode && - (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)) + if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) move_cnt = 3; qp->conn_wqes = move_cnt; @@ -1727,7 +1724,7 @@ int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) sq_ring_size = qp->sq_size << info->sq_shift; IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); if (info->first_sq_wq) { - irdma_setup_connection_wqes(qp, info); + irdma_setup_connection_wqes(qp); qp->swqe_polarity = 1; qp->first_sq_wq = true; } else { diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h index 008af1acc928..4dd3776a4cdd 100644 --- a/drivers/infiniband/hw/irdma/user.h +++ b/drivers/infiniband/hw/irdma/user.h @@ -563,7 +563,6 @@ struct irdma_qp_uk_init_info { u8 sq_shift; u8 rq_shift; int abi_ver; - bool legacy_mode; struct irdma_srq_uk *srq_uk; }; diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index b19edf31c800..284dfba4e984 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -634,7 +634,6 @@ static int irdma_setup_umode_qp(struct ib_udata *udata, iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; iwqp->user_mode = 1; if (req.user_wqe_bufs) { - info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode; spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, &ucontext->qp_reg_mem_list); @@ -2066,10 +2065,6 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - /* CQ resize not supported with legacy GEN_1 libi40iw */ - if (ucontext->legacy_mode) - return -EOPNOTSUPP; - if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return -EINVAL; @@ -2549,7 +2544,7 @@ static int irdma_create_cq(struct ib_cq *ibcq, cqmr = &iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & - IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { + IRDMA_FEATURE_CQ_RESIZE) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl_shadow = irdma_get_pbl( (unsigned long)req.user_shadow_area, From 4385ddd654d90245eeb83b3cb539670ab5c85ba4 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Tue, 2 Jun 2026 21:44:23 +0000 Subject: [PATCH 092/148] RDMA/irdma: Fix OOB read during CQ MR registration Sashiko pointed out an unrelated bug during a previous patch: https://sashiko.dev/#/patchset/20260512183852.614045-1-jmoroni%40google.com This change fixes the bug by eliminating the cqmr->split field which was not being set properly and instead just checks the CQ resize feature flag directly. The cqmr->split field essentially tracks whether IRDMA_FEATURE_CQ_RESIZE is set, but it was not being set until CQ creation time, which is _after_ CQ memory registration (the only other place where it is referenced). As a result, it would always be false during MR registration and would therefore cause irdma_handle_q_mem to populate cqmr->shadow even for GEN_2 HW and beyond: cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; The issue is that for GEN_2 and beyond, req->cq_pages may be exactly equal to iwmr->page_cnt and therefore equal to the size of arr, which would cause an OOB read by one. Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs") Link: https://patch.msgid.link/r/20260602214423.1315105-2-jmoroni@google.com Signed-off-by: Jacob Moroni Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/verbs.c | 4 ++-- drivers/infiniband/hw/irdma/verbs.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 284dfba4e984..231b2854bb14 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2557,7 +2557,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, } cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; - cqmr->split = true; } else { info.shadow_area_pa = cqmr->shadow; } @@ -2961,7 +2960,8 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev, case IRDMA_MEMREG_TYPE_CQ: hmc_p = &cqmr->cq_pbl; - if (!cqmr->split) + if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE)) cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; if (lvl) diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index aabbb3442098..289ebc9b23ca 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -65,7 +65,6 @@ struct irdma_hmc_pble { struct irdma_cq_mr { struct irdma_hmc_pble cq_pbl; dma_addr_t shadow; - bool split; }; struct irdma_srq_mr { From 1521d560b7a1a39e437d37fffd9b55435d329ad1 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Thu, 4 Jun 2026 15:41:04 +0000 Subject: [PATCH 093/148] RDMA/irdma: Initialize iwmr->access during MR registration Initialize iwmr->access during initial user mem registration so that it contains a valid value during a subsequent rereg_mr. Otherwise, a rereg_mr that doesn't set IB_MR_REREG_ACCESS (for example, one that only changes the PD) ends up clearing the access flags in HW since iwmr->access is zero-initialized, which is not intended. Fixes: 5ac388db27c4 ("RDMA/irdma: Add support to re-register a memory region") Link: https://patch.msgid.link/r/20260604154104.4035581-1-jmoroni@google.com Signed-off-by: Jacob Moroni Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/verbs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 231b2854bb14..b79f5afe68e5 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -3306,6 +3306,7 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, int err; lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; + iwmr->access = access; err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) From ea4f6f6c53577fb3f05dbd78b15e586772d49831 Mon Sep 17 00:00:00 2001 From: Bernard Metzler Date: Thu, 4 Jun 2026 18:08:08 +0200 Subject: [PATCH 094/148] RDMA/siw: Fix endpoint/socket association handling Disassociating a socket from an endpoint via siw_socket_disassoc() may release the last reference on that endpoint and free it. Therefore, don't clear the endpoints socket pointer after calling that function, but within. This fixes a: BUG: KASAN: slab-use-after-free in siw_cm_work_handler (drivers/infiniband/sw/siw/siw_cm.c:1053 drivers/infiniband/sw/siw/siw_cm.c:1075) which occurred after processing a malformed MPA request during connection establishment, causing the new endpoint to be closed. Fixes: 6c52fdc244b5c ("rdma/siw: connection management") Link: https://patch.msgid.link/r/20260604160808.30948-1-bernard.metzler@linux.dev Reported-by: Shuangpeng Bai Signed-off-by: Bernard Metzler Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/siw/siw_cm.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index f7ac81c0f267..87c79527ac09 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -138,6 +138,7 @@ static void siw_socket_disassoc(struct socket *s) cep = sk_to_cep(sk); if (cep) { siw_sk_restore_upcalls(sk, cep); + cep->sock = NULL; siw_cep_put(cep); } else { pr_warn("siw: cannot restore sk callbacks: no ep\n"); @@ -418,10 +419,11 @@ static void siw_free_cm_id(struct siw_cep *cep) static void siw_destroy_cep_sock(struct siw_cep *cep) { - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + struct socket *s = cep->sock; + + if (s) { + siw_socket_disassoc(s); + sock_release(s); } } @@ -1050,7 +1052,6 @@ static void siw_accept_newconn(struct siw_cep *cep) if (new_s) { siw_socket_disassoc(new_s); sock_release(new_s); - new_cep->sock = NULL; } siw_dbg_cep(cep, "error %d\n", rv); } @@ -1202,6 +1203,8 @@ static void siw_cm_work_handler(struct work_struct *w) WARN(1, "Undefined CM work type: %d\n", work->type); } if (release_cep) { + struct socket *s = cep->sock; + siw_dbg_cep(cep, "release: timer=%s, QP[%u]\n", cep->mpa_timer ? "y" : "n", @@ -1227,10 +1230,9 @@ static void siw_cm_work_handler(struct work_struct *w) cep->qp = NULL; siw_qp_put(qp); } - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + if (s) { + siw_socket_disassoc(s); + sock_release(s); } if (cep->cm_id) { siw_free_cm_id(cep); @@ -1561,7 +1563,6 @@ int siw_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) if (cep) { siw_socket_disassoc(s); sock_release(s); - cep->sock = NULL; cep->qp = NULL; @@ -1937,7 +1938,6 @@ int siw_create_listen(struct iw_cm_id *id, int backlog) siw_cep_set_inuse(cep); siw_free_cm_id(cep); - cep->sock = NULL; siw_socket_disassoc(s); cep->state = SIW_EPSTATE_CLOSED; @@ -1959,6 +1959,7 @@ static void siw_drop_listeners(struct iw_cm_id *id) */ list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) { struct siw_cep *cep = list_entry(p, struct siw_cep, listenq); + struct socket *s = cep->sock; list_del(p); @@ -1967,10 +1968,9 @@ static void siw_drop_listeners(struct iw_cm_id *id) siw_cep_set_inuse(cep); siw_free_cm_id(cep); - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + if (s) { + siw_socket_disassoc(s); + sock_release(s); } cep->state = SIW_EPSTATE_CLOSED; siw_cep_set_free_and_put(cep); From 55d339d200c908de83a408d023cfb7be779b0dd7 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:40 -0300 Subject: [PATCH 095/148] IB/mlx5: Don't take the rereg_mr fallback without a new translation Jumping to mlx5_ib_reg_user_mr() without IB_MR_REREG_TRANS set will use garbage values for start, length, and iova. Recovering the original mr parameters for ODP and DMABUF to properly recreate it is too hard in this flow, so just fail it. Fixes: ef3642c4f54d ("RDMA/mlx5: Fix error unwinds for rereg_mr") Link: https://patch.msgid.link/r/1-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 254e6aa4ccaf..0fa0a57240c0 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1198,7 +1198,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, } /* DM or ODP MR's don't have a normal umem so we can't re-use it */ if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) - goto recreate; + return ERR_PTR(-EOPNOTSUPP); /* * Only one active MR can refer to a umem at one time, revoke From 93e64c7a33ff6679d4d1c0a03021a2ff0e2b6c98 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:41 -0300 Subject: [PATCH 096/148] RDMA/mlx5: Create ODP EQ for non-pinned dmabuf MRs DMABUF generally relies on the ODP EQ mechanism to safely implement the move semantics. ODP requires a device-global one time startup of the ODP machinery when the first MR is created, and this was missed on the DMABUF path. Call mlx5r_odp_create_eq() when creating a ODP'able DMABUF. The core code prevents using IB_ACCESS_ON_DEMAND unless the driver advertises IB_ODP_SUPPORT, so until now, mlx5r_odp_create_eq() cannot be called unless the device has ODP support. However, DMABUF has no such protection and a second bug was allowing DMABUFs to be created on non-ODP capable HW. Add a guard at the start of mlx5r_odp_create_eq(). This is necessary here anyhow as the dev->odp_eq_mutex is not initialized without IB_ODP_SUPPORT. Link: https://patch.msgid.link/r/2-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 4 ++++ drivers/infiniband/hw/mlx5/odp.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 0fa0a57240c0..e75a3aa3e278 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -956,6 +956,10 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); umem_dmabuf->private = mr; if (!pinned_mode) { + err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); + if (err) + goto err_dereg_mr; + err = mlx5r_store_odp_mkey(dev, &mr->mmkey); if (err) goto err_dereg_mr; diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1119ce163ea7..10c11b72f412 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -1807,6 +1807,9 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) struct mlx5_eq_param param = {}; int err = 0; + if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) + return -EOPNOTSUPP; + mutex_lock(&dev->odp_eq_mutex); if (eq->core) goto unlock; From ee7a8335069150c3f1893a697ab30bbeca00d796 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:42 -0300 Subject: [PATCH 097/148] IB/mlx5: Properly support implicit ODP rereg_mr Due to all the child mkeys in the implicit ODP configuration we cannot change anything in place for the parent mkey. Instead the whole thing needs to be rebuilt if any change is requested. If the user does not specify a translation then force the implicit values which will then fall through the logic into mlx5_ib_reg_user_mr() to allocate a completely new MR. Since implicit children were also touching the mr->pd, this removes another case where the access was racy. Fixes: ef3642c4f54d ("RDMA/mlx5: Fix error unwinds for rereg_mr") Link: https://sashiko.dev/#/patchset/20260427-security-bug-fixes-v3-0-4621fa52de0e%40nvidia.com?part=4 Link: https://patch.msgid.link/r/3-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index e75a3aa3e278..17902a643ce5 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1188,6 +1188,21 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, if (!(flags & IB_MR_REREG_PD)) new_pd = ib_mr->pd; + if (mr->is_odp_implicit && !(flags & IB_MR_REREG_TRANS)) { + if (!(new_access_flags & IB_ACCESS_ON_DEMAND)) + return ERR_PTR(-EOPNOTSUPP); + + /* + * Due to all the child mkeys we cannot actually change an + * implicit MR in place. If the user did not specify a new + * translation then force the fixed implicit MR values. + */ + start = 0; + iova = 0; + length = U64_MAX; + flags |= IB_MR_REREG_TRANS; + } + if (!(flags & IB_MR_REREG_TRANS)) { struct ib_umem *umem; From 50d5c02ab8e62325548bd3a6e6b758a9dcd6e7c3 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:43 -0300 Subject: [PATCH 098/148] RDMA/nldev: Fix locking when accessing mr->pd Sashiko points out that, due to rereg_mr, the PD is actually variable and all the touches in nldev are racy. Use mr->device instead of mr->pd->device. Getting the PD restrack ID is more tricky. To avoid disturbing all the happy paths, add an rdma_restrack_sync() operation which is sort of like flush_workqueue() or synchronize_irq(): after it returns, all the old nldev touches to the mr are gone and everything sees the new PD. This makes it safe to reach into the PD pointer. Fixes: da5c85078215 ("RDMA/nldev: add driver-specific resource tracking") Link: https://patch.msgid.link/r/4-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/nldev.c | 15 +++++---- drivers/infiniband/core/restrack.c | 49 ++++++++++++++++++++++++++++ drivers/infiniband/core/restrack.h | 1 + drivers/infiniband/core/uverbs_cmd.c | 10 ++++-- include/rdma/ib_verbs.h | 5 +++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 5aaba2b9746b..02a0a9c0a4a6 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -695,7 +695,7 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (has_cap_net_admin) { if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey)) @@ -711,9 +711,12 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) return -EMSGSIZE; - if (!rdma_is_kernel_res(res) && - nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id)) - return -EMSGSIZE; + if (!rdma_is_kernel_res(res)) { + struct ib_pd *pd = READ_ONCE(mr->pd); + + if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, pd->res.id)) + return -EMSGSIZE; + } if (fill_res_name_pid(msg, res)) return -EMSGSIZE; @@ -727,7 +730,7 @@ static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (!dev->ops.fill_res_mr_entry_raw) return -EINVAL; @@ -1017,7 +1020,7 @@ static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) goto err; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index ac3688952cab..cfee2071586c 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -71,6 +71,8 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, xa_lock(&rt->xa); xas_for_each(&xas, e, U32_MAX) { + if (xa_is_zero(e)) + continue; if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) continue; cnt++; @@ -276,6 +278,53 @@ int rdma_restrack_put(struct rdma_restrack_entry *res) } EXPORT_SYMBOL(rdma_restrack_put); +/** + * rdma_restrack_sync() - Fence concurrent netlink dumps on an entry + * @res: resource entry + * + * After this returns any concurrent netlink dump threads will see the current + * value of the object. This is useful if the object has to be changed and there + * is not locking to protect the nl side. Eg for mr->pd. This effectively + * destroys the object from a kref/xarray perspective and then immediately + * restores it. The kref is acting like a lock to barrier concurrent nl threads. + * Callers must ensure rdma_restrack_del() is not concurrently called. + */ +void rdma_restrack_sync(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_entry *old; + struct rdma_restrack_root *rt; + struct task_struct *task; + struct ib_device *dev; + + if (!res->valid || res->no_track) + return; + + dev = res_to_dev(res); + if (WARN_ON(!dev)) + return; + + rt = &dev->res[res->type]; + if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) + return; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); + if (WARN_ON(old != res)) + return; + + task = res->task; + if (task) + get_task_struct(task); + rdma_restrack_put(res); + wait_for_completion(&res->comp); + reinit_completion(&res->comp); + if (task) + res->task = task; + kref_init(&res->kref); + + xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); +} +EXPORT_SYMBOL(rdma_restrack_sync); + /** * rdma_restrack_del() - delete object from the resource tracking database * @res: resource entry diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 6a04fc41f738..75b8d1005a98 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -27,6 +27,7 @@ int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); +void rdma_restrack_sync(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index e16eacf40d17..aca7c6ab55cd 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -47,6 +47,7 @@ #include "uverbs.h" #include "core_priv.h" +#include "restrack.h" /* * Copy a response to userspace. If the provided 'resp' is larger than the @@ -808,6 +809,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) ret = PTR_ERR(new_pd); goto put_uobjs; } + if (new_pd == orig_pd) { + uobj_put_obj_read(new_pd); + cmd.flags &= ~IB_MR_REREG_PD; + } } else { new_pd = mr->pd; } @@ -855,9 +860,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) mr = new_mr; } else { if (cmd.flags & IB_MR_REREG_PD) { - atomic_dec(&orig_pd->usecnt); - mr->pd = new_pd; atomic_inc(&new_pd->usecnt); + WRITE_ONCE(mr->pd, new_pd); + rdma_restrack_sync(&mr->res); + atomic_dec(&orig_pd->usecnt); } if (cmd.flags & IB_MR_REREG_TRANS) { mr->iova = cmd.hca_va; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 0daa5089d539..794746de8db0 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1977,6 +1977,11 @@ struct ib_dmah { struct ib_mr { struct ib_device *device; + /* + * Due to IB_MR_REREG_PD pd is not a fixed pointer and can change. For a + * user MR, this value should only be read from a system call that holds + * the uobject lock, or the driver should disable in-place REREG_PD. + */ struct ib_pd *pd; u32 lkey; u32 rkey; From 4910483bd7ee2b40d0cc8ebd537fa33c95562029 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:44 -0300 Subject: [PATCH 099/148] IB/mlx5: Remove unused mkc bits in mlx5r_umr_update_mr_page_shift() The HW only processes mkc fields selected by mkey_mask. pd, qpn and mkey_7_0 are never selected so they can be left as zero. This removes a racy read of mr->pd. Fixes: e73242aa14d2 ("RDMA/mlx5: Optimize DMABUF mkey page size") Link: https://patch.msgid.link/r/5-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/umr.c | 16 +++------------- drivers/infiniband/hw/mlx5/umr.h | 3 +-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 688d5246f284..f0ba8d91f81d 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -937,8 +937,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, * pinned and the HW can switch from 4K to huge-page alignment). */ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd) + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); struct mlx5r_umr_wqe wqe = {}; @@ -953,16 +952,8 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, /* Fill mkey segment with the new page size, keep the rest unchanged */ MLX5_SET(mkc, &wqe.mkey_seg, log_page_size, page_shift); - if (dd) - MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn); - else - MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); - MLX5_SET64(mkc, &wqe.mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, &wqe.mkey_seg, len, mr->ibmr.length); - MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); - MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, - mlx5_mkey_variant(mr->mmkey.key)); err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); if (!err) @@ -1049,7 +1040,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * non-present. */ if (*nblocks) { - err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift, dd); + err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift); if (err) { mr->page_shift = old_page_shift; return err; @@ -1114,8 +1105,7 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, goto err; } - err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift, - mr->data_direct); + err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift); if (err) goto err; err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks, diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index 7eeaf6a94c97..59809d4d7d72 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -106,8 +106,7 @@ int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags); int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int page_shift, int flags); int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd); + unsigned int page_shift); int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, unsigned int page_shift); From f387a9f06ea4f05e607a91b161963c0b19436652 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:45 -0300 Subject: [PATCH 100/148] IB/mlx5: Pull the pdn out of the depths of the umr machinery Instead of getting the pdn deep inside the umr code, pass it in from the top. to_mpd(mr->ibmr.pd)->pdn is not safe due to the rereg races, so all the call sites need some revision to obtain the pdn in a safe way. Mark them with mlx5_mr_pdn(); following patches will go through and remove these. Cases where the XLT flags are known and do not require the PDN can pass 0, such as for mlx5_ib_dmabuf_invalidate_cb(). Also extract the DMABUF data_direct special case from inside the UMR code and into the only place that needs it, pagefault_dmabuf_mr(). The actual mr was created directly without using the UMR flow. Ultimately this will be moved into mlx5_ib_init_dmabuf_mr(). Link: https://patch.msgid.link/r/6-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Assisted-by: Codex:gpt-5-5 Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 9 ++++ drivers/infiniband/hw/mlx5/mr.c | 8 +-- drivers/infiniband/hw/mlx5/odp.c | 12 +++-- drivers/infiniband/hw/mlx5/umr.c | 75 ++++++++++++++-------------- drivers/infiniband/hw/mlx5/umr.h | 6 +-- 5 files changed, 64 insertions(+), 46 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 1f03c05b0cbd..0ebf1010c709 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -331,6 +331,10 @@ struct mlx5_ib_flow_db { #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4 #define MLX5_IB_WR_UMR IB_WR_RESERVED1 +/* + * A valid pdn is required when flags include MLX5_IB_UPD_XLT_ENABLE, + * MLX5_IB_UPD_XLT_PD or MLX5_IB_UPD_XLT_ACCESS. + */ #define MLX5_IB_UPD_XLT_ZAP BIT(0) #define MLX5_IB_UPD_XLT_ENABLE BIT(1) #define MLX5_IB_UPD_XLT_ATOMIC BIT(2) @@ -1209,6 +1213,11 @@ static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) return container_of(ibpd, struct mlx5_ib_pd, ibpd); } +static inline u32 mlx5_mr_pdn(struct mlx5_ib_mr *mr) +{ + return to_mpd(mr->ibmr.pd)->pdn; +} + static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq) { return container_of(ibsrq, struct mlx5_ib_srq, ibsrq); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 17902a643ce5..47f6a13a5f85 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -781,7 +781,8 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, * configured properly but left disabled. It is safe to go ahead * and configure it again via UMR while enabling it. */ - err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); + err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE, + to_mpd(pd)->pdn); if (err) { mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); @@ -890,7 +891,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) if (!umem_dmabuf->sgt || !mr) return; - mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP); + /* MLX5_IB_UPD_XLT_ZAP does not change the pdn */ + mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP, 0); ib_umem_dmabuf_unmap_pages(umem_dmabuf); } @@ -1145,7 +1147,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, mr->ibmr.length = new_umem->length; mr->page_shift = order_base_2(page_size); mr->umem = new_umem; - err = mlx5r_umr_update_mr_pas(mr, upd_flags); + err = mlx5r_umr_update_mr_pas(mr, upd_flags, mlx5_mr_pdn(mr)); if (err) { /* * The MR is revoked at this point so there is no issue to free diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 10c11b72f412..d7feb49b28fb 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -833,12 +833,14 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, u32 *bytes_mapped, u32 flags) { struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); + struct mlx5_ib_dev *dev = mr_to_mdev(mr); int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT; unsigned int old_page_shift = mr->page_shift; unsigned int page_shift; unsigned long page_size; u32 xlt_flags = 0; + u32 pdn = 0; int err; if (flags & MLX5_PF_FLAGS_ENABLE) @@ -857,8 +859,12 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = -EINVAL; } else { page_shift = order_base_2(page_size); + if (mr->data_direct) + pdn = dev->ddr.pdn; + else + pdn = mlx5_mr_pdn(mr); if (page_shift != mr->page_shift && mr->dmabuf_faulted) { - err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, + err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, page_shift); } else { mr->page_shift = page_shift; @@ -866,8 +872,8 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = mlx5r_umr_update_data_direct_ksm_pas( mr, xlt_flags); else - err = mlx5r_umr_update_mr_pas(mr, - xlt_flags); + err = mlx5r_umr_update_mr_pas(mr, xlt_flags, + pdn); } } dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index f0ba8d91f81d..f3f428f5e1b6 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -603,11 +603,11 @@ mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg, static void mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev *dev, struct mlx5_mkey_seg *mkey_seg, - struct mlx5_ib_mr *mr, + struct mlx5_ib_mr *mr, u32 pdn, unsigned int page_shift) { mlx5r_umr_set_access_flags(dev, mkey_seg, mr->access_flags); - MLX5_SET(mkc, mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); + MLX5_SET(mkc, mkey_seg, pd, pdn); MLX5_SET64(mkc, mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, mkey_seg, len, mr->ibmr.length); MLX5_SET(mkc, mkey_seg, log_page_size, page_shift); @@ -670,23 +670,22 @@ static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev, wqe->data_seg.byte_count = cpu_to_be32(sg->length); } -static void -_mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, struct mlx5r_umr_wqe *wqe, - struct ib_sge *sg, unsigned int flags, - unsigned int page_shift, bool dd) +static void _mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, + struct mlx5r_umr_wqe *wqe, struct ib_sge *sg, + unsigned int flags, u32 pdn, + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); mlx5r_umr_set_update_xlt_ctrl_seg(&wqe->ctrl_seg, flags, sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, page_shift); - if (dd) /* Use the data direct internal kernel PD */ - MLX5_SET(mkc, &wqe->mkey_seg, pd, dev->ddr.pdn); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe->data_seg, sg); } -static int -_mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, - size_t start_block, size_t nblocks) +static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, + u32 pdn, bool dd, size_t start_block, + size_t nblocks) { size_t ent_size = dd ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -720,7 +719,7 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, orig_sg_length = sg.length; - _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, mr->page_shift, dd); + _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, pdn, mr->page_shift); /* Set initial translation offset to start_block */ offset = (u64)start_block * ent_size; @@ -811,7 +810,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, true, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + true, start_block, nblocks); } int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, @@ -821,12 +821,13 @@ int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, } int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks) + u32 pdn, size_t start_block, size_t nblocks) { if (WARN_ON(mr->umem->is_odp)) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, false, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, pdn, false, start_block, + nblocks); } /* @@ -834,9 +835,9 @@ int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP * flag may be used. */ -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn) { - return mlx5r_umr_update_mr_pas_range(mr, flags, 0, 0); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, 0, 0); } static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) @@ -861,6 +862,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, size_t orig_sg_length; size_t pages_iter; struct ib_sge sg; + u32 pdn = mlx5_mr_pdn(mr); int err = 0; void *xlt; @@ -895,7 +897,8 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, } mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, page_shift); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg); for (pages_mapped = 0; @@ -962,17 +965,18 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, return err; } -static inline int -_mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks, bool dd) +static inline int _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, + unsigned int flags, u32 pdn, + size_t start_block, + size_t nblocks, bool dd) { if (dd) return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, start_block, nblocks); else - return mlx5r_umr_update_mr_pas_range(mr, flags, start_block, - nblocks); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, + start_block, nblocks); } /** @@ -986,11 +990,9 @@ _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, * Return: On success, returns the number of entries that were zapped. * On error, returns a negative error code. */ -static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, - unsigned int flags, - unsigned int page_shift, - size_t *nblocks, - bool dd) +static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, unsigned int flags, + unsigned int page_shift, size_t *nblocks, + u32 pdn, bool dd) { unsigned int old_page_shift = mr->page_shift; struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -1030,7 +1032,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, */ if (*nblocks) mr->page_shift = max_page_shift; - err = _mlx5r_dmabuf_umr_update_pas(mr, flags, 0, *nblocks, dd); + err = _mlx5r_dmabuf_umr_update_pas(mr, flags, pdn, 0, *nblocks, dd); if (err) { mr->page_shift = old_page_shift; return err; @@ -1055,6 +1057,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * entries accordingly * @mr: The memory region to update * @xlt_flags: Translation table update flags + * @pdn: Protection domain number * @page_shift: The new (optimized) page shift to use * * This function updates the page size and mkey translation entries for a DMABUF @@ -1074,7 +1077,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * * Returns 0 on success or a negative error code on failure. */ -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift) { unsigned int old_page_shift = mr->page_shift; @@ -1083,7 +1086,7 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, int err; err = _mlx5r_umr_zap_mkey(mr, xlt_flags, page_shift, &zapped_blocks, - mr->data_direct); + pdn, mr->data_direct); if (err) return err; @@ -1096,10 +1099,8 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, * the page size in the mkey yet. */ err = _mlx5r_dmabuf_umr_update_pas( - mr, - xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, - zapped_blocks, - total_blocks - zapped_blocks, + mr, xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, pdn, + zapped_blocks, total_blocks - zapped_blocks, mr->data_direct); if (err) goto err; @@ -1108,7 +1109,7 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift); if (err) goto err; - err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks, + err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, pdn, 0, zapped_blocks, mr->data_direct); if (err) goto err; diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index 59809d4d7d72..99192ec67957 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -101,13 +101,13 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, size_t nblocks); int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int flags); int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks); -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags); + u32 pdn, size_t start_block, size_t nblocks); +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn); int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int page_shift, int flags); int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, unsigned int page_shift); -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift); #endif /* _MLX5_IB_UMR_H */ From 8e0a02a989c156ce17f06a645d5f075277e06b95 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:46 -0300 Subject: [PATCH 101/148] IB/mlx5: Don't mangle the mr->pd inside the rereg callback The rereg protocol expects the core code to change mr->pd and synchronize that change with the atomics and syncs. The driver should not touch it. mlx5 needed to update it in umr_rereg_pas() because mlx5r_umr_update_mr_pas() required the updated mr->pd to build the UMR. Simply switch mlx5r_umr_update_mr_pas() to use the pdn directly from the new pd and remove the mr->pd update. Fixes: 56e11d628c5d ("IB/mlx5: Added support for re-registration of MRs") Link: https://patch.msgid.link/r/7-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Assisted-by: Codex:gpt-5-5 Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 47f6a13a5f85..7e2d5219fa3e 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1134,10 +1134,8 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, if (err) return err; - if (flags & IB_MR_REREG_PD) { - mr->ibmr.pd = pd; + if (flags & IB_MR_REREG_PD) upd_flags |= MLX5_IB_UPD_XLT_PD; - } if (flags & IB_MR_REREG_ACCESS) { mr->access_flags = access_flags; upd_flags |= MLX5_IB_UPD_XLT_ACCESS; @@ -1147,7 +1145,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, mr->ibmr.length = new_umem->length; mr->page_shift = order_base_2(page_size); mr->umem = new_umem; - err = mlx5r_umr_update_mr_pas(mr, upd_flags, mlx5_mr_pdn(mr)); + err = mlx5r_umr_update_mr_pas(mr, upd_flags, to_mpd(pd)->pdn); if (err) { /* * The MR is revoked at this point so there is no issue to free From 23caeef1d06c01198d0620824bd4d8cb741c812b Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:47 -0300 Subject: [PATCH 102/148] IB/mlx5: Push pdn above mlx5r_umr_update_xlt() Keep pushing the pdn higher to remove more places touching mr->pd: - XLT combinations that don't use PDN can just pass 0 - Use local pd values instead of mr->pd - Implicit MR does not have inplace rereg, so the mr->pd is safe Link: https://patch.msgid.link/r/8-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Assisted-by: Codex:gpt-5-5 Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/odp.c | 40 +++++++++++++++++--------------- drivers/infiniband/hw/mlx5/umr.c | 3 +-- drivers/infiniband/hw/mlx5/umr.h | 2 +- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index d7feb49b28fb..50804b4c90e4 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -221,7 +221,8 @@ static void free_implicit_child_mr_work(struct work_struct *work) mutex_lock(&odp_imr->umem_mutex); mlx5r_umr_update_xlt(mr->parent, ib_umem_start(odp) >> mlx5_imr_mtt_shift, 1, 0, - MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC); + MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, + 0); mutex_unlock(&odp_imr->umem_mutex); mlx5_ib_dereg_mr(&mr->ibmr, NULL); @@ -318,10 +319,12 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, u64 umr_offset = idx & umr_block_mask; if (in_block && umr_offset == 0) { - mlx5r_umr_update_xlt(mr, blk_start_idx, - idx - blk_start_idx, 0, - MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ATOMIC); + mlx5r_umr_update_xlt( + mr, blk_start_idx, idx - blk_start_idx, + 0, + MLX5_IB_UPD_XLT_ZAP | + MLX5_IB_UPD_XLT_ATOMIC, + 0); in_block = 0; /* Count page invalidations */ invalidations += idx - blk_start_idx + 1; @@ -329,10 +332,9 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, } } if (in_block) { - mlx5r_umr_update_xlt(mr, blk_start_idx, - idx - blk_start_idx + 1, 0, - MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ATOMIC); + mlx5r_umr_update_xlt( + mr, blk_start_idx, idx - blk_start_idx + 1, 0, + MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ATOMIC, 0); /* Count page invalidations */ invalidations += idx - blk_start_idx + 1; } @@ -502,11 +504,9 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, */ refcount_set(&mr->mmkey.usecount, 2); - err = mlx5r_umr_update_xlt(mr, 0, - mlx5_imr_mtt_entries, - PAGE_SHIFT, - MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ENABLE); + err = mlx5r_umr_update_xlt(mr, 0, mlx5_imr_mtt_entries, PAGE_SHIFT, + MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ENABLE, + to_mpd(mr->ibmr.pd)->pdn); if (err) { ret = ERR_PTR(err); goto out_mr; @@ -647,7 +647,8 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, mlx5_imr_ksm_page_shift, MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ENABLE); + MLX5_IB_UPD_XLT_ENABLE, + pd->pdn); if (err) goto out_mr; @@ -720,7 +721,8 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, * No need to check whether the MTTs really belong to this MR, since * ib_umem_odp_map_dma_and_lock already checks this. */ - ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags); + ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags, + mlx5_mr_pdn(mr)); mutex_unlock(&odp->umem_mutex); if (ret < 0) { @@ -818,9 +820,9 @@ static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, * next pagefault handler will see the new information. */ mutex_lock(&odp_imr->umem_mutex); - err = mlx5r_umr_update_xlt(imr, upd_start_idx, upd_len, 0, - MLX5_IB_UPD_XLT_INDIRECT | - MLX5_IB_UPD_XLT_ATOMIC); + err = mlx5r_umr_update_xlt( + imr, upd_start_idx, upd_len, 0, + MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, 0); mutex_unlock(&odp_imr->umem_mutex); if (err) { mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index f3f428f5e1b6..48cae5cc1c1b 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -846,7 +846,7 @@ static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) } int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, - int page_shift, int flags) + int page_shift, int flags, u32 pdn) { int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) ? sizeof(struct mlx5_klm) @@ -862,7 +862,6 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, size_t orig_sg_length; size_t pages_iter; struct ib_sge sg; - u32 pdn = mlx5_mr_pdn(mr); int err = 0; void *xlt; diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index 99192ec67957..bda7123781a9 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -104,7 +104,7 @@ int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn, size_t start_block, size_t nblocks); int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn); int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, - int page_shift, int flags); + int page_shift, int flags, u32 pdn); int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, unsigned int page_shift); int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, From 62731d26bdeeaa9bec5893ca3331bd8a5ef6ea27 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:48 -0300 Subject: [PATCH 103/148] IB/mlx5: Push pdn above pagfault_real_mr() Remove the mlx5_mr_pdn() in pagefault_real_mr() by pushing the pdn up, all the callers use 0 since they don't pass MLX5_PF_FLAGS_ENABLE except the ioctl reg_mr path which can use the ioctl pd. Link: https://patch.msgid.link/r/9-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Assisted-by: Codex:gpt-5-5 Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 4 ++-- drivers/infiniband/hw/mlx5/mr.c | 2 +- drivers/infiniband/hw/mlx5/odp.c | 20 ++++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 0ebf1010c709..bb5383e8ded9 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1423,7 +1423,7 @@ int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, u32 flags, struct ib_sge *sg_list, u32 num_sge); -int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr); +int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd); int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr); #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; } @@ -1452,7 +1452,7 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, { return -EOPNOTSUPP; } -static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr) +static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { return -EOPNOTSUPP; } diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 7e2d5219fa3e..0c40c06618b6 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -842,7 +842,7 @@ static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_dereg_mr; - err = mlx5_ib_init_odp_mr(mr); + err = mlx5_ib_init_odp_mr(mr, pd); if (err) goto err_dereg_mr; return &mr->ibmr; diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 50804b4c90e4..2dcc4f5339f9 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -687,12 +687,16 @@ void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr) } } +/* + * pdn must be valid only when xlt_flags updates the mkey PD. In this path that + * is only MLX5_PF_FLAGS_ENABLE. DOWNGRADE and SNAPSHOT leave the PD masked out. + */ #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) #define MLX5_PF_FLAGS_SNAPSHOT BIT(2) #define MLX5_PF_FLAGS_ENABLE BIT(3) static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, u64 user_va, size_t bcnt, u32 *bytes_mapped, - u32 flags) + u32 flags, u32 pdn) { int page_shift, ret, np; bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; @@ -722,7 +726,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, * ib_umem_odp_map_dma_and_lock already checks this. */ ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags, - mlx5_mr_pdn(mr)); + pdn); mutex_unlock(&odp->umem_mutex); if (ret < 0) { @@ -788,7 +792,7 @@ static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, user_va; ret = pagefault_real_mr(mtt, umem_odp, user_va, len, - bytes_mapped, flags); + bytes_mapped, flags, 0); mlx5r_deref_odp_mkey(&mtt->mmkey); @@ -930,19 +934,20 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, ib_umem_end(odp) - user_va < bcnt)) return -EFAULT; return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, - flags); + flags, 0); } return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, flags); } -int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr) +int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { int ret; ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), mr->umem->address, mr->umem->length, NULL, - MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE); + MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE, + to_mpd(pd)->pdn); return ret >= 0 ? 0 : ret; } @@ -1575,8 +1580,7 @@ static void mlx5_ib_mr_memory_pfault_handler(struct mlx5_ib_dev *dev, ret = pagefault_mr(mr, prefetch_va, prefetch_size, NULL, 0, true); if (ret < 0) { ret = pagefault_mr(mr, pfault->memory.va, - pfault->memory.fault_byte_count, NULL, 0, - true); + pfault->memory.fault_byte_count, NULL, 0, true); if (ret < 0) goto err; } From c9bf47d2051ad7e0520b2f8d509de11079d2bf41 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 3 Jun 2026 22:27:49 -0300 Subject: [PATCH 104/148] IB/mlx5: Push pdn above pagefault_dmabuf_mr() Remove the mlx5_mr_pdn() inside pagefault_dmabuf_mr(), the only user of the pdn is the init path which is inside an ioctl. Link: https://patch.msgid.link/r/10-v1-29ebd2c229b5+fd5-ib_mr_pd_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 9 ++------- drivers/infiniband/hw/mlx5/mr.c | 2 +- drivers/infiniband/hw/mlx5/odp.c | 21 +++++++++++---------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index bb5383e8ded9..0550cdfacad4 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1213,11 +1213,6 @@ static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) return container_of(ibpd, struct mlx5_ib_pd, ibpd); } -static inline u32 mlx5_mr_pdn(struct mlx5_ib_mr *mr) -{ - return to_mpd(mr->ibmr.pd)->pdn; -} - static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq) { return container_of(ibsrq, struct mlx5_ib_srq, ibsrq); @@ -1424,7 +1419,7 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, u32 flags, struct ib_sge *sg_list, u32 num_sge); int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd); -int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr); +int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd); #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; } static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, @@ -1456,7 +1451,7 @@ static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { return -EOPNOTSUPP; } -static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr) +static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { return -EOPNOTSUPP; } diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 0c40c06618b6..14c1aec1edbb 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -969,7 +969,7 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, mr->data_direct = true; } - err = mlx5_ib_init_dmabuf_mr(mr); + err = mlx5_ib_init_dmabuf_mr(mr, pd); if (err) goto err_dereg_mr; return &mr->ibmr; diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 2dcc4f5339f9..1badec9bf527 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -836,17 +836,15 @@ static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, } static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, - u32 *bytes_mapped, u32 flags) + u32 *bytes_mapped, u32 flags, u32 pdn) { struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); - struct mlx5_ib_dev *dev = mr_to_mdev(mr); int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT; unsigned int old_page_shift = mr->page_shift; unsigned int page_shift; unsigned long page_size; u32 xlt_flags = 0; - u32 pdn = 0; int err; if (flags & MLX5_PF_FLAGS_ENABLE) @@ -865,10 +863,6 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = -EINVAL; } else { page_shift = order_base_2(page_size); - if (mr->data_direct) - pdn = dev->ddr.pdn; - else - pdn = mlx5_mr_pdn(mr); if (page_shift != mr->page_shift && mr->dmabuf_faulted) { err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, page_shift); @@ -915,7 +909,7 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, return -EFAULT; if (mr->umem->is_dmabuf) - return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags); + return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags, 0); if (!odp->is_implicit_odp) { u64 offset = io_virt < mr->ibmr.iova ? 0 : io_virt - mr->ibmr.iova; @@ -951,12 +945,19 @@ int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) return ret >= 0 ? 0 : ret; } -int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr) +int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { + struct mlx5_ib_dev *dev = mr_to_mdev(mr); + u32 pdn; int ret; + if (mr->data_direct) + pdn = dev->ddr.pdn; + else + pdn = to_mpd(pd)->pdn; + ret = pagefault_dmabuf_mr(mr, mr->umem->length, NULL, - MLX5_PF_FLAGS_ENABLE); + MLX5_PF_FLAGS_ENABLE, pdn); return ret >= 0 ? 0 : ret; } From 54bf38b27afc08a0eb6b732f9c14eb8a4bcb66b5 Mon Sep 17 00:00:00 2001 From: Aurelien DESBRIERES Date: Mon, 8 Jun 2026 15:47:15 +0200 Subject: [PATCH 105/148] RDMA/rtrs-srv: Fix integer underflow in process_read and process_write usr_len is read from a network-supplied message field (le16_to_cpu) and used to compute data_len = off - usr_len without validating that usr_len <= off. A malicious RDMA client can send usr_len > off causing an integer underflow, resulting in data_len wrapping to a huge size_t value which is then passed to the rdma_ev callback as a memory length, leading to out-of-bounds memory access. Fix by reading and validating usr_len <= off before rtrs_srv_get_ops_ids() in both process_read() and process_write(), ensuring the early return path acquires no reference and has no resource leak. Link: https://patch.msgid.link/r/20260608134802.5019-1-aurelien@hackers.camp Reported-by: Aurelien DESBRIERES Reviewed-by: Md Haris Iqbal Signed-off-by: Aurelien DESBRIERES Assisted-by: Claude Acked-by: Md Haris Iqbal Signed-off-by: Jason Gunthorpe --- drivers/infiniband/ulp/rtrs/rtrs-srv.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 6482ad859bd1..f2fd80c8a044 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -1059,6 +1059,11 @@ static void process_read(struct rtrs_srv_con *con, "Processing read request failed, invalid message\n"); return; } + usr_len = le16_to_cpu(msg->usr_len); + if (usr_len > off) { + pr_debug("rtrs-srv: Invalid usr_len %zu > off %u\n", usr_len, off); + return; + } rtrs_srv_get_ops_ids(srv_path); rtrs_srv_update_rdma_stats(srv_path->stats, off, READ); id = srv_path->ops_ids[buf_id]; @@ -1066,7 +1071,6 @@ static void process_read(struct rtrs_srv_con *con, id->dir = READ; id->msg_id = buf_id; id->rd_msg = msg; - usr_len = le16_to_cpu(msg->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, @@ -1112,6 +1116,11 @@ static void process_write(struct rtrs_srv_con *con, rtrs_srv_state_str(srv_path->state)); return; } + usr_len = le16_to_cpu(req->usr_len); + if (usr_len > off) { + pr_debug("rtrs-srv: Invalid usr_len %zu > off %u\n", usr_len, off); + return; + } rtrs_srv_get_ops_ids(srv_path); rtrs_srv_update_rdma_stats(srv_path->stats, off, WRITE); id = srv_path->ops_ids[buf_id]; @@ -1119,7 +1128,6 @@ static void process_write(struct rtrs_srv_con *con, id->dir = WRITE; id->msg_id = buf_id; - usr_len = le16_to_cpu(req->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, From bade9a3150d44ed20b8c6484c4c8a943b7289abb Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 5 Jun 2026 08:53:35 -0300 Subject: [PATCH 106/148] IB/mlx4: Fill in the access_flags if IB_MR_REREG_ACCESS is not specified Sashiko noticed mlx4 was using whatever random access flags were provided when IB_MR_REREG_ACCESS is not used. Since IB_MR_REREG_TRANS needs access_flags it used the random ones which means it doesn't work sensibly if userspace provides only IB_MR_REREG_TRANS. Keep track of the current access_flag of the MR and use it if the user does not specify one. Also fixup a little confusion around mmr.access, it is the HW access flags so the convert_access() was missing. But nothing reads this by the time rereg_mr can happen. Fixes: 9376932d0c26 ("IB/mlx4_ib: Add support for user MR re-registration") Link: https://patch.msgid.link/r/0-v1-29ca7a402625+ddd6-mlx4_rereg_flags_jgg@nvidia.com Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx4/mlx4_ib.h | 1 + drivers/infiniband/hw/mlx4/mr.c | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 598954dd0613..2b6cc011b25d 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -136,6 +136,7 @@ struct mlx4_ib_mr { dma_addr_t page_map; u32 npages; u32 max_pages; + int access_flags; struct mlx4_mr mmr; struct ib_umem *umem; size_t page_map_size; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 027ffb6f79d3..906f09a87704 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -181,6 +181,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_mr; + mr->access_flags = access_flags; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; mr->ibmr.page_size = 1U << shift; @@ -237,6 +238,8 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, if (err) goto release_mpt_entry; + } else { + mr_access_flags = mmr->access_flags; } if (flags & IB_MR_REREG_TRANS) { @@ -278,8 +281,10 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, * return a failure. But dereg_mr will free the resources. */ err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry); - if (!err && flags & IB_MR_REREG_ACCESS) - mmr->mmr.access = mr_access_flags; + if (!err && flags & IB_MR_REREG_ACCESS) { + mmr->access_flags = mr_access_flags; + mmr->mmr.access = convert_access(mr_access_flags); + } release_mpt_entry: mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry); From 8791501d394b7b66b36789b32ea370a38b17dd10 Mon Sep 17 00:00:00 2001 From: David Laight Date: Sat, 6 Jun 2026 21:26:04 +0100 Subject: [PATCH 107/148] RDMA/iwcm: User strscpy() to copy device name Link: https://patch.msgid.link/r/20260606202633.5018-10-david.laight.linux@gmail.com Signed-off-by: David Laight Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/iwcm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 9761d9365ffd..0b7246ec559e 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -518,8 +518,8 @@ static int iw_cm_map(struct iw_cm_id *cm_id, bool active) cm_id->m_local_addr = cm_id->local_addr; cm_id->m_remote_addr = cm_id->remote_addr; - strcpy(pm_reg_msg.dev_name, devname); - strcpy(pm_reg_msg.if_name, ifname); + strscpy(pm_reg_msg.dev_name, devname); + strscpy(pm_reg_msg.if_name, ifname); if (iwpm_register_pid(&pm_reg_msg, RDMA_NL_IWCM) || !iwpm_valid_pid()) From 16bf00323dbe5ac100b0426962ee4e76e3f2f8a0 Mon Sep 17 00:00:00 2001 From: David Laight Date: Sat, 6 Jun 2026 21:26:05 +0100 Subject: [PATCH 108/148] RDMA/usnic: User strscpy() to copy device name Link: https://patch.msgid.link/r/20260606202633.5018-11-david.laight.linux@gmail.com Signed-off-by: David Laight Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/usnic/usnic_fwd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/usnic/usnic_fwd.c b/drivers/infiniband/hw/usnic/usnic_fwd.c index 39cdd72eabf6..59c363db0355 100644 --- a/drivers/infiniband/hw/usnic/usnic_fwd.c +++ b/drivers/infiniband/hw/usnic/usnic_fwd.c @@ -93,7 +93,7 @@ struct usnic_fwd_dev *usnic_fwd_dev_alloc(struct pci_dev *pdev) ufdev->netdev = pci_get_drvdata(pdev); spin_lock_init(&ufdev->lock); BUILD_BUG_ON(sizeof(ufdev->name) != sizeof(ufdev->netdev->name)); - strcpy(ufdev->name, ufdev->netdev->name); + strscpy(ufdev->name, ufdev->netdev->name); return ufdev; } From 6eb287509dcf3508d2363934e82d9da7cf612f99 Mon Sep 17 00:00:00 2001 From: David Laight Date: Mon, 8 Jun 2026 10:54:57 +0100 Subject: [PATCH 109/148] RDMA/mlx5: Use strscpy() to copy strings into arrays Replacing strcpy() with strscpy() ensures that overflow of the target buffer cannot happen. Link: https://patch.msgid.link/r/20260608095500.2567-2-david.laight.linux@gmail.com Signed-off-by: David Laight Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/data_direct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 8e89dbe40c23..d57484245c38 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -88,7 +88,7 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) return -ENOMEM; reg->ibdev = ibdev; - strcpy(reg->vuid, vuid); + strscpy(reg->vuid, vuid); mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { From 7d4f515b66ebce2bb2ba09e8be4ff615a1579031 Mon Sep 17 00:00:00 2001 From: Tom Sela Date: Mon, 8 Jun 2026 08:37:36 +0000 Subject: [PATCH 110/148] RDMA/efa: Report 800 and 1600 Gbps link speed Add support for reporting 800 Gbps as 8X NDR and 1600 Gbps as 8X XDR link speeds. Link: https://patch.msgid.link/r/20260608083736.48454-1-tomsela@amazon.com Reviewed-by: Michael Margolin Reviewed-by: Yonatan Nachum Signed-off-by: Tom Sela Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_verbs.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 434d60235945..5cd34746e6a6 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -282,7 +282,13 @@ static void efa_link_gbps_to_speed_and_width(u16 gbps, enum ib_port_speed *speed, enum ib_port_width *width) { - if (gbps >= 400) { + if (gbps >= 1600) { + *width = IB_WIDTH_8X; + *speed = IB_SPEED_XDR; + } else if (gbps >= 800) { + *width = IB_WIDTH_8X; + *speed = IB_SPEED_NDR; + } else if (gbps >= 400) { *width = IB_WIDTH_8X; *speed = IB_SPEED_HDR; } else if (gbps >= 200) { From 20ff9350862468af21b46cae2c22d17d6ec637f9 Mon Sep 17 00:00:00 2001 From: Tom Sela Date: Mon, 8 Jun 2026 08:39:27 +0000 Subject: [PATCH 111/148] RDMA/efa: Implement the query port speed verb Implement the query port speed callback to report the port effective bandwidth directly in 100 Mb/s granularity. Link: https://patch.msgid.link/r/20260608083927.4116-1-tomsela@amazon.com Reviewed-by: Michael Margolin Reviewed-by: Yonatan Nachum Signed-off-by: Tom Sela Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa.h | 1 + drivers/infiniband/hw/efa/efa_com_cmd.c | 4 ++++ drivers/infiniband/hw/efa/efa_main.c | 1 + drivers/infiniband/hw/efa/efa_verbs.c | 13 ++++++++++--- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 00b19f2ba3da..f4586bb170c1 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -148,6 +148,7 @@ int efa_query_device(struct ib_device *ibdev, struct ib_udata *udata); int efa_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); +int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed); int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 63c7f07806a8..5db4f5805b59 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -6,6 +6,8 @@ #include "efa_com.h" #include "efa_com_cmd.h" +#define EFA_DEFAULT_LINK_SPEED_GBPS 100 + int efa_com_create_qp(struct efa_com_dev *edev, struct efa_com_create_qp_params *params, struct efa_com_create_qp_result *res) @@ -468,6 +470,8 @@ int efa_com_get_device_attr(struct efa_com_dev *edev, result->device_caps = resp.u.device_attr.device_caps; result->guid = resp.u.device_attr.guid; result->max_link_speed_gbps = resp.u.device_attr.max_link_speed_gbps; + if (!result->max_link_speed_gbps) + result->max_link_speed_gbps = EFA_DEFAULT_LINK_SPEED_GBPS; if (result->admin_api_version < 1) { ibdev_err_ratelimited( diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 03c237c8c81e..97da8e828e34 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -390,6 +390,7 @@ static const struct ib_device_ops efa_dev_ops = { .query_gid = efa_query_gid, .query_pkey = efa_query_pkey, .query_port = efa_query_port, + .query_port_speed = efa_query_port_speed, .query_qp = efa_query_qp, .reg_user_mr = efa_reg_mr, .reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf, diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 5cd34746e6a6..5bb00cb85775 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -90,8 +90,6 @@ static const struct rdma_stat_desc efa_port_stats_descs[] = { EFA_DEFINE_PORT_STATS(EFA_STATS_STR) }; -#define EFA_DEFAULT_LINK_SPEED_GBPS 100 - #define EFA_CHUNK_PAYLOAD_SHIFT 12 #define EFA_CHUNK_PAYLOAD_SIZE BIT(EFA_CHUNK_PAYLOAD_SHIFT) #define EFA_CHUNK_PAYLOAD_PTR_SIZE 8 @@ -332,7 +330,7 @@ int efa_query_port(struct ib_device *ibdev, u32 port, props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; props->gid_tbl_len = 1; props->pkey_tbl_len = 1; - link_gbps = dev->dev_attr.max_link_speed_gbps ?: EFA_DEFAULT_LINK_SPEED_GBPS; + link_gbps = dev->dev_attr.max_link_speed_gbps; efa_link_gbps_to_speed_and_width(link_gbps, &link_speed, &link_width); props->active_speed = link_speed; props->active_width = link_width; @@ -344,6 +342,15 @@ int efa_query_port(struct ib_device *ibdev, u32 port, return 0; } +int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed) +{ + struct efa_dev *dev = to_edev(ibdev); + + *speed = dev->dev_attr.max_link_speed_gbps * 10; + + return 0; +} + int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) From 2cde8282ff6e28c5e2fa3dbed25f154bd0e1cd7f Mon Sep 17 00:00:00 2001 From: Ruoyu Wang Date: Sat, 6 Jun 2026 12:06:44 +0800 Subject: [PATCH 112/148] RDMA/bnxt_re: Check debugfs parameter allocation for failure bnxt_re_debugfs_add_pdev() allocates per-file private data for the CC configuration debugfs entries. The loop that initializes those entries uses rdev->cc_config_params immediately, so allocation failure would lead to NULL pointer dereferences while setting up debugfs. Debugfs is best-effort. If the CC configuration private data cannot be allocated just stop. Link: https://patch.msgid.link/r/20260606040644.13-1-ruoyuw560@gmail.com Signed-off-by: Ruoyu Wang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/debugfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.c b/drivers/infiniband/hw/bnxt_re/debugfs.c index 5fed2cf66be3..143e9bfc6b79 100644 --- a/drivers/infiniband/hw/bnxt_re/debugfs.c +++ b/drivers/infiniband/hw/bnxt_re/debugfs.c @@ -498,6 +498,8 @@ void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev) bnxt_re_debugfs_add_info(rdev); rdev->cc_config_params = kzalloc_obj(*cc_params); + if (!rdev->cc_config_params) + return; for (i = 0; i < BNXT_RE_CC_PARAM_GEN0; i++) { struct bnxt_re_cc_param *tmp_params = &rdev->cc_config_params->gen0_parms[i]; From 942cd47faa2047b46dfd85745603eba9006973e6 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Tue, 9 Jun 2026 14:16:38 +0300 Subject: [PATCH 113/148] RDMA/core: Fix broadcast address falsely detected as local When rdma_resolve_addr() is invoked with a broadcast destination on an IPoIB interface, is_dst_local() inspects the resolved route and incorrectly concludes that the address is local. As a result, the resolution fails with -ENODEV. The issue stems from using '&' to compare rt_type with RTN_LOCAL. The RTN_* values form a sequential enum, not a bitmask (RTN_LOCAL=2, RTN_BROADCAST=3). Thus, "rt_type & RTN_LOCAL" yields a non-zero result for a broadcast route as well. Replace '&' with '==' when comparing rt_type against RTN_LOCAL. Link: https://patch.msgid.link/r/20260609-fix-rdma-resolve-addr-v1-1-449b8b4e6c09@nvidia.com Cc: stable@vger.kernel.org Fixes: c31e4038c97f ("RDMA/core: Use route entry flag to decide on loopback traffic") Signed-off-by: Maher Sanalla Reviewed-by: Vlad Dumitrescu Signed-off-by: Edward Srouji Reviewed-by: Parav Pandit Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 7e62b5b1ffaa..e9fb7ad4c377 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -438,7 +438,7 @@ static int addr6_resolve(struct sockaddr *src_sock, static bool is_dst_local(const struct dst_entry *dst) { if (dst->ops->family == AF_INET) - return !!(dst_rtable(dst)->rt_type & RTN_LOCAL); + return dst_rtable(dst)->rt_type == RTN_LOCAL; else if (dst->ops->family == AF_INET6) return !!(dst_rt6_info(dst)->rt6i_flags & RTF_LOCAL); else From a7b8dac8881dc6853afc46964e014a39d5cecad3 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:02 +0300 Subject: [PATCH 114/148] net/mlx5: Add UD and UC packet pacing caps Add the needed capabilities in mlx5_ifc to support packet pacing for UC and UD QPs. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-1-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- include/linux/mlx5/mlx5_ifc.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 49f3ad4b1a7c..f56de77cde3a 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1116,7 +1116,10 @@ struct mlx5_ifc_qos_cap_bits { u8 log_esw_max_sched_depth[0x4]; u8 reserved_at_10[0x10]; - u8 reserved_at_20[0x9]; + u8 reserved_at_20[0x2]; + u8 packet_pacing_req_ud[0x1]; + u8 packet_pacing_req_uc[0x1]; + u8 reserved_at_24[0x5]; u8 esw_cross_esw_sched[0x1]; u8 reserved_at_2a[0x1]; u8 log_max_qos_nic_queue_group[0x5]; @@ -3707,7 +3710,8 @@ struct mlx5_ifc_qpc_bits { u8 cur_retry_count[0x3]; u8 reserved_at_39b[0x5]; - u8 reserved_at_3a0[0x20]; + u8 reserved_at_3a0[0x10]; + u8 packet_pacing_rate_limit_index[0x10]; u8 reserved_at_3c0[0x8]; u8 next_send_psn[0x18]; From e1008f19fa977d5b7c3912420e3b46b0a33e5316 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:03 +0300 Subject: [PATCH 115/148] RDMA/mlx5: Refactor raw packet QP rate limit handling Refactor the raw packet QP modify path to extract rate limit configuration into a qp_rl_parse() helper that parses user attributes, and a qp_rl_prepare() helper that handles FW rate limit table adjustments before the SQ modify itself. Use qp_rl_commit() to commit changes to QP once FW call succeeds, and qp_rl_rollback() to rollback changes done to the FW rate limit table in the prepare stage, in case the modify operation fails. These helpers will be reused for extending rate limit support to additional QP types in the following patch. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-2-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 170 +++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 59 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index e8d34d54b435..6924fceb99c5 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -67,12 +67,19 @@ enum { MLX5_QP_RM_GO_BACK_N = 0x1, }; +struct mlx5_rate_limit_ctx { + struct mlx5_rate_limit rl_old; + struct mlx5_rate_limit rl_desired; + u16 rl_desired_index; + bool rl_changed; +}; + struct mlx5_modify_raw_qp_param { u16 operation; u32 set_mask; /* raw_qp_set_mask_map */ - struct mlx5_rate_limit rl; + struct mlx5_rate_limit_ctx rl_ctx; u8 rq_q_ctr_id; u32 port; @@ -3863,15 +3870,88 @@ static int modify_raw_packet_qp_rq( return err; } +static int qp_rl_parse(struct mlx5_ib_dev *dev, + const struct ib_qp_attr *attr, + const struct mlx5_ib_modify_qp *ucmd, + struct mlx5_rate_limit *rl_desired) +{ + rl_desired->rate = attr->rate_limit; + + if (ucmd->burst_info.max_burst_sz) { + if (!attr->rate_limit || + !MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) + return -EINVAL; + rl_desired->max_burst_sz = ucmd->burst_info.max_burst_sz; + } + + if (ucmd->burst_info.typical_pkt_sz) { + if (!attr->rate_limit || + !MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) + return -EINVAL; + rl_desired->typical_pkt_sz = ucmd->burst_info.typical_pkt_sz; + } + + return 0; +} + +static int qp_rl_prepare(struct mlx5_ib_dev *dev, + struct mlx5_ib_qp *qp, u16 op, + struct mlx5_rate_limit_ctx *ctx) +{ + int err; + + ctx->rl_old = qp->rl; + + if (!qp->sq.wqe_cnt) + return 0; + + if (op != MLX5_CMD_OP_RTR2RTS_QP && + op != MLX5_CMD_OP_RTS2RTS_QP) + return 0; + + ctx->rl_changed = true; + + if (ctx->rl_desired.rate) { + err = mlx5_rl_add_rate(dev->mdev, &ctx->rl_desired_index, + &ctx->rl_desired); + if (err) { + pr_err("Failed configuring rate limit(err %d): rate %u, max_burst_sz %u, typical_pkt_sz %u\n", + err, ctx->rl_desired.rate, + ctx->rl_desired.max_burst_sz, + ctx->rl_desired.typical_pkt_sz); + return err; + } + } + + return 0; +} + +static void qp_rl_rollback(struct mlx5_core_dev *dev, + struct mlx5_rate_limit_ctx *ctx) +{ + if (ctx->rl_desired_index) + mlx5_rl_remove_rate(dev, &ctx->rl_desired); +} + +static void qp_rl_commit(struct mlx5_core_dev *dev, + struct mlx5_ib_qp *qp, + struct mlx5_rate_limit_ctx *ctx) +{ + if (!ctx->rl_changed) + return; + + if (ctx->rl_old.rate) + mlx5_rl_remove_rate(dev, &ctx->rl_old); + + qp->rl = ctx->rl_desired; +} + static int modify_raw_packet_qp_sq( struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state, const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd) { + const struct mlx5_rate_limit_ctx *rl_ctx = &raw_qp_param->rl_ctx; struct mlx5_ib_qp *ibqp = sq->base.container_mibqp; - struct mlx5_rate_limit old_rl = ibqp->rl; - struct mlx5_rate_limit new_rl = old_rl; - bool new_rate_added = false; - u16 rl_index = 0; void *in; void *sqc; int inlen; @@ -3889,49 +3969,26 @@ static int modify_raw_packet_qp_sq( MLX5_SET(sqc, sqc, state, new_state); if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) { - if (new_state != MLX5_SQC_STATE_RDY) + if (new_state != MLX5_SQC_STATE_RDY) { pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n", __func__); - else - new_rl = raw_qp_param->rl; - } - - if (!mlx5_rl_are_equal(&old_rl, &new_rl)) { - if (new_rl.rate) { - err = mlx5_rl_add_rate(dev, &rl_index, &new_rl); - if (err) { - pr_err("Failed configuring rate limit(err %d): \ - rate %u, max_burst_sz %u, typical_pkt_sz %u\n", - err, new_rl.rate, new_rl.max_burst_sz, - new_rl.typical_pkt_sz); - - goto out; - } - new_rate_added = true; + } else if (rl_ctx->rl_changed) { + MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); + /* index 0 means no limit */ + MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, + rl_ctx->rl_desired_index); } - - MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); - /* index 0 means no limit */ - MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index); } err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in); - if (err) { - /* Remove new rate from table if failed */ - if (new_rate_added) - mlx5_rl_remove_rate(dev, &new_rl); + if (err) goto out; + + if (new_state != MLX5_SQC_STATE_RDY) { + mlx5_rl_remove_rate(dev, &ibqp->rl); + memset(&ibqp->rl, 0, sizeof(ibqp->rl)); } - /* Only remove the old rate after new rate was set */ - if ((old_rl.rate && !mlx5_rl_are_equal(&old_rl, &new_rl)) || - (new_state != MLX5_SQC_STATE_RDY)) { - mlx5_rl_remove_rate(dev, &old_rl); - if (new_state != MLX5_SQC_STATE_RDY) - memset(&new_rl, 0, sizeof(new_rl)); - } - - ibqp->rl = new_rl; sq->state = new_state; out: @@ -4406,34 +4463,29 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, raw_qp_param.port = attr->port_num; if (attr_mask & IB_QP_RATE_LIMIT) { - raw_qp_param.rl.rate = attr->rate_limit; + err = qp_rl_parse(dev, qp, attr, ucmd, + &raw_qp_param.rl_ctx.rl_desired); + if (err) + goto out; - if (ucmd->burst_info.max_burst_sz) { - if (attr->rate_limit && - MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) { - raw_qp_param.rl.max_burst_sz = - ucmd->burst_info.max_burst_sz; - } else { - err = -EINVAL; + if (!mlx5_rl_are_equal(&raw_qp_param.rl_ctx.rl_desired, + &qp->rl)) { + err = qp_rl_prepare(dev, qp, op, + &raw_qp_param.rl_ctx); + if (err) goto out; - } - } - - if (ucmd->burst_info.typical_pkt_sz) { - if (attr->rate_limit && - MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) { - raw_qp_param.rl.typical_pkt_sz = - ucmd->burst_info.typical_pkt_sz; - } else { - err = -EINVAL; - goto out; - } } raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT; } err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity); + if (err) { + qp_rl_rollback(dev->mdev, &raw_qp_param.rl_ctx); + goto out; + } + + qp_rl_commit(dev->mdev, qp, &raw_qp_param.rl_ctx); } else { if (udata) { /* For the kernel flows, the resp will stay zero */ From 29e0a014ddfbdbd6b02684304dddbcf9161034ae Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:04 +0300 Subject: [PATCH 116/148] RDMA/mlx5: Add support for rate limit in UD and UC QPs Rate limiting is currently supported only for raw packet QPs, where the packet pacing index is programmed into the SQC during SQ modify. Extend rate limit support to UD and UC QPs by setting the pacing index in the QPC during RTR2RTS and RTS2RTS transitions. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-3-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 111 ++++++++++++++++++++++---------- include/linux/mlx5/qp.h | 1 + 2 files changed, 78 insertions(+), 34 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 6924fceb99c5..1f8d6dd6057c 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -2754,6 +2754,10 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, if (err) mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n", base->mqp.qpn); + if (qp->rl.rate) { + mlx5_rl_remove_rate(dev->mdev, &qp->rl); + memset(&qp->rl, 0, sizeof(qp->rl)); + } } destroy_qp(dev, qp, base, udata); @@ -3703,8 +3707,10 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q MLX5_QP_OPTPAR_RNR_TIMEOUT, [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | MLX5_QP_OPTPAR_RWE | - MLX5_QP_OPTPAR_PM_STATE, - [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY, + MLX5_QP_OPTPAR_PM_STATE | + MLX5_QP_OPTPAR_PP_INDEX, + [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY | + MLX5_QP_OPTPAR_PP_INDEX, [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE | @@ -3723,10 +3729,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q MLX5_QP_OPTPAR_ALT_ADDR_PATH, [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_PM_STATE | - MLX5_QP_OPTPAR_ALT_ADDR_PATH, + MLX5_QP_OPTPAR_ALT_ADDR_PATH | + MLX5_QP_OPTPAR_PP_INDEX, [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY | MLX5_QP_OPTPAR_SRQN | - MLX5_QP_OPTPAR_CQN_RCV, + MLX5_QP_OPTPAR_CQN_RCV | + MLX5_QP_OPTPAR_PP_INDEX, [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE | MLX5_QP_OPTPAR_RWE | @@ -3870,11 +3878,31 @@ static int modify_raw_packet_qp_rq( return err; } +static bool qp_rate_limit_supported(struct mlx5_ib_dev *dev, + struct mlx5_ib_qp *qp) +{ + if (qp->type == IB_QPT_RAW_PACKET || + qp->flags & IB_QP_CREATE_SOURCE_QPN) + return true; + + if (qp->type == IB_QPT_UD) + return MLX5_CAP_QOS(dev->mdev, packet_pacing_req_ud); + + if (qp->type == IB_QPT_UC) + return MLX5_CAP_QOS(dev->mdev, packet_pacing_req_uc); + + return false; +} + static int qp_rl_parse(struct mlx5_ib_dev *dev, + struct mlx5_ib_qp *qp, const struct ib_qp_attr *attr, const struct mlx5_ib_modify_qp *ucmd, struct mlx5_rate_limit *rl_desired) { + if (!qp_rate_limit_supported(dev, qp)) + return -EOPNOTSUPP; + rl_desired->rate = attr->rate_limit; if (ucmd->burst_info.max_burst_sz) { @@ -3935,15 +3963,20 @@ static void qp_rl_rollback(struct mlx5_core_dev *dev, static void qp_rl_commit(struct mlx5_core_dev *dev, struct mlx5_ib_qp *qp, - struct mlx5_rate_limit_ctx *ctx) + struct mlx5_rate_limit_ctx *ctx, + enum ib_qp_state new_state) { - if (!ctx->rl_changed) - return; + if (ctx->rl_changed) { + if (ctx->rl_old.rate) + mlx5_rl_remove_rate(dev, &ctx->rl_old); + qp->rl = ctx->rl_desired; + } - if (ctx->rl_old.rate) - mlx5_rl_remove_rate(dev, &ctx->rl_old); - - qp->rl = ctx->rl_desired; + if (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR) { + if (qp->rl.rate) + mlx5_rl_remove_rate(dev, &qp->rl); + memset(&qp->rl, 0, sizeof(qp->rl)); + } } static int modify_raw_packet_qp_sq( @@ -4250,6 +4283,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, struct mlx5_ib_qp *qp = to_mqp(ibqp); struct mlx5_ib_qp_base *base = &qp->trans_qp.base; struct mlx5_ib_cq *send_cq, *recv_cq; + struct mlx5_rate_limit_ctx rl_ctx = {}; struct mlx5_ib_pd *pd; enum mlx5_qp_state mlx5_cur, mlx5_new; void *qpc, *pri_path, *alt_path; @@ -4440,20 +4474,31 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, goto out; } - op = optab[mlx5_cur][mlx5_new]; - optpar |= ib_mask_to_mlx5_opt(attr_mask); - optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st]; - - if (attr_mask & IB_QP_RATE_LIMIT && qp->type != IB_QPT_RAW_PACKET) { - err = -EOPNOTSUPP; - goto out; + if (attr_mask & IB_QP_RATE_LIMIT) { + err = qp_rl_parse(dev, qp, attr, ucmd, &rl_ctx.rl_desired); + if (err) + goto out; + } else { + rl_ctx.rl_desired = qp->rl; } + op = optab[mlx5_cur][mlx5_new]; + if (!mlx5_rl_are_equal(&rl_ctx.rl_desired, &qp->rl)) { + err = qp_rl_prepare(dev, qp, op, &rl_ctx); + if (err) + goto out; + } + optpar |= ib_mask_to_mlx5_opt(attr_mask); + if (rl_ctx.rl_changed) + optpar |= MLX5_QP_OPTPAR_PP_INDEX; + optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st]; + if (qp->type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { struct mlx5_modify_raw_qp_param raw_qp_param = {}; raw_qp_param.operation = op; + raw_qp_param.rl_ctx = rl_ctx; if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { raw_qp_param.rq_q_ctr_id = set_id; raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID; @@ -4462,22 +4507,8 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, if (attr_mask & IB_QP_PORT) raw_qp_param.port = attr->port_num; - if (attr_mask & IB_QP_RATE_LIMIT) { - err = qp_rl_parse(dev, qp, attr, ucmd, - &raw_qp_param.rl_ctx.rl_desired); - if (err) - goto out; - - if (!mlx5_rl_are_equal(&raw_qp_param.rl_ctx.rl_desired, - &qp->rl)) { - err = qp_rl_prepare(dev, qp, op, - &raw_qp_param.rl_ctx); - if (err) - goto out; - } - + if (rl_ctx.rl_changed) raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT; - } err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity); if (err) { @@ -4485,8 +4516,13 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, goto out; } - qp_rl_commit(dev->mdev, qp, &raw_qp_param.rl_ctx); + qp_rl_commit(dev->mdev, qp, &raw_qp_param.rl_ctx, new_state); } else { + if (rl_ctx.rl_changed) { + MLX5_SET(qpc, qpc, packet_pacing_rate_limit_index, + rl_ctx.rl_desired_index); + } + if (udata) { /* For the kernel flows, the resp will stay zero */ resp->ece_options = @@ -4496,6 +4532,13 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, } err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp, &resp->ece_options); + + if (err) { + qp_rl_rollback(dev->mdev, &rl_ctx); + goto out; + } + + qp_rl_commit(dev->mdev, qp, &rl_ctx, new_state); } if (err) diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index d67aedc6ea68..40f889403b07 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -72,6 +72,7 @@ enum mlx5_qp_optpar { MLX5_QP_OPTPAR_CQN_RCV = 1 << 19, MLX5_QP_OPTPAR_DC_HS = 1 << 20, MLX5_QP_OPTPAR_DC_KEY = 1 << 21, + MLX5_QP_OPTPAR_PP_INDEX = 1 << 22, MLX5_QP_OPTPAR_COUNTER_SET_ID = 1 << 25, }; From 915bbc8578e3f784b35bd9e5fa06a3b0720eb292 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:05 +0300 Subject: [PATCH 117/148] RDMA/mlx5: Support deferred rate limit configuration Allow passing a rate limit attribute in modify QP flows even when the QP is in a state that does not support packet pacing programming in the lower layers. When the user sets a rate limit during a QP transition that is not to RTS, store the value in the mlx5 QP struct and program it to FW when the QP later transitions to RTS, which is the state that allows configuring the rate limit index in the QP context. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-4-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + drivers/infiniband/hw/mlx5/qp.c | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 0550cdfacad4..522984d958bb 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -540,6 +540,7 @@ struct mlx5_ib_qp { struct list_head cq_recv_list; struct list_head cq_send_list; struct mlx5_rate_limit rl; + struct mlx5_rate_limit rl_desired; u32 underlay_qpn; u32 flags_en; /* diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 1f8d6dd6057c..0725d1161c40 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3976,7 +3976,11 @@ static void qp_rl_commit(struct mlx5_core_dev *dev, if (qp->rl.rate) mlx5_rl_remove_rate(dev, &qp->rl); memset(&qp->rl, 0, sizeof(qp->rl)); + memset(&qp->rl_desired, 0, sizeof(qp->rl_desired)); + return; } + + qp->rl_desired = ctx->rl_desired; } static int modify_raw_packet_qp_sq( @@ -4020,6 +4024,7 @@ static int modify_raw_packet_qp_sq( if (new_state != MLX5_SQC_STATE_RDY) { mlx5_rl_remove_rate(dev, &ibqp->rl); memset(&ibqp->rl, 0, sizeof(ibqp->rl)); + memset(&ibqp->rl_desired, 0, sizeof(ibqp->rl_desired)); } sq->state = new_state; @@ -4479,7 +4484,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, if (err) goto out; } else { - rl_ctx.rl_desired = qp->rl; + rl_ctx.rl_desired = qp->rl_desired; } op = optab[mlx5_cur][mlx5_new]; @@ -4863,6 +4868,13 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, attr_mask); goto out; } + } else if (attr_mask == IB_QP_RATE_LIMIT && cur_state != IB_QPS_RTS) { + struct mlx5_rate_limit rl_desired = {}; + + err = qp_rl_parse(dev, qp, attr, &ucmd, &rl_desired); + if (!err) + qp->rl_desired = rl_desired; + goto out; } else if (qp_type != MLX5_IB_QPT_REG_UMR && qp_type != MLX5_IB_QPT_DCI && !ib_modify_qp_is_ok(cur_state, new_state, qp_type, From 1f307090e9575005ae3fc5cbb9d9520dcaf5086d Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:06 +0300 Subject: [PATCH 118/148] RDMA/mlx5: Report packet pacing capabilities when querying device When querying device, report packet pacing capabilities for UD and UC QPs when device supports it. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-5-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/main.c | 37 +++++++++++++++++++------------ 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index a558ac5bb219..38b2481aaae1 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1214,20 +1214,29 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } } - if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen && - raw_support) { - if (MLX5_CAP_QOS(mdev, packet_pacing) && - MLX5_CAP_GEN(mdev, qos)) { - resp.packet_pacing_caps.qp_rate_limit_max = - MLX5_CAP_QOS(mdev, packet_pacing_max_rate); - resp.packet_pacing_caps.qp_rate_limit_min = - MLX5_CAP_QOS(mdev, packet_pacing_min_rate); - resp.packet_pacing_caps.supported_qpts |= - 1 << IB_QPT_RAW_PACKET; - if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) && - MLX5_CAP_QOS(mdev, packet_pacing_typical_size)) - resp.packet_pacing_caps.cap_flags |= - MLX5_IB_PP_SUPPORT_BURST; + if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen) { + if (MLX5_CAP_GEN(mdev, qos)) { + if (MLX5_CAP_QOS(mdev, packet_pacing) && raw_support) + resp.packet_pacing_caps.supported_qpts |= + BIT(IB_QPT_RAW_PACKET); + if (MLX5_CAP_QOS(mdev, packet_pacing_req_ud)) + resp.packet_pacing_caps.supported_qpts |= + BIT(IB_QPT_UD); + if (MLX5_CAP_QOS(mdev, packet_pacing_req_uc)) + resp.packet_pacing_caps.supported_qpts |= + BIT(IB_QPT_UC); + + if (resp.packet_pacing_caps.supported_qpts) { + resp.packet_pacing_caps.qp_rate_limit_max = + MLX5_CAP_QOS(mdev, packet_pacing_max_rate); + resp.packet_pacing_caps.qp_rate_limit_min = + MLX5_CAP_QOS(mdev, packet_pacing_min_rate); + + if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) && + MLX5_CAP_QOS(mdev, packet_pacing_typical_size)) + resp.packet_pacing_caps.cap_flags |= + MLX5_IB_PP_SUPPORT_BURST; + } } resp.response_length += sizeof(resp.packet_pacing_caps); } From 8cca27313636c3e2cfc5f8071abfbbfdcf3138f8 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:07 +0300 Subject: [PATCH 119/148] RDMA/bnxt_re: Validate rate limit attribute in modify QP Rate limit transition validation for RC QPs currently relies on the IB core qp_state_table. Add a driver-level helper to validate the rate limit attribute directly during QP modify, ensuring it is only accepted for RC QPs in INIT->RTR, RTR->RTS and RTS->RTS transitions. This makes the driver responsible for rate limit validation and prepares for a follow-up IB core change that delegates IB_QP_RATE_LIMIT and all future non-standard modify attributes handling to individual vendor drivers. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-6-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index b1c489867fc7..13e765f10db1 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2406,6 +2406,23 @@ static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, return rc; } +static bool bnxt_re_is_modify_ok(enum ib_qp_attr_mask ext_mask, + enum ib_qp_type type, enum ib_qp_state cur, + enum ib_qp_state next) +{ + if (!ext_mask) + return true; + + if (ext_mask & ~IB_QP_RATE_LIMIT) + return false; + + /* Rate limit is only supported for RC QPs during specific transitions */ + return type == IB_QPT_RC && + ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || + (cur == IB_QPS_RTR && next == IB_QPS_RTS) || + (cur == IB_QPS_RTS && next == IB_QPS_RTS)); +} + int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata) { @@ -2430,7 +2447,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); new_qp_state = qp_attr->qp_state; if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, - ib_qp->qp_type, qp_attr_mask)) { + ib_qp->qp_type, qp_attr_mask) || + !bnxt_re_is_modify_ok(qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS, + ib_qp->qp_type, curr_qp_state, + new_qp_state)) { ibdev_err(&rdev->ibdev, "Invalid attribute mask: %#x specified ", qp_attr_mask); From c2cd90ab881c69ac0df3dbd0228656216d0e7aa8 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:08 +0300 Subject: [PATCH 120/148] RDMA/ionic: Validate rate limit attribute in modify QP Rate limit transition validation for RC QPs currently relies on the IB core qp_state_table. Add a driver-level helper to validate the rate limit attribute directly during QP modify, ensuring it is only accepted for RC QPs in INIT->RTR, RTR->RTS and RTS->RTS transitions. This makes the driver responsible for rate limit validation and prepares for a follow-up IB core change that delegates IB_QP_RATE_LIMIT and all future non-standard modify attributes handling to individual vendor drivers. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-7-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- .../infiniband/hw/ionic/ionic_controlpath.c | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index db18c315cc21..9d91f7667d4f 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -2535,6 +2535,23 @@ static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state, return false; } +static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask, + enum ib_qp_type type, enum ib_qp_state cur, + enum ib_qp_state next) +{ + if (!ext_mask) + return true; + + if (ext_mask & ~IB_QP_RATE_LIMIT) + return false; + + /* Rate limit is only supported for RC QPs during specific transitions */ + return type == IB_QPT_RC && + ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || + (cur == IB_QPS_RTR && next == IB_QPS_RTS) || + (cur == IB_QPS_RTS && next == IB_QPS_RTS)); +} + static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, int mask) { @@ -2547,7 +2564,9 @@ static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state)) return -EINVAL; - if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask)) + if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) || + !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS, + qp->ibqp.qp_type, cur_state, next_state)) return -EINVAL; /* unprivileged qp not allowed privileged qkey */ From 8b0c66fa058b34015fcc7cafc9dea86dd91d8db9 Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Sun, 24 May 2026 18:38:09 +0300 Subject: [PATCH 121/148] IB/core: Delegate IB_QP_RATE_LIMIT validation to drivers Remove IB_QP_RATE_LIMIT from the qp_state_table and instead pass it through ib_modify_qp_is_ok() unconditionally. This delegates rate limit attribute validation to the individual drivers that support it. As rate limit support expands to additional QP types and transitions across different vendors, centralizing this policy in the core becomes impractical. Each driver is better positioned to enforce its own supported QP types and transitions over non-standard attributes. Future support for non-standard attributes will be handled per vendor driver instead of in generic IB core qp_state_table. Signed-off-by: Maher Sanalla Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260524-packet-pacing-v1-8-3d79439f8d08@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/verbs.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index c0ba517297b8..3b613b57e269 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1540,8 +1540,7 @@ static const struct { IB_QP_PKEY_INDEX), [IB_QPT_RC] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | - IB_QP_PKEY_INDEX | - IB_QP_RATE_LIMIT), + IB_QP_PKEY_INDEX), [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX), @@ -1589,8 +1588,7 @@ static const struct { IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_MIN_RNR_TIMER | - IB_QP_PATH_MIG_STATE | - IB_QP_RATE_LIMIT), + IB_QP_PATH_MIG_STATE), [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | @@ -1604,7 +1602,6 @@ static const struct { IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), - [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT, } } }, @@ -1624,8 +1621,7 @@ static const struct { IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE | - IB_QP_MIN_RNR_TIMER | - IB_QP_RATE_LIMIT), + IB_QP_MIN_RNR_TIMER), [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | @@ -1639,7 +1635,6 @@ static const struct { IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), - [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT, } }, [IB_QPS_SQD] = { @@ -1777,7 +1772,7 @@ bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, if ((mask & req_param) != req_param) return false; - if (mask & ~(req_param | opt_param | IB_QP_STATE)) + if (mask & ~(req_param | opt_param | IB_QP_STATE | IB_QP_RATE_LIMIT)) return false; return true; From c70fcfa9881207659ad193ae10a3bd56b2ae3f8a Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:37 +0300 Subject: [PATCH 122/148] RDMA/mlx5: Fix mkey creation error flow rollback Fix the indices of mkeys destroyed in case of an error in batch mkey creation. Fixes: 36680ef7bceb ("RDMA/mlx5: Switch from MR cache to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-2-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 14c1aec1edbb..a0f23339387b 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -294,7 +294,7 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, free_in: kfree(in); if (err) - for (; i > 0; i--) + for (i--; i >= 0; i--) mlx5_core_destroy_mkey(dev->mdev, handles[i]); return err; } From fe683274fee497834d2e6b54b7342642e1f21892 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:38 +0300 Subject: [PATCH 123/148] RDMA/mlx5: Fix TPH extraction in FRMR pool key Fix reading the PH value from the FRMR pool key by shifting the pool key to the relevant bits. Fixes: 36680ef7bceb ("RDMA/mlx5: Switch from MR cache to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-3-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index a0f23339387b..6e7de9d2f0bd 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -31,6 +31,7 @@ * SOFTWARE. */ +#include #include #include #include @@ -163,9 +164,8 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, #define MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED \ MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT 16 -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK 0xFF0000 -#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK 0xFFFF +#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16) +#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0) static struct mlx5_ib_mr * _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, @@ -194,7 +194,8 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, ph ^= MLX5_IB_NO_PH; mr->ibmr.frmr.key.kernel_vendor_key = - st_index | (ph << MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT); + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) | + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph); err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr); if (err) { kfree(mr); @@ -271,9 +272,10 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, get_mkc_octo_size(access_mode, key->num_dma_blocks)); MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - st_index = key->kernel_vendor_key & - MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK; - ph = key->kernel_vendor_key & MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK; + st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, + key->kernel_vendor_key); + ph = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, + key->kernel_vendor_key); if (ph) { /* Normalize ph: swap MLX5_IB_NO_PH for 0 */ if (ph == MLX5_IB_NO_PH) From 3d7fd88aeff73f25ee740b3a65a3b4dd38ad7783 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:39 +0300 Subject: [PATCH 124/148] RDMA/core: Fix skipped usage for driver built FRMR key When creating FRMR handles following a netlink command to pin handles, use the key after driver callback instead of using the key passed directly from user. Fixes: 020d189d16a6 ("RDMA/core: Add pinned handles to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-4-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/frmr_pools.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 5e992ff3d7cf..6170466ea958 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -426,7 +426,7 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key, if (!handles) return -ENOMEM; - ret = pools->pool_ops->create_frmrs(device, key, handles, + ret = pools->pool_ops->create_frmrs(device, &driver_key, handles, needed_handles); if (ret) { kfree(handles); From c6936506ed556ce3ccad36ab999baf2764dd7d25 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:40 +0300 Subject: [PATCH 125/148] RDMA/core: Fix FRMR aging push to queue error flow Aging pools with pinned handles requires moving handles from the active queue to a non-empty inactive queue that might fail on new page allocation, we are currently not handling the fault and leaking any mkey that fails the push. Fix by Introducing push_queue_to_queue_locked() that fills the destination's partial tail page from the source and then splices the remaining source pages onto the destination, performing no allocation. Replace the per-handle move loop in age_pinned_pool() and the open-coded splice in pool_aging_work() with calls to the helper. As the helper cannot fail under memory pressure, removing a class of GFP_ATOMIC allocations under the pool lock and simplifying the error flow. Fixes: 020d189d16a6 ("RDMA/core: Add pinned handles to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-5-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/frmr_pools.c | 53 ++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 6170466ea958..927642c06f3a 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -97,13 +97,44 @@ static void destroy_all_handles_in_queue(struct ib_device *device, } } +/* + * Bulk-move all handles from @src into @dst without allocating new pages. + * If @dst has a partial tail page, fill it handle-by-handle from @src first + * to preserve the invariant that only the tail page is partial, then splice + * the remaining @src pages onto @dst. On return @src is empty. + * + * Caller must hold the lock protecting both queues. + */ +static void splice_frmr_queue_locked(struct frmr_queue *dst, + struct frmr_queue *src) +{ + u32 free_in_tail = dst->ci % NUM_HANDLES_PER_PAGE; + u32 handle; + + if (free_in_tail) { + free_in_tail = NUM_HANDLES_PER_PAGE - free_in_tail; + while (free_in_tail && src->ci) { + handle = pop_handle_from_queue_locked(src); + push_handle_to_queue_locked(dst, handle); + free_in_tail--; + } + } + + if (src->ci > 0) { + list_splice_tail_init(&src->pages_list, &dst->pages_list); + dst->num_pages += src->num_pages; + dst->ci += src->ci; + src->num_pages = 0; + src->ci = 0; + } +} + static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) { struct ib_frmr_pools *pools = device->frmr_pools; u32 total, to_destroy, destroyed = 0; bool has_work = false; u32 *handles; - u32 handle; spin_lock(&pool->lock); total = pool->queue.ci + pool->inactive_queue.ci + pool->in_use; @@ -112,7 +143,7 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) return false; } - to_destroy = total - pool->pinned_handles; + to_destroy = min(total - pool->pinned_handles, pool->inactive_queue.ci); handles = kcalloc(to_destroy, sizeof(*handles), GFP_ATOMIC); if (!handles) { @@ -121,15 +152,13 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) } /* Destroy all excess handles in the inactive queue */ - while (pool->inactive_queue.ci && destroyed < to_destroy) { - handles[destroyed++] = pop_handle_from_queue_locked( + for (; destroyed < to_destroy; destroyed++) + handles[destroyed] = pop_handle_from_queue_locked( &pool->inactive_queue); - } /* Move all handles from regular queue to inactive queue */ - while (pool->queue.ci) { - handle = pop_handle_from_queue_locked(&pool->queue); - push_handle_to_queue_locked(&pool->inactive_queue, handle); + if (pool->queue.ci > 0) { + splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue); has_work = true; } @@ -158,13 +187,7 @@ static void pool_aging_work(struct work_struct *work) /* Move all pages from regular queue to inactive queue */ spin_lock(&pool->lock); if (pool->queue.ci > 0) { - list_splice_tail_init(&pool->queue.pages_list, - &pool->inactive_queue.pages_list); - pool->inactive_queue.num_pages = pool->queue.num_pages; - pool->inactive_queue.ci = pool->queue.ci; - - pool->queue.num_pages = 0; - pool->queue.ci = 0; + splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue); has_work = true; } spin_unlock(&pool->lock); From 41a707d0275cdec9ac125e826dd6836fa9623cbc Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:41 +0300 Subject: [PATCH 126/148] RDMA/core: Fix FRMR set pinned push error path Add destruction of FRMR handles in case the push to the pool fails. This prevents resources leak in case pool page allocation fails. Fixes: 020d189d16a6 ("RDMA/core: Add pinned handles to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-6-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Reviewed-by: Tao Cui Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/frmr_pools.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 927642c06f3a..1cfdddc3fcda 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -461,11 +461,16 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key, ret = push_handle_to_queue_locked(&pool->queue, handles[i]); if (ret) - goto end; + break; + } + spin_unlock(&pool->lock); + + if (ret) { + /* Destroy handles created but never pushed to the pool. */ + pools->pool_ops->destroy_frmrs(device, &handles[i], + needed_handles - i); } -end: - spin_unlock(&pool->lock); kfree(handles); schedule_aging: From 3937243095b5cfed6556bd1ea170790223f3eeb0 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:42 +0300 Subject: [PATCH 127/148] RDMA/core: Avoid NULL dereference on FRMR bad usage In case a driver calls FRMR pop operation without a successful init, return after triggering a warning to avoid the NULL dereference. Fixes: ce5df0b891ed ("IB/core: Introduce FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-7-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/frmr_pools.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 1cfdddc3fcda..892aedfe03be 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -529,7 +529,9 @@ int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr) struct ib_frmr_pools *pools = device->frmr_pools; struct ib_frmr_pool *pool; - WARN_ON_ONCE(!device->frmr_pools); + if (WARN_ON_ONCE(!pools)) + return -EINVAL; + pool = ib_frmr_pool_find(pools, &mr->frmr.key); if (!pool) { pool = create_frmr_pool(device, &mr->frmr.key); From 8c76126b866649d8e8acc09a06f2b03b6ff88900 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:43 +0300 Subject: [PATCH 128/148] RDMA/core: Fix FRMR handle leak on push failure Failure to push a handle to the pool, caused by ENOMEM on queue page allocation, will trigger missing in_use counter update, skewing pool state indefinitely. Fix that by moving the handling of handle destruction in such case into the FRMR code, ensuring the handle is either pushed to the pool or destroyed inside the same function. Adjust mlx5_ib call site accordingly. Fixes: ce5df0b891ed ("IB/core: Introduce FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-8-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/frmr_pools.c | 21 ++++++++++++--------- drivers/infiniband/hw/mlx5/mr.c | 5 +++-- include/rdma/frmr_pools.h | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 892aedfe03be..e214a8273df8 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -549,9 +549,8 @@ EXPORT_SYMBOL(ib_frmr_pool_pop); * @device: The device to push the FRMR handle to. * @mr: The MR containing the FRMR handle to push back to the pool. * - * Returns 0 on success, negative error code on failure. */ -int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) +void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) { struct ib_frmr_pool *pool = mr->frmr.pool; struct ib_frmr_pools *pools = device->frmr_pools; @@ -559,19 +558,23 @@ int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) int ret; spin_lock(&pool->lock); - /* Schedule aging every time an empty pool becomes non-empty */ - if (pool->queue.ci == 0) - schedule_aging = true; + pool->in_use--; ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle); - if (ret == 0) - pool->in_use--; + + /* Schedule aging every time an empty pool becomes non-empty */ + if (!ret && pool->queue.ci == 1) + schedule_aging = true; spin_unlock(&pool->lock); - if (ret == 0 && schedule_aging) + if (ret) { + pools->pool_ops->destroy_frmrs(device, &mr->frmr.handle, 1); + return; + } + + if (schedule_aging) queue_delayed_work(pools->aging_wq, &pool->aging_work, secs_to_jiffies(READ_ONCE(pools->aging_period_sec))); - return ret; } EXPORT_SYMBOL(ib_frmr_pool_push); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 6e7de9d2f0bd..46cbdc86321f 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1398,9 +1398,10 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) bool is_odp = is_odp_mr(mr); int ret; - if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr) && - !ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr)) + if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr)) { + ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); return 0; + } if (is_odp) mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex); diff --git a/include/rdma/frmr_pools.h b/include/rdma/frmr_pools.h index af1b88801fa4..5b57bafa3636 100644 --- a/include/rdma/frmr_pools.h +++ b/include/rdma/frmr_pools.h @@ -34,6 +34,6 @@ int ib_frmr_pools_init(struct ib_device *device, const struct ib_frmr_pool_ops *pool_ops); void ib_frmr_pools_cleanup(struct ib_device *device); int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr); -int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); +void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); #endif /* FRMR_POOLS_H */ From ddbc251be18fb82884ee6e9af634cc9f1171a4d6 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:44 +0300 Subject: [PATCH 129/148] RDMA/core: Add ib_frmr_pool_drop for unrecoverable handles A driver that has popped a handle from an FRMR pool can hit failures that leave the handle in a state where it can't safely be returned for reuse. The driver destroys the handle itself, but the pool has no way to learn about it, so the in_use counter drifts upward. Add ib_frmr_pool_drop to balance the pool's accounting in this case. Every pop is now balanced by exactly one push or drop. Fixes: 36680ef7bceb ("RDMA/mlx5: Switch from MR cache to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-9-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/frmr_pools.c | 15 +++++++++++++++ include/rdma/frmr_pools.h | 1 + 2 files changed, 16 insertions(+) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index e214a8273df8..ce8ae4305b9c 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -578,3 +578,18 @@ void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) } EXPORT_SYMBOL(ib_frmr_pool_push); + +/* + * Drop a handle previously popped from the pool without returning it for + * reuse. The caller is responsible for destroying the underlying hardware + * resource. + */ +void ib_frmr_pool_drop(struct ib_mr *mr) +{ + struct ib_frmr_pool *pool = mr->frmr.pool; + + spin_lock(&pool->lock); + pool->in_use--; + spin_unlock(&pool->lock); +} +EXPORT_SYMBOL(ib_frmr_pool_drop); diff --git a/include/rdma/frmr_pools.h b/include/rdma/frmr_pools.h index 5b57bafa3636..aed4d69d3841 100644 --- a/include/rdma/frmr_pools.h +++ b/include/rdma/frmr_pools.h @@ -35,5 +35,6 @@ int ib_frmr_pools_init(struct ib_device *device, void ib_frmr_pools_cleanup(struct ib_device *device); int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr); void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); +void ib_frmr_pool_drop(struct ib_mr *mr); #endif /* FRMR_POOLS_H */ From c37d79dd967d450ea02e0ee2b6438b8534bbd044 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 10 Jun 2026 03:01:45 +0300 Subject: [PATCH 130/148] RDMA/mlx5: Drop FRMR pool handle on UMR revoke failure When UMR revoke fails during MR cleanup, the handle is left in an unknown state and cannot be returned to the pool. The driver already destroys the mkey via the fallback path, but the pool's in_use counter is never decremented, drifting upward over time. Call ib_frmr_pool_drop on the revoke-failure path so the pool's accounting stays consistent with the handles it has handed out. Fixes: 36680ef7bceb ("RDMA/mlx5: Switch from MR cache to FRMR pools") Link: https://patch.msgid.link/r/20260610000145.820592-10-michaelgur@nvidia.com Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 46cbdc86321f..499bfd201831 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1398,9 +1398,11 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) bool is_odp = is_odp_mr(mr); int ret; - if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr)) { - ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); - return 0; + if (mr->ibmr.frmr.pool) { + if (!mlx5_umr_revoke_mr_with_lock(mr)) { + ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); + return 0; + } } if (is_odp) @@ -1422,6 +1424,10 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) dma_resv_unlock( to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv); } + + if (mr->ibmr.frmr.pool && !ret) + ib_frmr_pool_drop(&mr->ibmr); + return ret; } From b136a7af41f796a48665afc6a55907488a3d5500 Mon Sep 17 00:00:00 2001 From: Patrisious Haddad Date: Sun, 7 Jun 2026 21:18:08 +0300 Subject: [PATCH 131/148] RDMA/mlx5: Remove DCT restrack tracking DCT restrack tracking wasn't working to begin with as it was only tracking the first DCT which was added, since at creation the DCT number isn't yet initialized because the DCT FW object is only created during modify. The following DCT additions were failing silently. Since the fix isn't trivial and there were no users that required or complained about this issue we are dropping this for now instead of fixing. Fixes: fd3af5e21866 ("RDMA/mlx5: Track DCT, DCI and REG_UMR QPs as diver_detail resources.") Link: https://patch.msgid.link/r/20260607-restrack-uaf-fix-v1-1-d72e45eb76c2@nvidia.com Signed-off-by: Patrisious Haddad Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 1 + drivers/infiniband/hw/mlx5/restrack.c | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 0725d1161c40..a9575219cdd1 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3146,6 +3146,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, switch (qp->type) { case MLX5_IB_QPT_DCT: + rdma_restrack_no_track(&qp->ibqp.res); err = create_dct(dev, pd, qp, params); break; case MLX5_IB_QPT_DCI: diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c index 67841922c7b8..00a9bcb2603f 100644 --- a/drivers/infiniband/hw/mlx5/restrack.c +++ b/drivers/infiniband/hw/mlx5/restrack.c @@ -178,9 +178,6 @@ static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "REG_UMR"); break; - case MLX5_IB_QPT_DCT: - ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT"); - break; case MLX5_IB_QPT_DCI: ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI"); break; From 666031fed8f0fdfc29b20d125a628c1b0a04cdaf Mon Sep 17 00:00:00 2001 From: Patrisious Haddad Date: Sun, 7 Jun 2026 21:18:09 +0300 Subject: [PATCH 132/148] RDMA/mlx5: Remove raw RSS QP restrack tracking Raw RSS QP restrack tracking wasn't working to begin with as it was only tracking the first raw RSS QP which was added, since at creation the raw RSS QP number is reserved so the QP number for this qp type was always zero. The following raw RSS QP additions were always failing silently. Since the fix isn't trivial and there were no users that required or complained about this issue we are dropping this for now instead of fixing. Fixes: 968f0b6f9c01 ("RDMA/mlx5: Consolidate into special function all create QP calls") Link: https://patch.msgid.link/r/20260607-restrack-uaf-fix-v1-2-d72e45eb76c2@nvidia.com Signed-off-by: Patrisious Haddad Reviewed-by: Michael Guralnik Signed-off-by: Edward Srouji Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index a9575219cdd1..d195131d06f2 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3140,6 +3140,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, int err; if (params->is_rss_raw) { + rdma_restrack_no_track(&qp->ibqp.res); err = create_rss_raw_qp_tir(dev, pd, qp, params); goto out; } From d881d60223aac8fdc12b227d89c76e131e92a9cd Mon Sep 17 00:00:00 2001 From: Maher Sanalla Date: Thu, 11 Jun 2026 15:50:42 +0300 Subject: [PATCH 133/148] RDMA/mlx5: Fix undefined shift of user RQ WQE size set_rq_size() computes the RQ WQE size as "1 << rq_wqe_shift" based on the user-provided rq_wqe_shift, which is only checked to be greater than 32, so shifts of 32 are still accepted. A shift of 31 also overflows a signed integer, leading to undefined behavior. Use check_shl_overflow() to compute the RQ WQE size and reject any invalid values. Fixes: e126ba97dba9 ("mlx5: Add driver for Mellanox Connect-IB adapters") Link: https://patch.msgid.link/r/20260611-maher-sec-fixes-v1-1-cd8eb2542869@nvidia.com Signed-off-by: Maher Sanalla Signed-off-by: Edward Srouji Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index d195131d06f2..7ff02d89c31d 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -461,16 +461,13 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, if (ucmd) { qp->rq.wqe_cnt = ucmd->rq_wqe_count; - if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift)) - return -EINVAL; qp->rq.wqe_shift = ucmd->rq_wqe_shift; - if ((1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) < - wq_sig) + if (check_shl_overflow(1, qp->rq.wqe_shift, &wqe_size)) + return -EINVAL; + if (wqe_size / sizeof(struct mlx5_wqe_data_seg) < wq_sig) return -EINVAL; qp->rq.max_gs = - (1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) - + wqe_size / sizeof(struct mlx5_wqe_data_seg) - wq_sig; qp->rq.max_post = qp->rq.wqe_cnt; } else { From 449ae7927152e46acbe5f19f97eafdae6d3a96b1 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Thu, 11 Jun 2026 13:20:15 +0300 Subject: [PATCH 134/148] =?UTF-8?q?RDMA/mlx5:=20Release=20the=20HW?= =?UTF-8?q?=E2=80=91provided=20UAR=20index=20rather=20than=20the=20SW=20on?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Free the UAR index returned by the hardware. Fixes: 4ed131d0bb15 ("IB/mlx5: Expose dynamic mmap allocation") Link: https://patch.msgid.link/r/20260611-fix-uar-release-v1-1-f5464d845dbf@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 38b2481aaae1..02809114fc79 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2683,7 +2683,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd, if (!dyn_uar) return err; - mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid); + mlx5_cmd_uar_dealloc(dev->mdev, uar_index, context->devx_uid); free_bfreg: mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx); From c695e969269e33e8030487d1436cafa678928dde Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Fri, 12 Jun 2026 14:26:12 +0200 Subject: [PATCH 135/148] ABI: sysfs-class-infiniband: minor cleanup Close parenthesis with ')'. Add '-': 64-bit counter. Link: https://patch.msgid.link/r/20260612122611.183127-2-manuelebner@mailbox.org Signed-off-by: Manuel Ebner Signed-off-by: Jason Gunthorpe --- Documentation/ABI/stable/sysfs-class-infiniband | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband index 694f23a03a28..7ba116103429 100644 --- a/Documentation/ABI/stable/sysfs-class-infiniband +++ b/Documentation/ABI/stable/sysfs-class-infiniband @@ -148,17 +148,17 @@ Description: **Data info**: port_xmit_data: (RO) Total number of data octets, divided by 4 - (lanes), transmitted on all VLs. This is 64 bit counter + (lanes), transmitted on all VLs. This is a 64-bit counter port_rcv_data: (RO) Total number of data octets, divided by 4 - (lanes), received on all VLs. This is 64 bit counter. + (lanes), received on all VLs. This is a 64-bit counter. port_xmit_packets: (RO) Total number of packets transmitted on all VLs from this port. This may include packets with errors. - This is 64 bit counter. + This is a 64-bit counter. port_rcv_packets: (RO) Total number of packets (this may include - packets containing Errors. This is 64 bit counter. + packets containing Errors). This is a 64-bit counter. link_downed: (RO) Total number of times the Port Training state machine has failed the link error recovery process and downed From 978b27d6ce538bb832ccd69e45802824e4301c4b Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:37 -0700 Subject: [PATCH 136/148] RDMA/bnxt_re: Initialize dpi variable to zero dpi is initialized only for BNXT_RE_ALLOC_WC_PAGE, but copied for all the cases. So initialize the dpi to 0. Fixes: eee6268421a2 ("RDMA/bnxt_re: Move the UAPI methods to a dedicated file") Fixes: 360da60d6c6e ("RDMA/bnxt_re: Enable low latency push") Link: https://patch.msgid.link/r/20260615224751.232802-2-selvin.xavier@broadcom.com Reviewed-by: Anantha Prabhu Signed-off-by: Kalesh AP Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/uapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 9e68b4a7e952..b9922360f11b 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -76,8 +76,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * struct ib_ucontext *ib_uctx; struct bnxt_re_dev *rdev; u64 mmap_offset; + u32 dpi = 0; u32 length; - u32 dpi; u64 addr; int err; From 131e2918b9b0529687e67e2e58047304027f095a Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:38 -0700 Subject: [PATCH 137/148] RDMA/bnxt_re: Free SRQ toggle page after firmware teardown Free the toggle page only after firmware teardown completes so that an NQ interrupt arriving during bnxt_qplib_destroy_srq() won't write the toggle values to an already-freed page. Move free_page() after bnxt_qplib_destroy_srq(). Fixes: 181028a0d84c ("RDMA/bnxt_re: Share a page to expose per SRQ info with userspace") Link: https://patch.msgid.link/r/20260615224751.232802-3-selvin.xavier@broadcom.com Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 13e765f10db1..e48ae2e62be8 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2149,11 +2149,11 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) if (ret) return ret; - if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) { - free_page((unsigned long)srq->uctx_srq_page); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) hash_del(&srq->hash_entry); - } bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) + free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); return ib_respond_empty_udata(udata); From bb45e06f9914ca64ac95341a80a0c20bb8dd46a9 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:39 -0700 Subject: [PATCH 138/148] RDMA/bnxt_re: Free CQ toggle page after firmware teardown Free the toggle page only after firmware teardown completes so that an NQ interrupt arriving during bnxt_qplib_destroy_cq() won't write the toggle value to an already-freed page. Move free_page() after bnxt_qplib_destroy_cq. Fixes: e275919d9669 ("RDMA/bnxt_re: Share a page to expose per CQ info with userspace") Link: https://patch.msgid.link/r/20260615224751.232802-4-selvin.xavier@broadcom.com Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index e48ae2e62be8..35905ae7d168 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3472,11 +3472,11 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (ret) return ret; - if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) { - free_page((unsigned long)cq->uctx_cq_page); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) hash_del(&cq->hash_entry); - } bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) + free_page((unsigned long)cq->uctx_cq_page); bnxt_re_put_nq(rdev, nq); From 7d70c704a06f620d5d421ab76bac5e225bfb4308 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:44 -0700 Subject: [PATCH 139/148] RDMA/bnxt_re: Avoid displaying the kernel pointer While dumping the info on MR using the rdma tool, we dump the mr_hwq which is a kernel pointer. There is no need to expose this value for end user. So avoid it. Fixes: 7363eb76b7f3 ("RDMA/bnxt_re: Support driver specific data collection using rdma tool") Link: https://patch.msgid.link/r/20260615224751.232802-9-selvin.xavier@broadcom.com Reviewed-by: Kalesh AP Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/main.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index a892f1172917..d25fdc458120 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -1093,8 +1093,6 @@ static int bnxt_re_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr) goto err; if (rdma_nl_put_driver_u32(msg, "element_size", mr_hwq->element_size)) goto err; - if (rdma_nl_put_driver_u64_hex(msg, "hwq", (unsigned long)mr_hwq)) - goto err; if (rdma_nl_put_driver_u64_hex(msg, "va", mr->qplib_mr.va)) goto err; From dc95931b7e1326dacae547874bf38c092e5960d8 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:45 -0700 Subject: [PATCH 140/148] RDMA/bnxt_re: Add a max slot check for SQ The variable WQE mode must be validated against the maximum slots supported by HW. The max supported value is 64K. Adding a max and min check and fail if user supplied value is more than the max supported and zero. Fixes: d8ea645d6984 ("RDMA/bnxt_re: Handle variable WQE support for user applications") Link: https://patch.msgid.link/r/20260615224751.232802-10-selvin.xavier@broadcom.com Reviewed-by: Sriharsha Basavapatna Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 3 +++ drivers/infiniband/hw/bnxt_re/qplib_sp.h | 1 + 2 files changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 35905ae7d168..c3570bf0204f 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1748,6 +1748,9 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, return qptype; qplqp->type = (u8)qptype; qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx); + if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE && + (!ureq->sq_slots || ureq->sq_slots > BNXT_RE_MAX_SQ_SLOTS)) + return -EINVAL; if (fixed_que_attr) { if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE) return -EOPNOTSUPP; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 9fadd637cb5b..c4193ae75b54 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -369,6 +369,7 @@ int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res); #define BNXT_VAR_MAX_SLOT_ALIGN 256 #define BNXT_VAR_MAX_SGE 13 #define BNXT_RE_MAX_RQ_WQES 65536 +#define BNXT_RE_MAX_SQ_SLOTS 65536 #define BNXT_STATIC_MAX_SGE 6 From 87267803a8c824616eb147c5dad7030a5db6f878 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:46 -0700 Subject: [PATCH 141/148] RDMA/bnxt_re: Proper rollback if the ioremap fails bnxt_qplib_alloc_dpi returns success even if ioremap fails. Add the proper rollback when the ioremap fails and return -ENOMEM status. Fixes: 0ac20faf5d83 ("RDMA/bnxt_re: Reorg the bar mapping") Fixes: 360da60d6c6e ("RDMA/bnxt_re: Enable low latency push") Link: https://patch.msgid.link/r/20260615224751.232802-11-selvin.xavier@broadcom.com Reviewed-by: Sriharsha Basavapatna Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/qplib_res.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 95e0489c53c3..756f8b5f042a 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -764,9 +764,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, break; case BNXT_QPLIB_DPI_TYPE_WC: dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; default: dpi->dbr = ioremap(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; } @@ -774,6 +778,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, mutex_unlock(&res->dpi_tbl_lock); return 0; +fail_ioremap: + /* Roll back the bit we just claimed. */ + set_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = NULL; + mutex_unlock(&res->dpi_tbl_lock); + return -ENOMEM; + } int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, From 441baa79043431807115fd030d7d0bb14ed441a0 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:47 -0700 Subject: [PATCH 142/148] RDMA/bnxt_re: Avoid repeated requests to allocate WC pages Applications can request multiple WC pages for the same ucontext. As of now, only 1 WC page per ucontext is supported. Add a lock to avoid concurrent access and a check to fail repeated requests. Also, if the mmap entry insert fails for the WC, free the Doorbell page index mapped for the WC page. Fixes: eee6268421a2 ("RDMA/bnxt_re: Move the UAPI methods to a dedicated file") Fixes: 360da60d6c6e ("RDMA/bnxt_re: Enable low latency push") Link: https://patch.msgid.link/r/20260615224751.232802-12-selvin.xavier@broadcom.com Reviewed-by: Kalesh AP Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 1 + drivers/infiniband/hw/bnxt_re/ib_verbs.h | 1 + drivers/infiniband/hw/bnxt_re/uapi.c | 33 +++++++++++++++++++----- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index c3570bf0204f..565762529007 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -4770,6 +4770,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) goto fail; } spin_lock_init(&uctx->sh_lock); + mutex_init(&uctx->wcdpi_lock); resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; chip_met_rev_num = rdev->chip_ctx->chip_num; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 4d6d1259a795..22bf81668cfb 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -143,6 +143,7 @@ struct bnxt_re_ucontext { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_qplib_dpi wcdpi; + struct mutex wcdpi_lock; /* serialises WC DPI alloc/free */ void *shpg; spinlock_t sh_lock; /* protect shpg */ struct rdma_user_mmap_entry *shpage_mmap; diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index b9922360f11b..7e5a55d29076 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -98,14 +98,23 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * switch (alloc_type) { case BNXT_RE_ALLOC_WC_PAGE: - if (cctx->modes.db_push) { + if (cctx->modes.db_push) { + mutex_lock(&uctx->wcdpi_lock); + /* already allocated — one WC page per context */ + if (uctx->wcdpi.dbr) { + mutex_unlock(&uctx->wcdpi_lock); + return -EEXIST; + } if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi, - uctx, BNXT_QPLIB_DPI_TYPE_WC)) + uctx, BNXT_QPLIB_DPI_TYPE_WC)) { + mutex_unlock(&uctx->wcdpi_lock); return -ENOMEM; + } length = PAGE_SIZE; dpi = uctx->wcdpi.dpi; addr = (u64)uctx->wcdpi.umdbr; mmap_flag = BNXT_RE_MMAP_WC_DB; + mutex_unlock(&uctx->wcdpi_lock); } else { return -EINVAL; } @@ -128,8 +137,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * } entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); - if (!entry) + if (!entry) { + if (mmap_flag == BNXT_RE_MMAP_WC_DB) { + mutex_lock(&uctx->wcdpi_lock); + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + mutex_unlock(&uctx->wcdpi_lock); + } return -ENOMEM; + } uobj->object = entry; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); @@ -160,11 +176,16 @@ static int alloc_page_obj_cleanup(struct ib_uobject *uobject, switch (entry->mmap_flag) { case BNXT_RE_MMAP_WC_DB: - if (uctx && uctx->wcdpi.dbr) { + if (uctx) { struct bnxt_re_dev *rdev = uctx->rdev; - bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); - uctx->wcdpi.dbr = NULL; + mutex_lock(&uctx->wcdpi_lock); + if (uctx->wcdpi.dbr) { + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + } + mutex_unlock(&uctx->wcdpi_lock); } break; case BNXT_RE_MMAP_DBR_BAR: From a57592c6392a8e333c9c2731701297a5a279313a Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:50 -0700 Subject: [PATCH 143/148] RDMA/bnxt_re: Fail DBR related page allocation UAPIs if the feature is disabled No need to support the DBR related page allocations if the pacing feature is disabled. Fail the request if pacing is disabled. Fixes: ea2224857882 ("RDMA/bnxt_re: Update alloc_page uapi for pacing") Link: https://patch.msgid.link/r/20260615224751.232802-15-selvin.xavier@broadcom.com Reviewed-by: Sriharsha Basavapatna Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/uapi.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 7e5a55d29076..19f92d6a46c0 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -121,12 +121,16 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * break; case BNXT_RE_ALLOC_DBR_BAR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_bar_addr; mmap_flag = BNXT_RE_MMAP_DBR_BAR; break; case BNXT_RE_ALLOC_DBR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_page; mmap_flag = BNXT_RE_MMAP_DBR_PAGE; From 33a215f499b0643cd88a32ab5b8de1547be419c6 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 15 Jun 2026 15:47:51 -0700 Subject: [PATCH 144/148] RDMA/bnxt_re: Reject GET_TOGGLE_MEM when toggle page was not allocated If a user calls BNXT_RE_METHOD_GET_TOGGLE_MEM on a device that does not support the CQ/SRQ toggle feature, uctx_cq_page or uctx_srq_page will be NULL. Add an explicit -EOPNOTSUPP return after capturing the address from uctx_cq_page / uctx_srq_page if the address is zero. Fixes: e275919d9669 ("RDMA/bnxt_re: Share a page to expose per CQ info with userspace") Fixes: 181028a0d84c ("RDMA/bnxt_re: Share a page to expose per SRQ info with userspace") Link: https://patch.msgid.link/r/20260615224751.232802-16-selvin.xavier@broadcom.com Signed-off-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/uapi.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 19f92d6a46c0..263238a6e4cd 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -277,6 +277,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)cq->uctx_cq_page; + if (!addr) + return -EOPNOTSUPP; break; case BNXT_RE_SRQ_TOGGLE_MEM: srq = bnxt_re_search_for_srq(rdev, res_id); @@ -284,6 +286,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)srq->uctx_srq_page; + if (!addr) + return -EOPNOTSUPP; break; default: From 9b28231f0e5a6031cabe3b98dc6f28d3e0e5e437 Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Mon, 15 Jun 2026 17:20:23 -0700 Subject: [PATCH 145/148] docs: infiniband: correct name of option to enable the ib_uverbs module The Infiniband documentation states that CONFIG_INFINIBAND_USER_VERBS should be used to enable the ib_uverbs module. However, this option was renamed to CONFIG_INFINIBAND_USER_ACCESS in commit 17781cd6186c ("[PATCH] IB: clean up user access config options"). Update the documentation to reflect this. Link: https://patch.msgid.link/r/20260616002027.67925-1-enelsonmoore@gmail.com Signed-off-by: Ethan Nelson-Moore Reviewed-by: Dongliang Mu Signed-off-by: Jason Gunthorpe --- Documentation/infiniband/user_verbs.rst | 2 +- Documentation/translations/zh_CN/infiniband/user_verbs.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/infiniband/user_verbs.rst b/Documentation/infiniband/user_verbs.rst index 8ddc4b1cfef2..96bcd1bd37ad 100644 --- a/Documentation/infiniband/user_verbs.rst +++ b/Documentation/infiniband/user_verbs.rst @@ -2,7 +2,7 @@ Userspace verbs access ====================== - The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_VERBS, + The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_ACCESS, enables direct userspace access to IB hardware via "verbs," as described in chapter 11 of the InfiniBand Architecture Specification. diff --git a/Documentation/translations/zh_CN/infiniband/user_verbs.rst b/Documentation/translations/zh_CN/infiniband/user_verbs.rst index 970bc1a4e396..31534681654b 100644 --- a/Documentation/translations/zh_CN/infiniband/user_verbs.rst +++ b/Documentation/translations/zh_CN/infiniband/user_verbs.rst @@ -17,7 +17,7 @@ 用户空间verbs访问 ================= - ib_uverbs模块，通过启用CONFIG_INFINIBAND_USER_VERBS构建，使用户空间 + ib_uverbs模块，通过启用CONFIG_INFINIBAND_USER_ACCESS构建，使用户空间 通过“verbs”直接访问IB硬件，如InfiniBand架构规范第11章所述。 要使用verbs，需要libibverbs库，可从https://github.com/linux-rdma/rdma-core。 From 963af8d97a8c6a117134a8d0db1415e0489200b1 Mon Sep 17 00:00:00 2001 From: Zhenhao Wan Date: Fri, 12 Jun 2026 01:15:54 +0800 Subject: [PATCH 146/148] RDMA/rtrs-srv: Bound RDMA-Write length to chunk size in rdma_write_sg When the server answers an RTRS READ, rdma_write_sg() builds the source scatter/gather entry for the IB_WR_RDMA_WRITE that returns data to the peer. Its length is taken directly from the wire descriptor: plist->length = le32_to_cpu(id->rd_msg->desc[0].len); rd_msg points into the chunk buffer that the remote peer filled via RDMA-WRITE-WITH-IMM (rtrs_srv_rdma_done() -> process_io_req() -> process_read()), so desc[0].len is attacker-controlled and, before this change, was only rejected when zero. The source address is the fixed chunk start (dma_addr[msg_id]) and the source lkey is the PD-wide local_dma_lkey, which is not tied to the chunk's MR mapping, so the verbs layer does not constrain the transfer length to max_chunk_size. msg_id and off are bounded against queue_depth and max_chunk_size in rtrs_srv_rdma_done(), but desc[0].len is a separate field that was not checked against the chunk size. A peer that advertises desc[0].len larger than max_chunk_size can make the posted RDMA write read past the chunk's mapped region. The resulting behaviour depends on the IOMMU configuration: with no IOMMU or in passthrough mode the read may extend into memory adjacent to the chunk and be returned to the peer, which can disclose host memory; with a translating IOMMU the out-of-range access is expected to fault and abort the connection. In either case the transfer exceeds what the protocol permits and is driven by a remote peer. Reject a descriptor length above max_chunk_size, mirroring the existing off >= max_chunk_size bound in rtrs_srv_rdma_done(). Legitimate clients do not exceed it: the client sets desc[0].len to its MR length, which is capped at the negotiated max_io_size (max_chunk_size - MAX_HDR_SIZE). Fixes: 9cb837480424 ("RDMA/rtrs: server: main functionality") Link: https://patch.msgid.link/r/20260612-master-v1-1-70cde5c6fdc9@gmail.com Reported-by: Yuhao Jiang Cc: stable@vger.kernel.org Signed-off-by: Zhenhao Wan Reviewed-by: Md Haris Iqbal Signed-off-by: Jason Gunthorpe --- drivers/infiniband/ulp/rtrs/rtrs-srv.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index f2fd80c8a044..7d8e4422cc57 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -225,8 +225,9 @@ static int rdma_write_sg(struct rtrs_srv_op *id) /* WR will fail with length error * if this is 0 */ - if (plist->length == 0) { - rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); + if (plist->length == 0 || plist->length > max_chunk_size) { + rtrs_err(s, "Invalid RDMA-Write sg list length %u\n", + plist->length); return -EINVAL; } From c0bd03b850d81a8914168d87ddf7f6ffa58875ef Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Sat, 13 Jun 2026 18:20:45 +0800 Subject: [PATCH 147/148] RDMA/hns: Fix memory leak of bonding resources In a corner case of concurrent driver removal and driver reset, bonding resource is first released in hns_roce_hw_v2_exit() during driver removal, and then is allocated again in hns_roce_register_device() during driver reset. This leads to memory leak because the release timing has already passed. This may also lead to a kernel panic as below because of the leaked notifier callback: Call trace: 0xffffa20fccc04978 (P) raw_notifier_call_chain+0x20/0x38 call_netdevice_notifiers_info+0x60/0xb8 netdev_lower_state_changed+0x4c/0xb8 As Sashiko suggested, the teardown order of bonding resources should be inverted to make sure the resources are released when the driver is removed. Fixes: b37ad2e290fc ("RDMA/hns: Initialize bonding resources") Link: https://patch.msgid.link/r/20260613102045.811623-1-huangjunxian6@hisilicon.com Signed-off-by: Junxian Huang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 332a4816f2ca..2b3a1cafd1b2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -7654,8 +7654,8 @@ static int __init hns_roce_hw_v2_init(void) static void __exit hns_roce_hw_v2_exit(void) { - hns_roce_dealloc_bond_grp(); hnae3_unregister_client(&hns_roce_hw_v2_client); + hns_roce_dealloc_bond_grp(); hns_roce_cleanup_debugfs(); } From d9c8c45e6d2f438a3c8e643ae78b59454fa0fadd Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Tue, 16 Jun 2026 15:56:01 +0000 Subject: [PATCH 148/148] RDMA/irdma: Replace waitqueue and flag with completion The driver previously used a waitqueue along with an explicit request_done flag, but without proper barriers around request_done. An earlier patch by Gui-Dong Han attempted to fix this by adding the missing memory barriers. Rather than adding the barriers, this patch replaces the waitqueue+flag with a completion, which is designed for this exact purpose. Fixes: 44d9e52977a1 ("RDMA/irdma: Implement device initialization definitions") Fixes: 915cc7ac0f8e ("RDMA/irdma: Add miscellaneous utility definitions") Link: https://patch.msgid.link/r/20260616155601.1081448-1-jmoroni@google.com Signed-off-by: Jacob Moroni Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/irdma/hw.c | 7 +++---- drivers/infiniband/hw/irdma/main.h | 3 +-- drivers/infiniband/hw/irdma/utils.c | 12 +++++------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index f9be467d137f..c345cc654256 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -235,8 +235,7 @@ static void irdma_complete_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { if (cqp_request->waiting) { - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } else if (cqp_request->callback_fcn) { cqp_request->callback_fcn(cqp_request); } @@ -1107,9 +1106,9 @@ static int irdma_create_cqp(struct irdma_pci_f *rf) INIT_LIST_HEAD(&cqp->cqp_avail_reqs); INIT_LIST_HEAD(&cqp->cqp_pending_reqs); - /* init the waitqueue of the cqp_requests and add them to the list */ + /* init the completion of the cqp_requests and add them to the list */ for (i = 0; i < sqsize; i++) { - init_waitqueue_head(&cqp->cqp_requests[i].waitq); + init_completion(&cqp->cqp_requests[i].comp); list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); } init_waitqueue_head(&cqp->remove_wq); diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h index 3d49bd57bae7..8c17a201c1fd 100644 --- a/drivers/infiniband/hw/irdma/main.h +++ b/drivers/infiniband/hw/irdma/main.h @@ -161,13 +161,12 @@ struct irdma_cqp_compl_info { struct irdma_cqp_request { struct cqp_cmds_info info; - wait_queue_head_t waitq; + struct completion comp; struct list_head list; refcount_t refcnt; void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; - bool request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; bool dynamic:1; bool pending:1; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 7cc7b107db3c..e4037d5ef899 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -442,7 +442,7 @@ struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, if (cqp_request) { cqp_request->dynamic = true; if (wait) - init_waitqueue_head(&cqp_request->waitq); + init_completion(&cqp_request->comp); } } if (!cqp_request) { @@ -480,7 +480,7 @@ void irdma_free_cqp_request(struct irdma_cqp *cqp, if (cqp_request->dynamic) { kfree(cqp_request); } else { - WRITE_ONCE(cqp_request->request_done, false); + reinit_completion(&cqp_request->comp); cqp_request->callback_fcn = NULL; cqp_request->waiting = false; cqp_request->pending = false; @@ -515,8 +515,7 @@ irdma_free_pending_cqp_request(struct irdma_cqp *cqp, { if (cqp_request->waiting) { cqp_request->compl_info.error = true; - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } wait_event_timeout(cqp->remove_wq, refcount_read(&cqp_request->refcnt) == 1, 1000); @@ -609,9 +608,8 @@ static int irdma_wait_event(struct irdma_pci_f *rf, cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); - if (wait_event_timeout(cqp_request->waitq, - READ_ONCE(cqp_request->request_done), - msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) + if (wait_for_completion_timeout(&cqp_request->comp, + msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) break; if (cqp_request->pending)