diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband index 694f23a03a28..7ba116103429 100644 --- a/Documentation/ABI/stable/sysfs-class-infiniband +++ b/Documentation/ABI/stable/sysfs-class-infiniband @@ -148,17 +148,17 @@ Description: **Data info**: port_xmit_data: (RO) Total number of data octets, divided by 4 - (lanes), transmitted on all VLs. This is 64 bit counter + (lanes), transmitted on all VLs. This is a 64-bit counter port_rcv_data: (RO) Total number of data octets, divided by 4 - (lanes), received on all VLs. This is 64 bit counter. + (lanes), received on all VLs. This is a 64-bit counter. port_xmit_packets: (RO) Total number of packets transmitted on all VLs from this port. This may include packets with errors. - This is 64 bit counter. + This is a 64-bit counter. port_rcv_packets: (RO) Total number of packets (this may include - packets containing Errors. This is 64 bit counter. + packets containing Errors). This is a 64-bit counter. link_downed: (RO) Total number of times the Port Training state machine has failed the link error recovery process and downed diff --git a/Documentation/infiniband/user_verbs.rst b/Documentation/infiniband/user_verbs.rst index 8ddc4b1cfef2..96bcd1bd37ad 100644 --- a/Documentation/infiniband/user_verbs.rst +++ b/Documentation/infiniband/user_verbs.rst @@ -2,7 +2,7 @@ Userspace verbs access ====================== - The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_VERBS, + The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_ACCESS, enables direct userspace access to IB hardware via "verbs," as described in chapter 11 of the InfiniBand Architecture Specification. diff --git a/Documentation/translations/zh_CN/infiniband/user_verbs.rst b/Documentation/translations/zh_CN/infiniband/user_verbs.rst index 970bc1a4e396..31534681654b 100644 --- a/Documentation/translations/zh_CN/infiniband/user_verbs.rst +++ b/Documentation/translations/zh_CN/infiniband/user_verbs.rst @@ -17,7 +17,7 @@ 用户空间verbs访问 ================= - ib_uverbs模块，通过启用CONFIG_INFINIBAND_USER_VERBS构建，使用户空间 + ib_uverbs模块，通过启用CONFIG_INFINIBAND_USER_ACCESS构建，使用户空间 通过“verbs”直接访问IB硬件，如InfiniBand架构规范第11章所述。 要使用verbs，需要libibverbs库，可从https://github.com/linux-rdma/rdma-core。 diff --git a/MAINTAINERS b/MAINTAINERS index e67d929facbd..2d7e28b98f3f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12937,7 +12937,6 @@ F: include/linux/avf/virtchnl.h F: include/linux/net/intel/*/ INTEL ETHERNET PROTOCOL DRIVER FOR RDMA -M: Krzysztof Czurylo M: Tatyana Nikolova L: linux-rdma@vger.kernel.org S: Supported diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index a7e3f29dc037..086195758a8a 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -37,6 +37,10 @@ config INFINIBAND_USER_ACCESS libibverbs, libibcm and a hardware driver library from rdma-core . +config INFINIBAND_USER_ACCESS_CORE + bool + default y if INFINIBAND_USER_ACCESS != n + config INFINIBAND_USER_MEM bool depends on INFINIBAND_USER_ACCESS != n diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index dce798d8cfe6..ab7a2197bc86 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -5,19 +5,22 @@ user_access-$(CONFIG_INFINIBAND_ADDR_TRANS) := rdma_ucm.o obj-$(CONFIG_INFINIBAND) += ib_core.o ib_cm.o iw_cm.o \ $(infiniband-y) obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o -obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y) +obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o \ + $(user_access-y) \ + ib_uverbs_support.o ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \ device.o cache.o netlink.o \ roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \ multicast.o mad.o smi.o agent.o mad_rmpp.o \ - nldev.o restrack.o counters.o ib_core_uverbs.o \ + nldev.o restrack.o counters.o \ trace.o lag.o iter.o frmr_pools.o ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o +ib_core-$(CONFIG_INFINIBAND_USER_ACCESS_CORE) += ib_core_uverbs.o ib_cm-y := cm.o cm_trace.o @@ -33,7 +36,7 @@ rdma_ucm-y := ucma.o ib_umad-y := user_mad.o ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ - rdma_core.o uverbs_std_types.o uverbs_ioctl.o \ + uverbs_std_types.o uverbs_ioctl.o \ uverbs_std_types_cq.o \ uverbs_std_types_dmabuf.o \ uverbs_std_types_dmah.o \ @@ -43,5 +46,8 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types_async_fd.o \ uverbs_std_types_srq.o \ uverbs_std_types_wq.o \ - uverbs_std_types_qp.o \ - ucaps.o + uverbs_std_types_qp.o + +ib_uverbs_support-y := rdma_core.o \ + ucaps.o \ + uverbs_flow.o diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 27992c38ad90..e9fb7ad4c377 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -438,7 +438,7 @@ static int addr6_resolve(struct sockaddr *src_sock, static bool is_dst_local(const struct dst_entry *dst) { if (dst->ops->family == AF_INET) - return !!(dst_rtable(dst)->rt_type & RTN_LOCAL); + return dst_rtable(dst)->rt_type == RTN_LOCAL; else if (dst->ops->family == AF_INET6) return !!(dst_rt6_info(dst)->rt6i_flags & RTF_LOCAL); else @@ -850,7 +850,7 @@ static struct notifier_block nb = { int addr_init(void) { - addr_wq = alloc_ordered_workqueue("ib_addr", 0); + addr_wq = alloc_workqueue("ib_addr", WQ_UNBOUND, 0); if (!addr_wq) return -ENOMEM; diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 6ab9a0aee1ec..1a2c2775b14d 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -530,6 +530,7 @@ static int cm_init_av_by_path(struct sa_path_rec *path, struct rdma_ah_attr new_ah_attr; struct cm_device *cm_dev; struct cm_port *port; + u16 pkey_index; int ret; port = get_cm_port_from_path(path, sgid_attr); @@ -538,12 +539,10 @@ static int cm_init_av_by_path(struct sa_path_rec *path, cm_dev = port->cm_dev; ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num, - be16_to_cpu(path->pkey), &av->pkey_index); + be16_to_cpu(path->pkey), &pkey_index); if (ret) return ret; - cm_set_av_port(av, port); - /* * av->ah_attr might be initialized based on wc or during * request processing time which might have reference to sgid_attr. @@ -558,6 +557,8 @@ static int cm_init_av_by_path(struct sa_path_rec *path, if (ret) return ret; + av->pkey_index = pkey_index; + cm_set_av_port(av, port); av->timeout = path->packet_life_time + 1; rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); return 0; @@ -2184,8 +2185,10 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->av.ah_attr.roce.dmac); work->path[0].hop_limit = grh->hop_limit; - /* This destroy call is needed to pair with cm_init_av_for_response */ - cm_destroy_av(&cm_id_priv->av); + /* + * cm_init_av_by_path() will internally pair with the above + * cm_init_av_for_response() if it succeeds. + */ ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av); if (ret) { int err; diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c index 819927ce4f0e..891e52afb8f4 100644 --- a/drivers/infiniband/core/cma_configfs.c +++ b/drivers/infiniband/core/cma_configfs.c @@ -255,7 +255,7 @@ static void release_cma_ports_group(struct config_item *item) cma_dev_group->ports = NULL; }; -static struct configfs_item_operations cma_ports_item_ops = { +static const struct configfs_item_operations cma_ports_item_ops = { .release = release_cma_ports_group }; @@ -264,7 +264,7 @@ static const struct config_item_type cma_ports_group_type = { .ct_owner = THIS_MODULE }; -static struct configfs_item_operations cma_device_item_ops = { +static const struct configfs_item_operations cma_device_item_ops = { .release = release_cma_dev }; @@ -327,7 +327,7 @@ static void drop_cma_dev(struct config_group *cgroup, struct config_item *item) config_item_put(item); } -static struct configfs_group_operations cma_subsys_group_ops = { +static const struct configfs_group_operations cma_subsys_group_ops = { .make_group = make_cma_dev, .drop_item = drop_cma_dev, }; diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index a2c36666e6fc..19104c542b27 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -321,7 +321,7 @@ void nldev_exit(void); struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata, + struct uverbs_attr_bundle *uattrs, struct ib_uqp_object *uobj, const char *caller); void ib_qp_usecnt_inc(struct ib_qp *qp); diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index c3aa6d7fc66b..a9e189194c13 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -198,12 +198,20 @@ static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port, ret = __rdma_counter_bind_qp(counter, qp, port); if (ret) - goto err_mode; + goto err_bind; rdma_restrack_parent_name(&counter->res, &qp->res); rdma_restrack_add(&counter->res); return counter; +err_bind: + mutex_lock(&port_counter->lock); + port_counter->num_counters--; + if (!port_counter->num_counters && + port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL) + __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, + false); + mutex_unlock(&port_counter->lock); err_mode: rdma_free_hw_stats_struct(counter->stats); err_stats: @@ -661,7 +669,7 @@ void rdma_counter_init(struct ib_device *dev) fail: for (i = port; i >= rdma_start_port(dev); i--) { - port_counter = &dev->port_data[port].port_counter; + port_counter = &dev->port_data[i].port_counter; rdma_free_hw_stats_struct(port_counter->hstats); port_counter->hstats = NULL; mutex_destroy(&port_counter->lock); diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index b89efaaa81ec..b8193e077a74 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -1245,7 +1246,6 @@ static int assign_name(struct ib_device *device, const char *name) */ static int setup_device(struct ib_device *device) { - struct ib_udata uhw = {.outlen = 0, .inlen = 0}; int ret; ib_device_check_mandatory(device); @@ -1257,7 +1257,7 @@ static int setup_device(struct ib_device *device) } memset(&device->attrs, 0, sizeof(device->attrs)); - ret = device->ops.query_device(device, &device->attrs, &uhw); + ret = device->ops.query_device(device, &device->attrs, NULL); if (ret) { dev_warn(&device->dev, "Couldn't query the device attributes\n"); @@ -1419,6 +1419,14 @@ int ib_register_device(struct ib_device *device, const char *name, */ WARN_ON(dma_device && !dma_device->dma_parms); device->dma_device = dma_device; + /* + * In a CoCo guest every device is currently assumed to be untrusted + * (T=0) and therefore subject to DMA bouncing. Once trusted (T=1) + * device detection is wired up, narrow this check to exclude such + * devices. + */ + if (dma_device && cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) + device->cc_dma_bounce = 1; ret = setup_device(device); if (ret) diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c index 5e992ff3d7cf..ce8ae4305b9c 100644 --- a/drivers/infiniband/core/frmr_pools.c +++ b/drivers/infiniband/core/frmr_pools.c @@ -97,13 +97,44 @@ static void destroy_all_handles_in_queue(struct ib_device *device, } } +/* + * Bulk-move all handles from @src into @dst without allocating new pages. + * If @dst has a partial tail page, fill it handle-by-handle from @src first + * to preserve the invariant that only the tail page is partial, then splice + * the remaining @src pages onto @dst. On return @src is empty. + * + * Caller must hold the lock protecting both queues. + */ +static void splice_frmr_queue_locked(struct frmr_queue *dst, + struct frmr_queue *src) +{ + u32 free_in_tail = dst->ci % NUM_HANDLES_PER_PAGE; + u32 handle; + + if (free_in_tail) { + free_in_tail = NUM_HANDLES_PER_PAGE - free_in_tail; + while (free_in_tail && src->ci) { + handle = pop_handle_from_queue_locked(src); + push_handle_to_queue_locked(dst, handle); + free_in_tail--; + } + } + + if (src->ci > 0) { + list_splice_tail_init(&src->pages_list, &dst->pages_list); + dst->num_pages += src->num_pages; + dst->ci += src->ci; + src->num_pages = 0; + src->ci = 0; + } +} + static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) { struct ib_frmr_pools *pools = device->frmr_pools; u32 total, to_destroy, destroyed = 0; bool has_work = false; u32 *handles; - u32 handle; spin_lock(&pool->lock); total = pool->queue.ci + pool->inactive_queue.ci + pool->in_use; @@ -112,7 +143,7 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) return false; } - to_destroy = total - pool->pinned_handles; + to_destroy = min(total - pool->pinned_handles, pool->inactive_queue.ci); handles = kcalloc(to_destroy, sizeof(*handles), GFP_ATOMIC); if (!handles) { @@ -121,15 +152,13 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool) } /* Destroy all excess handles in the inactive queue */ - while (pool->inactive_queue.ci && destroyed < to_destroy) { - handles[destroyed++] = pop_handle_from_queue_locked( + for (; destroyed < to_destroy; destroyed++) + handles[destroyed] = pop_handle_from_queue_locked( &pool->inactive_queue); - } /* Move all handles from regular queue to inactive queue */ - while (pool->queue.ci) { - handle = pop_handle_from_queue_locked(&pool->queue); - push_handle_to_queue_locked(&pool->inactive_queue, handle); + if (pool->queue.ci > 0) { + splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue); has_work = true; } @@ -158,13 +187,7 @@ static void pool_aging_work(struct work_struct *work) /* Move all pages from regular queue to inactive queue */ spin_lock(&pool->lock); if (pool->queue.ci > 0) { - list_splice_tail_init(&pool->queue.pages_list, - &pool->inactive_queue.pages_list); - pool->inactive_queue.num_pages = pool->queue.num_pages; - pool->inactive_queue.ci = pool->queue.ci; - - pool->queue.num_pages = 0; - pool->queue.ci = 0; + splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue); has_work = true; } spin_unlock(&pool->lock); @@ -426,7 +449,7 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key, if (!handles) return -ENOMEM; - ret = pools->pool_ops->create_frmrs(device, key, handles, + ret = pools->pool_ops->create_frmrs(device, &driver_key, handles, needed_handles); if (ret) { kfree(handles); @@ -438,11 +461,16 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key, ret = push_handle_to_queue_locked(&pool->queue, handles[i]); if (ret) - goto end; + break; + } + spin_unlock(&pool->lock); + + if (ret) { + /* Destroy handles created but never pushed to the pool. */ + pools->pool_ops->destroy_frmrs(device, &handles[i], + needed_handles - i); } -end: - spin_unlock(&pool->lock); kfree(handles); schedule_aging: @@ -501,7 +529,9 @@ int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr) struct ib_frmr_pools *pools = device->frmr_pools; struct ib_frmr_pool *pool; - WARN_ON_ONCE(!device->frmr_pools); + if (WARN_ON_ONCE(!pools)) + return -EINVAL; + pool = ib_frmr_pool_find(pools, &mr->frmr.key); if (!pool) { pool = create_frmr_pool(device, &mr->frmr.key); @@ -519,9 +549,8 @@ EXPORT_SYMBOL(ib_frmr_pool_pop); * @device: The device to push the FRMR handle to. * @mr: The MR containing the FRMR handle to push back to the pool. * - * Returns 0 on success, negative error code on failure. */ -int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) +void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) { struct ib_frmr_pool *pool = mr->frmr.pool; struct ib_frmr_pools *pools = device->frmr_pools; @@ -529,19 +558,38 @@ int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr) int ret; spin_lock(&pool->lock); - /* Schedule aging every time an empty pool becomes non-empty */ - if (pool->queue.ci == 0) - schedule_aging = true; + pool->in_use--; ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle); - if (ret == 0) - pool->in_use--; + + /* Schedule aging every time an empty pool becomes non-empty */ + if (!ret && pool->queue.ci == 1) + schedule_aging = true; spin_unlock(&pool->lock); - if (ret == 0 && schedule_aging) + if (ret) { + pools->pool_ops->destroy_frmrs(device, &mr->frmr.handle, 1); + return; + } + + if (schedule_aging) queue_delayed_work(pools->aging_wq, &pool->aging_work, secs_to_jiffies(READ_ONCE(pools->aging_period_sec))); - return ret; } EXPORT_SYMBOL(ib_frmr_pool_push); + +/* + * Drop a handle previously popped from the pool without returning it for + * reuse. The caller is responsible for destroying the underlying hardware + * resource. + */ +void ib_frmr_pool_drop(struct ib_mr *mr) +{ + struct ib_frmr_pool *pool = mr->frmr.pool; + + spin_lock(&pool->lock); + pool->in_use--; + spin_unlock(&pool->lock); +} +EXPORT_SYMBOL(ib_frmr_pool_drop); diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 8a0e6fa2a528..dbbc0875132a 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -398,7 +398,7 @@ EXPORT_SYMBOL(rdma_user_mmap_entry_insert); * The struct ib_device that is handling the uverbs call. Must not be called if * udata is NULL. The result can be NULL. */ -struct ib_device *rdma_udata_to_dev(struct ib_udata *udata) +static struct ib_device *rdma_udata_to_dev(struct ib_udata *udata) { struct uverbs_attr_bundle *bundle = rdma_udata_to_uverbs_attr_bundle(udata); @@ -415,10 +415,9 @@ struct ib_device *rdma_udata_to_dev(struct ib_udata *udata) return srcu_dereference(bundle->ufile->device->ib_dev, &bundle->ufile->device->disassociate_srcu); } -EXPORT_SYMBOL(rdma_udata_to_dev); -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) -uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata) +typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs); +static uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata) { struct uverbs_attr_bundle *bundle = rdma_udata_to_uverbs_attr_bundle(udata); @@ -502,4 +501,258 @@ int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len) return -EFAULT; } EXPORT_SYMBOL(_ib_respond_udata); -#endif + +/* + * Must be called with the ufile->device->disassociate_srcu held, and the lock + * must be held until use of the ucontext is finished. + */ +struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile) +{ + /* + * We do not hold the hw_destroy_rwsem lock for this flow, instead + * srcu is used. It does not matter if someone races this with + * get_context, we get NULL or valid ucontext. + */ + struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext); + + if (!srcu_dereference(ufile->device->ib_dev, + &ufile->device->disassociate_srcu)) + return ERR_PTR(-EIO); + + if (!ucontext) + return ERR_PTR(-EINVAL); + + return ucontext; +} +EXPORT_SYMBOL(ib_uverbs_get_ucontext_file); + +int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) +{ + return 0; +} +EXPORT_SYMBOL(uverbs_destroy_def_handler); + +/* + * When calling a destroy function during an error unwind we need to pass in + * the udata that is sanitized of all user arguments. Ie from the driver + * perspective it looks like no udata was passed. + */ +struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) +{ + attrs->driver_udata = (struct ib_udata){}; + return &attrs->driver_udata; +} +EXPORT_SYMBOL_NS_GPL(uverbs_get_cleared_udata, "rdma_core"); + +/** + * _uverbs_alloc() - Quickly allocate memory for use with a bundle + * @bundle: The bundle + * @size: Number of bytes to allocate + * @flags: Allocator flags + * + * The bundle allocator is intended for allocations that are connected with + * processing the system call related to the bundle. The allocated memory is + * always freed once the system call completes, and cannot be freed any other + * way. + * + * This tries to use a small pool of pre-allocated memory for performance. + */ +__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, + gfp_t flags) +{ + struct bundle_priv *pbundle = + container_of(&bundle->hdr, struct bundle_priv, bundle); + size_t new_used; + void *res; + + if (check_add_overflow(size, pbundle->internal_used, &new_used)) + return ERR_PTR(-EOVERFLOW); + + if (new_used > pbundle->internal_avail) { + struct bundle_alloc_head *buf; + + buf = kvmalloc_flex(*buf, data, size, flags); + if (!buf) + return ERR_PTR(-ENOMEM); + buf->next = pbundle->allocated_mem; + pbundle->allocated_mem = buf; + return buf->data; + } + + res = (void *)pbundle->internal_buffer + pbundle->internal_used; + pbundle->internal_used = + ALIGN(new_used, sizeof(*pbundle->internal_buffer)); + if (want_init_on_alloc(flags)) + memset(res, 0, size); + return res; +} +EXPORT_SYMBOL(_uverbs_alloc); + +int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, + const void *from, size_t size) +{ + const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); + size_t min_size; + + if (IS_ERR(attr)) + return PTR_ERR(attr); + + min_size = min_t(size_t, attr->ptr_attr.len, size); + if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) + return -EFAULT; + + return uverbs_set_output(bundle, attr); +} +EXPORT_SYMBOL(uverbs_copy_to); + +int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, + size_t idx, const void *from, size_t size) +{ + const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); + + if (IS_ERR(attr)) + return PTR_ERR(attr); + + if (size < attr->ptr_attr.len) { + if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, + attr->ptr_attr.len - size)) + return -EFAULT; + } + return uverbs_copy_to(bundle, idx, from, size); +} +EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); + +int _uverbs_get_const_unsigned(u64 *to, + const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 upper_bound, u64 *def_val) +{ + const struct uverbs_attr *attr; + + attr = uverbs_attr_get(attrs_bundle, idx); + if (IS_ERR(attr)) { + if ((PTR_ERR(attr) != -ENOENT) || !def_val) + return PTR_ERR(attr); + + *to = *def_val; + } else { + *to = attr->ptr_attr.data; + } + + if (*to > upper_bound) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(_uverbs_get_const_unsigned); + +int _uverbs_get_const_signed(s64 *to, + const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, s64 lower_bound, u64 upper_bound, + s64 *def_val) +{ + const struct uverbs_attr *attr; + + attr = uverbs_attr_get(attrs_bundle, idx); + if (IS_ERR(attr)) { + if ((PTR_ERR(attr) != -ENOENT) || !def_val) + return PTR_ERR(attr); + + *to = *def_val; + } else { + *to = attr->ptr_attr.data; + } + + if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(_uverbs_get_const_signed); + +int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits) +{ + const struct uverbs_attr *attr; + u64 flags; + + attr = uverbs_attr_get(attrs_bundle, idx); + /* Missing attribute means 0 flags */ + if (IS_ERR(attr)) { + *to = 0; + return 0; + } + + /* + * New userspace code should use 8 bytes to pass flags, but we + * transparently support old userspaces that were using 4 bytes as + * well. + */ + if (attr->ptr_attr.len == 8) + flags = attr->ptr_attr.data; + else if (attr->ptr_attr.len == 4) + flags = *(u32 *)&attr->ptr_attr.data; + else + return -EINVAL; + + if (flags & ~allowed_bits) + return -EINVAL; + + *to = flags; + return 0; +} +EXPORT_SYMBOL(uverbs_get_flags64); + +int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx, u64 allowed_bits) +{ + u64 flags; + int ret; + + ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); + if (ret) + return ret; + + if (flags > U32_MAX) + return -EINVAL; + *to = flags; + + return 0; +} +EXPORT_SYMBOL(uverbs_get_flags32); + +/** + * uverbs_get_buffer_desc - Read a buffer descriptor from a uverbs attr. + * @attrs_bundle: uverbs attribute bundle. + * @attr_id: id of an UVERBS_ATTR_UMEM-typed attribute. + * @desc: descriptor to fill. + * + * Return: 0 on success, -ENOENT if @attr_id is not set, -EINVAL on a + * malformed descriptor, or any other negative errno propagated from + * uverbs_copy_from() (notably -EFAULT on copy_from_user() failure). + */ +int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle, + u16 attr_id, struct ib_uverbs_buffer_desc *desc) +{ + int ret; + + ret = uverbs_copy_from(desc, attrs_bundle, attr_id); + if (ret) + return ret; + if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK) + return -EINVAL; + desc->optional_flags &= IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK; + return 0; +} +EXPORT_SYMBOL(uverbs_get_buffer_desc); + +/* Once called an abort will call through to the type's destroy_hw() */ +void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, + u16 idx) +{ + struct bundle_priv *pbundle = + container_of(&bundle->hdr, struct bundle_priv, bundle); + + __set_bit(uapi_bkey_attr(uapi_key_attr(idx)), + pbundle->uobj_hw_obj_valid); +} +EXPORT_SYMBOL(uverbs_finalize_uobj_create); diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 9761d9365ffd..0b7246ec559e 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -518,8 +518,8 @@ static int iw_cm_map(struct iw_cm_id *cm_id, bool active) cm_id->m_local_addr = cm_id->local_addr; cm_id->m_remote_addr = cm_id->remote_addr; - strcpy(pm_reg_msg.dev_name, devname); - strcpy(pm_reg_msg.if_name, ifname); + strscpy(pm_reg_msg.dev_name, devname); + strscpy(pm_reg_msg.if_name, ifname); if (iwpm_register_pid(&pm_reg_msg, RDMA_NL_IWCM) || !iwpm_valid_pid()) diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 5aaba2b9746b..02a0a9c0a4a6 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -695,7 +695,7 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (has_cap_net_admin) { if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey)) @@ -711,9 +711,12 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) return -EMSGSIZE; - if (!rdma_is_kernel_res(res) && - nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id)) - return -EMSGSIZE; + if (!rdma_is_kernel_res(res)) { + struct ib_pd *pd = READ_ONCE(mr->pd); + + if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, pd->res.id)) + return -EMSGSIZE; + } if (fill_res_name_pid(msg, res)) return -EMSGSIZE; @@ -727,7 +730,7 @@ static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (!dev->ops.fill_res_mr_entry_raw) return -EINVAL; @@ -1017,7 +1020,7 @@ static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) goto err; diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 5018ec837056..fd5651c003ae 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -42,6 +42,40 @@ #include "core_priv.h" #include "rdma_core.h" +static void release_ufile_idr_uobject(struct ib_uverbs_file *ufile); + +void ib_uverbs_release_file(struct kref *ref) +{ + struct ib_uverbs_file *file = + container_of(ref, struct ib_uverbs_file, ref); + struct ib_device *ib_dev; + int srcu_key; + + release_ufile_idr_uobject(file); + + srcu_key = srcu_read_lock(&file->device->disassociate_srcu); + ib_dev = srcu_dereference(file->device->ib_dev, + &file->device->disassociate_srcu); + if (ib_dev && !ib_dev->ops.disassociate_ucontext) + module_put(ib_dev->ops.owner); + srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); + + if (refcount_dec_and_test(&file->device->refcount)) + ib_uverbs_comp_dev(file->device); + + if (file->default_async_file) + uverbs_uobject_put(&file->default_async_file->uobj); + put_device(&file->device->dev); + + if (file->disassociate_page) + __free_pages(file->disassociate_page, 0); + mutex_destroy(&file->disassociation_lock); + mutex_destroy(&file->umap_lock); + mutex_destroy(&file->ucontext_lock); + kfree(file); +} +EXPORT_SYMBOL_NS_GPL(ib_uverbs_release_file, "rdma_core"); + static void uverbs_uobject_free(struct kref *ref) { kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu); @@ -214,6 +248,7 @@ int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs) up_read(&ufile->hw_destroy_rwsem); return ret; } +EXPORT_SYMBOL_NS_GPL(uobj_destroy, "rdma_core"); /* * uobj_get_destroy destroys the HW object and returns a handle to the uobj @@ -239,6 +274,7 @@ struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj, return uobj; } +EXPORT_SYMBOL_NS_GPL(__uobj_get_destroy, "rdma_core"); /* * Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success @@ -255,6 +291,7 @@ int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, uobj_put_destroy(uobj); return 0; } +EXPORT_SYMBOL_NS_GPL(__uobj_perform_destroy, "rdma_core"); /* alloc_uobj must be undone by uverbs_destroy_uobject() */ static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs, @@ -420,6 +457,7 @@ struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj, uverbs_uobject_put(uobj); return ERR_PTR(ret); } +EXPORT_SYMBOL_NS_GPL(rdma_lookup_get_uobject, "rdma_core"); static struct ib_uobject * alloc_begin_idr_uobject(const struct uverbs_api_object *obj, @@ -465,8 +503,8 @@ alloc_begin_fd_uobject(const struct uverbs_api_object *obj, fd_type = container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); - if (WARN_ON(fd_type->fops && fd_type->fops->release != &uverbs_uobject_fd_release && - fd_type->fops->release != &uverbs_async_event_release)) { + if (WARN_ON(fd_type->fops && + fd_type->fops->release != &uverbs_uobject_fd_release)) { ret = ERR_PTR(-EINVAL); goto err_fd; } @@ -522,6 +560,7 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, } return ret; } +EXPORT_SYMBOL_NS_GPL(rdma_alloc_begin_uobject, "rdma_core"); static void alloc_abort_idr_uobject(struct ib_uobject *uobj) { @@ -668,6 +707,7 @@ void rdma_alloc_commit_uobject(struct ib_uobject *uobj, /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); } +EXPORT_SYMBOL_NS_GPL(rdma_alloc_commit_uobject, "rdma_core"); /* * new_uobj will be assigned to the handle currently used by to_uobj, and @@ -697,6 +737,7 @@ void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj */ uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs); } +EXPORT_SYMBOL_NS_GPL(rdma_assign_uobject, "rdma_core"); /* * This consumes the kref for uobj. It is up to the caller to unwind the HW @@ -727,6 +768,7 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); } +EXPORT_SYMBOL_NS_GPL(rdma_alloc_abort_uobject, "rdma_core"); static void lookup_put_idr_uobject(struct ib_uobject *uobj, enum rdma_lookup_mode mode) @@ -770,13 +812,15 @@ void rdma_lookup_put_uobject(struct ib_uobject *uobj, /* Pairs with the kref obtained by type->lookup_get */ uverbs_uobject_put(uobj); } +EXPORT_SYMBOL_NS_GPL(rdma_lookup_put_uobject, "rdma_core"); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile) { xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC); } +EXPORT_SYMBOL_NS_GPL(setup_ufile_idr_uobject, "rdma_core"); -void release_ufile_idr_uobject(struct ib_uverbs_file *ufile) +static void release_ufile_idr_uobject(struct ib_uverbs_file *ufile) { struct ib_uobject *entry; unsigned long id; @@ -839,6 +883,7 @@ int uverbs_uobject_release(struct ib_uobject *uobj) uverbs_uobject_put(uobj); return 0; } +EXPORT_SYMBOL_NS_GPL(uverbs_uobject_release, "rdma_core"); /* * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct @@ -846,51 +891,42 @@ int uverbs_uobject_release(struct ib_uobject *uobj) */ int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) { + void (*release_cleanup)(struct ib_uobject *uobj) = NULL; + struct ib_uobject *uobj = filp->private_data; + const struct uverbs_obj_type *type_attrs; + int ret; + /* * This can only happen if the fput came from alloc_abort_fd_uobject() */ - if (!filp->private_data) + if (!uobj) return 0; - return uverbs_uobject_release(filp->private_data); + /* + * uverbs_disassociate_api() can NULL type_attrs after disassociate, but + * it won't if release_cleanup is used. + */ + type_attrs = READ_ONCE(uobj->uapi_object->type_attrs); + if (type_attrs) + release_cleanup = container_of(type_attrs, + struct uverbs_obj_fd_type, type) + ->release_cleanup; + if (release_cleanup) + uverbs_uobject_get(uobj); + + ret = uverbs_uobject_release(uobj); + + if (release_cleanup) { + release_cleanup(uobj); + uverbs_uobject_put(uobj); + } + + return ret; } EXPORT_SYMBOL(uverbs_uobject_fd_release); -/* - * Drop the ucontext off the ufile and completely disconnect it from the - * ib_device - */ -static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile, - enum rdma_remove_reason reason) -{ - struct ib_ucontext *ucontext = ufile->ucontext; - struct ib_device *ib_dev = ucontext->device; - - /* - * If we are closing the FD then the user mmap VMAs must have - * already been destroyed as they hold on to the filep, otherwise - * they need to be zap'd. - */ - if (reason == RDMA_REMOVE_DRIVER_REMOVE) { - uverbs_user_mmap_disassociate(ufile); - if (ib_dev->ops.disassociate_ucontext) - ib_dev->ops.disassociate_ucontext(ucontext); - } - - ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev, - RDMACG_RESOURCE_HCA_HANDLE); - - rdma_restrack_del(&ucontext->res); - - ib_dev->ops.dealloc_ucontext(ucontext); - WARN_ON(!xa_empty(&ucontext->mmap_xa)); - kfree(ucontext); - - ufile->ucontext = NULL; -} - -static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, - enum rdma_remove_reason reason) +int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason) { struct uverbs_attr_bundle attrs = { .ufile = ufile }; struct ib_ucontext *ucontext = ufile->ucontext; @@ -931,36 +967,7 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, } return ret; } - -/* - * Destroy the ucontext and every uobject associated with it. - * - * This is internally locked and can be called in parallel from multiple - * contexts. - */ -void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, - enum rdma_remove_reason reason) -{ - down_write(&ufile->hw_destroy_rwsem); - - /* - * If a ucontext was never created then we can't have any uobjects to - * cleanup, nothing to do. - */ - if (!ufile->ucontext) - goto done; - - while (!list_empty(&ufile->uobjects) && - !__uverbs_cleanup_ufile(ufile, reason)) { - } - - if (WARN_ON(!list_empty(&ufile->uobjects))) - __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE); - ufile_destroy_ucontext(ufile, reason); - -done: - up_write(&ufile->hw_destroy_rwsem); -} +EXPORT_SYMBOL_NS_GPL(__uverbs_cleanup_ufile, "rdma_core"); const struct uverbs_obj_type_class uverbs_fd_class = { .alloc_begin = alloc_begin_fd_uobject, @@ -998,6 +1005,7 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, return ERR_PTR(-EOPNOTSUPP); } } +EXPORT_SYMBOL_NS_GPL(uverbs_get_uobject_from_file, "rdma_core"); void uverbs_finalize_object(struct ib_uobject *uobj, enum uverbs_obj_access access, bool hw_obj_valid, @@ -1030,6 +1038,7 @@ void uverbs_finalize_object(struct ib_uobject *uobj, WARN_ON(true); } } +EXPORT_SYMBOL_NS_GPL(uverbs_finalize_object, "rdma_core"); /** * rdma_uattrs_has_raw_cap() - Returns whether a rdma device linked to the @@ -1059,3 +1068,6 @@ bool rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs) return has_cap; } EXPORT_SYMBOL(rdma_uattrs_has_raw_cap); + +MODULE_DESCRIPTION("InfiniBand uverbs objects"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 269b393799ab..56121103e9f4 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -70,9 +70,15 @@ void uverbs_finalize_object(struct ib_uobject *uobj, int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile); -void release_ufile_idr_uobject(struct ib_uverbs_file *ufile); +#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs); +#else +static inline struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) +{ + return NULL; +} +#endif /* * This is the runtime description of the uverbs API, used by the syscall @@ -151,9 +157,6 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, unsigned int num_attrs); void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile); -typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs); -uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata); - extern const struct uapi_definition uverbs_def_obj_async_fd[]; extern const struct uapi_definition uverbs_def_obj_counters[]; extern const struct uapi_definition uverbs_def_obj_cq[]; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index ac3688952cab..cfee2071586c 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -71,6 +71,8 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, xa_lock(&rt->xa); xas_for_each(&xas, e, U32_MAX) { + if (xa_is_zero(e)) + continue; if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) continue; cnt++; @@ -276,6 +278,53 @@ int rdma_restrack_put(struct rdma_restrack_entry *res) } EXPORT_SYMBOL(rdma_restrack_put); +/** + * rdma_restrack_sync() - Fence concurrent netlink dumps on an entry + * @res: resource entry + * + * After this returns any concurrent netlink dump threads will see the current + * value of the object. This is useful if the object has to be changed and there + * is not locking to protect the nl side. Eg for mr->pd. This effectively + * destroys the object from a kref/xarray perspective and then immediately + * restores it. The kref is acting like a lock to barrier concurrent nl threads. + * Callers must ensure rdma_restrack_del() is not concurrently called. + */ +void rdma_restrack_sync(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_entry *old; + struct rdma_restrack_root *rt; + struct task_struct *task; + struct ib_device *dev; + + if (!res->valid || res->no_track) + return; + + dev = res_to_dev(res); + if (WARN_ON(!dev)) + return; + + rt = &dev->res[res->type]; + if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) + return; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); + if (WARN_ON(old != res)) + return; + + task = res->task; + if (task) + get_task_struct(task); + rdma_restrack_put(res); + wait_for_completion(&res->comp); + reinit_completion(&res->comp); + if (task) + res->task = task; + kref_init(&res->kref); + + xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); +} +EXPORT_SYMBOL(rdma_restrack_sync); + /** * rdma_restrack_del() - delete object from the resource tracking database * @res: resource entry diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 6a04fc41f738..75b8d1005a98 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -27,6 +27,7 @@ int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); +void rdma_restrack_sync(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/ucaps.c b/drivers/infiniband/core/ucaps.c index 5155ff0e538e..03c78ade0289 100644 --- a/drivers/infiniband/core/ucaps.c +++ b/drivers/infiniband/core/ucaps.c @@ -46,12 +46,7 @@ static const struct file_operations ucaps_cdev_fops = { .open = simple_open, }; -/** - * ib_cleanup_ucaps - cleanup all API resources and class. - * - * This is called once, when removing the ib_uverbs module. - */ -void ib_cleanup_ucaps(void) +static __exit void ib_cleanup_ucaps(void) { mutex_lock(&ucaps_mutex); if (!ucaps_class_is_registered) { @@ -263,3 +258,6 @@ int ib_get_ucaps(int *fds, int fd_count, uint64_t *idx_mask) mutex_unlock(&ucaps_mutex); return ret; } +EXPORT_SYMBOL_NS_GPL(ib_get_ucaps, "rdma_core"); + +module_exit(ib_cleanup_ucaps); diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 4b055712b0d0..73498723a5d5 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -84,14 +84,17 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d */ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt) + u64 virt) { unsigned long curr_len = 0; dma_addr_t curr_base = ~0; - unsigned long va, pgoff; + unsigned long pgoff; struct scatterlist *sg; - dma_addr_t mask; + unsigned long mask = 0; + unsigned int bits; dma_addr_t end; + u64 last_va; + u64 va; int i; umem->iova = va = virt; @@ -109,9 +112,12 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, * number of required pages. Compute the largest page size that could * work based on VA address bits that don't change. */ - mask = pgsz_bitmap & - GENMASK(BITS_PER_LONG - 1, - bits_per((umem->length - 1 + virt) ^ virt)); + if (check_add_overflow(umem->length - 1, virt, &last_va)) + return 0; + bits = bits_per(virt ^ last_va); + if (bits < BITS_PER_LONG) + mask = pgsz_bitmap & GENMASK(BITS_PER_LONG - 1, bits); + /* offset into first SGL */ pgoff = umem->address & ~PAGE_MASK; @@ -153,16 +159,9 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, } EXPORT_SYMBOL(ib_umem_find_best_pgsz); -/** - * ib_umem_get - Pin and DMA map userspace memory. - * - * @device: IB device to connect UMEM - * @addr: userspace virtual address to start at - * @size: length of region to pin - * @access: IB_ACCESS_xxx flags for memory being pinned - */ -struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, - size_t size, int access) +static struct ib_umem *__ib_umem_get_va(struct ib_device *device, + unsigned long addr, size_t size, + int access) { struct ib_umem *umem; struct page **page_list; @@ -174,6 +173,9 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, int pinned, ret; unsigned int gup_flags = FOLL_LONGTERM; + if (device->cc_dma_bounce) + return ERR_PTR(-EOPNOTSUPP); + /* * If the combination of the addr and size requested for this memory * region causes an integer overflow, return error. @@ -275,10 +277,342 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, } return ret ? ERR_PTR(ret) : umem; } -EXPORT_SYMBOL(ib_umem_get); /** - * ib_umem_release - release memory pinned with ib_umem_get + * ib_umem_get_desc - Pin a umem from a buffer descriptor. + * @device: IB device. + * @desc: buffer descriptor (VA or DMABUF). + * @access: IB access flags. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access) +{ + struct ib_umem_dmabuf *umem_dmabuf; + + if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK) + return ERR_PTR(-EINVAL); + + if (overflows_type(desc->addr, unsigned long) || + overflows_type(desc->length, size_t)) + return ERR_PTR(-EOVERFLOW); + + switch (desc->type) { + case IB_UVERBS_BUFFER_TYPE_DMABUF: + umem_dmabuf = ib_umem_dmabuf_get_pinned(device, desc->addr, + desc->length, desc->fd, + access); + if (IS_ERR(umem_dmabuf)) + return ERR_CAST(umem_dmabuf); + return &umem_dmabuf->umem; + case IB_UVERBS_BUFFER_TYPE_VA: + return __ib_umem_get_va(device, desc->addr, desc->length, + access); + default: + return ERR_PTR(-EINVAL); + } +} +EXPORT_SYMBOL(ib_umem_get_desc); + +/* + * Per-command legacy buffer-desc filler. + * Returns 0 on success (desc filled), -ENODATA if no legacy attrs apply, + * negative errno on validation failure. + */ +typedef int (*ib_umem_buf_desc_filler_t)(const struct uverbs_attr_bundle *attrs, + struct ib_uverbs_buffer_desc *desc); + +/* + * ib_umem_resolve_desc - Resolve a buffer descriptor from a per-command UMEM + * attribute and/or a legacy attr filler. + * + * Return: + * 0 @desc filled. + * -ENOENT no source produced a buffer. + * -EINVAL both the UMEM attribute and the legacy filler produced a buffer. + * -errno propagated from attr read / filler validation. + */ +static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs, + u16 attr_id, + ib_umem_buf_desc_filler_t legacy_filler, + struct ib_uverbs_buffer_desc *desc) +{ + bool have_desc = false; + int ret; + + if (!attrs) + return -ENOENT; + + ret = uverbs_get_buffer_desc(attrs, attr_id, desc); + if (!ret) + have_desc = true; + else if (ret != -ENOENT) + return ret; + + if (legacy_filler) { + struct ib_uverbs_buffer_desc legacy_desc = {}; + + ret = legacy_filler(attrs, &legacy_desc); + if (!ret) { + if (have_desc) + return -EINVAL; + *desc = legacy_desc; + have_desc = true; + } else if (ret != -ENODATA) { + return ret; + } + } + + return have_desc ? 0 : -ENOENT; +} + +/* + * ib_umem_get_desc_check - Pin a umem from @desc and verify it meets + * @min_size. + */ +static struct ib_umem * +ib_umem_get_desc_check(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + size_t min_size, int access) +{ + struct ib_umem *umem; + + umem = ib_umem_get_desc(device, desc, access); + if (IS_ERR(umem)) + return umem; + if (umem->length < min_size) { + ib_umem_release(umem); + return ERR_PTR(-EINVAL); + } + return umem; +} + +/* + * ib_umem_get_from_attrs - Pin a umem from a per-command UMEM attribute + * and/or a legacy attr filler. + * + * Return: caller-owned umem on success; NULL when no source supplied a + * buffer; ERR_PTR(...) on error. + */ +static struct ib_umem * +ib_umem_get_from_attrs(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler, + size_t size, int access) +{ + struct ib_uverbs_buffer_desc desc = {}; + int ret; + + ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc); + if (ret == -ENOENT) + return NULL; + if (ret) + return ERR_PTR(ret); + return ib_umem_get_desc_check(device, &desc, size, access); +} + +/* + * ib_umem_get_from_attrs_or_va - Pin a umem from a per-command UMEM + * attribute and/or a legacy attr filler, + * falling back to a UHW VA when no source + * matched. + * + * @size is always consumed: it is the length to pin on the VA fallback + * path AND the post-pin minimum-length check on the attr / legacy paths. + * Callers must always pass a meaningful, validated value. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +static struct ib_umem * +ib_umem_get_from_attrs_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, + ib_umem_buf_desc_filler_t legacy_filler, + u64 addr, size_t size, int access) +{ + struct ib_uverbs_buffer_desc desc = {}; + int ret; + + ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc); + if (ret == -ENOENT) + desc = (struct ib_uverbs_buffer_desc){ + .type = IB_UVERBS_BUFFER_TYPE_VA, + .addr = addr, + .length = size, + }; + else if (ret) + return ERR_PTR(ret); + return ib_umem_get_desc_check(device, &desc, size, access); +} + +/** + * ib_umem_get_attr - Pin a umem from a per-command UMEM attribute. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @attr_id: per-command UMEM attribute id. + * @size: minimum required umem length. + * @access: IB access flags. + * + * Return: caller-owned umem on success; NULL when no source supplied + * a buffer; ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_attr(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, size_t size, int access) +{ + return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size, + access); +} +EXPORT_SYMBOL(ib_umem_get_attr); + +/** + * ib_umem_get_attr_or_va - Pin a umem from a per-command UMEM attribute, + * falling back to a UHW VA. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @attr_id: per-command UMEM attribute id. + * @addr: UHW user VA used when no per-command attribute matched. + * @size: on the attr / legacy paths, the minimum required umem length + * validated post-pin; on the VA fallback path, the length to pin. + * @access: IB access flags. + * + * Like ib_umem_get_attr(), but pins @addr/@size when no per-command + * UMEM attribute is supplied. + * + * IMPORTANT: @size is always consumed. On the attr / legacy paths it is + * used as the post-pin minimum-length check; on the VA fallback path it + * is the length to pin. Callers MUST pass a meaningful, validated value + * even when they expect an attribute-supplied buffer to be used. + * + * Every in-tree caller passes the same value for the two roles of @size + * because no driver today distinguishes a user-passed buffer length from + * a driver-computed minimum. Drivers that currently accept a user-supplied + * length without cross-checking it against a driver minimum (vmw_pvrdma + * CQ/QP/SRQ, qedr CQ/QP/SRQ, mana WQ/QP, ionic CQ/QP), once tightened to + * compute and check a real minimum, will want to introduce a separate + * helper that passes these as distinct values. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, u64 addr, size_t size, + int access) +{ + return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr, + size, access); +} +EXPORT_SYMBOL(ib_umem_get_attr_or_va); + +static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *attrs, + struct ib_uverbs_buffer_desc *desc) +{ + struct ib_device *ib_dev = attrs->context->device; + int ret; + + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) { + ret = uverbs_copy_from(&desc->addr, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_VA); + if (ret) + return ret; + ret = uverbs_copy_from(&desc->length, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); + if (ret) + return ret; + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) || + uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || + !ib_dev->ops.create_user_cq) + return -EINVAL; + desc->type = IB_UVERBS_BUFFER_TYPE_VA; + return 0; + } + + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) { + ret = uverbs_get_raw_fd(&desc->fd, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_FD); + if (ret) + return ret; + + ret = uverbs_copy_from(&desc->addr, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET); + if (ret) + return ret; + ret = uverbs_copy_from(&desc->length, attrs, + UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); + if (ret) + return ret; + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) || + !ib_dev->ops.create_user_cq) + return -EINVAL; + desc->type = IB_UVERBS_BUFFER_TYPE_DMABUF; + return 0; + } + + if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || + uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH)) + return -EINVAL; + return -ENODATA; +} + +/** + * ib_umem_get_cq_buf - Pin a CQ buffer umem from per-command attributes. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @size: minimum required CQ buffer length. + * @access: IB access flags. + * + * Resolves the CQ buffer from the new UMEM attribute or the legacy + * CQ buffer attributes. There is no UHW VA fallback, so the caller + * must arrange its own backing (typically an in-kernel allocation) + * when no source is available. + * + * Return: caller-owned umem on success; NULL when no source supplied + * a buffer; ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + size_t size, int access) +{ + return ib_umem_get_from_attrs(device, attrs, + UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + uverbs_create_cq_get_buffer_desc, + size, access); +} +EXPORT_SYMBOL(ib_umem_get_cq_buf); + +/** + * ib_umem_get_cq_buf_or_va - Pin a CQ buffer umem with UHW VA fallback. + * @device: IB device. + * @attrs: uverbs attribute bundle (may be NULL). + * @addr: UHW user VA used when no per-command attribute matched. + * @size: on the attr / legacy paths, the minimum required umem length + * validated post-pin; on the VA fallback path, the length to pin. + * @access: IB access flags. + * + * Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the + * UMEM attribute nor the legacy CQ buffer attributes are supplied. + * + * See ib_umem_get_attr_or_va() for the note on @size's dual role and + * the migration path for drivers that would distinguish a user-supplied + * length from a driver-computed minimum. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u64 addr, size_t size, int access) +{ + return ib_umem_get_from_attrs_or_va(device, attrs, + UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + uverbs_create_cq_get_buffer_desc, + addr, size, access); +} +EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va); + +/** + * ib_umem_release - release pinned memory * @umem: umem struct to release */ void ib_umem_release(struct ib_umem *umem) diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index f2e192b51e60..c64dd6b94e10 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -203,7 +203,6 @@ void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue); void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file); void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue); void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res); -int uverbs_async_event_release(struct inode *inode, struct file *filp); int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs); int ib_init_ucontext(struct uverbs_attr_bundle *attrs); @@ -263,6 +262,21 @@ struct bundle_priv { u64 internal_buffer[32]; }; +static inline int uverbs_set_output(const struct uverbs_attr_bundle *bundle, + const struct uverbs_attr *attr) +{ + struct bundle_priv *pbundle = + container_of(&bundle->hdr, struct bundle_priv, bundle); + u16 flags; + + flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | + UVERBS_ATTR_F_VALID_OUTPUT; + if (put_user(flags, + &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) + return -EFAULT; + return 0; +} + long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); struct ib_uverbs_flow_spec { @@ -345,4 +359,13 @@ static inline void ib_uverbs_dmabuf_done(struct kref *kref) complete(&priv->comp); } +int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason); + +static inline void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) +{ + complete(&dev->comp); +} + + #endif /* UVERBS_H */ diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 91a62d2ade4d..aca7c6ab55cd 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -47,6 +47,7 @@ #include "uverbs.h" #include "core_priv.h" +#include "restrack.h" /* * Copy a response to userspace. If the provided 'resp' is larger than the @@ -163,17 +164,6 @@ static int uverbs_request_finish(struct uverbs_req_iter *iter) return 0; } -/* - * When calling a destroy function during an error unwind we need to pass in - * the udata that is sanitized of all user arguments. Ie from the driver - * perspective it looks like no udata was passed. - */ -struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) -{ - attrs->driver_udata = (struct ib_udata){}; - return &attrs->driver_udata; -} - static struct ib_uverbs_completion_event_file * _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs) { @@ -819,6 +809,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) ret = PTR_ERR(new_pd); goto put_uobjs; } + if (new_pd == orig_pd) { + uobj_put_obj_read(new_pd); + cmd.flags &= ~IB_MR_REREG_PD; + } } else { new_pd = mr->pd; } @@ -866,9 +860,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) mr = new_mr; } else { if (cmd.flags & IB_MR_REREG_PD) { - atomic_dec(&orig_pd->usecnt); - mr->pd = new_pd; atomic_inc(&new_pd->usecnt); + WRITE_ONCE(mr->pd, new_pd); + rdma_restrack_sync(&mr->res); + atomic_dec(&orig_pd->usecnt); } if (cmd.flags & IB_MR_REREG_TRANS) { mr->iova = cmd.hca_va; @@ -1084,7 +1079,6 @@ static int create_cq(struct uverbs_attr_bundle *attrs, return uverbs_response(attrs, &resp, sizeof(resp)); err_free: - ib_umem_release(cq->umem); rdma_restrack_put(&cq->res); kfree(cq); err_file: @@ -1462,8 +1456,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, attr.source_qpn = cmd->source_qpn; } - qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj, - KBUILD_MODNAME); + qp = ib_create_qp_user(device, pd, &attr, attrs, obj, KBUILD_MODNAME); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; @@ -2594,82 +2587,6 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) return ret; } -struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) -{ - struct ib_uflow_resources *resources; - - resources = kzalloc_obj(*resources); - - if (!resources) - return NULL; - - if (!num_specs) - goto out; - - resources->counters = - kzalloc_objs(*resources->counters, num_specs); - resources->collection = - kzalloc_objs(*resources->collection, num_specs); - - if (!resources->counters || !resources->collection) - goto err; - -out: - resources->max = num_specs; - return resources; - -err: - kfree(resources->counters); - kfree(resources); - - return NULL; -} -EXPORT_SYMBOL(flow_resources_alloc); - -void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) -{ - unsigned int i; - - if (!uflow_res) - return; - - for (i = 0; i < uflow_res->collection_num; i++) - atomic_dec(&uflow_res->collection[i]->usecnt); - - for (i = 0; i < uflow_res->counters_num; i++) - atomic_dec(&uflow_res->counters[i]->usecnt); - - kfree(uflow_res->collection); - kfree(uflow_res->counters); - kfree(uflow_res); -} -EXPORT_SYMBOL(ib_uverbs_flow_resources_free); - -void flow_resources_add(struct ib_uflow_resources *uflow_res, - enum ib_flow_spec_type type, - void *ibobj) -{ - WARN_ON(uflow_res->num >= uflow_res->max); - - switch (type) { - case IB_FLOW_SPEC_ACTION_HANDLE: - atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); - uflow_res->collection[uflow_res->collection_num++] = - (struct ib_flow_action *)ibobj; - break; - case IB_FLOW_SPEC_ACTION_COUNT: - atomic_inc(&((struct ib_counters *)ibobj)->usecnt); - uflow_res->counters[uflow_res->counters_num++] = - (struct ib_counters *)ibobj; - break; - default: - WARN_ON(1); - } - - uflow_res->num++; -} -EXPORT_SYMBOL(flow_resources_add); - static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs, struct ib_uverbs_flow_spec *kern_spec, union ib_flow_spec *ib_spec, @@ -3661,6 +3578,8 @@ static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs) resp.timestamp_mask = attr.timestamp_mask; resp.hca_core_clock = attr.hca_core_clock; resp.device_cap_flags_ex = attr.device_cap_flags; + if (ib_dev->cc_dma_bounce) + resp.device_cap_flags_ex |= IB_UVERBS_DEVICE_CC_DMA_BOUNCE; resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; resp.rss_caps.max_rwq_indirection_tables = attr.rss_caps.max_rwq_indirection_tables; diff --git a/drivers/infiniband/core/uverbs_flow.c b/drivers/infiniband/core/uverbs_flow.c new file mode 100644 index 000000000000..1528a294f7f8 --- /dev/null +++ b/drivers/infiniband/core/uverbs_flow.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +#include "uverbs.h" + +struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) +{ + struct ib_uflow_resources *resources; + + resources = kzalloc_obj(*resources); + + if (!resources) + return NULL; + + if (!num_specs) + goto out; + + resources->counters = + kzalloc_objs(*resources->counters, num_specs); + resources->collection = + kzalloc_objs(*resources->collection, num_specs); + + if (!resources->counters || !resources->collection) + goto err; + +out: + resources->max = num_specs; + return resources; + +err: + kfree(resources->counters); + kfree(resources); + + return NULL; +} +EXPORT_SYMBOL(flow_resources_alloc); + +void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) +{ + unsigned int i; + + if (!uflow_res) + return; + + for (i = 0; i < uflow_res->collection_num; i++) + atomic_dec(&uflow_res->collection[i]->usecnt); + + for (i = 0; i < uflow_res->counters_num; i++) + atomic_dec(&uflow_res->counters[i]->usecnt); + + kfree(uflow_res->collection); + kfree(uflow_res->counters); + kfree(uflow_res); +} +EXPORT_SYMBOL(ib_uverbs_flow_resources_free); + +void flow_resources_add(struct ib_uflow_resources *uflow_res, + enum ib_flow_spec_type type, + void *ibobj) +{ + WARN_ON(uflow_res->num >= uflow_res->max); + + switch (type) { + case IB_FLOW_SPEC_ACTION_HANDLE: + atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); + uflow_res->collection[uflow_res->collection_num++] = + (struct ib_flow_action *)ibobj; + break; + case IB_FLOW_SPEC_ACTION_COUNT: + atomic_inc(&((struct ib_counters *)ibobj)->usecnt); + uflow_res->counters[uflow_res->counters_num++] = + (struct ib_counters *)ibobj; + break; + default: + WARN_ON(1); + } + + uflow_res->num++; +} +EXPORT_SYMBOL(flow_resources_add); diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c index 2552a7efe2fb..6a78288e27a1 100644 --- a/drivers/infiniband/core/uverbs_ioctl.c +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -58,50 +58,6 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE); } -/** - * _uverbs_alloc() - Quickly allocate memory for use with a bundle - * @bundle: The bundle - * @size: Number of bytes to allocate - * @flags: Allocator flags - * - * The bundle allocator is intended for allocations that are connected with - * processing the system call related to the bundle. The allocated memory is - * always freed once the system call completes, and cannot be freed any other - * way. - * - * This tries to use a small pool of pre-allocated memory for performance. - */ -__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, - gfp_t flags) -{ - struct bundle_priv *pbundle = - container_of(&bundle->hdr, struct bundle_priv, bundle); - size_t new_used; - void *res; - - if (check_add_overflow(size, pbundle->internal_used, &new_used)) - return ERR_PTR(-EOVERFLOW); - - if (new_used > pbundle->internal_avail) { - struct bundle_alloc_head *buf; - - buf = kvmalloc_flex(*buf, data, size, flags); - if (!buf) - return ERR_PTR(-ENOMEM); - buf->next = pbundle->allocated_mem; - pbundle->allocated_mem = buf; - return buf->data; - } - - res = (void *)pbundle->internal_buffer + pbundle->internal_used; - pbundle->internal_used = - ALIGN(new_used, sizeof(*pbundle->internal_buffer)); - if (want_init_on_alloc(flags)) - memset(res, 0, size); - return res; -} -EXPORT_SYMBOL(_uverbs_alloc); - static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, u16 len) { @@ -113,21 +69,6 @@ static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, 0, uattr->len - len); } -static int uverbs_set_output(const struct uverbs_attr_bundle *bundle, - const struct uverbs_attr *attr) -{ - struct bundle_priv *pbundle = - container_of(&bundle->hdr, struct bundle_priv, bundle); - u16 flags; - - flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | - UVERBS_ATTR_F_VALID_OUTPUT; - if (put_user(flags, - &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) - return -EFAULT; - return 0; -} - static int uverbs_process_idrs_array(struct bundle_priv *pbundle, const struct uverbs_api_attr *attr_uapi, struct uverbs_objs_arr_attr *attr, @@ -616,57 +557,6 @@ long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return err; } -int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, u64 allowed_bits) -{ - const struct uverbs_attr *attr; - u64 flags; - - attr = uverbs_attr_get(attrs_bundle, idx); - /* Missing attribute means 0 flags */ - if (IS_ERR(attr)) { - *to = 0; - return 0; - } - - /* - * New userspace code should use 8 bytes to pass flags, but we - * transparently support old userspaces that were using 4 bytes as - * well. - */ - if (attr->ptr_attr.len == 8) - flags = attr->ptr_attr.data; - else if (attr->ptr_attr.len == 4) - flags = *(u32 *)&attr->ptr_attr.data; - else - return -EINVAL; - - if (flags & ~allowed_bits) - return -EINVAL; - - *to = flags; - return 0; -} -EXPORT_SYMBOL(uverbs_get_flags64); - -int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, u64 allowed_bits) -{ - u64 flags; - int ret; - - ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); - if (ret) - return ret; - - if (flags > U32_MAX) - return -EINVAL; - *to = flags; - - return 0; -} -EXPORT_SYMBOL(uverbs_get_flags32); - /* * Fill a ib_udata struct (core or uhw) using the given attribute IDs. * This is primarily used to convert the UVERBS_ATTR_UHW() into the @@ -707,24 +597,6 @@ void uverbs_fill_udata(struct uverbs_attr_bundle *bundle, } } -int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, - const void *from, size_t size) -{ - const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); - size_t min_size; - - if (IS_ERR(attr)) - return PTR_ERR(attr); - - min_size = min_t(size_t, attr->ptr_attr.len, size); - if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) - return -EFAULT; - - return uverbs_set_output(bundle, attr); -} -EXPORT_SYMBOL(uverbs_copy_to); - - /* * This is only used if the caller has directly used copy_to_use to write the * data. It signals to user space that the buffer is filled in. @@ -738,79 +610,3 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx) return uverbs_set_output(bundle, attr); } - -int _uverbs_get_const_signed(s64 *to, - const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, s64 lower_bound, u64 upper_bound, - s64 *def_val) -{ - const struct uverbs_attr *attr; - - attr = uverbs_attr_get(attrs_bundle, idx); - if (IS_ERR(attr)) { - if ((PTR_ERR(attr) != -ENOENT) || !def_val) - return PTR_ERR(attr); - - *to = *def_val; - } else { - *to = attr->ptr_attr.data; - } - - if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) - return -EINVAL; - - return 0; -} -EXPORT_SYMBOL(_uverbs_get_const_signed); - -int _uverbs_get_const_unsigned(u64 *to, - const struct uverbs_attr_bundle *attrs_bundle, - size_t idx, u64 upper_bound, u64 *def_val) -{ - const struct uverbs_attr *attr; - - attr = uverbs_attr_get(attrs_bundle, idx); - if (IS_ERR(attr)) { - if ((PTR_ERR(attr) != -ENOENT) || !def_val) - return PTR_ERR(attr); - - *to = *def_val; - } else { - *to = attr->ptr_attr.data; - } - - if (*to > upper_bound) - return -EINVAL; - - return 0; -} -EXPORT_SYMBOL(_uverbs_get_const_unsigned); - -int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, - size_t idx, const void *from, size_t size) -{ - const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); - - if (IS_ERR(attr)) - return PTR_ERR(attr); - - if (size < attr->ptr_attr.len) { - if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, - attr->ptr_attr.len - size)) - return -EFAULT; - } - return uverbs_copy_to(bundle, idx, from, size); -} -EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); - -/* Once called an abort will call through to the type's destroy_hw() */ -void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, - u16 idx) -{ - struct bundle_priv *pbundle = - container_of(&bundle->hdr, struct bundle_priv, bundle); - - __set_bit(uapi_bkey_attr(uapi_key_attr(idx)), - pbundle->uobj_hw_obj_valid); -} -EXPORT_SYMBOL(uverbs_finalize_uobj_create); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index f5837da47299..3ccf58e96aed 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -61,6 +61,7 @@ MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace verbs access"); MODULE_LICENSE("Dual BSD/GPL"); +MODULE_IMPORT_NS("rdma_core"); enum { IB_UVERBS_MAJOR = 231, @@ -91,30 +92,6 @@ static const struct class uverbs_class = { .devnode = uverbs_devnode, }; -/* - * Must be called with the ufile->device->disassociate_srcu held, and the lock - * must be held until use of the ucontext is finished. - */ -struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile) -{ - /* - * We do not hold the hw_destroy_rwsem lock for this flow, instead - * srcu is used. It does not matter if someone races this with - * get_context, we get NULL or valid ucontext. - */ - struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext); - - if (!srcu_dereference(ufile->device->ib_dev, - &ufile->device->disassociate_srcu)) - return ERR_PTR(-EIO); - - if (!ucontext) - return ERR_PTR(-EINVAL); - - return ucontext; -} -EXPORT_SYMBOL(ib_uverbs_get_ucontext_file); - int uverbs_dealloc_mw(struct ib_mw *mw) { struct ib_pd *pd = mw->pd; @@ -189,42 +166,6 @@ void ib_uverbs_detach_umcast(struct ib_qp *qp, } } -static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) -{ - complete(&dev->comp); -} - -void ib_uverbs_release_file(struct kref *ref) -{ - struct ib_uverbs_file *file = - container_of(ref, struct ib_uverbs_file, ref); - struct ib_device *ib_dev; - int srcu_key; - - release_ufile_idr_uobject(file); - - srcu_key = srcu_read_lock(&file->device->disassociate_srcu); - ib_dev = srcu_dereference(file->device->ib_dev, - &file->device->disassociate_srcu); - if (ib_dev && !ib_dev->ops.disassociate_ucontext) - module_put(ib_dev->ops.owner); - srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); - - if (refcount_dec_and_test(&file->device->refcount)) - ib_uverbs_comp_dev(file->device); - - if (file->default_async_file) - uverbs_uobject_put(&file->default_async_file->uobj); - put_device(&file->device->dev); - - if (file->disassociate_page) - __free_pages(file->disassociate_page, 0); - mutex_destroy(&file->disassociation_lock); - mutex_destroy(&file->umap_lock); - mutex_destroy(&file->ucontext_lock); - kfree(file); -} - static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, struct file *filp, char __user *buf, size_t count, loff_t *pos, @@ -362,7 +303,7 @@ const struct file_operations uverbs_async_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_async_event_read, .poll = ib_uverbs_async_event_poll, - .release = uverbs_async_event_release, + .release = uverbs_uobject_fd_release, .fasync = ib_uverbs_async_event_fasync, }; @@ -1009,6 +950,69 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp) return ret; } +/* + * Drop the ucontext off the ufile and completely disconnect it from the + * ib_device + */ +static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason) +{ + struct ib_ucontext *ucontext = ufile->ucontext; + struct ib_device *ib_dev = ucontext->device; + + /* + * If we are closing the FD then the user mmap VMAs must have + * already been destroyed as they hold on to the filep, otherwise + * they need to be zap'd. + */ + if (reason == RDMA_REMOVE_DRIVER_REMOVE) { + uverbs_user_mmap_disassociate(ufile); + if (ib_dev->ops.disassociate_ucontext) + ib_dev->ops.disassociate_ucontext(ucontext); + } + + ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev, + RDMACG_RESOURCE_HCA_HANDLE); + + rdma_restrack_del(&ucontext->res); + + ib_dev->ops.dealloc_ucontext(ucontext); + WARN_ON(!xa_empty(&ucontext->mmap_xa)); + kfree(ucontext); + + ufile->ucontext = NULL; +} + +/* + * Destroy the ucontext and every uobject associated with it. + * + * This is internally locked and can be called in parallel from multiple + * contexts. + */ +void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, + enum rdma_remove_reason reason) +{ + down_write(&ufile->hw_destroy_rwsem); + + /* + * If a ucontext was never created then we can't have any uobjects to + * cleanup, nothing to do. + */ + if (!ufile->ucontext) + goto done; + + while (!list_empty(&ufile->uobjects) && + !__uverbs_cleanup_ufile(ufile, reason)) { + } + + if (WARN_ON(!list_empty(&ufile->uobjects))) + __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE); + ufile_destroy_ucontext(ufile, reason); + +done: + up_write(&ufile->hw_destroy_rwsem); +} + static int ib_uverbs_close(struct inode *inode, struct file *filp) { struct ib_uverbs_file *file = filp->private_data; @@ -1346,7 +1350,6 @@ static void __exit ib_uverbs_cleanup(void) IB_UVERBS_NUM_FIXED_MINOR); unregister_chrdev_region(dynamic_uverbs_dev, IB_UVERBS_NUM_DYNAMIC_MINOR); - ib_cleanup_ucaps(); mmu_notifier_synchronize(); } diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index 13776a66e2e4..e160786e1df1 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -165,12 +165,6 @@ uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, ib_uverbs_free_event_queue(&file->ev_queue); } -int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) -{ - return 0; -} -EXPORT_SYMBOL(uverbs_destroy_def_handler); - DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_COMP_CHANNEL, UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file), diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c index cc24cfdf7aee..671f510bca49 100644 --- a/drivers/infiniband/core/uverbs_std_types_async_fd.c +++ b/drivers/infiniband/core/uverbs_std_types_async_fd.c @@ -32,14 +32,9 @@ static void uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, NULL, NULL); } -int uverbs_async_event_release(struct inode *inode, struct file *filp) +static void uverbs_async_event_free_event_queue(struct ib_uobject *uobj) { struct ib_uverbs_async_event_file *event_file; - struct ib_uobject *uobj = filp->private_data; - int ret; - - if (!uobj) - return uverbs_uobject_fd_release(inode, filp); event_file = container_of(uobj, struct ib_uverbs_async_event_file, uobj); @@ -50,11 +45,7 @@ int uverbs_async_event_release(struct inode *inode, struct file *filp) * release. The user knows it has reached the end of the event stream * when it sees IB_EVENT_DEVICE_FATAL. */ - uverbs_uobject_get(uobj); - ret = uverbs_uobject_fd_release(inode, filp); ib_uverbs_free_event_queue(&event_file->ev_queue); - uverbs_uobject_put(uobj); - return ret; } DECLARE_UVERBS_NAMED_METHOD( @@ -66,11 +57,12 @@ DECLARE_UVERBS_NAMED_METHOD( DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_ASYNC_EVENT, - UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file), - uverbs_async_event_destroy_uobj, - &uverbs_async_event_fops, - "[infinibandevent]", - O_RDONLY), + UVERBS_TYPE_ALLOC_FD_RELEASE(sizeof(struct ib_uverbs_async_event_file), + uverbs_async_event_destroy_uobj, + uverbs_async_event_free_event_queue, + &uverbs_async_event_fops, + "[infinibandevent]", + O_RDONLY), &UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC)); const struct uapi_definition uverbs_def_obj_async_fd[] = { diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index 79b51f60ce2a..148cdd180dab 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -66,16 +66,10 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( typeof(*obj), uevent.uobject); struct ib_uverbs_completion_event_file *ev_file = NULL; struct ib_device *ib_dev = attrs->context->device; - struct ib_umem_dmabuf *umem_dmabuf; struct ib_cq_init_attr attr = {}; struct ib_uobject *ev_file_uobj; - struct ib_umem *umem = NULL; - u64 buffer_length; - u64 buffer_offset; struct ib_cq *cq; u64 user_handle; - u64 buffer_va; - int buffer_fd; int ret; if ((!ib_dev->ops.create_cq && !ib_dev->ops.create_user_cq) || @@ -122,65 +116,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( INIT_LIST_HEAD(&obj->comp_list); INIT_LIST_HEAD(&obj->uevent.event_list); - if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) { - - ret = uverbs_copy_from(&buffer_va, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA); - if (ret) - goto err_event_file; - - ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); - if (ret) - goto err_event_file; - - if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) || - uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || - !ib_dev->ops.create_user_cq) { - ret = -EINVAL; - goto err_event_file; - } - - umem = ib_umem_get(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - ret = PTR_ERR(umem); - goto err_event_file; - } - } else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) { - - ret = uverbs_get_raw_fd(&buffer_fd, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD); - if (ret) - goto err_event_file; - - ret = uverbs_copy_from(&buffer_offset, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET); - if (ret) - goto err_event_file; - - ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH); - if (ret) - goto err_event_file; - - if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) || - !ib_dev->ops.create_user_cq) { - ret = -EINVAL; - goto err_event_file; - } - - umem_dmabuf = ib_umem_dmabuf_get_pinned(ib_dev, buffer_offset, buffer_length, - buffer_fd, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem_dmabuf)) { - ret = PTR_ERR(umem_dmabuf); - goto err_event_file; - } - umem = &umem_dmabuf->umem; - } else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) || - uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH)) { - ret = -EINVAL; - goto err_event_file; - } - cq = rdma_zalloc_drv_obj(ib_dev, ib_cq); if (!cq) { ret = -ENOMEM; - ib_umem_release(umem); goto err_event_file; } @@ -189,11 +127,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; - /* - * If UMEM is not provided here, legacy drivers will set it during - * CQ creation based on their internal udata. - */ - cq->umem = umem; atomic_set(&cq->usecnt, 0); rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ); @@ -206,9 +139,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( if (ret) goto err_free; - /* Check that driver didn't overrun existing umem */ - WARN_ON(umem && cq->umem != umem); - obj->uevent.uobject.object = cq; obj->uevent.uobject.user_handle = user_handle; rdma_restrack_add(&cq->res); @@ -219,7 +149,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( return ret; err_free: - ib_umem_release(cq->umem); rdma_restrack_put(&cq->res); kfree(cq); err_event_file: @@ -269,6 +198,8 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET, UVERBS_ATTR_TYPE(u64), UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + UA_OPTIONAL), UVERBS_ATTR_UHW()); static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)( diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index be0730e8509e..5767607dd420 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -248,8 +248,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( set_caps(&attr, &cap, true); mutex_init(&obj->mcast_lock); - qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj, - KBUILD_MODNAME); + qp = ib_create_qp_user(device, pd, &attr, attrs, obj, KBUILD_MODNAME); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; @@ -340,6 +339,12 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_QP_NUM, UVERBS_ATTR_TYPE(u32), UA_MANDATORY), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_BUF_UMEM, + UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + UA_OPTIONAL), UVERBS_ATTR_UHW()); static int UVERBS_HANDLER(UVERBS_METHOD_QP_DESTROY)( diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 31b248295854..4e2e556c8119 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -718,12 +718,25 @@ void uverbs_disassociate_api(struct uverbs_api *uapi) if (uapi_key_is_object(iter.index)) { struct uverbs_api_object *object_elm = rcu_dereference_protected(*slot, true); + const struct uverbs_obj_type *type_attrs = + object_elm->type_attrs; /* * Some type_attrs are in the driver module. We don't * bother to keep track of which since there should be * no use of this after disassociate. + * + * release_cleanup is the exception because + * uverbs_uobject_fd_release() needs it. In this case + * the module reference held by the fops will guarentee + * the type_class remains valid too. */ + if (type_attrs && + type_attrs->type_class == &uverbs_fd_class && + container_of(type_attrs, struct uverbs_obj_fd_type, + type)->release_cleanup) + continue; + object_elm->type_attrs = NULL; } else if (uapi_key_is_attr(iter.index)) { struct uverbs_api_attr *elm = diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index bac87de9cc67..3b613b57e269 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -53,6 +53,7 @@ #include #include +#include "rdma_core.h" #include "core_priv.h" #include @@ -1265,10 +1266,9 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata, + struct uverbs_attr_bundle *uattrs, struct ib_uqp_object *uobj, const char *caller) { - struct ib_udata dummy = {}; struct ib_qp *qp; int ret; @@ -1301,9 +1301,10 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, qp->recv_cq = attr->recv_cq; rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP); - WARN_ONCE(!udata && !caller, "Missing kernel QP owner"); - rdma_restrack_set_name(&qp->res, udata ? NULL : caller); - ret = dev->ops.create_qp(qp, attr, udata); + WARN_ONCE(!uattrs && !caller, "Missing kernel QP owner"); + rdma_restrack_set_name(&qp->res, uattrs ? NULL : caller); + ret = dev->ops.create_qp(qp, attr, + uattrs ? &uattrs->driver_udata : NULL); if (ret) goto err_create; @@ -1322,7 +1323,8 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, return qp; err_security: - qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL); + qp->device->ops.destroy_qp( + qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL); err_create: rdma_restrack_put(&qp->res); kfree(qp); @@ -1338,13 +1340,13 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, * @attr: A list of initial attributes required to create the * QP. If QP creation succeeds, then the attributes are updated to * the actual capabilities of the created QP. - * @udata: User data + * @uattrs: User ioctl attributes and udata * @uobj: uverbs obect * @caller: caller's build-time module name */ struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata, + struct uverbs_attr_bundle *uattrs, struct ib_uqp_object *uobj, const char *caller) { struct ib_qp *qp, *xrc_qp; @@ -1352,7 +1354,7 @@ struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, if (attr->qp_type == IB_QPT_XRC_TGT) qp = create_qp(dev, pd, attr, NULL, NULL, caller); else - qp = create_qp(dev, pd, attr, udata, uobj, NULL); + qp = create_qp(dev, pd, attr, uattrs, uobj, NULL); if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(qp)) return qp; @@ -1538,8 +1540,7 @@ static const struct { IB_QP_PKEY_INDEX), [IB_QPT_RC] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | - IB_QP_PKEY_INDEX | - IB_QP_RATE_LIMIT), + IB_QP_PKEY_INDEX), [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX), @@ -1587,8 +1588,7 @@ static const struct { IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | IB_QP_MIN_RNR_TIMER | - IB_QP_PATH_MIG_STATE | - IB_QP_RATE_LIMIT), + IB_QP_PATH_MIG_STATE), [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | IB_QP_ALT_PATH | IB_QP_ACCESS_FLAGS | @@ -1602,7 +1602,6 @@ static const struct { IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), - [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT, } } }, @@ -1622,8 +1621,7 @@ static const struct { IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE | - IB_QP_MIN_RNR_TIMER | - IB_QP_RATE_LIMIT), + IB_QP_MIN_RNR_TIMER), [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS | IB_QP_ALT_PATH | @@ -1637,7 +1635,6 @@ static const struct { IB_QP_QKEY), [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY), - [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT, } }, [IB_QPS_SQD] = { @@ -1775,7 +1772,7 @@ bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, if ((mask & req_param) != req_param) return false; - if (mask & ~(req_param | opt_param | IB_QP_STATE)) + if (mask & ~(req_param | opt_param | IB_QP_STATE | IB_QP_RATE_LIMIT)) return false; return true; @@ -2221,12 +2218,6 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, kfree(cq); return ERR_PTR(ret); } - /* - * We are in kernel verbs flow and drivers are not allowed - * to set umem pointer, it needs to stay NULL. - */ - WARN_ON_ONCE(cq->umem); - rdma_restrack_add(&cq->res); return cq; } @@ -2257,7 +2248,6 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (ret) return ret; - ib_umem_release(cq->umem); rdma_restrack_del(&cq->res); kfree(cq); return ret; diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c index 17d7cc3aa11d..50156c300b33 100644 --- a/drivers/infiniband/hw/bng_re/bng_fw.c +++ b/drivers/infiniband/hw/bng_re/bng_fw.c @@ -123,7 +123,6 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw, bool is_waiter_alive; struct pci_dev *pdev; u32 wait_cmds = 0; - int rc = 0; pdev = rcfw->pdev; switch (qp_event->event) { @@ -152,7 +151,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw, "rcfw timedout: cookie = %#x, free_slots = %d", cookie, crsqe->free_slots); spin_unlock(&hwq->lock); - return rc; + return 0; } if (crsqe->is_waiter_alive) { @@ -182,7 +181,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw, spin_unlock(&hwq->lock); } *num_wait += wait_cmds; - return rc; + return 0; } /* function events */ diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.c b/drivers/infiniband/hw/bnxt_re/debugfs.c index 5fed2cf66be3..143e9bfc6b79 100644 --- a/drivers/infiniband/hw/bnxt_re/debugfs.c +++ b/drivers/infiniband/hw/bnxt_re/debugfs.c @@ -498,6 +498,8 @@ void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev) bnxt_re_debugfs_add_info(rdev); rdev->cc_config_params = kzalloc_obj(*cc_params); + if (!rdev->cc_config_params) + return; for (i = 0; i < BNXT_RE_CC_PARAM_GEN0; i++) { struct bnxt_re_cc_param *tmp_params = &rdev->cc_config_params->gen0_parms[i]; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 365ec2767d25..565762529007 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -265,7 +265,10 @@ int bnxt_re_query_device(struct ib_device *ibdev, resp.packet_pacing_caps.supported_qpts = 1 << IB_QPT_RC; } - return ib_respond_udata(udata, resp); + + if (udata) + return ib_respond_udata(udata, resp); + return 0; } int bnxt_re_modify_device(struct ib_device *ibdev, @@ -1024,6 +1027,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (rc) ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); + if (qp->dbr_obj) + kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release); + if (rdma_is_kernel_res(&qp->ib_qp.res)) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); @@ -1136,23 +1142,42 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, return 0; } -static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, - struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, - struct bnxt_re_qp_req *ureq) +static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, + struct ib_umem *umem, + struct bnxt_qplib_sg_info *sginfo) { - struct bnxt_qplib_qp *qplib_qp; - int bytes = 0, psn_sz; - struct ib_umem *umem; - int psn_nume; + unsigned long page_size; - qplib_qp = &qp->qplib_qp; + if (!umem) + return -EINVAL; - bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); - /* Consider mapping PSN search memory only for RC QPs. */ - if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { + page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); + if (!page_size || page_size != SZ_4K) + return -EINVAL; + + sginfo->umem = umem; + sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); + sginfo->pgsize = page_size; + sginfo->pgshft = __builtin_ctz(page_size); + return 0; +} + +static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *cntx, + struct bnxt_qplib_qp *qplib_qp, + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) +{ + int psn_sz, psn_nume; + + if (rdev->dev_attr && + _is_host_msn_table(rdev->dev_attr->dev_cap_flags2)) + psn_sz = sizeof(struct sq_msn_search); + else psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? - sizeof(struct sq_psn_search_ext) : - sizeof(struct sq_psn_search); + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (!fixed_que_attr) { if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { psn_nume = ureq->sq_slots; } else { @@ -1162,43 +1187,80 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, } if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) psn_nume = roundup_pow_of_two(psn_nume); - bytes += (psn_nume * psn_sz); + } else { + psn_nume = ureq->sq_npsn; } + return psn_nume * psn_sz; +} + +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, + struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr, + struct bnxt_re_dbr_obj *dbr_obj, + struct uverbs_attr_bundle *attrs) +{ + struct bnxt_qplib_qp *qplib_qp; + struct ib_umem *umem; + int bytes; + int rc; + + qplib_qp = &qp->qplib_qp; + + bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + /* Consider mapping PSN search memory only for RC QPs. */ + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) + bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr); + bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_attr_or_va(&rdev->ibdev, attrs, + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + ureq->qpsva, bytes, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) return PTR_ERR(umem); qp->sumem = umem; - qplib_qp->sq.sg_info.umem = umem; - qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; - qplib_qp->qp_handle = ureq->qp_handle; + rc = bnxt_re_setup_sginfo(rdev, qp->sumem, &qplib_qp->sq.sg_info); + if (rc) + goto fail; - if (!qp->qplib_qp.srq) { - bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); - bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) - goto rqfail; - qp->rumem = umem; - qplib_qp->rq.sg_info.umem = umem; - qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; + if (qp->qplib_qp.srq) + goto done; + + bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_attr_or_va(&rdev->ibdev, attrs, + UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + ureq->qprva, bytes, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + goto fail; } - qplib_qp->dpi = &cntx->dpi; + qp->rumem = umem; + rc = bnxt_re_setup_sginfo(rdev, qp->rumem, &qplib_qp->rq.sg_info); + if (rc) + goto rqfail; + +done: + if (dbr_obj) + qplib_qp->dpi = &dbr_obj->dpi; + else + qplib_qp->dpi = &cntx->dpi; + qplib_qp->qp_handle = ureq->qp_handle; qplib_qp->is_user = true; return 0; + rqfail: + ib_umem_release(qp->rumem); + qp->rumem = NULL; + memset(&qplib_qp->rq.sg_info, 0, sizeof(qplib_qp->rq.sg_info)); +fail: ib_umem_release(qp->sumem); qp->sumem = NULL; memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); - - return PTR_ERR(umem); + return rc; } static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah @@ -1297,7 +1359,7 @@ static int bnxt_re_qp_alloc_init_xrrq(struct bnxt_re_qp *qp) return rc; } -static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) +static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp, bool fixed_que_attr) { struct bnxt_qplib_res *res = &qp->rdev->qplib_res; struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; @@ -1311,12 +1373,17 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) hwq_attr.res = res; hwq_attr.sginfo = &sq->sg_info; hwq_attr.stride = bnxt_qplib_get_stride(); - hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true); hwq_attr.aux_stride = qplib_qp->psn_sz; - hwq_attr.aux_depth = (qplib_qp->psn_sz) ? - bnxt_qplib_set_sq_size(sq, wqe_mode) : 0; - if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) + if (!fixed_que_attr) { + hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true); + hwq_attr.aux_depth = (qplib_qp->psn_sz) ? + bnxt_qplib_set_sq_size(sq, wqe_mode) : 0; + if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) + hwq_attr.aux_depth = qplib_qp->msn_tbl_sz; + } else { + hwq_attr.depth = sq->max_wqe; hwq_attr.aux_depth = qplib_qp->msn_tbl_sz; + } hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); if (rc) @@ -1327,6 +1394,9 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) CMDQ_CREATE_QP_SQ_LVL_SFT); sq->hwq.pg_sz_lvl = pg_sz_lvl; + if (qplib_qp->srq) + goto done; + hwq_attr.res = res; hwq_attr.sginfo = &rq->sg_info; hwq_attr.stride = bnxt_qplib_get_stride(); @@ -1343,6 +1413,7 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp) CMDQ_CREATE_QP_RQ_LVL_SFT); rq->hwq.pg_sz_lvl = pg_sz_lvl; +done: if (qplib_qp->psn_sz) { rc = bnxt_re_qp_alloc_init_xrrq(qp); if (rc) @@ -1411,7 +1482,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; qp->qplib_qp.dpi = &rdev->dpi_privileged; - rc = bnxt_re_setup_qp_hwqs(qp); + rc = bnxt_re_setup_qp_hwqs(qp, false); if (rc) goto fail; @@ -1436,7 +1507,8 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, - struct bnxt_re_ucontext *uctx) + struct bnxt_re_ucontext *uctx, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1461,12 +1533,16 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, init_attr->cap.max_recv_sge = rq->max_sge; rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge, dev_attr->max_qp_sges); - /* Allocate 1 more than what's provided so posting max doesn't - * mean empty. - */ - rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, - dev_attr->max_qp_wqes + 1, - uctx); + if (!fixed_que_attr) { + /* Allocate 1 more than what's provided so posting max doesn't + * mean empty. + */ + rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, + dev_attr->max_qp_wqes + 1, + uctx); + } else { + rq->max_wqe = init_attr->cap.max_recv_wr; + } rq->max_sw_wqe = rq->max_wqe; rq->q_full_delta = 0; rq->sg_info.pgsize = PAGE_SIZE; @@ -1497,7 +1573,8 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1538,13 +1615,18 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, sq->max_sw_wqe = sq->max_wqe; } - sq->q_full_delta = diff + 1; - /* - * Reserving one slot for Phantom WQE. Application can - * post one extra entry in this case. But allowing this to avoid - * unexpected Queue full condition - */ - qplqp->sq.q_full_delta -= 1; + if (!fixed_que_attr) { + sq->q_full_delta = diff + 1; + /* + * Reserving one slot for Phantom WQE. Application can + * post one extra entry in this case. But allowing this to avoid + * unexpected Queue full condition + */ + qplqp->sq.q_full_delta -= 1; + } else { + sq->q_full_delta = 0; + } + qplqp->sq.sg_info.pgsize = PAGE_SIZE; qplqp->sq.sg_info.pgshft = PAGE_SHIFT; @@ -1597,7 +1679,9 @@ static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, return qptype; } -static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) +static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp, + bool fixed_que_attr, + struct bnxt_re_qp_req *req) { struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; struct bnxt_qplib_q *sq = &qplib_qp->sq; @@ -1620,12 +1704,17 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) /* Update msn tbl size */ if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) { - if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); - else - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2; + if (!fixed_que_attr) { + if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); + else + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) + / 2; + } else { + qplib_qp->msn_tbl_sz = req->sq_npsn; + } qplib_qp->msn = 0; } } @@ -1633,7 +1722,10 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + struct bnxt_re_dbr_obj *dbr_obj, + bool fixed_que_attr, + struct uverbs_attr_bundle *attrs) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1656,6 +1748,16 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, return qptype; qplqp->type = (u8)qptype; qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx); + if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE && + (!ureq->sq_slots || ureq->sq_slots > BNXT_RE_MAX_SQ_SLOTS)) + return -EINVAL; + if (fixed_que_attr) { + if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE) + return -EOPNOTSUPP; + if (!ureq->sq_npsn || + ureq->sq_npsn > roundup_pow_of_two(ureq->sq_slots / 2)) + return -EINVAL; + } qplqp->dev_cap_flags = dev_attr->dev_cap_flags; qplqp->cctx = rdev->chip_ctx; if (init_attr->qp_type == IB_QPT_RC) { @@ -1685,28 +1787,29 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, } /* Setup RQ/SRQ */ - rc = bnxt_re_init_rq_attr(qp, init_attr, uctx); + rc = bnxt_re_init_rq_attr(qp, init_attr, uctx, fixed_que_attr); if (rc) return rc; if (init_attr->qp_type == IB_QPT_GSI) bnxt_re_adjust_gsi_rq_attr(qp); /* Setup SQ */ - rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq); + rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq, fixed_que_attr); if (rc) return rc; if (init_attr->qp_type == IB_QPT_GSI) bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); if (uctx) { /* This will update DPI and qp_handle */ - rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq); + rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr, + dbr_obj, attrs); if (rc) return rc; } - bnxt_re_qp_calculate_msn_psn_size(qp); + bnxt_re_qp_calculate_msn_psn_size(qp, fixed_que_attr, ureq); - rc = bnxt_re_setup_qp_hwqs(qp); + rc = bnxt_re_setup_qp_hwqs(qp, fixed_que_attr); if (rc) goto free_umem; @@ -1835,8 +1938,11 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev) int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { + struct uverbs_attr_bundle *attrs = NULL; + struct bnxt_re_dbr_obj *dbr_obj = NULL; struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_re_ucontext *uctx; + bool fixed_que_attr = false; struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; struct bnxt_re_pd *pd; @@ -1853,9 +1959,23 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); if (udata) { - rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0); + rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR); if (rc) return rc; + + attrs = rdma_udata_to_uverbs_attr_bundle(udata); + if (uverbs_attr_is_valid(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) { + dbr_obj = uverbs_attr_get_obj(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE); + if (IS_ERR(dbr_obj)) + return PTR_ERR(dbr_obj); + kref_get(&dbr_obj->usecnt); + qp->dbr_obj = dbr_obj; + } + if (ureq.comp_mask & BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR) + fixed_que_attr = true; } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1865,7 +1985,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } qp->rdev = rdev; - rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq); + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, + dbr_obj, fixed_que_attr, attrs); if (rc) goto fail; @@ -1884,7 +2005,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } if (udata) { - struct bnxt_re_qp_resp resp; + struct bnxt_re_qp_resp resp = {}; resp.qpid = qp->qplib_qp.id; resp.rsvd = 0; @@ -1935,6 +2056,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); bnxt_re_qp_free_umem(qp); fail: + if (dbr_obj) + kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release); return rc; } @@ -2029,11 +2152,11 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) if (ret) return ret; - if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) { - free_page((unsigned long)srq->uctx_srq_page); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) hash_del(&srq->hash_entry); - } bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) + free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); return ib_respond_empty_udata(udata); @@ -2058,8 +2181,8 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&rdev->ibdev, ureq.srqva, bytes, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) return PTR_ERR(umem); @@ -2286,6 +2409,23 @@ static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, return rc; } +static bool bnxt_re_is_modify_ok(enum ib_qp_attr_mask ext_mask, + enum ib_qp_type type, enum ib_qp_state cur, + enum ib_qp_state next) +{ + if (!ext_mask) + return true; + + if (ext_mask & ~IB_QP_RATE_LIMIT) + return false; + + /* Rate limit is only supported for RC QPs during specific transitions */ + return type == IB_QPT_RC && + ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || + (cur == IB_QPS_RTR && next == IB_QPS_RTS) || + (cur == IB_QPS_RTS && next == IB_QPS_RTS)); +} + int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata) { @@ -2310,7 +2450,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); new_qp_state = qp_attr->qp_state; if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, - ib_qp->qp_type, qp_attr_mask)) { + ib_qp->qp_type, qp_attr_mask) || + !bnxt_re_is_modify_ok(qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS, + ib_qp->qp_type, curr_qp_state, + new_qp_state)) { ibdev_err(&rdev->ibdev, "Invalid attribute mask: %#x specified ", qp_attr_mask); @@ -3332,39 +3475,20 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (ret) return ret; - if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) { - free_page((unsigned long)cq->uctx_cq_page); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) hash_del(&cq->hash_entry); - } bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) + free_page((unsigned long)cq->uctx_cq_page); bnxt_re_put_nq(rdev, nq); atomic_dec(&rdev->stats.res.cq_count); kfree(cq->cql); + ib_umem_release(cq->umem); return ib_respond_empty_udata(udata); } -static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, - struct ib_umem *umem, - struct bnxt_qplib_sg_info *sginfo) -{ - unsigned long page_size; - - if (!umem) - return -EINVAL; - - page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); - if (!page_size || page_size != SZ_4K) - return -EINVAL; - - sginfo->umem = umem; - sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); - sginfo->pgsize = page_size; - sginfo->pgshft = __builtin_ctz(page_size); - return 0; -} - int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { @@ -3402,17 +3526,15 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att entries = bnxt_re_init_depth(attr->cqe + 1, dev_attr->max_cq_wqes + 1, uctx); - if (!ibcq->umem) { - ibcq->umem = ib_umem_get(&rdev->ibdev, req.cq_va, - entries * sizeof(struct cq_base), - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(ibcq->umem)) - return PTR_ERR(ibcq->umem); - } + cq->umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, attrs, req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->umem)) + return PTR_ERR(cq->umem); - rc = bnxt_re_setup_sginfo(rdev, ibcq->umem, &cq->qplib_cq.sg_info); + rc = bnxt_re_setup_sginfo(rdev, cq->umem, &cq->qplib_cq.sg_info); if (rc) - return rc; + goto free_umem; cq->qplib_cq.dpi = &uctx->dpi; cq->qplib_cq.max_wqe = entries; @@ -3422,7 +3544,7 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); if (rc) - return rc; + goto free_umem; cq->ib_cq.cqe = entries; cq->cq_period = cq->qplib_cq.period; @@ -3435,8 +3557,10 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id); /* Allocate a page */ cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL); - if (!cq->uctx_cq_page) - return -ENOMEM; + if (!cq->uctx_cq_page) { + rc = -ENOMEM; + goto destroy_cq; + } resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT; } @@ -3444,15 +3568,17 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att resp.tail = cq->qplib_cq.hwq.cons; resp.phase = cq->qplib_cq.period; rc = ib_respond_udata(udata, resp); - if (rc) { - bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) goto free_mem; - } return 0; free_mem: free_page((unsigned long)cq->uctx_cq_page); +destroy_cq: + bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); +free_umem: + ib_umem_release(cq->umem); return rc; } @@ -3516,8 +3642,8 @@ static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) cq->qplib_cq.max_wqe = cq->resize_cqe; if (cq->resize_umem) { - ib_umem_release(cq->ib_cq.umem); - cq->ib_cq.umem = cq->resize_umem; + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; cq->resize_umem = NULL; cq->resize_cqe = 0; } @@ -3562,12 +3688,12 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; - cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va, - entries * sizeof(struct cq_base), - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { rc = PTR_ERR(cq->resize_umem); - ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %pe\n", + ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n", __func__, cq->resize_umem); cq->resize_umem = NULL; goto fail; @@ -4113,7 +4239,7 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) /* User CQ; the only processing we do is to * complete any pending CQ resize operation. */ - if (cq->ib_cq.umem) { + if (cq->umem) { if (cq->resize_umem) bnxt_re_resize_cq_complete(cq); return 0; @@ -4577,7 +4703,7 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, if (dmah) return ERR_PTR(-EOPNOTSUPP); - umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); + umem = ib_umem_get_va(&rdev->ibdev, start, length, mr_access_flags); if (IS_ERR(umem)) return ERR_CAST(umem); @@ -4644,6 +4770,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) goto fail; } spin_lock_init(&uctx->sh_lock); + mutex_init(&uctx->wcdpi_lock); resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; chip_met_rev_num = rdev->chip_ctx->chip_num; @@ -4886,6 +5013,10 @@ void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry) bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry, rdma_entry); + if (bnxt_entry->dpi_valid) + bnxt_qplib_free_uc_dpi(&bnxt_entry->uctx->rdev->qplib_res, + &bnxt_entry->dpi); + kfree(bnxt_entry); } diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 08f71a94d55d..22bf81668cfb 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -96,6 +96,7 @@ struct bnxt_re_qp { struct bnxt_re_cq *scq; struct bnxt_re_cq *rcq; struct dentry *dentry; + struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */ }; struct bnxt_re_cq { @@ -108,6 +109,7 @@ struct bnxt_re_cq { struct bnxt_qplib_cqe *cql; #define MAX_CQL_PER_POLL 1024 u32 max_cql; + struct ib_umem *umem; struct ib_umem *resize_umem; int resize_cqe; void *uctx_cq_page; @@ -141,6 +143,7 @@ struct bnxt_re_ucontext { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_qplib_dpi wcdpi; + struct mutex wcdpi_lock; /* serialises WC DPI alloc/free */ void *shpg; spinlock_t sh_lock; /* protect shpg */ struct rdma_user_mmap_entry *shpage_mmap; @@ -161,13 +164,15 @@ struct bnxt_re_user_mmap_entry { struct bnxt_re_ucontext *uctx; u64 mem_offset; u8 mmap_flag; + bool dpi_valid; + struct bnxt_qplib_dpi dpi; }; struct bnxt_re_dbr_obj { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_re_user_mmap_entry *entry; - atomic_t usecnt; /* QPs using this dbr */ + struct kref usecnt; /* 1 (uobject) + n (QPs using this dbr) */ }; struct bnxt_re_flow { @@ -308,4 +313,5 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); struct bnxt_re_user_mmap_entry* bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset, enum bnxt_re_mmap_flag mmap_flag, u64 *offset); +void bnxt_re_dbr_kref_release(struct kref *ref); #endif /* __BNXT_RE_IB_VERBS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index a892f1172917..d25fdc458120 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -1093,8 +1093,6 @@ static int bnxt_re_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr) goto err; if (rdma_nl_put_driver_u32(msg, "element_size", mr_hwq->element_size)) goto err; - if (rdma_nl_put_driver_u64_hex(msg, "hwq", (unsigned long)mr_hwq)) - goto err; if (rdma_nl_put_driver_u64_hex(msg, "va", mr->qplib_mr.va)) goto err; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 95e0489c53c3..756f8b5f042a 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -764,9 +764,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, break; case BNXT_QPLIB_DPI_TYPE_WC: dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; default: dpi->dbr = ioremap(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; } @@ -774,6 +778,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, mutex_unlock(&res->dpi_tbl_lock); return 0; +fail_ioremap: + /* Roll back the bit we just claimed. */ + set_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = NULL; + mutex_unlock(&res->dpi_tbl_lock); + return -ENOMEM; + } int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 9fadd637cb5b..c4193ae75b54 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -369,6 +369,7 @@ int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res); #define BNXT_VAR_MAX_SLOT_ALIGN 256 #define BNXT_VAR_MAX_SGE 13 #define BNXT_RE_MAX_RQ_WQES 65536 +#define BNXT_RE_MAX_SQ_SLOTS 65536 #define BNXT_STATIC_MAX_SGE 6 diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 3eaee7101615..263238a6e4cd 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -76,8 +76,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * struct ib_ucontext *ib_uctx; struct bnxt_re_dev *rdev; u64 mmap_offset; + u32 dpi = 0; u32 length; - u32 dpi; u64 addr; int err; @@ -98,26 +98,39 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * switch (alloc_type) { case BNXT_RE_ALLOC_WC_PAGE: - if (cctx->modes.db_push) { + if (cctx->modes.db_push) { + mutex_lock(&uctx->wcdpi_lock); + /* already allocated — one WC page per context */ + if (uctx->wcdpi.dbr) { + mutex_unlock(&uctx->wcdpi_lock); + return -EEXIST; + } if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi, - uctx, BNXT_QPLIB_DPI_TYPE_WC)) + uctx, BNXT_QPLIB_DPI_TYPE_WC)) { + mutex_unlock(&uctx->wcdpi_lock); return -ENOMEM; + } length = PAGE_SIZE; dpi = uctx->wcdpi.dpi; addr = (u64)uctx->wcdpi.umdbr; mmap_flag = BNXT_RE_MMAP_WC_DB; + mutex_unlock(&uctx->wcdpi_lock); } else { return -EINVAL; } break; case BNXT_RE_ALLOC_DBR_BAR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_bar_addr; mmap_flag = BNXT_RE_MMAP_DBR_BAR; break; case BNXT_RE_ALLOC_DBR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_page; mmap_flag = BNXT_RE_MMAP_DBR_PAGE; @@ -128,8 +141,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * } entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); - if (!entry) + if (!entry) { + if (mmap_flag == BNXT_RE_MMAP_WC_DB) { + mutex_lock(&uctx->wcdpi_lock); + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + mutex_unlock(&uctx->wcdpi_lock); + } return -ENOMEM; + } uobj->object = entry; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); @@ -160,11 +180,16 @@ static int alloc_page_obj_cleanup(struct ib_uobject *uobject, switch (entry->mmap_flag) { case BNXT_RE_MMAP_WC_DB: - if (uctx && uctx->wcdpi.dbr) { + if (uctx) { struct bnxt_re_dev *rdev = uctx->rdev; - bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); - uctx->wcdpi.dbr = NULL; + mutex_lock(&uctx->wcdpi_lock); + if (uctx->wcdpi.dbr) { + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + } + mutex_unlock(&uctx->wcdpi_lock); } break; case BNXT_RE_MMAP_DBR_BAR: @@ -252,6 +277,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)cq->uctx_cq_page; + if (!addr) + return -EOPNOTSUPP; break; case BNXT_RE_SRQ_TOGGLE_MEM: srq = bnxt_re_search_for_srq(rdev, res_id); @@ -259,6 +286,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)srq->uctx_srq_page; + if (!addr) + return -EOPNOTSUPP; break; default: @@ -368,7 +397,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a goto free_dpi; } + /* Save DPI info to the mmap entry so that bnxt_re_mmap_free() + * can free the DPI slot only after the last reference to the + * mmap entry is released. + */ + obj->entry->dpi = *dpi; + obj->entry->dpi_valid = true; + obj->rdev = rdev; + kref_init(&obj->usecnt); uobj->object = obj; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE); @@ -391,15 +428,35 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a return ret; } +void bnxt_re_dbr_kref_release(struct kref *ref) +{ + struct bnxt_re_dbr_obj *obj = + container_of(ref, struct bnxt_re_dbr_obj, usecnt); + + /* Drop the driver's reference to the mmap entry (_remove()). + * The DPI slot gets freed from bnxt_re_mmap_free() only + * when there's no VMA mapping reference to it. + */ + rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); + kfree(obj); +} + static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { struct bnxt_re_dbr_obj *obj = uobject->object; - struct bnxt_re_dev *rdev = obj->rdev; - rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); - bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + /* If it is being destroyed explicitly while QPs still hold a + * reference (> 1), reject it with EBUSY. If no QP references + * or implicit teardown (process exit, driver removal), drop + * the uobject reference unconditionally. The object gets freed + * (bnxt_re_dbr_kref_release) when the usecnt goes to zero. + */ + if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1) + return -EBUSY; + + kref_put(&obj->usecnt, bnxt_re_dbr_kref_release); return 0; } @@ -459,11 +516,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR)); +ADD_UVERBS_ATTRIBUTES_SIMPLE( + bnxt_re_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE, + BNXT_RE_OBJECT_DBR, + UVERBS_ACCESS_READ, + UA_OPTIONAL)); + +const struct uapi_definition bnxt_re_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create), + {}, +}; + const struct uapi_definition bnxt_re_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR), + UAPI_DEF_CHAIN(bnxt_re_create_qp_defs), {} }; diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index e31fb9134aa8..d1517f2560b9 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -1004,7 +1004,7 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device); struct c4iw_cq *chp = to_c4iw_cq(ibcq); struct c4iw_create_cq ucmd; - struct c4iw_create_cq_resp uresp; + struct c4iw_create_cq_resp uresp = {}; int ret, wr_len; size_t memsize, hwentries; struct c4iw_mm_entry *mm, *mm2; @@ -1102,7 +1102,6 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (!mm2) goto err_free_mm; - memset(&uresp, 0, sizeof(uresp)); uresp.qid_mask = rhp->rdev.cqmask; uresp.cqid = chp->cq.cqid; uresp.size = chp->cq.size; @@ -1115,13 +1114,11 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, /* communicate to the userspace that * kernel driver supports 64B CQE */ - uresp.flags |= C4IW_64B_CQE; + if (!ucontext->is_32b_cqe) + uresp.flags |= C4IW_64B_CQE; spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, - ucontext->is_32b_cqe ? - sizeof(uresp) - sizeof(uresp.flags) : - sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_mm2; diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 9fde78b74690..cd1b01014198 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -530,7 +530,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mhp->rhp = rhp; - mhp->umem = ib_umem_get(pd->device, start, length, acc); + mhp->umem = ib_umem_get_va(pd->device, start, length, acc); if (IS_ERR(mhp->umem)) goto err_free_skb; diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 616019ac1da5..e1eec37ee822 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "iw_cxgb4.h" @@ -79,7 +80,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext, struct ib_device *ibdev = ucontext->device; struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext); struct c4iw_dev *rhp = to_c4iw_dev(ibdev); - struct c4iw_alloc_ucontext_resp uresp; + struct c4iw_alloc_ucontext_resp uresp = {}; int ret = 0; struct c4iw_mm_entry *mm = NULL; @@ -105,8 +106,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext, context->key += PAGE_SIZE; spin_unlock(&context->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, - sizeof(uresp) - sizeof(uresp.reserved)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_mm; @@ -209,8 +209,9 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) { struct c4iw_pd *php = to_c4iw_pd(pd); struct ib_device *ibdev = pd->device; - u32 pdid; struct c4iw_dev *rhp; + u32 pdid; + int ret; pr_debug("ibdev %p\n", ibdev); rhp = (struct c4iw_dev *) ibdev; @@ -223,9 +224,10 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata) if (udata) { struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid}; - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + ret = ib_respond_udata(udata, uresp); + if (ret) { c4iw_deallocate_pd(&php->ibpd, udata); - return -EFAULT; + return ret; } } mutex_lock(&rhp->rdev.stats.lock); @@ -257,11 +259,13 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro { struct c4iw_dev *dev; + int err; pr_debug("ibdev %p\n", ibdev); - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; dev = to_c4iw_dev(ibdev); addrconf_addr_eui48((u8 *)&props->sys_image_guid, @@ -296,7 +300,7 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro props->max_fast_reg_page_list_len = t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl); - return 0; + return ib_respond_empty_udata(uhw); } static int c4iw_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index d9a86e4c5461..e295f79e0cd3 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -2120,7 +2120,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, struct c4iw_pd *php; struct c4iw_cq *schp; struct c4iw_cq *rchp; - struct c4iw_create_qp_resp uresp; + struct c4iw_create_qp_resp uresp = {}; unsigned int sqsize, rqsize = 0; struct c4iw_ucontext *ucontext = rdma_udata_to_drv_context( udata, struct c4iw_ucontext, ibucontext); @@ -2242,7 +2242,6 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, goto err_free_sq_db_key; } } - memset(&uresp, 0, sizeof(uresp)); if (t4_sq_onchip(&qhp->wq.sq)) { ma_sync_key_mm = kmalloc_obj(*ma_sync_key_mm); if (!ma_sync_key_mm) { @@ -2280,7 +2279,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, ucontext->key += PAGE_SIZE; } spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_ma_sync_key; sq_key_mm->key = uresp.sq_key; @@ -2686,7 +2685,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, struct c4iw_dev *rhp; struct c4iw_srq *srq = to_c4iw_srq(ib_srq); struct c4iw_pd *php; - struct c4iw_create_srq_resp uresp; + struct c4iw_create_srq_resp uresp = {}; struct c4iw_ucontext *ucontext; struct c4iw_mm_entry *srq_key_mm, *srq_db_key_mm; int rqsize; @@ -2764,7 +2763,6 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, ret = -ENOMEM; goto err_free_srq_key_mm; } - memset(&uresp, 0, sizeof(uresp)); uresp.flags = srq->flags; uresp.qid_mask = rhp->rdev.qpmask; uresp.srqid = srq->wq.qid; @@ -2777,7 +2775,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs, uresp.srq_db_gts_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_srq_db_key_mm; srq_key_mm->key = uresp.srq_key; diff --git a/drivers/infiniband/hw/efa/Kconfig b/drivers/infiniband/hw/efa/Kconfig index 457e18ba1d57..ff7f7c0870b3 100644 --- a/drivers/infiniband/hw/efa/Kconfig +++ b/drivers/infiniband/hw/efa/Kconfig @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. # # Amazon fabric device configuration # @@ -8,6 +8,7 @@ config INFINIBAND_EFA tristate "Amazon Elastic Fabric Adapter (EFA) support" depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN depends on INFINIBAND_USER_ACCESS + select CRC16 help This driver supports Amazon Elastic Fabric Adapter (EFA). diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 00b19f2ba3da..f4586bb170c1 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -148,6 +148,7 @@ int efa_query_device(struct ib_device *ibdev, struct ib_udata *udata); int efa_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); +int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed); int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index ad34ea5da6b0..826790ca9d83 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -6,9 +6,6 @@ #ifndef _EFA_ADMIN_CMDS_H_ #define _EFA_ADMIN_CMDS_H_ -#define EFA_ADMIN_API_VERSION_MAJOR 0 -#define EFA_ADMIN_API_VERSION_MINOR 1 - /* EFA admin queue opcodes */ enum efa_admin_aq_opcode { EFA_ADMIN_CREATE_QP = 1, diff --git a/drivers/infiniband/hw/efa/efa_admin_defs.h b/drivers/infiniband/hw/efa/efa_admin_defs.h index 35700c93e639..02f86edabed8 100644 --- a/drivers/infiniband/hw/efa/efa_admin_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_defs.h @@ -1,18 +1,20 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* - * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_ADMIN_H_ #define _EFA_ADMIN_H_ +#define EFA_ADMIN_API_VERSION_MAJOR 0 +#define EFA_ADMIN_API_VERSION_MINOR 2 + enum efa_admin_aq_completion_status { EFA_ADMIN_SUCCESS = 0, EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1, EFA_ADMIN_BAD_OPCODE = 2, EFA_ADMIN_UNSUPPORTED_OPCODE = 3, EFA_ADMIN_MALFORMED_REQUEST = 4, - /* Additional status is provided in ACQ entry extended_status */ EFA_ADMIN_ILLEGAL_PARAMETER = 5, EFA_ADMIN_UNKNOWN_ERROR = 6, EFA_ADMIN_RESOURCE_BUSY = 7, @@ -78,13 +80,10 @@ struct efa_admin_acq_common_desc { */ u8 flags; - u16 extended_status; + /* Poly 0x8005 CRC16 with initial value 0xFFFF and final XOR of 0xFFFF */ + u16 checksum; - /* - * indicates to the driver which AQ entry has been consumed by the - * device and could be reused - */ - u16 sq_head_indx; + u16 reserved; }; struct efa_admin_acq_entry { diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index e97b5f0d7003..7cc3f4af0bb9 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -3,6 +3,7 @@ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ +#include #include #include "efa_com.h" @@ -22,6 +23,14 @@ #define EFA_CTRL_MINOR 0 #define EFA_CTRL_SUB_MINOR 1 +#define EFA_CRC16_INIT_VAL 0xffff + +#define EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR 0 +#define EFA_CRC_MIN_ADMIN_API_VERSION_MINOR 2 + +#define EFA_MIN_ADMIN_API_VERSION_MAJOR 0 +#define EFA_MIN_ADMIN_API_VERSION_MINOR 1 + enum efa_cmd_status { EFA_CMD_UNUSED, EFA_CMD_ALLOCATED, @@ -167,9 +176,8 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev) struct efa_com_admin_queue *aq = &edev->aq; struct efa_com_admin_cq *cq = &aq->cq; u16 size = aq->depth * sizeof(*cq->entries); - u32 acq_caps = 0; - u32 addr_high; - u32 addr_low; + u32 acq_caps = 0, crc_min_ver = 0; + u32 addr_high, addr_low; cq->entries = dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); @@ -178,6 +186,11 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev) spin_lock_init(&cq->lock); + EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR); + EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MINOR); + if (edev->dev_api_ver >= crc_min_ver) + cq->validate_checksum = true; + cq->cc = 0; cq->phase = 1; @@ -409,12 +422,35 @@ static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, return 0; } +static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq, + struct efa_admin_acq_entry *cqe) +{ + u16 cqe_checksum = cqe->acq_common_descriptor.checksum; + u16 calc_checksum; + + cqe->acq_common_descriptor.checksum = 0; + + calc_checksum = crc16(EFA_CRC16_INIT_VAL, (u8 *)cqe, sizeof(*cqe)) ^ EFA_CRC16_INIT_VAL; + if (calc_checksum != cqe_checksum) { + ibdev_err(aq->efa_dev, + "Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n", + cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc, + aq->cq.cc); + return false; + } + + return true; +} + static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, struct efa_admin_acq_entry *cqe) { struct efa_comp_ctx *comp_ctx; u16 cmd_id; + if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe)) + return -EINVAL; + cmd_id = EFA_GET(&cqe->acq_common_descriptor.command, EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID); @@ -954,16 +990,16 @@ int efa_com_validate_version(struct efa_com_dev *edev) EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION), EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION)); - EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION, - EFA_ADMIN_API_VERSION_MAJOR); - EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION, - EFA_ADMIN_API_VERSION_MINOR); + EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MAJOR); + EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MINOR); if (ver < min_ver) { ibdev_err(edev->efa_dev, "EFA version is lower than the minimal version the driver supports\n"); return -EOPNOTSUPP; } + edev->dev_api_ver = ver; + ibdev_dbg( edev->efa_dev, "efa controller version: %d.%d.%d implementation version %d\n", diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h index 4d9ca97e4296..f8c692b0e092 100644 --- a/drivers/infiniband/hw/efa/efa_com.h +++ b/drivers/infiniband/hw/efa/efa_com.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_COM_H_ @@ -25,6 +25,7 @@ struct efa_com_admin_cq { struct efa_admin_acq_entry *entries; dma_addr_t dma_addr; spinlock_t lock; /* Protects ACQ */ + bool validate_checksum; u16 cc; /* consumer counter */ u8 phase; @@ -112,6 +113,7 @@ struct efa_com_dev { u32 supported_features; u32 dma_addr_bits; + u32 dev_api_ver; struct efa_com_mmio_read mmio_read; }; diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 63c7f07806a8..5db4f5805b59 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -6,6 +6,8 @@ #include "efa_com.h" #include "efa_com_cmd.h" +#define EFA_DEFAULT_LINK_SPEED_GBPS 100 + int efa_com_create_qp(struct efa_com_dev *edev, struct efa_com_create_qp_params *params, struct efa_com_create_qp_result *res) @@ -468,6 +470,8 @@ int efa_com_get_device_attr(struct efa_com_dev *edev, result->device_caps = resp.u.device_attr.device_caps; result->guid = resp.u.device_attr.guid; result->max_link_speed_gbps = resp.u.device_attr.max_link_speed_gbps; + if (!result->max_link_speed_gbps) + result->max_link_speed_gbps = EFA_DEFAULT_LINK_SPEED_GBPS; if (result->admin_api_version < 1) { ibdev_err_ratelimited( diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 03c237c8c81e..97da8e828e34 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -390,6 +390,7 @@ static const struct ib_device_ops efa_dev_ops = { .query_gid = efa_query_gid, .query_pkey = efa_query_pkey, .query_port = efa_query_port, + .query_port_speed = efa_query_port_speed, .query_qp = efa_query_qp, .reg_user_mr = efa_reg_mr, .reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf, diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 9b2b652800e4..06d3365aeb56 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -90,8 +90,6 @@ static const struct rdma_stat_desc efa_port_stats_descs[] = { EFA_DEFINE_PORT_STATS(EFA_STATS_STR) }; -#define EFA_DEFAULT_LINK_SPEED_GBPS 100 - #define EFA_CHUNK_PAYLOAD_SHIFT 12 #define EFA_CHUNK_PAYLOAD_SIZE BIT(EFA_CHUNK_PAYLOAD_SHIFT) #define EFA_CHUNK_PAYLOAD_PTR_SIZE 8 @@ -218,12 +216,9 @@ int efa_query_device(struct ib_device *ibdev, struct efa_dev *dev = to_edev(ibdev); int err; - if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(ibdev, - "Incompatible ABI params, udata not cleared\n"); - return -EINVAL; - } + err = ib_is_udata_in_empty(udata); + if (err) + return err; dev_attr = &dev->dev_attr; @@ -273,13 +268,9 @@ int efa_query_device(struct ib_device *ibdev, if (dev->neqs) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(ibdev, - "Failed to copy udata for query_device\n"); + err = ib_respond_udata(udata, resp); + if (err) return err; - } } return 0; @@ -289,7 +280,13 @@ static void efa_link_gbps_to_speed_and_width(u16 gbps, enum ib_port_speed *speed, enum ib_port_width *width) { - if (gbps >= 400) { + if (gbps >= 1600) { + *width = IB_WIDTH_8X; + *speed = IB_SPEED_XDR; + } else if (gbps >= 800) { + *width = IB_WIDTH_8X; + *speed = IB_SPEED_NDR; + } else if (gbps >= 400) { *width = IB_WIDTH_8X; *speed = IB_SPEED_HDR; } else if (gbps >= 200) { @@ -333,7 +330,7 @@ int efa_query_port(struct ib_device *ibdev, u32 port, props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; props->gid_tbl_len = 1; props->pkey_tbl_len = 1; - link_gbps = dev->dev_attr.max_link_speed_gbps ?: EFA_DEFAULT_LINK_SPEED_GBPS; + link_gbps = dev->dev_attr.max_link_speed_gbps; efa_link_gbps_to_speed_and_width(link_gbps, &link_speed, &link_width); props->active_speed = link_speed; props->active_width = link_width; @@ -345,6 +342,15 @@ int efa_query_port(struct ib_device *ibdev, u32 port, return 0; } +int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed) +{ + struct efa_dev *dev = to_edev(ibdev); + + *speed = dev->dev_attr.max_link_speed_gbps * 10; + + return 0; +} + int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) @@ -433,13 +439,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct efa_pd *pd = to_epd(ibpd); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, - "Incompatible ABI params, udata not cleared\n"); - err = -EINVAL; + err = ib_is_udata_in_empty(udata); + if (err) goto err_out; - } err = efa_com_alloc_pd(&dev->edev, &result); if (err) @@ -449,13 +451,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) resp.pdn = result.pdn; if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&dev->ibdev, - "Failed to copy udata for alloc_pd\n"); + err = ib_respond_udata(udata, resp); + if (err) goto err_dealloc_pd; - } } ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn); @@ -798,14 +796,9 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, qp->max_inline_data = init_attr->cap.max_inline_data; if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&dev->ibdev, - "Failed to copy udata for qp[%u]\n", - create_qp_resp.qp_num); + err = ib_respond_udata(udata, resp); + if (err) goto err_remove_mmap_entries; - } } ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num); @@ -991,12 +984,9 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, - "Incompatible ABI params, udata not cleared\n"); - return -EINVAL; - } + err = ib_is_udata_in_empty(udata); + if (err) + return err; cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state : qp->state; @@ -1081,6 +1071,7 @@ int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (cq->cpu_addr) efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); + ib_umem_release(cq->umem); return 0; } @@ -1133,6 +1124,7 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct efa_ibv_create_cq cmd; struct efa_cq *cq = to_ecq(ibcq); int entries = attr->cqe; + struct ib_umem *umem; bool set_src_addr; int err; @@ -1181,26 +1173,29 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->ucontext = ucontext; cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs); - if (ibcq->umem) { - if (ibcq->umem->length < cq->size) { - ibdev_dbg(&dev->ibdev, "External memory too small\n"); - err = -EINVAL; - goto err_out; - } + umem = ib_umem_get_cq_buf(ibcq->device, attrs, cq->size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + err = PTR_ERR(umem); + goto err_out; + } - if (!ib_umem_is_contiguous(ibcq->umem)) { + cq->umem = umem; + + if (umem) { + if (!ib_umem_is_contiguous(umem)) { ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n"); err = -EINVAL; - goto err_out; + goto err_release_umem; } - cq->dma_addr = ib_umem_start_dma_addr(ibcq->umem); + cq->dma_addr = ib_umem_start_dma_addr(umem); } else { cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size, DMA_FROM_DEVICE); if (!cq->cpu_addr) { err = -ENOMEM; - goto err_out; + goto err_release_umem; } } @@ -1245,13 +1240,9 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(ibdev, - "Failed to copy udata for create_cq\n"); + err = ib_respond_udata(udata, resp); + if (err) goto err_xa_erase; - } } ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", @@ -1271,6 +1262,8 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (cq->cpu_addr) efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); +err_release_umem: + ib_umem_release(cq->umem); err_out: atomic64_inc(&dev->stats.create_cq_err); return err; @@ -1621,13 +1614,11 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, struct efa_dev *dev = to_edev(ibpd->device); int supp_access_flags; struct efa_mr *mr; + int ret; - if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, - "Incompatible ABI params, udata not cleared\n"); - return ERR_PTR(-EINVAL); - } + ret = ib_is_udata_in_empty(udata); + if (ret) + return ERR_PTR(ret); supp_access_flags = IB_ACCESS_LOCAL_WRITE | @@ -1783,7 +1774,7 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, goto err_out; } - mr->umem = ib_umem_get(ibpd->device, start, length, access_flags); + mr->umem = ib_umem_get_va(ibpd->device, start, length, access_flags); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); ibdev_dbg(&dev->ibdev, @@ -1956,8 +1947,7 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) resp.max_tx_batch = dev->dev_attr.max_tx_batch; resp.min_sq_wr = dev->dev_attr.min_sq_depth; - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); + err = ib_respond_udata(udata, resp); if (err) goto err_dealloc_uar; @@ -2091,12 +2081,9 @@ int efa_create_ah(struct ib_ah *ibah, goto err_out; } - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - err = -EINVAL; + err = ib_is_udata_in_empty(udata); + if (err) goto err_out; - } memcpy(params.dest_addr, ah_attr->grh.dgid.raw, sizeof(params.dest_addr)); @@ -2111,13 +2098,9 @@ int efa_create_ah(struct ib_ah *ibah, resp.efa_address_handle = result.ah; if (udata->outlen) { - err = ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&dev->ibdev, - "Failed to copy udata for create_ah response\n"); + err = ib_respond_udata(udata, resp); + if (err) goto err_destroy_ah; - } } ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 5523b4e151e1..74afe6eb18b0 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -315,9 +315,14 @@ erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, } int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, - struct ib_udata *unused) + struct ib_udata *udata) { struct erdma_dev *dev = to_edev(ibdev); + int err; + + err = ib_is_udata_in_empty(udata); + if (err) + return err; memset(attr, 0, sizeof(*attr)); @@ -358,7 +363,7 @@ int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, addrconf_addr_eui48((u8 *)&attr->sys_image_guid, dev->netdev->dev_addr); - return 0; + return ib_respond_empty_udata(udata); } int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, @@ -829,7 +834,7 @@ static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, { int ret = 0; - mem->umem = ib_umem_get(&dev->ibdev, start, len, access); + mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access); if (IS_ERR(mem->umem)) { ret = PTR_ERR(mem->umem); mem->umem = NULL; @@ -896,8 +901,8 @@ static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, page->va = (dbrecords_va & PAGE_MASK); page->refcnt = 0; - page->umem = ib_umem_get(ctx->ibucontext.device, - dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); + page->umem = ib_umem_get_va(ctx->ibucontext.device, + dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); if (IS_ERR(page->umem)) { rv = PTR_ERR(page->umem); kfree(page); @@ -996,7 +1001,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, struct erdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct erdma_ucontext, ibucontext); struct erdma_ureq_create_qp ureq; - struct erdma_uresp_create_qp uresp; + struct erdma_uresp_create_qp uresp = {}; void *old_entry; int ret = 0; @@ -1048,14 +1053,12 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (ret) goto err_out_xa; - memset(&uresp, 0, sizeof(uresp)); - uresp.num_sqe = qp->attrs.sq_size; uresp.num_rqe = qp->attrs.rq_size; uresp.qp_id = QP_ID(qp); uresp.rq_offset = qp->user_qp.rq_offset; - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_out_cmd; } else { @@ -1571,7 +1574,7 @@ int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) uresp.dev_id = dev->pdev->device; - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_put_mmap_entries; @@ -1977,7 +1980,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (!rdma_is_kernel_res(&ibcq->res)) { struct erdma_ureq_create_cq ureq; - struct erdma_uresp_create_cq uresp; + struct erdma_uresp_create_cq uresp = {}; ret = ib_copy_validate_udata_in(udata, ureq, rsvd0); if (ret) @@ -1990,8 +1993,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, uresp.cq_id = cq->cqn; uresp.num_cqe = depth; - ret = ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen)); + ret = ib_respond_udata(udata, uresp); if (ret) goto err_free_res; } else { diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 8b5a5b32b0fa..b7fd8b1fbbbd 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1206,6 +1206,7 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra) { struct hfi1_devdata *dd; + struct ib_device *ibdev; int ret, nports; /* extra is * number of ports */ @@ -1227,7 +1228,17 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, "Could not allocate unit ID: error %d\n", -ret); goto bail; } - rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit); + + /* + * FIXME: rvt and its users want to touch the ibdev before + * registration and have things like the name work. We don't have the + * infrastructure in the core to support this directly today, hack it + * to work by setting the name manually here. + */ + ibdev = &dd->verbs_dev.rdi.ibdev; + dev_set_name(&ibdev->dev, "%s_%d", class_name(), dd->unit); + strscpy(ibdev->name, dev_name(&ibdev->dev), IB_DEVICE_NAME_MAX); + /* * If the BIOS does not have the NUMA node information set, select * NUMA 0 so we get consistent performance. diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 8a605da8a93c..925ddf15b681 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -32,6 +32,7 @@ #include #include +#include #include "hns_roce_device.h" #include "hns_roce_hw_v2.h" @@ -112,8 +113,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, resp.priority = ah->av.sl; resp.tc_mode = tc_mode; memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); + ret = ib_respond_udata(udata, resp); } err_out: diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 621568e11405..1dd0efb5620d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -174,9 +174,9 @@ static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC, hr_cq->cqn); if (ret) - ibdev_err(ibdev, - "failed to send create cmd for CQ(0x%lx), ret = %d.\n", - hr_cq->cqn, ret); + ibdev_err_ratelimited(ibdev, + "failed to send create cmd for CQ(0x%lx), ret = %d.\n", + hr_cq->cqn, ret); hns_roce_free_cmd_mailbox(hr_dev, mailbox); @@ -452,8 +452,7 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, if (udata) { resp.cqn = hr_cq->cqn; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); + ret = ib_respond_udata(udata, resp); if (ret) goto err_cqc; } diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index f64023f5cf0a..5e008a1700ef 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -29,8 +29,8 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, refcount_set(&page->refcount, 1); page->user_virt = page_addr; - page->umem = ib_umem_get(context->ibucontext.device, page_addr, - PAGE_SIZE, 0); + page->umem = ib_umem_get_va(context->ibucontext.device, page_addr, + PAGE_SIZE, 0); if (IS_ERR(page->umem)) { ret = PTR_ERR(page->umem); kfree(page); diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index b869cdc54118..05630f7c9155 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -3,11 +3,13 @@ * Copyright (c) 2023 Hisilicon Limited. */ +#include #include #include #include #include "hns_roce_device.h" +#include "hns_roce_hw_v2.h" static struct dentry *hns_roce_dbgfs_root; @@ -18,25 +20,34 @@ static int hns_debugfs_seqfile_open(struct inode *inode, struct file *f) return single_open(f, seqfile->read, seqfile->data); } +static ssize_t hns_debugfs_seqfile_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct hns_debugfs_seqfile *seqfile = file_inode(file)->i_private; + char buf[16] = {}; + + if (!seqfile->write) + return -EOPNOTSUPP; + + if (count >= sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, buffer, count)) + return -EFAULT; + + return seqfile->write(buf, count, seqfile->data); +} + static const struct file_operations hns_debugfs_seqfile_fops = { .owner = THIS_MODULE, .open = hns_debugfs_seqfile_open, .release = single_release, .read = seq_read, + .write = hns_debugfs_seqfile_write, .llseek = seq_lseek }; -static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq, - const char *name, struct dentry *parent, - int (*read_fn)(struct seq_file *, void *), - void *data) -{ - debugfs_create_file(name, 0400, parent, seq, &hns_debugfs_seqfile_fops); - - seq->read = read_fn; - seq->data = data; -} - static const char * const sw_stat_info[] = { [HNS_ROCE_DFX_AEQE_CNT] = "aeqe", [HNS_ROCE_DFX_CEQE_CNT] = "ceqe", @@ -76,10 +87,256 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev, { struct hns_sw_stat_debugfs *dbgfs = &hr_dev->dbgfs.sw_stat_root; - dbgfs->root = debugfs_create_dir("sw_stat", parent); + dbgfs->sw_stat.read = sw_stat_debugfs_show; + dbgfs->sw_stat.data = hr_dev; - init_debugfs_seqfile(&dbgfs->sw_stat, "sw_stat", dbgfs->root, - sw_stat_debugfs_show, hr_dev); + dbgfs->root = debugfs_create_dir("sw_stat", parent); + debugfs_create_file("sw_stat", 0400, dbgfs->root, &dbgfs->sw_stat, + &hns_debugfs_seqfile_fops); +} + +#define __HNS_SCC_ATTR(_name, _type, _offset, _size, _min, _max) { \ + .name = _name, \ + .algo_type = _type, \ + .offset = _offset, \ + .size = _size, \ + .min = _min, \ + .max = _max, \ +} + +#define HNS_DCQCN_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DCQCN, \ + HNS_ROCE_DCQCN_##NAME##_OFS, \ + HNS_ROCE_DCQCN_##NAME##_SZ, \ + 0, HNS_ROCE_DCQCN_##NAME##_MAX) + +#define HNS_LDCP_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_LDCP, \ + HNS_ROCE_LDCP_##NAME##_OFS, \ + HNS_ROCE_LDCP_##NAME##_SZ, \ + 0, HNS_ROCE_LDCP_##NAME##_MAX) + +#define HNS_HC3_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_HC3, \ + HNS_ROCE_HC3_##NAME##_OFS, \ + HNS_ROCE_HC3_##NAME##_SZ, \ + HNS_ROCE_HC3_##NAME##_MIN, \ + HNS_ROCE_HC3_##NAME##_MAX) + +#define HNS_DIP_CC_ATTR_RW(_name, NAME) \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DIP, \ + HNS_ROCE_DIP_##NAME##_OFS, \ + HNS_ROCE_DIP_##NAME##_SZ, \ + 0, HNS_ROCE_DIP_##NAME##_MAX) + +static const struct hns_roce_cong_attr { + enum hns_roce_cong_type cong_type; + const char *name; + struct hns_roce_cc_param_attr params[HNS_ROCE_CC_PARAM_MAX_NUM]; +} cong_attrs[] = { + { CONG_TYPE_DCQCN, "dcqcn_cc_param", + { + HNS_DCQCN_CC_ATTR_RW("ai", AI), + HNS_DCQCN_CC_ATTR_RW("f", F), + HNS_DCQCN_CC_ATTR_RW("tkp", TKP), + HNS_DCQCN_CC_ATTR_RW("tmp", TMP), + HNS_DCQCN_CC_ATTR_RW("alp", ALP), + HNS_DCQCN_CC_ATTR_RW("max_speed", MAX_SPEED), + HNS_DCQCN_CC_ATTR_RW("g", G), + HNS_DCQCN_CC_ATTR_RW("al", AL), + HNS_DCQCN_CC_ATTR_RW("cnp_time", CNP_TIME), + HNS_DCQCN_CC_ATTR_RW("ashift", ASHIFT), + } + }, + { CONG_TYPE_LDCP, "ldcp_cc_param", + { + HNS_LDCP_CC_ATTR_RW("cwd0", CWD0), + HNS_LDCP_CC_ATTR_RW("alpha", ALPHA), + HNS_LDCP_CC_ATTR_RW("gamma", GAMMA), + HNS_LDCP_CC_ATTR_RW("beta", BETA), + HNS_LDCP_CC_ATTR_RW("eta", ETA), + } + }, + { CONG_TYPE_HC3, "hc3_cc_param", + { + HNS_HC3_CC_ATTR_RW("initial_window", INITIAL_WINDOW), + HNS_HC3_CC_ATTR_RW("bandwidth", BANDWIDTH), + HNS_HC3_CC_ATTR_RW("qlen_shift", QLEN_SHIFT), + HNS_HC3_CC_ATTR_RW("port_usage_shift", PORT_USAGE_SHIFT), + HNS_HC3_CC_ATTR_RW("over_period", OVER_PERIOD), + HNS_HC3_CC_ATTR_RW("max_stage", MAX_STAGE), + HNS_HC3_CC_ATTR_RW("gamma_shift", GAMMA_SHIFT), + } + }, + { CONG_TYPE_DIP, "dip_cc_param", + { + HNS_DIP_CC_ATTR_RW("ai", AI), + HNS_DIP_CC_ATTR_RW("f", F), + HNS_DIP_CC_ATTR_RW("tkp", TKP), + HNS_DIP_CC_ATTR_RW("tmp", TMP), + HNS_DIP_CC_ATTR_RW("alp", ALP), + HNS_DIP_CC_ATTR_RW("max_speed", MAX_SPEED), + HNS_DIP_CC_ATTR_RW("g", G), + HNS_DIP_CC_ATTR_RW("al", AL), + HNS_DIP_CC_ATTR_RW("cnp_time", CNP_TIME), + HNS_DIP_CC_ATTR_RW("ashift", ASHIFT), + } + } +}; + +static int cc_param_debugfs_show(struct seq_file *file, void *offset) +{ + struct hns_cc_param_seqfile *param_seqfile = file->private; + const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr; + int algo_type = param_attr->algo_type; + int index = param_seqfile->index; + struct hns_roce_dev *hr_dev = + container_of(param_seqfile, struct hns_roce_dev, + dbgfs.cc_param_root[algo_type].params[index]); + struct hns_roce_scc_param *scc_param; + __le32 val = 0; + + scc_param = &hr_dev->scc_param[algo_type]; + + scoped_guard(mutex, &scc_param->scc_mutex) { + memcpy(&val, + (void *)scc_param->param + param_attr->offset, + param_attr->size); + } + + seq_printf(file, "%u\n", le32_to_cpu(val)); + + return 0; +} + +static ssize_t cc_param_debugfs_store(char *buf, size_t count, void *data) +{ + struct hns_cc_param_seqfile *param_seqfile = data; + const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr; + int algo_type = param_attr->algo_type; + int index = param_seqfile->index; + struct hns_roce_dev *hr_dev = + container_of(param_seqfile, struct hns_roce_dev, + dbgfs.cc_param_root[algo_type].params[index]); + struct hns_roce_scc_param *scc_param; + __le32 attr_val, old_val = 0; + u32 val; + int ret; + + if (kstrtou32(buf, 0, &val)) + return -EINVAL; + + if (val > param_attr->max || val < param_attr->min) + return -EINVAL; + + attr_val = cpu_to_le32(val); + scc_param = &hr_dev->scc_param[algo_type]; + guard(mutex)(&scc_param->scc_mutex); + + memcpy(&old_val, (void *)scc_param->param + param_attr->offset, + param_attr->size); + memcpy((void *)scc_param->param + param_attr->offset, &attr_val, + param_attr->size); + + ret = hr_dev->hw->config_scc_param(hr_dev, algo_type); + if (ret) { + memcpy((void *)scc_param->param + param_attr->offset, &old_val, + param_attr->size); + return ret; + } + + return count; +} + +static void get_default_scc_param(struct hns_roce_dev *hr_dev) +{ + int ret; + int i; + + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) { + ret = hr_dev->hw->query_scc_param(hr_dev, i); + if (ret && ret != -EOPNOTSUPP) + ibdev_warn_ratelimited(&hr_dev->ib_dev, + "failed to get default parameters of scc algo %d, ret = %d.\n", + i, ret); + } +} + +static int hns_roce_alloc_scc_param(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_scc_param *scc_param; + int i; + + scc_param = kvcalloc(HNS_ROCE_SCC_ALGO_TOTAL, sizeof(*scc_param), + GFP_KERNEL); + if (!scc_param) + return -ENOMEM; + + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) { + scc_param[i].algo_type = i; + scc_param[i].hr_dev = hr_dev; + mutex_init(&scc_param[i].scc_mutex); + } + + hr_dev->scc_param = scc_param; + + get_default_scc_param(hr_dev); + + return 0; +} + +static void hns_roce_dealloc_scc_param(struct hns_roce_dev *hr_dev) +{ + int i; + + if (!hr_dev->scc_param) + return; + + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) + mutex_destroy(&hr_dev->scc_param[i].scc_mutex); + + kvfree(hr_dev->scc_param); +} + +static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev, + struct dentry *parent) +{ + const struct hns_roce_cong_attr *cong_attr; + struct hns_cc_param_debugfs *dbgfs; + int i, j; + int ret; + + if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 || + !(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) || + hr_dev->is_vf) + return; + + ret = hns_roce_alloc_scc_param(hr_dev); + if (ret) + return; + + for (i = 0; i < CONG_TYPE_MAX_NUM; i++) { + cong_attr = &cong_attrs[i]; + if (!test_bit(cong_attr->cong_type, + (unsigned long *)&hr_dev->caps.cong_cap)) + continue; + + dbgfs = &hr_dev->dbgfs.cc_param_root[i]; + dbgfs->root = debugfs_create_dir(cong_attr->name, parent); + for (j = 0; j < HNS_ROCE_CC_PARAM_MAX_NUM; j++) { + if (!cong_attr->params[j].name) + break; + dbgfs->params[j].param_attr = &cong_attr->params[j]; + dbgfs->params[j].index = j; + dbgfs->params[j].seqfile.read = cc_param_debugfs_show; + dbgfs->params[j].seqfile.write = cc_param_debugfs_store; + dbgfs->params[j].seqfile.data = &dbgfs->params[j]; + debugfs_create_file(cong_attr->params[j].name, 0600, + dbgfs->root, + &dbgfs->params[j].seqfile, + &hns_debugfs_seqfile_fops); + } + } } /* debugfs for device */ @@ -91,11 +348,13 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) hns_roce_dbgfs_root); create_sw_stat_debugfs(hr_dev, dbgfs->root); + create_cc_param_debugfs(hr_dev, dbgfs->root); } void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev) { debugfs_remove_recursive(hr_dev->dbgfs.root); + hns_roce_dealloc_scc_param(hr_dev); } /* debugfs for hns module */ diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h index 98e87bd3161e..116b1e8b6677 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h @@ -9,6 +9,7 @@ /* debugfs seqfile */ struct hns_debugfs_seqfile { int (*read)(struct seq_file *seq, void *data); + ssize_t (*write)(char *buf, size_t count, void *data); void *data; }; @@ -17,10 +18,35 @@ struct hns_sw_stat_debugfs { struct hns_debugfs_seqfile sw_stat; }; +struct hns_roce_cc_param_attr { + const char *name; + int algo_type; + u32 offset; + u32 size; + u32 max; + u32 min; +}; + +struct hns_cc_param_seqfile { + struct hns_debugfs_seqfile seqfile; + const struct hns_roce_cc_param_attr *param_attr; + int index; +}; + +#define HNS_ROCE_CC_PARAM_MAX_NUM 11 + +struct hns_cc_param_debugfs { + struct dentry *root; + struct hns_cc_param_seqfile params[HNS_ROCE_CC_PARAM_MAX_NUM]; +}; + +#define CONG_TYPE_MAX_NUM 4 + /* Debugfs for device */ struct hns_roce_dev_debugfs { struct dentry *root; struct hns_sw_stat_debugfs sw_stat_root; + struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM]; }; struct hns_roce_dev; diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 3f032b8038af..eb7b1865e4c7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -726,6 +726,14 @@ struct hns_roce_eq_table { struct hns_roce_eq *eq; }; +enum hns_roce_scc_algo { + HNS_ROCE_SCC_ALGO_DCQCN = 0, + HNS_ROCE_SCC_ALGO_LDCP, + HNS_ROCE_SCC_ALGO_HC3, + HNS_ROCE_SCC_ALGO_DIP, + HNS_ROCE_SCC_ALGO_TOTAL, +}; + struct hns_roce_caps { u64 fw_ver; u8 num_ports; @@ -963,6 +971,18 @@ struct hns_roce_hw { u8 *tc_mode, u8 *priority); const struct ib_device_ops *hns_roce_dev_ops; const struct ib_device_ops *hns_roce_dev_srq_ops; + int (*config_scc_param)(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo algo); + int (*query_scc_param)(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo alog); +}; + +#define HNS_ROCE_SCC_PARAM_SIZE 4 +struct hns_roce_scc_param { + __le32 param[HNS_ROCE_SCC_PARAM_SIZE]; + enum hns_roce_scc_algo algo_type; + struct hns_roce_dev *hr_dev; + struct mutex scc_mutex; /* protect @param */ }; struct hns_roce_dev { @@ -1026,6 +1046,7 @@ struct hns_roce_dev { u64 dwqe_page; struct hns_roce_dev_debugfs dbgfs; atomic64_t *dfx_cnt; + struct hns_roce_scc_param *scc_param; }; enum hns_roce_trace_type { diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index e7c9e30ad2d8..7041a8e9134b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -314,14 +314,14 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev, bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); switch (bt_num) { case 3: - index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l1 = (u64)l0_idx * chunk_ba_num + l1_idx; index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num * chunk_ba_num + - l1_idx * chunk_ba_num + l2_idx; + index->buf = (u64)l0_idx * chunk_ba_num * chunk_ba_num + + (u64)l1_idx * chunk_ba_num + l2_idx; break; case 2: index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num + l1_idx; + index->buf = (u64)l0_idx * chunk_ba_num + l1_idx; break; case 1: index->buf = l0_idx; @@ -1269,8 +1269,6 @@ setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, root_hem = list_first_entry(&head->root, struct hns_roce_hem_item, list); - if (!root_hem) - return -ENOMEM; total = 0; for (i = 0; i < region_cnt && total <= max_ba_num; i++) { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index fa36700d0db2..2b3a1cafd1b2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -31,6 +31,7 @@ */ #include +#include #include #include #include @@ -6192,9 +6193,9 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn); hns_roce_free_cmd_mailbox(hr_dev, mailbox); if (ret) - ibdev_err(&hr_dev->ib_dev, - "failed to handle cmd of modifying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to handle cmd of modifying SRQ, ret = %d.\n", + ret); } out: @@ -6220,9 +6221,9 @@ static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC, srq->srqn); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd of querying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd of querying SRQ, ret = %d.\n", + ret); goto out; } @@ -6328,9 +6329,9 @@ static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, key_to_hw_index(key)); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd when querying MPT, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd when querying MPT, ret = %d.\n", + ret); goto err_mailbox; } @@ -7196,6 +7197,57 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) kfree(eq_table->eq); } +static const enum hns_roce_opcode_type scc_opcode[] = { + HNS_ROCE_OPC_CFG_DCQCN_PARAM, + HNS_ROCE_OPC_CFG_LDCP_PARAM, + HNS_ROCE_OPC_CFG_HC3_PARAM, + HNS_ROCE_OPC_CFG_DIP_PARAM, +}; + +static int hns_roce_v2_config_scc_param(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo algo) +{ + struct hns_roce_scc_param *scc_param = &hr_dev->scc_param[algo]; + struct hns_roce_cmq_desc desc; + int ret; + + lockdep_assert_held(&scc_param->scc_mutex); + + hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], false); + memcpy(&desc.data, scc_param->param, sizeof(scc_param->param)); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to configure scc param, opcode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + + return ret; +} + +static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev, + enum hns_roce_scc_algo algo) +{ + struct hns_roce_scc_param *scc_param; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to query scc param, opcode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + return ret; + } + + scc_param = &hr_dev->scc_param[algo]; + scoped_guard(mutex, &scc_param->scc_mutex) + memcpy(scc_param->param, &desc.data, sizeof(scc_param->param)); + + return 0; +} + static const struct ib_device_ops hns_roce_v2_dev_ops = { .destroy_qp = hns_roce_v2_destroy_qp, .modify_cq = hns_roce_v2_modify_cq, @@ -7246,19 +7298,35 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .get_dscp = hns_roce_hw_v2_get_dscp, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, + .config_scc_param = hns_roce_v2_config_scc_param, + .query_scc_param = hns_roce_v2_query_scc_param, }; static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0}, - {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), - HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, + { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), + .driver_data = 0, + }, { + PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), + .driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS, + }, /* required last entry */ - {0, } + { } }; MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); @@ -7586,8 +7654,8 @@ static int __init hns_roce_hw_v2_init(void) static void __exit hns_roce_hw_v2_exit(void) { - hns_roce_dealloc_bond_grp(); hnae3_unregister_client(&hns_roce_hw_v2_client); + hns_roce_dealloc_bond_grp(); hns_roce_cleanup_debugfs(); } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 285fe0875fac..8c7856aa24dc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -202,6 +202,10 @@ enum { /* CMQ command */ enum hns_roce_opcode_type { HNS_QUERY_FW_VER = 0x0001, + HNS_ROCE_OPC_CFG_DCQCN_PARAM = 0x1A80, + HNS_ROCE_OPC_CFG_LDCP_PARAM = 0x1A81, + HNS_ROCE_OPC_CFG_HC3_PARAM = 0x1A82, + HNS_ROCE_OPC_CFG_DIP_PARAM = 0x1A83, HNS_ROCE_OPC_QUERY_HW_VER = 0x8000, HNS_ROCE_OPC_CFG_GLOBAL_PARAM = 0x8001, HNS_ROCE_OPC_ALLOC_PF_RES = 0x8004, @@ -1459,6 +1463,127 @@ struct hns_roce_wqe_atomic_seg { __le64 cmp_data; }; +#define HNS_ROCE_DCQCN_AI_OFS 0 +#define HNS_ROCE_DCQCN_AI_SZ sizeof(u16) +#define HNS_ROCE_DCQCN_AI_MAX ((u16)(~0U)) +#define HNS_ROCE_DCQCN_F_OFS (HNS_ROCE_DCQCN_AI_OFS + HNS_ROCE_DCQCN_AI_SZ) +#define HNS_ROCE_DCQCN_F_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_F_MAX ((u8)(~0U)) +#define HNS_ROCE_DCQCN_TKP_OFS (HNS_ROCE_DCQCN_F_OFS + HNS_ROCE_DCQCN_F_SZ) +#define HNS_ROCE_DCQCN_TKP_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_TKP_MAX 10 +#define HNS_ROCE_DCQCN_TMP_OFS (HNS_ROCE_DCQCN_TKP_OFS + HNS_ROCE_DCQCN_TKP_SZ) +#define HNS_ROCE_DCQCN_TMP_SZ sizeof(u16) +#define HNS_ROCE_DCQCN_TMP_MAX 15 +#define HNS_ROCE_DCQCN_ALP_OFS (HNS_ROCE_DCQCN_TMP_OFS + HNS_ROCE_DCQCN_TMP_SZ) +#define HNS_ROCE_DCQCN_ALP_SZ sizeof(u16) +#define HNS_ROCE_DCQCN_ALP_MAX ((u16)(~0U)) +#define HNS_ROCE_DCQCN_MAX_SPEED_OFS (HNS_ROCE_DCQCN_ALP_OFS + \ + HNS_ROCE_DCQCN_ALP_SZ) +#define HNS_ROCE_DCQCN_MAX_SPEED_SZ sizeof(u32) +#define HNS_ROCE_DCQCN_MAX_SPEED_MAX ((u32)(~0U)) +#define HNS_ROCE_DCQCN_G_OFS (HNS_ROCE_DCQCN_MAX_SPEED_OFS + \ + HNS_ROCE_DCQCN_MAX_SPEED_SZ) +#define HNS_ROCE_DCQCN_G_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_G_MAX 15 +#define HNS_ROCE_DCQCN_AL_OFS (HNS_ROCE_DCQCN_G_OFS + HNS_ROCE_DCQCN_G_SZ) +#define HNS_ROCE_DCQCN_AL_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_AL_MAX ((u8)(~0U)) +#define HNS_ROCE_DCQCN_CNP_TIME_OFS (HNS_ROCE_DCQCN_AL_OFS + \ + HNS_ROCE_DCQCN_AL_SZ) +#define HNS_ROCE_DCQCN_CNP_TIME_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_CNP_TIME_MAX ((u8)(~0U)) +#define HNS_ROCE_DCQCN_ASHIFT_OFS (HNS_ROCE_DCQCN_CNP_TIME_OFS + \ + HNS_ROCE_DCQCN_CNP_TIME_SZ) +#define HNS_ROCE_DCQCN_ASHIFT_SZ sizeof(u8) +#define HNS_ROCE_DCQCN_ASHIFT_MAX 15 + +#define HNS_ROCE_LDCP_CWD0_OFS 0 +#define HNS_ROCE_LDCP_CWD0_SZ sizeof(u32) +#define HNS_ROCE_LDCP_CWD0_MAX ((u32)(~0U)) +#define HNS_ROCE_LDCP_ALPHA_OFS (HNS_ROCE_LDCP_CWD0_OFS + HNS_ROCE_LDCP_CWD0_SZ) +#define HNS_ROCE_LDCP_ALPHA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_ALPHA_MAX ((u8)(~0U)) +#define HNS_ROCE_LDCP_GAMMA_OFS (HNS_ROCE_LDCP_ALPHA_OFS + \ + HNS_ROCE_LDCP_ALPHA_SZ) +#define HNS_ROCE_LDCP_GAMMA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_GAMMA_MAX 7 +#define HNS_ROCE_LDCP_BETA_OFS (HNS_ROCE_LDCP_GAMMA_OFS + \ + HNS_ROCE_LDCP_GAMMA_SZ) +#define HNS_ROCE_LDCP_BETA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_BETA_MAX 7 +#define HNS_ROCE_LDCP_ETA_OFS (HNS_ROCE_LDCP_BETA_OFS + HNS_ROCE_LDCP_BETA_SZ) +#define HNS_ROCE_LDCP_ETA_SZ sizeof(u8) +#define HNS_ROCE_LDCP_ETA_MAX 7 + +#define HNS_ROCE_HC3_INITIAL_WINDOW_OFS 0 +#define HNS_ROCE_HC3_INITIAL_WINDOW_SZ sizeof(u32) +#define HNS_ROCE_HC3_INITIAL_WINDOW_MIN 0 +#define HNS_ROCE_HC3_INITIAL_WINDOW_MAX ((u32)(~0U)) +#define HNS_ROCE_HC3_BANDWIDTH_OFS (HNS_ROCE_HC3_INITIAL_WINDOW_OFS + \ + HNS_ROCE_HC3_INITIAL_WINDOW_SZ) +#define HNS_ROCE_HC3_BANDWIDTH_SZ sizeof(u32) +#define HNS_ROCE_HC3_BANDWIDTH_MIN 1000 +#define HNS_ROCE_HC3_BANDWIDTH_MAX ((u32)(~0U)) +#define HNS_ROCE_HC3_QLEN_SHIFT_OFS (HNS_ROCE_HC3_BANDWIDTH_OFS + \ + HNS_ROCE_HC3_BANDWIDTH_SZ) +#define HNS_ROCE_HC3_QLEN_SHIFT_SZ sizeof(u8) +#define HNS_ROCE_HC3_QLEN_SHIFT_MIN 0 +#define HNS_ROCE_HC3_QLEN_SHIFT_MAX 31 +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS (HNS_ROCE_HC3_QLEN_SHIFT_OFS + \ + HNS_ROCE_HC3_QLEN_SHIFT_SZ) +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ sizeof(u8) +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MIN 0 +#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MAX 100 +#define HNS_ROCE_HC3_OVER_PERIOD_OFS (HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS + \ + HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ) +#define HNS_ROCE_HC3_OVER_PERIOD_SZ sizeof(u8) +#define HNS_ROCE_HC3_OVER_PERIOD_MIN 0 +#define HNS_ROCE_HC3_OVER_PERIOD_MAX ((u8)(~0U)) +#define HNS_ROCE_HC3_MAX_STAGE_OFS (HNS_ROCE_HC3_OVER_PERIOD_OFS + \ + HNS_ROCE_HC3_OVER_PERIOD_SZ) +#define HNS_ROCE_HC3_MAX_STAGE_SZ sizeof(u8) +#define HNS_ROCE_HC3_MAX_STAGE_MIN 0 +#define HNS_ROCE_HC3_MAX_STAGE_MAX ((u8)(~0U)) +#define HNS_ROCE_HC3_GAMMA_SHIFT_OFS (HNS_ROCE_HC3_MAX_STAGE_OFS + \ + HNS_ROCE_HC3_MAX_STAGE_SZ) +#define HNS_ROCE_HC3_GAMMA_SHIFT_SZ sizeof(u8) +#define HNS_ROCE_HC3_GAMMA_SHIFT_MIN 0 +#define HNS_ROCE_HC3_GAMMA_SHIFT_MAX 15 + +#define HNS_ROCE_DIP_AI_OFS 0 +#define HNS_ROCE_DIP_AI_SZ sizeof(u16) +#define HNS_ROCE_DIP_AI_MAX ((u16)(~0U)) +#define HNS_ROCE_DIP_F_OFS (HNS_ROCE_DIP_AI_OFS + HNS_ROCE_DIP_AI_SZ) +#define HNS_ROCE_DIP_F_SZ sizeof(u8) +#define HNS_ROCE_DIP_F_MAX ((u8)(~0U)) +#define HNS_ROCE_DIP_TKP_OFS (HNS_ROCE_DIP_F_OFS + HNS_ROCE_DIP_F_SZ) +#define HNS_ROCE_DIP_TKP_SZ sizeof(u8) +#define HNS_ROCE_DIP_TKP_MAX 10 +#define HNS_ROCE_DIP_TMP_OFS (HNS_ROCE_DIP_TKP_OFS + HNS_ROCE_DIP_TKP_SZ) +#define HNS_ROCE_DIP_TMP_SZ sizeof(u16) +#define HNS_ROCE_DIP_TMP_MAX 15 +#define HNS_ROCE_DIP_ALP_OFS (HNS_ROCE_DIP_TMP_OFS + HNS_ROCE_DIP_TMP_SZ) +#define HNS_ROCE_DIP_ALP_SZ sizeof(u16) +#define HNS_ROCE_DIP_ALP_MAX ((u16)(~0U)) +#define HNS_ROCE_DIP_MAX_SPEED_OFS (HNS_ROCE_DIP_ALP_OFS + HNS_ROCE_DIP_ALP_SZ) +#define HNS_ROCE_DIP_MAX_SPEED_SZ sizeof(u32) +#define HNS_ROCE_DIP_MAX_SPEED_MAX ((u32)(~0U)) +#define HNS_ROCE_DIP_G_OFS (HNS_ROCE_DIP_MAX_SPEED_OFS + \ + HNS_ROCE_DIP_MAX_SPEED_SZ) +#define HNS_ROCE_DIP_G_SZ sizeof(u8) +#define HNS_ROCE_DIP_G_MAX 15 +#define HNS_ROCE_DIP_AL_OFS (HNS_ROCE_DIP_G_OFS + HNS_ROCE_DIP_G_SZ) +#define HNS_ROCE_DIP_AL_SZ sizeof(u8) +#define HNS_ROCE_DIP_AL_MAX ((u8)(~0U)) +#define HNS_ROCE_DIP_CNP_TIME_OFS (HNS_ROCE_DIP_AL_OFS + HNS_ROCE_DIP_AL_SZ) +#define HNS_ROCE_DIP_CNP_TIME_SZ sizeof(u8) +#define HNS_ROCE_DIP_CNP_TIME_MAX ((u8)(~0U)) +#define HNS_ROCE_DIP_ASHIFT_OFS (HNS_ROCE_DIP_CNP_TIME_OFS + \ + HNS_ROCE_DIP_CNP_TIME_SZ) +#define HNS_ROCE_DIP_ASHIFT_SZ sizeof(u8) +#define HNS_ROCE_DIP_ASHIFT_MAX 15 + struct hns_roce_sccc_clr { __le32 qpn; __le32 rsv[5]; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 0dbe99aab6ad..c6f633bd5a34 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -221,6 +221,11 @@ static int hns_roce_query_device(struct ib_device *ib_dev, struct ib_udata *uhw) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + int ret; + + ret = ib_is_udata_in_empty(uhw); + if (ret) + return ret; memset(props, 0, sizeof(*props)); @@ -274,7 +279,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) props->device_cap_flags |= IB_DEVICE_XRC; - return 0; + return ib_respond_empty_udata(uhw); } static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num, @@ -477,8 +482,7 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, resp.cqe_size = hr_dev->caps.cqe_sz; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); + ret = ib_respond_udata(udata, resp); if (ret) goto error_fail_copy_to_udata; @@ -1113,7 +1117,7 @@ static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) unsigned long flags; spin_lock_irqsave(&hr_cq->lock, flags); - if (cq->comp_handler) { + if (cq->comp_handler && hr_cq->ib_cq.poll_ctx != IB_POLL_DIRECT) { if (!hr_cq->is_armed) { hr_cq->is_armed = 1; list_add_tail(&hr_cq->node, cq_list); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 25bfd3970f5b..7d41ae897458 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -173,7 +173,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx & (hr_dev->caps.num_mtpts - 1)); if (ret) { - dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + dev_err_ratelimited(dev, "failed to create mpt, ret = %d.\n", ret); goto err_page; } @@ -319,7 +319,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT, mtpt_idx); if (ret) - ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); + ibdev_warn_ratelimited(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); mr->enabled = 0; mr->iova = virt_addr; @@ -350,7 +350,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx); if (ret) { - ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret); + ibdev_err_ratelimited(ib_dev, "failed to create MPT, ret = %d.\n", ret); goto free_cmd_mbox; } @@ -595,8 +595,8 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, if (udata) { mtr->kmem = NULL; - mtr->umem = ib_umem_get(ibdev, user_addr, total_size, - buf_attr->user_access); + mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, + buf_attr->user_access); if (IS_ERR(mtr->umem)) { ibdev_err(ibdev, "failed to get umem, ret = %pe.\n", mtr->umem); diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index 225c3e328e0e..73bb000574c5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include #include "hns_roce_device.h" void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev) @@ -61,12 +62,9 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) if (udata) { struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd->pdn}; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); - if (ret) { + ret = ib_respond_udata(udata, resp); + if (ret) ida_free(&pd_ida->ida, id); - ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret); - } } return ret; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index bf04ee84a943..e333a8c4acb5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1236,12 +1236,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (udata) { resp.cap_flags = hr_qp->en_flags; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); - if (ret) { - ibdev_err(ibdev, "copy qp resp failed!\n"); + ret = ib_respond_udata(udata, resp); + if (ret) goto err_flow_ctrl; - } } if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { @@ -1494,11 +1491,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (udata && udata->outlen) { resp.tc_mode = hr_qp->tc_mode; resp.priority = hr_qp->sl; - ret = ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp))); - if (ret) - ibdev_err_ratelimited(&hr_dev->ib_dev, - "failed to copy modify qp resp.\n"); + ret = ib_respond_udata(udata, resp); } out: diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 8b94cbdfa54d..4a54394f96be 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -103,7 +103,7 @@ static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ, srq->srqn); if (ret) - ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, "failed to config SRQC, ret = %d.\n", ret); err_mbox: hns_roce_free_cmd_mailbox(hr_dev, mailbox); @@ -477,11 +477,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, if (udata) { resp.cap_flags = srq->cap_flags; resp.srqn = srq->srqn; - if (ib_copy_to_udata(udata, &resp, - min(udata->outlen, sizeof(resp)))) { - ret = -EFAULT; + ret = ib_respond_udata(udata, resp); + if (ret) goto err_srqc; - } } return 0; diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 7051a81cca94..9d91f7667d4f 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -110,8 +110,8 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (rc) goto err_qdesc; - cq->umem = ib_umem_get(&dev->ibdev, req_cq->addr, req_cq->size, - IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr, + req_cq->size, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { rc = PTR_ERR(cq->umem); goto err_qdesc; @@ -414,7 +414,7 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) if (dev->lif_cfg.rq_expdb) resp.expdb_qtypes |= IONIC_EXPDB_RQ; - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; @@ -752,7 +752,7 @@ int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, if (udata) { resp.ahid = ah->ahid; - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; } @@ -895,7 +895,7 @@ struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); - mr->umem = ib_umem_get(&dev->ibdev, start, length, access); + mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access); if (IS_ERR(mr->umem)) { rc = PTR_ERR(mr->umem); goto err_umem; @@ -1263,7 +1263,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (udata) { resp.udma_mask = vcq->udma_mask; - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; } @@ -1837,7 +1837,7 @@ static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, qp->sq_meta = NULL; qp->sq_msn_idx = NULL; - qp->sq_umem = ib_umem_get(&dev->ibdev, sq->addr, sq->size, 0); + qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0); if (IS_ERR(qp->sq_umem)) return PTR_ERR(qp->sq_umem); } else { @@ -2050,7 +2050,7 @@ static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, qp->rq_meta = NULL; - qp->rq_umem = ib_umem_get(&dev->ibdev, rq->addr, rq->size, 0); + qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0); if (IS_ERR(qp->rq_umem)) return PTR_ERR(qp->rq_umem); } else { @@ -2315,7 +2315,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, resp.rq_cmb = qp->rq_cmb; } - rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; } @@ -2535,6 +2535,23 @@ static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state, return false; } +static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask, + enum ib_qp_type type, enum ib_qp_state cur, + enum ib_qp_state next) +{ + if (!ext_mask) + return true; + + if (ext_mask & ~IB_QP_RATE_LIMIT) + return false; + + /* Rate limit is only supported for RC QPs during specific transitions */ + return type == IB_QPT_RC && + ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || + (cur == IB_QPS_RTR && next == IB_QPS_RTS) || + (cur == IB_QPS_RTS && next == IB_QPS_RTS)); +} + static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, int mask) { @@ -2547,7 +2564,9 @@ static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state)) return -EINVAL; - if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask)) + if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) || + !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS, + qp->ibqp.qp_type, cur_state, next_state)) return -EINVAL; /* unprivileged qp not allowed privileged qkey */ diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index 73a616ae3502..b0449c75f893 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -25,6 +25,11 @@ static int ionic_query_device(struct ib_device *ibdev, { struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); struct net_device *ndev; + int err; + + err = ib_is_udata_in_empty(udata); + if (err) + return err; ndev = ib_device_get_netdev(ibdev, 1); addrconf_ifid_eui48((u8 *)&attr->sys_image_guid, ndev); @@ -69,7 +74,7 @@ static int ionic_query_device(struct ib_device *ibdev, attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; attr->max_pkeys = IONIC_PKEY_TBL_LEN; - return 0; + return ib_respond_empty_udata(udata); } static int ionic_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index f9be467d137f..c345cc654256 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -235,8 +235,7 @@ static void irdma_complete_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { if (cqp_request->waiting) { - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } else if (cqp_request->callback_fcn) { cqp_request->callback_fcn(cqp_request); } @@ -1107,9 +1106,9 @@ static int irdma_create_cqp(struct irdma_pci_f *rf) INIT_LIST_HEAD(&cqp->cqp_avail_reqs); INIT_LIST_HEAD(&cqp->cqp_pending_reqs); - /* init the waitqueue of the cqp_requests and add them to the list */ + /* init the completion of the cqp_requests and add them to the list */ for (i = 0; i < sqsize; i++) { - init_waitqueue_head(&cqp->cqp_requests[i].waitq); + init_completion(&cqp->cqp_requests[i].comp); list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); } init_waitqueue_head(&cqp->remove_wq); diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h index 3d49bd57bae7..8c17a201c1fd 100644 --- a/drivers/infiniband/hw/irdma/main.h +++ b/drivers/infiniband/hw/irdma/main.h @@ -161,13 +161,12 @@ struct irdma_cqp_compl_info { struct irdma_cqp_request { struct cqp_cmds_info info; - wait_queue_head_t waitq; + struct completion comp; struct list_head list; refcount_t refcnt; void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; - bool request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; bool dynamic:1; bool pending:1; diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c index 4718acf6c6fd..a34883fe9983 100644 --- a/drivers/infiniband/hw/irdma/uk.c +++ b/drivers/infiniband/hw/irdma/uk.c @@ -1568,15 +1568,12 @@ static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = { * irdma_setup_connection_wqes - setup WQEs necessary to complete * connection. * @qp: hw qp (user and kernel) - * @info: qp initialization info */ -static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, - struct irdma_qp_uk_init_info *info) +static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp) { u16 move_cnt = 1; - if (!info->legacy_mode && - (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)) + if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) move_cnt = 3; qp->conn_wqes = move_cnt; @@ -1727,7 +1724,7 @@ int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) sq_ring_size = qp->sq_size << info->sq_shift; IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); if (info->first_sq_wq) { - irdma_setup_connection_wqes(qp, info); + irdma_setup_connection_wqes(qp); qp->swqe_polarity = 1; qp->first_sq_wq = true; } else { diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h index 008af1acc928..4dd3776a4cdd 100644 --- a/drivers/infiniband/hw/irdma/user.h +++ b/drivers/infiniband/hw/irdma/user.h @@ -563,7 +563,6 @@ struct irdma_qp_uk_init_info { u8 sq_shift; u8 rq_shift; int abi_ver; - bool legacy_mode; struct irdma_srq_uk *srq_uk; }; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 495e5daff4b4..e4037d5ef899 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -442,7 +442,7 @@ struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, if (cqp_request) { cqp_request->dynamic = true; if (wait) - init_waitqueue_head(&cqp_request->waitq); + init_completion(&cqp_request->comp); } } if (!cqp_request) { @@ -480,7 +480,7 @@ void irdma_free_cqp_request(struct irdma_cqp *cqp, if (cqp_request->dynamic) { kfree(cqp_request); } else { - WRITE_ONCE(cqp_request->request_done, false); + reinit_completion(&cqp_request->comp); cqp_request->callback_fcn = NULL; cqp_request->waiting = false; cqp_request->pending = false; @@ -515,8 +515,7 @@ irdma_free_pending_cqp_request(struct irdma_cqp *cqp, { if (cqp_request->waiting) { cqp_request->compl_info.error = true; - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } wait_event_timeout(cqp->remove_wq, refcount_read(&cqp_request->refcnt) == 1, 1000); @@ -609,9 +608,8 @@ static int irdma_wait_event(struct irdma_pci_f *rf, cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); - if (wait_event_timeout(cqp_request->waitq, - READ_ONCE(cqp_request->request_done), - msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) + if (wait_for_completion_timeout(&cqp_request->comp, + msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) break; if (cqp_request->pending) @@ -2442,7 +2440,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp) cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; sw_wqe = qp->sq_base[wqe_idx].elem; get_64bit_val(sw_wqe, 24, &wqe_qword); - cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); + cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; /* remove the SQ WR by moving SQ tail*/ IRDMA_RING_SET_TAIL(*sq_ring, diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 8cd427532805..cb54c7c8fcd8 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -16,9 +16,11 @@ static int irdma_query_device(struct ib_device *ibdev, struct irdma_pci_f *rf = iwdev->rf; struct pci_dev *pcidev = iwdev->rf->pcidev; struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; + int err; - if (udata->inlen || udata->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(udata); + if (err) + return err; memset(props, 0, sizeof(*props)); addrconf_addr_eui48((u8 *)&props->sys_image_guid, @@ -74,7 +76,7 @@ static int irdma_query_device(struct ib_device *ibdev, if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; - return 0; + return ib_respond_empty_udata(udata); } /** @@ -325,9 +327,9 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx, uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; uresp.kernel_ver = req.userspace_ver; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) - return -EFAULT; + ret = ib_respond_udata(udata, uresp); + if (ret) + return ret; } else { u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; @@ -354,10 +356,10 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx, uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta; uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) { + ret = ib_respond_udata(udata, uresp); + if (ret) { rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); - return -EFAULT; + return ret; } } @@ -420,11 +422,9 @@ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) ibucontext); irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); uresp.pd_id = pd_id; - if (ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen))) { - err = -EFAULT; + err = ib_respond_udata(udata, uresp); + if (err) goto error; - } } else { irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); } @@ -634,7 +634,6 @@ static int irdma_setup_umode_qp(struct ib_udata *udata, iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; iwqp->user_mode = 1; if (req.user_wqe_bufs) { - info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode; spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, &ucontext->qp_reg_mem_list); @@ -1124,10 +1123,8 @@ static int irdma_create_qp(struct ib_qp *ibqp, uresp.qp_id = qp_num; uresp.qp_caps = qp->qp_uk.qp_caps; - err_code = ib_copy_to_udata(udata, &uresp, - min(sizeof(uresp), udata->outlen)); + err_code = ib_respond_udata(udata, uresp); if (err_code) { - ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); irdma_destroy_qp(&iwqp->ibqp, udata); return err_code; } @@ -1612,12 +1609,9 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, uresp.push_valid = 1; uresp.push_offset = iwqp->sc_qp.push_offset; } - ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), - udata->outlen)); + ret = ib_respond_udata(udata, uresp); if (ret) { irdma_remove_push_mmap_entries(iwqp); - ibdev_dbg(&iwdev->ibdev, - "VERBS: copy_to_udata failed\n"); return ret; } } @@ -1860,12 +1854,9 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, uresp.push_offset = iwqp->sc_qp.push_offset; } - err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), - udata->outlen)); + err = ib_respond_udata(udata, uresp); if (err) { irdma_remove_push_mmap_entries(iwqp); - ibdev_dbg(&iwdev->ibdev, - "VERBS: copy_to_udata failed\n"); return err; } } @@ -2074,10 +2065,6 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - /* CQ resize not supported with legacy GEN_1 libi40iw */ - if (ucontext->legacy_mode) - return -EOPNOTSUPP; - if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return -EINVAL; @@ -2418,11 +2405,9 @@ static int irdma_create_srq(struct ib_srq *ibsrq, resp.srq_id = iwsrq->srq_num; resp.srq_size = ukinfo->srq_size; - if (ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen))) { - err_code = -EPROTO; + err_code = ib_respond_udata(udata, resp); + if (err_code) goto srq_destroy; - } } return 0; @@ -2559,7 +2544,7 @@ static int irdma_create_cq(struct ib_cq *ibcq, cqmr = &iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & - IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { + IRDMA_FEATURE_CQ_RESIZE) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl_shadow = irdma_get_pbl( (unsigned long)req.user_shadow_area, @@ -2572,7 +2557,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, } cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; - cqmr->split = true; } else { info.shadow_area_pa = cqmr->shadow; } @@ -2664,13 +2648,9 @@ static int irdma_create_cq(struct ib_cq *ibcq, resp.cq_id = info.cq_uk_init_info.cq_id; resp.cq_size = info.cq_uk_init_info.cq_size; - if (ib_copy_to_udata(udata, &resp, - min(sizeof(resp), udata->outlen))) { - ibdev_dbg(&iwdev->ibdev, - "VERBS: copy to user data\n"); - err_code = -EPROTO; + err_code = ib_respond_udata(udata, resp); + if (err_code) goto cq_destroy; - } } init_completion(&iwcq->free_cq); @@ -2781,10 +2761,11 @@ static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, * irdma_copy_user_pgaddrs - copy user page address to pble's os locally * @iwmr: iwmr for IB's user page addresses * @pbl: ple pointer to save 1 level or 0 level pble + * @pbl_len: Max number of PBL entries to populate * @level: indicated level 0, 1 or 2 */ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, - enum irdma_pble_level level) + u32 pbl_len, enum irdma_pble_level level) { struct ib_umem *region = iwmr->region; struct irdma_pbl *iwpbl = &iwmr->iwpbl; @@ -2792,7 +2773,9 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, struct irdma_pble_info *pinfo; struct ib_block_iter biter; u32 idx = 0; - u32 pbl_cnt = 0; + + if (!pbl_len) + return; pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; @@ -2801,7 +2784,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { *pbl = rdma_block_iter_dma_address(&biter); - if (++pbl_cnt == palloc->total_cnt) + if (!--pbl_len) break; pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); } @@ -2877,6 +2860,7 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, u64 *pbl; int status; enum irdma_pble_level level = PBLE_LEVEL_1; + u32 pbl_len; if (lvl) { status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, @@ -2884,16 +2868,18 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, if (status) return status; + pbl_len = palloc->total_cnt; iwpbl->pbl_allocated = true; level = palloc->level; pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf; pbl = pinfo->addr; } else { + pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR; pbl = iwmr->pgaddrmem; } - irdma_copy_user_pgaddrs(iwmr, pbl, level); + irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); if (lvl) iwmr->pgaddrmem[0] = *pbl; @@ -2974,7 +2960,8 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev, case IRDMA_MEMREG_TYPE_CQ: hmc_p = &cqmr->cq_pbl; - if (!cqmr->split) + if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE)) cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; if (lvl) @@ -3319,6 +3306,7 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, int err; lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; + iwmr->access = access; err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) @@ -3537,7 +3525,7 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) return ERR_PTR(-EINVAL); - region = ib_umem_get(pd->device, start, len, access); + region = ib_umem_get_va(pd->device, start, len, access); if (IS_ERR(region)) { ibdev_dbg(&iwdev->ibdev, @@ -3739,7 +3727,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, struct ib_umem *region; int err; - region = ib_umem_get(pd->device, start, len, iwmr->access); + region = ib_umem_get_va(pd->device, start, len, iwmr->access); if (IS_ERR(region)) return PTR_ERR(region); @@ -5334,7 +5322,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah, mutex_unlock(&iwdev->rf->ah_tbl_lock); uresp.ah_id = ah->sc_ah.ah_info.ah_idx; - err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); + err = ib_respond_udata(udata, uresp); if (err) irdma_destroy_ah(ibah, attr->flags); diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index aabbb3442098..289ebc9b23ca 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -65,7 +65,6 @@ struct irdma_hmc_pble { struct irdma_cq_mr { struct irdma_hmc_pble cq_pbl; dma_addr_t shadow; - bool split; }; struct irdma_srq_mr { diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 2d682428ef20..f2547989f422 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -79,11 +79,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (udata) { resp.cqid = cq->queue.id; - err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + err = ib_respond_udata(udata, resp); + if (err) goto err_remove_cq_cb; - } } spin_lock_init(&cq->cq_lock); diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 142047847f38..efe2935bda29 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -262,7 +262,7 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, queue->id = INVALID_QUEUE_ID; queue->gdma_region = GDMA_INVALID_DMA_REGION; - umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); return PTR_ERR(umem); @@ -573,6 +573,11 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, { struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)->dev); + int err; + + err = ib_is_udata_in_empty(uhw); + if (err) + return err; memset(props, 0, sizeof(*props)); props->vendor_id = pdev->vendor; @@ -600,7 +605,7 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, if (!mana_ib_is_rnic(dev)) props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM; - return 0; + return ib_respond_empty_udata(uhw); } int mana_ib_query_port(struct ib_device *ibdev, u32 port, @@ -624,8 +629,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } - props->active_width = IB_WIDTH_4X; - props->active_speed = IB_SPEED_EDR; + ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { props->gid_tbl_len = 16; diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c index 8092a7bb785b..030bfdcfff3c 100644 --- a/drivers/infiniband/hw/mana/mr.c +++ b/drivers/infiniband/hw/mana/mr.c @@ -127,7 +127,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, if (!mr) return ERR_PTR(-ENOMEM); - mr->umem = ib_umem_get(ibdev, start, length, access_flags); + mr->umem = ib_umem_get_va(ibdev, start, length, access_flags); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); ibdev_dbg(ibdev, diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index d3ee30b64f53..60926f39ab9d 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -243,13 +243,9 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, if (ret) goto free_vport; - ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); - if (ret) { - ibdev_dbg(&mdev->ib_dev, - "Failed to copy to udata create rss-qp, %d\n", - ret); + ret = ib_respond_udata(udata, resp); + if (ret) goto err_disable_vport_rx; - } kfree(mana_ind_table); @@ -398,13 +394,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, resp.cqid = send_cq->queue.id; resp.tx_vp_offset = pd->tx_vp_offset; - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); - if (err) { - ibdev_dbg(&mdev->ib_dev, - "Failed copy udata for create qp-raw, %d\n", - err); + err = ib_respond_udata(udata, resp); + if (err) goto err_remove_cq_cb; - } return 0; @@ -602,11 +594,9 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, resp.queue_id[j] = qp->rc_qp.queues[i].id; j++; } - err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + err = ib_respond_udata(udata, resp); + if (err) goto destroy_qp; - } } err = mana_table_store_qp(mdev, qp); diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c index c6e1b9e4122a..fc23960cf1fd 100644 --- a/drivers/infiniband/hw/mlx4/alias_GUID.c +++ b/drivers/infiniband/hw/mlx4/alias_GUID.c @@ -437,7 +437,7 @@ static void aliasguid_query_handler(int status, queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq, &dev->sriov.alias_guid.ports_guid[port_index]. alias_guid_work, - msecs_to_jiffies(resched_delay_sec * 1000)); + secs_to_jiffies(resched_delay_sec)); } if (cb_ctx->sa_query) { list_del(&cb_ctx->list); diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 7a6eb602d4a6..887912469742 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -142,6 +142,7 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, { struct ib_udata *udata = &attrs->driver_udata; struct ib_device *ibdev = ibcq->device; + struct mlx4_ib_create_cq_resp uresp = {}; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx4_ib_dev *dev = to_mdev(ibdev); @@ -172,32 +173,40 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, if (err) goto err_cq; - if (ibcq->umem && - (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT)) - return -EOPNOTSUPP; + cq->umem = ib_umem_get_cq_buf(&dev->ib_dev, attrs, entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->umem)) { + err = PTR_ERR(cq->umem); + goto err_cq; + } + if (cq->umem) { + if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT) { + err = -EOPNOTSUPP; + goto err_umem; + } + } else { + cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->umem)) { + err = PTR_ERR(cq->umem); + goto err_cq; + } + } buf_addr = (void *)(unsigned long)ucmd.buf_addr; - if (!ibcq->umem) - ibcq->umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(ibcq->umem)) { - err = PTR_ERR(ibcq->umem); - goto err_cq; - } - - shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->ibcq.umem, 0, &n); + shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->umem, 0, &n); if (shift < 0) { err = shift; - goto err_cq; + goto err_umem; } err = mlx4_mtt_init(dev->dev, n, shift, &cq->buf.mtt); if (err) - goto err_cq; + goto err_umem; - err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->ibcq.umem); + err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem); if (err) goto err_mtt; @@ -219,10 +228,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, cq->mcq.event = mlx4_ib_cq_event; cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS; - if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { - err = -EFAULT; + uresp.cqn = cq->mcq.cqn; + err = ib_respond_udata(udata, uresp); + if (err) goto err_cq_free; - } return 0; @@ -234,7 +243,9 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, err_mtt: mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); - /* UMEM is released by ib_core */ + +err_umem: + ib_umem_release(cq->umem); err_cq: return err; @@ -343,9 +354,9 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq if (!cq->resize_buf) return -ENOMEM; - cq->resize_umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { err = PTR_ERR(cq->resize_umem); goto err_buf; @@ -471,8 +482,8 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, unsigned int entries, if (ibcq->uobject) { cq->buf = cq->resize_buf->buf; cq->ibcq.cqe = cq->resize_buf->cqe; - ib_umem_release(cq->ibcq.umem); - cq->ibcq.umem = cq->resize_umem; + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; kfree(cq->resize_buf); cq->resize_buf = NULL; @@ -532,7 +543,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) struct mlx4_ib_ucontext, ibucontext), &mcq->db); - /* UMEM is released by ib_core */ + ib_umem_release(mcq->umem); } else { mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); mlx4_db_free(dev->dev, &mcq->db); diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c index 8ba86b1e4e46..22ae728834fc 100644 --- a/drivers/infiniband/hw/mlx4/doorbell.c +++ b/drivers/infiniband/hw/mlx4/doorbell.c @@ -64,8 +64,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt, page->user_virt = (virt & PAGE_MASK); page->refcnt = 0; - page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, - PAGE_SIZE, 0); + page->umem = ib_umem_get_va(context->ibucontext.device, + virt & PAGE_MASK, PAGE_SIZE, 0); if (IS_ERR(page->umem)) { err = PTR_ERR(page->umem); kfree(page); diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 464c9ab42516..17073e8f105a 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -444,8 +444,9 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, struct mlx4_uverbs_ex_query_device cmd; struct mlx4_uverbs_ex_query_device_resp resp = {}; struct mlx4_clock_params clock_params; + size_t uhw_outlen = uhw ? uhw->outlen : 0; - if (uhw->inlen) { + if (uhw && uhw->inlen) { err = ib_copy_validate_udata_in_cm(uhw, cmd, reserved, 0); if (err) return err; @@ -572,7 +573,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, props->cq_caps.max_cq_moderation_count = MLX4_MAX_CQ_COUNT; props->cq_caps.max_cq_moderation_period = MLX4_MAX_CQ_PERIOD; - if (uhw->outlen >= resp.response_length + sizeof(resp.hca_core_clock_offset)) { + if (uhw_outlen >= resp.response_length + sizeof(resp.hca_core_clock_offset)) { resp.response_length += sizeof(resp.hca_core_clock_offset); if (!mlx4_get_internal_clock_params(dev->dev, &clock_params)) { resp.comp_mask |= MLX4_IB_QUERY_DEV_RESP_MASK_CORE_CLOCK_OFFSET; @@ -580,14 +581,14 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, } } - if (uhw->outlen >= resp.response_length + + if (uhw_outlen >= resp.response_length + sizeof(resp.max_inl_recv_sz)) { resp.response_length += sizeof(resp.max_inl_recv_sz); resp.max_inl_recv_sz = dev->dev->caps.max_rq_sg * sizeof(struct mlx4_wqe_data_seg); } - if (offsetofend(typeof(resp), rss_caps) <= uhw->outlen) { + if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) { if (props->rss_caps.supported_qpts) { resp.rss_caps.rx_hash_function = MLX4_IB_RX_HASH_FUNC_TOEPLITZ; @@ -611,7 +612,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, sizeof(resp.rss_caps); } - if (offsetofend(typeof(resp), tso_caps) <= uhw->outlen) { + if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) { if (dev->dev->caps.max_gso_sz && ((mlx4_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) || @@ -625,8 +626,8 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, sizeof(resp.tso_caps); } - if (uhw->outlen) { - err = ib_copy_to_udata(uhw, &resp, resp.response_length); + if (uhw_outlen) { + err = ib_respond_udata(uhw, resp); if (err) goto out; } @@ -1090,8 +1091,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, struct ib_device *ibdev = uctx->device; struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_ucontext *context = to_mucontext(uctx); - struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3; - struct mlx4_ib_alloc_ucontext_resp resp; + struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3 = {}; + struct mlx4_ib_alloc_ucontext_resp resp = {}; int err; if (!dev->ib_active) @@ -1121,16 +1122,16 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, mutex_init(&context->wqn_ranges_mutex); if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) - err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); + err = ib_respond_udata(udata, resp_v3); else - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); + err = ib_respond_udata(udata, resp); if (err) { mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar); - return -EFAULT; + return err; } - return err; + return 0; } static void mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) @@ -1199,9 +1200,14 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) if (err) return err; - if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) { - mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); - return -EFAULT; + if (udata) { + struct mlx4_ib_alloc_pd_resp uresp = { .pdn = pd->pdn }; + + err = ib_respond_udata(udata, uresp); + if (err) { + mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn); + return err; + } } return 0; } @@ -1696,9 +1702,9 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp, (flow_attr->type != IB_FLOW_ATTR_NORMAL)) return ERR_PTR(-EOPNOTSUPP); - if (udata && - udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen)) - return ERR_PTR(-EOPNOTSUPP); + err = ib_is_udata_in_empty(udata); + if (err) + return ERR_PTR(err); memset(type, 0, sizeof(type)); diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 5a799d6df93e..2b6cc011b25d 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -121,6 +121,7 @@ struct mlx4_ib_cq { struct mlx4_db db; spinlock_t lock; struct mutex resize_mutex; + struct ib_umem *umem; struct ib_umem *resize_umem; /* List of qps that it serves.*/ struct list_head send_qp_list; @@ -135,6 +136,7 @@ struct mlx4_ib_mr { dma_addr_t page_map; u32 npages; u32 max_pages; + int access_flags; struct mlx4_mr mmr; struct ib_umem *umem; size_t page_map_size; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 6747bca30677..761e2c05dd0f 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -110,7 +110,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start, /* * Force registering the memory as writable if the underlying pages * are writable. This is so rereg can change the access permissions - * from readable to writable without having to run through ib_umem_get + * from readable to writable without having to run through ib_umem_get_va * again */ if (!ib_access_writable(access_flags)) { @@ -135,7 +135,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start, mmap_read_unlock(current->mm); } - return ib_umem_get(device, start, length, access_flags); + return ib_umem_get_va(device, start, length, access_flags); } struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, @@ -181,6 +181,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_mr; + mr->access_flags = access_flags; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; mr->ibmr.page_size = 1U << shift; @@ -241,6 +242,8 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, if (err) goto release_mpt_entry; + } else { + mr_access_flags = mmr->access_flags; } if (flags & IB_MR_REREG_TRANS) { @@ -282,8 +285,10 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, * return a failure. But dereg_mr will free the resources. */ err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry); - if (!err && flags & IB_MR_REREG_ACCESS) - mmr->mmr.access = mr_access_flags; + if (!err && flags & IB_MR_REREG_ACCESS) { + mmr->access_flags = mr_access_flags; + mmr->mmr.access = convert_access(mr_access_flags); + } release_mpt_entry: mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry); diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 790be09d985a..effc6bcceb76 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -897,7 +897,7 @@ static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + (qp->sq.wqe_cnt << qp->sq.wqe_shift); - qp->umem = ib_umem_get(pd->device, wq.buf_addr, qp->buf_size, 0); + qp->umem = ib_umem_get_va(pd->device, wq.buf_addr, qp->buf_size, 0); if (IS_ERR(qp->umem)) { err = PTR_ERR(qp->umem); goto err; @@ -1080,7 +1080,7 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, goto err; qp->umem = - ib_umem_get(pd->device, ucmd.buf_addr, qp->buf_size, 0); + ib_umem_get_va(pd->device, ucmd.buf_addr, qp->buf_size, 0); if (IS_ERR(qp->umem)) { err = PTR_ERR(qp->umem); goto err; @@ -4297,10 +4297,9 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table, size_t min_resp_len; int i, err = 0; - if (udata->inlen > 0 && - !ib_is_udata_cleared(udata, 0, - udata->inlen)) - return -EOPNOTSUPP; + err = ib_is_udata_in_empty(udata); + if (err) + return err; min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); if (udata->outlen && udata->outlen < min_resp_len) @@ -4332,7 +4331,7 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table, if (udata->outlen) { resp.response_length = offsetof(typeof(resp), response_length) + sizeof(resp.response_length); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); } return err; diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index 767840736d58..7bcb7e225662 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -116,7 +116,7 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, return err; srq->umem = - ib_umem_get(ib_srq->device, ucmd.buf_addr, buf_size, 0); + ib_umem_get_va(ib_srq->device, ucmd.buf_addr, buf_size, 0); if (IS_ERR(srq->umem)) return PTR_ERR(srq->umem); @@ -191,11 +191,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, srq->msrq.event = mlx4_ib_srq_event; srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; - if (udata) - if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { - err = -EFAULT; + if (udata) { + struct mlx4_ib_create_srq_resp uresp = { + .srqn = srq->msrq.srqn + }; + + err = ib_respond_udata(udata, uresp); + if (err) goto err_srq; - } + } init_attr->attr.max_wr = srq->msrq.max - 1; diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c index b8fa4ecfc961..e688ad66a895 100644 --- a/drivers/infiniband/hw/mlx4/sysfs.c +++ b/drivers/infiniband/hw/mlx4/sysfs.c @@ -636,12 +636,6 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) p->port_num = port_num; p->slave = slave; - ret = kobject_init_and_add(&p->kobj, &port_type, - kobject_get(dev->dev_ports_parent[slave]), - "%d", port_num); - if (ret) - goto err_alloc; - p->pkey_group.name = "pkey_idx"; p->pkey_group.attrs = alloc_group_attrs(show_port_pkey, @@ -649,13 +643,9 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) dev->dev->caps.pkey_table_len[port_num]); if (!p->pkey_group.attrs) { ret = -ENOMEM; - goto err_alloc; + goto err_free_port; } - ret = sysfs_create_group(&p->kobj, &p->pkey_group); - if (ret) - goto err_free_pkey; - p->gid_group.name = "gid_idx"; p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); if (!p->gid_group.attrs) { @@ -663,28 +653,47 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) goto err_free_pkey; } + ret = kobject_init_and_add(&p->kobj, &port_type, + kobject_get(dev->dev_ports_parent[slave]), + "%d", port_num); + if (ret) + goto err_put; + + ret = sysfs_create_group(&p->kobj, &p->pkey_group); + if (ret) + goto err_del; + ret = sysfs_create_group(&p->kobj, &p->gid_group); if (ret) - goto err_free_gid; + goto err_remove_pkey; ret = add_vf_smi_entries(p); if (ret) - goto err_free_gid; + goto err_remove_gid; list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); return 0; -err_free_gid: - kfree(p->gid_group.attrs[0]); - kfree(p->gid_group.attrs); +err_remove_gid: + sysfs_remove_group(&p->kobj, &p->gid_group); + +err_remove_pkey: + sysfs_remove_group(&p->kobj, &p->pkey_group); + +err_del: + kobject_del(&p->kobj); + +err_put: + kobject_put(dev->dev_ports_parent[slave]); + kobject_put(&p->kobj); + return ret; err_free_pkey: for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) kfree(p->pkey_group.attrs[i]); kfree(p->pkey_group.attrs); -err_alloc: - kobject_put(dev->dev_ports_parent[slave]); +err_free_port: kfree(p); return ret; } diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c index 531a57f9ee7e..a3aa700d0835 100644 --- a/drivers/infiniband/hw/mlx5/ah.c +++ b/drivers/infiniband/hw/mlx5/ah.c @@ -121,7 +121,7 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, resp.response_length = min_resp_len; memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) return err; } diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index a76b7a36087d..49b4bf148a4a 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -169,7 +169,8 @@ enum { static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, struct mlx5_ib_qp *qp) { - enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1); + enum rdma_link_layer ll = + rdma_port_get_link_layer(qp->ibqp.device, qp->port); struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); struct mlx5_ib_srq *srq = NULL; struct mlx5_ib_wq *wq; @@ -745,31 +746,33 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, *cqe_size = ucmd.cqe_size; - if (!cq->ibcq.umem) - cq->ibcq.umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - entries * ucmd.cqe_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(cq->ibcq.umem)) - return PTR_ERR(cq->ibcq.umem); + cq->buf.umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, attrs, + ucmd.buf_addr, + entries * ucmd.cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->buf.umem)) + return PTR_ERR(cq->buf.umem); page_size = mlx5_umem_find_best_cq_quantized_pgoff( - cq->ibcq.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT, + cq->buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64, &page_offset_quantized); if (!page_size) { err = -EINVAL; goto err_umem; } - err = mlx5_ib_db_map_user(context, ucmd.db_addr, &cq->db); + err = mlx5_ib_db_map_user(context, attrs, + MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, + ucmd.db_addr, &cq->db); if (err) goto err_umem; - ncont = ib_umem_num_dma_blocks(cq->ibcq.umem, page_size); + ncont = ib_umem_num_dma_blocks(cq->buf.umem, page_size); mlx5_ib_dbg( dev, "addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n", ucmd.buf_addr, entries * ucmd.cqe_size, - ib_umem_num_pages(cq->ibcq.umem), page_size, ncont); + ib_umem_num_pages(cq->buf.umem), page_size, ncont); *inlen = MLX5_ST_SZ_BYTES(create_cq_in) + MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont; @@ -780,7 +783,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, } pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas); - mlx5_ib_populate_pas(cq->ibcq.umem, page_size, pas, 0); + mlx5_ib_populate_pas(cq->buf.umem, page_size, pas, 0); cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context); MLX5_SET(cqc, cqc, log_page_size, @@ -853,7 +856,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, mlx5_ib_db_unmap_user(context, &cq->db); err_umem: - /* UMEM is released by ib_core */ + ib_umem_release(cq->buf.umem); return err; } @@ -863,6 +866,7 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata) udata, struct mlx5_ib_ucontext, ibucontext); mlx5_ib_db_unmap_user(context, &cq->db); + ib_umem_release(cq->buf.umem); } static void init_cq_frag_buf(struct mlx5_ib_cq_buf *buf) @@ -949,6 +953,7 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq, { struct ib_udata *udata = &attrs->driver_udata; struct ib_device *ibdev = ibcq->device; + struct mlx5_ib_create_cq_resp uresp = {}; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx5_ib_dev *dev = to_mdev(ibdev); @@ -1015,10 +1020,10 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq, INIT_LIST_HEAD(&cq->wc_list); - if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { - err = -EFAULT; + uresp.cqn = cq->mcq.cqn; + err = ib_respond_udata(udata, uresp); + if (err) goto err_cmd; - } kvfree(cqb); return 0; @@ -1240,9 +1245,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1) return -EINVAL; - umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, - (size_t)ucmd.cqe_size * entries, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, + (size_t)ucmd.cqe_size * entries, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { err = PTR_ERR(umem); return err; @@ -1434,8 +1439,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, unsigned int entries, if (udata) { cq->ibcq.cqe = entries - 1; - ib_umem_release(cq->ibcq.umem); - cq->ibcq.umem = cq->resize_umem; + ib_umem_release(cq->buf.umem); + cq->buf.umem = cq->resize_umem; cq->resize_umem = NULL; } else { struct mlx5_ib_cq_buf tbuf; @@ -1518,7 +1523,9 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE( UVERBS_ATTR_PTR_IN( MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX, UVERBS_ATTR_TYPE(u32), - UA_OPTIONAL)); + UA_OPTIONAL), + UVERBS_ATTR_UMEM(MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, + UA_OPTIONAL)); const struct uapi_definition mlx5_ib_create_cq_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_CQ, &mlx5_ib_cq_create), diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 8e89dbe40c23..d57484245c38 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -88,7 +88,7 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) return -ENOMEM; reg->ibdev = ibdev; - strcpy(reg->vuid, vuid); + strscpy(reg->vuid, vuid); mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 645ebcc0832d..fc6f793dcc65 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -1913,6 +1913,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( return err; } +static bool devx_key_in_sub_list(struct list_head *list, u32 key_level1) +{ + struct devx_event_subscription *s; + + list_for_each_entry(s, list, event_list) + if (s->xa_key_level1 == key_level1) + return true; + + return false; +} + static void subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table, u32 key_level1, @@ -2160,10 +2171,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( event_sub = kzalloc_obj(*event_sub); if (!event_sub) { + if (!devx_key_in_sub_list(&sub_list, key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + key_level1, + obj, + obj_id); err = -ENOMEM; goto err; } + event_sub->ev_file = ev_file; + event_sub->xa_key_level1 = key_level1; list_add_tail(&event_sub->event_list, &sub_list); uverbs_uobject_get(&ev_file->uobj); if (use_eventfd) { @@ -2178,9 +2196,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( } event_sub->cookie = cookie; - event_sub->ev_file = ev_file; - /* May be needed upon cleanup the devx object/subscription */ - event_sub->xa_key_level1 = key_level1; event_sub->xa_key_level2 = obj_id; INIT_LIST_HEAD(&event_sub->obj_list); } @@ -2225,10 +2240,11 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { list_del(&event_sub->event_list); - subscribe_event_xa_dealloc(devx_event_table, - event_sub->xa_key_level1, - obj, - obj_id); + if (!devx_key_in_sub_list(&sub_list, event_sub->xa_key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + event_sub->xa_key_level1, + obj, + obj_id); if (event_sub->eventfd) eventfd_ctx_put(event_sub->eventfd); @@ -2271,7 +2287,7 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, return PTR_ERR(umem_dmabuf); obj->umem = &umem_dmabuf->umem; } else { - obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags); + obj->umem = ib_umem_get_va(&dev->ib_dev, addr, size, access_flags); if (IS_ERR(obj->umem)) return PTR_ERR(obj->umem); } @@ -2512,7 +2528,7 @@ static int deliver_event(struct devx_event_subscription *event_sub, const void *data) { struct devx_async_event_file *ev_file; - struct devx_async_event_data *event_data; + struct devx_async_event_data *event_data, *to_free; unsigned long flags; ev_file = event_sub->ev_file; @@ -2543,12 +2559,17 @@ static int deliver_event(struct devx_event_subscription *event_sub, event_data->hdr.cookie = event_sub->cookie; memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe)); + to_free = NULL; + spin_lock_irqsave(&ev_file->lock, flags); if (!ev_file->is_destroyed) list_add_tail(&event_data->list, &ev_file->event_list); else - kfree(event_data); + to_free = event_data; spin_unlock_irqrestore(&ev_file->lock, flags); + + kfree(to_free); + wake_up_interruptible(&ev_file->poll_wait); return 0; @@ -2942,6 +2963,7 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, uobj); struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; struct devx_async_data *entry, *tmp; + LIST_HEAD(tmp_list); spin_lock_irq(&ev_queue->lock); ev_queue->is_destroyed = 1; @@ -2951,12 +2973,15 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); spin_lock_irq(&comp_ev_file->ev_queue.lock); - list_for_each_entry_safe(entry, tmp, - &comp_ev_file->ev_queue.event_list, list) { + /* Move all entries to a temporary list and free them outside lock */ + list_splice_init(&comp_ev_file->ev_queue.event_list, &tmp_list); + spin_unlock_irq(&comp_ev_file->ev_queue.lock); + + /* Free memory outside of critical section */ + list_for_each_entry_safe(entry, tmp, &tmp_list, list) { list_del(&entry->list); kvfree(entry); } - spin_unlock_irq(&comp_ev_file->ev_queue.lock); }; static void devx_async_event_destroy_uobj(struct ib_uobject *uobj, @@ -2966,7 +2991,9 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj, container_of(uobj, struct devx_async_event_file, uobj); struct devx_event_subscription *event_sub, *event_sub_tmp; + struct devx_async_event_data *entry, *tmp; struct mlx5_ib_dev *dev = ev_file->dev; + LIST_HEAD(tmp_list); spin_lock_irq(&ev_file->lock); ev_file->is_destroyed = 1; @@ -2980,18 +3007,19 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj, list_del_init(&event_sub->event_list); } else { - struct devx_async_event_data *entry, *tmp; - - list_for_each_entry_safe(entry, tmp, &ev_file->event_list, - list) { - list_del(&entry->list); - kfree(entry); - } + /* Move all entries to a temporary list */ + list_splice_init(&ev_file->event_list, &tmp_list); } spin_unlock_irq(&ev_file->lock); wake_up_interruptible(&ev_file->poll_wait); + /* Free event data outside of critical section */ + list_for_each_entry_safe(entry, tmp, &tmp_list, list) { + list_del(&entry->list); + kfree(entry); + } + mutex_lock(&dev->devx_event_table.event_xa_lock); /* delete the subscriptions which are related to this FD */ list_for_each_entry_safe(event_sub, event_sub_tmp, diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c index bd68fcf011f4..3108894534a8 100644 --- a/drivers/infiniband/hw/mlx5/doorbell.c +++ b/drivers/infiniband/hw/mlx5/doorbell.c @@ -37,50 +37,95 @@ #include "mlx5_ib.h" +#define MLX5_IB_DBR_SIZE (sizeof(__be32) * 2) + struct mlx5_ib_user_db_page { struct list_head list; struct ib_umem *umem; - unsigned long user_virt; + struct ib_uverbs_buffer_desc desc; int refcnt; struct mm_struct *mm; }; -int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, - struct mlx5_db *db) +static int mlx5_ib_db_map_user_desc(struct mlx5_ib_ucontext *context, + const struct ib_uverbs_buffer_desc *desc, + struct mlx5_db *db) { struct mlx5_ib_user_db_page *page; + struct ib_umem *umem; int err = 0; + if (desc->length < MLX5_IB_DBR_SIZE) + return -EINVAL; + /* + * For VA descriptors the umem is normalized to a single PAGE_SIZE + * region, so reject offsets that would place the 8-byte DBR + * straddling the page boundary. + */ + if (desc->type == IB_UVERBS_BUFFER_TYPE_VA && + (desc->addr & ~PAGE_MASK) > PAGE_SIZE - MLX5_IB_DBR_SIZE) + return -EINVAL; + mutex_lock(&context->db_page_mutex); - list_for_each_entry(page, &context->db_page_list, list) - if ((current->mm == page->mm) && - (page->user_virt == (virt & PAGE_MASK))) - goto found; + /* + * Only VA-typed descriptors are eligible to share a per-page + * doorbell umem; FD-typed descriptors are pinned individually. + */ + if (desc->type == IB_UVERBS_BUFFER_TYPE_VA) { + list_for_each_entry(page, &context->db_page_list, list) { + if (current->mm != page->mm) + continue; + if (page->desc.addr == (desc->addr & PAGE_MASK)) + goto found; + } + } - page = kmalloc_obj(*page); + page = kzalloc_obj(*page); if (!page) { err = -ENOMEM; goto out; } - page->user_virt = (virt & PAGE_MASK); - page->refcnt = 0; - page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK, - PAGE_SIZE, 0); - if (IS_ERR(page->umem)) { - err = PTR_ERR(page->umem); + page->desc = *desc; + + /* + * Normalize VA descriptors to a page-aligned PAGE_SIZE region so + * multiple DBRs that fall in the same user page share one umem. + */ + if (page->desc.type == IB_UVERBS_BUFFER_TYPE_VA) { + page->desc.addr &= PAGE_MASK; + page->desc.length = PAGE_SIZE; + } + + umem = ib_umem_get_desc(context->ibucontext.device, &page->desc, 0); + if (IS_ERR(umem)) { + err = PTR_ERR(umem); kfree(page); goto out; } - mmgrab(current->mm); - page->mm = current->mm; + /* + * The 8-byte DBR is programmed to the device as one DMA address, + * so it must live in a single contiguous DMA segment. + */ + if (!ib_umem_is_contiguous(umem)) { + ib_umem_release(umem); + kfree(page); + err = -EINVAL; + goto out; + } + + page->umem = umem; + if (page->desc.type == IB_UVERBS_BUFFER_TYPE_VA) { + mmgrab(current->mm); + page->mm = current->mm; + } list_add(&page->list, &context->db_page_list); found: db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + - (virt & ~PAGE_MASK); + (desc->addr & ~PAGE_MASK); db->u.user_page = page; ++page->refcnt; @@ -90,13 +135,35 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, return err; } +int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + unsigned long virt, struct mlx5_db *db) +{ + struct ib_uverbs_buffer_desc desc = { + .type = IB_UVERBS_BUFFER_TYPE_VA, + .addr = virt, + .length = MLX5_IB_DBR_SIZE, + }; + + if (attrs) { + int err; + + err = uverbs_get_buffer_desc(attrs, attr_id, &desc); + if (err && err != -ENOENT) + return err; + } + + return mlx5_ib_db_map_user_desc(context, &desc, db); +} + void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db) { mutex_lock(&context->db_page_mutex); if (!--db->u.user_page->refcnt) { list_del(&db->u.user_page->list); - mmdrop(db->u.user_page->mm); + if (db->u.user_page->mm) + mmdrop(db->u.user_page->mm); ib_umem_release(db->u.user_page->umem); kfree(db->u.user_page); } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 61078281953d..02809114fc79 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -965,8 +965,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, resp.response_length = resp_len; - if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen)) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; memset(props, 0, sizeof(*props)); err = mlx5_query_system_image_guid(ibdev, @@ -1213,20 +1214,29 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } } - if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen && - raw_support) { - if (MLX5_CAP_QOS(mdev, packet_pacing) && - MLX5_CAP_GEN(mdev, qos)) { - resp.packet_pacing_caps.qp_rate_limit_max = - MLX5_CAP_QOS(mdev, packet_pacing_max_rate); - resp.packet_pacing_caps.qp_rate_limit_min = - MLX5_CAP_QOS(mdev, packet_pacing_min_rate); - resp.packet_pacing_caps.supported_qpts |= - 1 << IB_QPT_RAW_PACKET; - if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) && - MLX5_CAP_QOS(mdev, packet_pacing_typical_size)) - resp.packet_pacing_caps.cap_flags |= - MLX5_IB_PP_SUPPORT_BURST; + if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen) { + if (MLX5_CAP_GEN(mdev, qos)) { + if (MLX5_CAP_QOS(mdev, packet_pacing) && raw_support) + resp.packet_pacing_caps.supported_qpts |= + BIT(IB_QPT_RAW_PACKET); + if (MLX5_CAP_QOS(mdev, packet_pacing_req_ud)) + resp.packet_pacing_caps.supported_qpts |= + BIT(IB_QPT_UD); + if (MLX5_CAP_QOS(mdev, packet_pacing_req_uc)) + resp.packet_pacing_caps.supported_qpts |= + BIT(IB_QPT_UC); + + if (resp.packet_pacing_caps.supported_qpts) { + resp.packet_pacing_caps.qp_rate_limit_max = + MLX5_CAP_QOS(mdev, packet_pacing_max_rate); + resp.packet_pacing_caps.qp_rate_limit_min = + MLX5_CAP_QOS(mdev, packet_pacing_min_rate); + + if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) && + MLX5_CAP_QOS(mdev, packet_pacing_typical_size)) + resp.packet_pacing_caps.cap_flags |= + MLX5_IB_PP_SUPPORT_BURST; + } } resp.response_length += sizeof(resp.packet_pacing_caps); } @@ -1355,7 +1365,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } if (uhw_outlen) { - err = ib_copy_to_udata(uhw, &resp, resp.response_length); + err = ib_respond_udata(uhw, resp); if (err) return err; @@ -2023,6 +2033,9 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) dev->lb.qps == 1) { if (!dev->lb.enabled) { err = mlx5_nic_vport_update_local_lb(dev->mdev, true); + if (err) + goto err_rollback; + dev->lb.enabled = true; } } @@ -2030,6 +2043,14 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) mutex_unlock(&dev->lb.mutex); return err; + +err_rollback: + if (td) + dev->lb.user_td--; + if (qp) + dev->lb.qps--; + mutex_unlock(&dev->lb.mutex); + return err; } void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) @@ -2069,9 +2090,13 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn, if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) || (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) && !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - return err; + return 0; - return mlx5_ib_enable_lb(dev, true, false); + err = mlx5_ib_enable_lb(dev, true, false); + if (err) + mlx5_cmd_dealloc_transport_domain(dev->mdev, *tdn, uid); + + return err; } static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, @@ -2280,7 +2305,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, goto out_mdev; resp.response_length = min(udata->outlen, sizeof(resp)); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto out_mdev; @@ -2658,7 +2683,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd, if (!dyn_uar) return err; - mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid); + mlx5_cmd_uar_dealloc(dev->mdev, uar_index, context->devx_uid); free_bfreg: mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx); @@ -2772,7 +2797,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct mlx5_ib_pd *pd = to_mpd(ibpd); struct ib_device *ibdev = ibpd->device; - struct mlx5_ib_alloc_pd_resp resp; + struct mlx5_ib_alloc_pd_resp resp = {}; int err; u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; @@ -2791,9 +2816,10 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) pd->uid = uid; if (udata) { resp.pdn = pd->pdn; - if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { + err = ib_respond_udata(udata, resp); + if (err) { mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid); - return -EFAULT; + return err; } } @@ -4450,6 +4476,7 @@ static const struct uapi_definition mlx5_ib_defs[] = { UAPI_DEF_CHAIN(mlx5_ib_std_types_defs), UAPI_DEF_CHAIN(mlx5_ib_dm_defs), UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs), + UAPI_DEF_CHAIN(mlx5_ib_create_qp_defs), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr), @@ -4488,6 +4515,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) dev->port[i].roce.last_port_state = IB_PORT_DOWN; } + mutex_init(&dev->lb.mutex); + err = mlx5r_cmd_query_special_mkeys(dev); if (err) return err; @@ -4788,11 +4817,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) if (err) return err; - if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && - (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) || - MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - mutex_init(&dev->lb.mutex); - if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { err = mlx5_ib_init_var_region(dev); @@ -5112,7 +5136,7 @@ int __mlx5_ib_add(struct mlx5_ib_dev *dev, if (profile->stage[i].cleanup) profile->stage[i].cleanup(dev); } - return -ENOMEM; + return err; } static const struct mlx5_ib_profile pf_profile = { diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e156dc4d7529..522984d958bb 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -331,6 +331,10 @@ struct mlx5_ib_flow_db { #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4 #define MLX5_IB_WR_UMR IB_WR_RESERVED1 +/* + * A valid pdn is required when flags include MLX5_IB_UPD_XLT_ENABLE, + * MLX5_IB_UPD_XLT_PD or MLX5_IB_UPD_XLT_ACCESS. + */ #define MLX5_IB_UPD_XLT_ZAP BIT(0) #define MLX5_IB_UPD_XLT_ENABLE BIT(1) #define MLX5_IB_UPD_XLT_ATOMIC BIT(2) @@ -536,6 +540,7 @@ struct mlx5_ib_qp { struct list_head cq_recv_list; struct list_head cq_send_list; struct mlx5_rate_limit rl; + struct mlx5_rate_limit rl_desired; u32 underlay_qpn; u32 flags_en; /* @@ -1259,8 +1264,9 @@ to_mmmap(struct rdma_user_mmap_entry *rdma_entry) int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev); int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev); -int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, - struct mlx5_db *db); +int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + unsigned long virt, struct mlx5_db *db); void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db); void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq); void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq); @@ -1413,8 +1419,8 @@ int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, u32 flags, struct ib_sge *sg_list, u32 num_sge); -int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr); -int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr); +int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd); +int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd); #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; } static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, @@ -1442,11 +1448,11 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, { return -EOPNOTSUPP; } -static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr) +static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { return -EOPNOTSUPP; } -static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr) +static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { return -EOPNOTSUPP; } @@ -1510,6 +1516,7 @@ extern const struct uapi_definition mlx5_ib_flow_defs[]; extern const struct uapi_definition mlx5_ib_qos_defs[]; extern const struct uapi_definition mlx5_ib_std_types_defs[]; extern const struct uapi_definition mlx5_ib_create_cq_defs[]; +extern const struct uapi_definition mlx5_ib_create_qp_defs[]; static inline int is_qp1(enum ib_qp_type qp_type) { diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index fb40b44496f4..e6b74955d95d 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -31,6 +31,7 @@ * SOFTWARE. */ +#include #include #include #include @@ -163,9 +164,8 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, #define MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED \ MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT 16 -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK 0xFF0000 -#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK 0xFFFF +#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16) +#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0) static struct mlx5_ib_mr * _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, @@ -194,7 +194,8 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, ph ^= MLX5_IB_NO_PH; mr->ibmr.frmr.key.kernel_vendor_key = - st_index | (ph << MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT); + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) | + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph); err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr); if (err) { kfree(mr); @@ -271,9 +272,10 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, get_mkc_octo_size(access_mode, key->num_dma_blocks)); MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - st_index = key->kernel_vendor_key & - MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK; - ph = key->kernel_vendor_key & MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK; + st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, + key->kernel_vendor_key); + ph = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, + key->kernel_vendor_key); if (ph) { /* Normalize ph: swap MLX5_IB_NO_PH for 0 */ if (ph == MLX5_IB_NO_PH) @@ -294,7 +296,7 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, free_in: kfree(in); if (err) - for (; i > 0; i--) + for (i--; i >= 0; i--) mlx5_core_destroy_mkey(dev->mdev, handles[i]); return err; } @@ -781,7 +783,8 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, * configured properly but left disabled. It is safe to go ahead * and configure it again via UMR while enabling it. */ - err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); + err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE, + to_mpd(pd)->pdn); if (err) { mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); @@ -841,7 +844,7 @@ static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_dereg_mr; - err = mlx5_ib_init_odp_mr(mr); + err = mlx5_ib_init_odp_mr(mr, pd); if (err) goto err_dereg_mr; return &mr->ibmr; @@ -874,7 +877,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (access_flags & IB_ACCESS_ON_DEMAND) return create_user_odp_mr(pd, start, length, iova, access_flags, udata); - umem = ib_umem_get(&dev->ib_dev, start, length, access_flags); + umem = ib_umem_get_va(&dev->ib_dev, start, length, access_flags); if (IS_ERR(umem)) return ERR_CAST(umem); return create_real_mr(pd, umem, iova, access_flags, dmah); @@ -890,7 +893,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) if (!umem_dmabuf->sgt || !mr) return; - mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP); + /* MLX5_IB_UPD_XLT_ZAP does not change the pdn */ + mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP, 0); ib_umem_dmabuf_unmap_pages(umem_dmabuf); } @@ -956,6 +960,10 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); umem_dmabuf->private = mr; if (!pinned_mode) { + err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); + if (err) + goto err_dereg_mr; + err = mlx5r_store_odp_mkey(dev, &mr->mmkey); if (err) goto err_dereg_mr; @@ -963,7 +971,7 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, mr->data_direct = true; } - err = mlx5_ib_init_dmabuf_mr(mr); + err = mlx5_ib_init_dmabuf_mr(mr, pd); if (err) goto err_dereg_mr; return &mr->ibmr; @@ -1128,10 +1136,8 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, if (err) return err; - if (flags & IB_MR_REREG_PD) { - mr->ibmr.pd = pd; + if (flags & IB_MR_REREG_PD) upd_flags |= MLX5_IB_UPD_XLT_PD; - } if (flags & IB_MR_REREG_ACCESS) { mr->access_flags = access_flags; upd_flags |= MLX5_IB_UPD_XLT_ACCESS; @@ -1141,7 +1147,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, mr->ibmr.length = new_umem->length; mr->page_shift = order_base_2(page_size); mr->umem = new_umem; - err = mlx5r_umr_update_mr_pas(mr, upd_flags); + err = mlx5r_umr_update_mr_pas(mr, upd_flags, to_mpd(pd)->pdn); if (err) { /* * The MR is revoked at this point so there is no issue to free @@ -1188,6 +1194,21 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, if (!(flags & IB_MR_REREG_PD)) new_pd = ib_mr->pd; + if (mr->is_odp_implicit && !(flags & IB_MR_REREG_TRANS)) { + if (!(new_access_flags & IB_ACCESS_ON_DEMAND)) + return ERR_PTR(-EOPNOTSUPP); + + /* + * Due to all the child mkeys we cannot actually change an + * implicit MR in place. If the user did not specify a new + * translation then force the fixed implicit MR values. + */ + start = 0; + iova = 0; + length = U64_MAX; + flags |= IB_MR_REREG_TRANS; + } + if (!(flags & IB_MR_REREG_TRANS)) { struct ib_umem *umem; @@ -1202,7 +1223,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, } /* DM or ODP MR's don't have a normal umem so we can't re-use it */ if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) - goto recreate; + return ERR_PTR(-EOPNOTSUPP); /* * Only one active MR can refer to a umem at one time, revoke @@ -1231,8 +1252,8 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, struct ib_umem *new_umem; unsigned long page_size; - new_umem = ib_umem_get(&dev->ib_dev, start, length, - new_access_flags); + new_umem = ib_umem_get_va(&dev->ib_dev, start, length, + new_access_flags); if (IS_ERR(new_umem)) return ERR_CAST(new_umem); @@ -1381,9 +1402,12 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) bool is_odp = is_odp_mr(mr); int ret; - if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr) && - !ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr)) - return 0; + if (mr->ibmr.frmr.pool) { + if (!mlx5_umr_revoke_mr_with_lock(mr)) { + ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); + return 0; + } + } if (is_odp) mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex); @@ -1404,6 +1428,10 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) dma_resv_unlock( to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv); } + + if (mr->ibmr.frmr.pool && !ret) + ib_frmr_pool_drop(&mr->ibmr); + return ret; } @@ -1813,7 +1841,7 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) resp.response_length = min(offsetofend(typeof(resp), response_length), udata->outlen); if (resp.response_length) { - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto free_mkey; } diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1119ce163ea7..1badec9bf527 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -221,7 +221,8 @@ static void free_implicit_child_mr_work(struct work_struct *work) mutex_lock(&odp_imr->umem_mutex); mlx5r_umr_update_xlt(mr->parent, ib_umem_start(odp) >> mlx5_imr_mtt_shift, 1, 0, - MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC); + MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, + 0); mutex_unlock(&odp_imr->umem_mutex); mlx5_ib_dereg_mr(&mr->ibmr, NULL); @@ -318,10 +319,12 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, u64 umr_offset = idx & umr_block_mask; if (in_block && umr_offset == 0) { - mlx5r_umr_update_xlt(mr, blk_start_idx, - idx - blk_start_idx, 0, - MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ATOMIC); + mlx5r_umr_update_xlt( + mr, blk_start_idx, idx - blk_start_idx, + 0, + MLX5_IB_UPD_XLT_ZAP | + MLX5_IB_UPD_XLT_ATOMIC, + 0); in_block = 0; /* Count page invalidations */ invalidations += idx - blk_start_idx + 1; @@ -329,10 +332,9 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, } } if (in_block) { - mlx5r_umr_update_xlt(mr, blk_start_idx, - idx - blk_start_idx + 1, 0, - MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ATOMIC); + mlx5r_umr_update_xlt( + mr, blk_start_idx, idx - blk_start_idx + 1, 0, + MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ATOMIC, 0); /* Count page invalidations */ invalidations += idx - blk_start_idx + 1; } @@ -502,11 +504,9 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, */ refcount_set(&mr->mmkey.usecount, 2); - err = mlx5r_umr_update_xlt(mr, 0, - mlx5_imr_mtt_entries, - PAGE_SHIFT, - MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ENABLE); + err = mlx5r_umr_update_xlt(mr, 0, mlx5_imr_mtt_entries, PAGE_SHIFT, + MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ENABLE, + to_mpd(mr->ibmr.pd)->pdn); if (err) { ret = ERR_PTR(err); goto out_mr; @@ -647,7 +647,8 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, mlx5_imr_ksm_page_shift, MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ZAP | - MLX5_IB_UPD_XLT_ENABLE); + MLX5_IB_UPD_XLT_ENABLE, + pd->pdn); if (err) goto out_mr; @@ -686,12 +687,16 @@ void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr) } } +/* + * pdn must be valid only when xlt_flags updates the mkey PD. In this path that + * is only MLX5_PF_FLAGS_ENABLE. DOWNGRADE and SNAPSHOT leave the PD masked out. + */ #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) #define MLX5_PF_FLAGS_SNAPSHOT BIT(2) #define MLX5_PF_FLAGS_ENABLE BIT(3) static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, u64 user_va, size_t bcnt, u32 *bytes_mapped, - u32 flags) + u32 flags, u32 pdn) { int page_shift, ret, np; bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; @@ -720,7 +725,8 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, * No need to check whether the MTTs really belong to this MR, since * ib_umem_odp_map_dma_and_lock already checks this. */ - ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags); + ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags, + pdn); mutex_unlock(&odp->umem_mutex); if (ret < 0) { @@ -786,7 +792,7 @@ static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, user_va; ret = pagefault_real_mr(mtt, umem_odp, user_va, len, - bytes_mapped, flags); + bytes_mapped, flags, 0); mlx5r_deref_odp_mkey(&mtt->mmkey); @@ -818,9 +824,9 @@ static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, * next pagefault handler will see the new information. */ mutex_lock(&odp_imr->umem_mutex); - err = mlx5r_umr_update_xlt(imr, upd_start_idx, upd_len, 0, - MLX5_IB_UPD_XLT_INDIRECT | - MLX5_IB_UPD_XLT_ATOMIC); + err = mlx5r_umr_update_xlt( + imr, upd_start_idx, upd_len, 0, + MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, 0); mutex_unlock(&odp_imr->umem_mutex); if (err) { mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); @@ -830,7 +836,7 @@ static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, } static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, - u32 *bytes_mapped, u32 flags) + u32 *bytes_mapped, u32 flags, u32 pdn) { struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : @@ -858,7 +864,7 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, } else { page_shift = order_base_2(page_size); if (page_shift != mr->page_shift && mr->dmabuf_faulted) { - err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, + err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, page_shift); } else { mr->page_shift = page_shift; @@ -866,8 +872,8 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = mlx5r_umr_update_data_direct_ksm_pas( mr, xlt_flags); else - err = mlx5r_umr_update_mr_pas(mr, - xlt_flags); + err = mlx5r_umr_update_mr_pas(mr, xlt_flags, + pdn); } } dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); @@ -903,7 +909,7 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, return -EFAULT; if (mr->umem->is_dmabuf) - return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags); + return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags, 0); if (!odp->is_implicit_odp) { u64 offset = io_virt < mr->ibmr.iova ? 0 : io_virt - mr->ibmr.iova; @@ -922,28 +928,36 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, ib_umem_end(odp) - user_va < bcnt)) return -EFAULT; return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, - flags); + flags, 0); } return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, flags); } -int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr) +int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { int ret; ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), mr->umem->address, mr->umem->length, NULL, - MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE); + MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE, + to_mpd(pd)->pdn); return ret >= 0 ? 0 : ret; } -int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr) +int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) { + struct mlx5_ib_dev *dev = mr_to_mdev(mr); + u32 pdn; int ret; + if (mr->data_direct) + pdn = dev->ddr.pdn; + else + pdn = to_mpd(pd)->pdn; + ret = pagefault_dmabuf_mr(mr, mr->umem->length, NULL, - MLX5_PF_FLAGS_ENABLE); + MLX5_PF_FLAGS_ENABLE, pdn); return ret >= 0 ? 0 : ret; } @@ -1567,8 +1581,7 @@ static void mlx5_ib_mr_memory_pfault_handler(struct mlx5_ib_dev *dev, ret = pagefault_mr(mr, prefetch_va, prefetch_size, NULL, 0, true); if (ret < 0) { ret = pagefault_mr(mr, pfault->memory.va, - pfault->memory.fault_byte_count, NULL, 0, - true); + pfault->memory.fault_byte_count, NULL, 0, true); if (ret < 0) goto err; } @@ -1807,6 +1820,9 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) struct mlx5_eq_param param = {}; int err = 0; + if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) + return -EOPNOTSUPP; + mutex_lock(&dev->odp_eq_mutex); if (eq->core) goto unlock; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8fd05532c09c..7ff02d89c31d 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -44,6 +44,9 @@ #include "qp.h" #include "wr.h" +#define UVERBS_MODULE_NAME mlx5_ib +#include + enum { MLX5_IB_ACK_REQ_FREQ = 8, }; @@ -64,12 +67,19 @@ enum { MLX5_QP_RM_GO_BACK_N = 0x1, }; +struct mlx5_rate_limit_ctx { + struct mlx5_rate_limit rl_old; + struct mlx5_rate_limit rl_desired; + u16 rl_desired_index; + bool rl_changed; +}; + struct mlx5_modify_raw_qp_param { u16 operation; u32 set_mask; /* raw_qp_set_mask_map */ - struct mlx5_rate_limit rl; + struct mlx5_rate_limit_ctx rl_ctx; u8 rq_q_ctr_id; u32 port; @@ -451,16 +461,13 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, if (ucmd) { qp->rq.wqe_cnt = ucmd->rq_wqe_count; - if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift)) - return -EINVAL; qp->rq.wqe_shift = ucmd->rq_wqe_shift; - if ((1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) < - wq_sig) + if (check_shl_overflow(1, qp->rq.wqe_shift, &wqe_size)) + return -EINVAL; + if (wqe_size / sizeof(struct mlx5_wqe_data_seg) < wq_sig) return -EINVAL; qp->rq.max_gs = - (1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) - + wqe_size / sizeof(struct mlx5_wqe_data_seg) - wq_sig; qp->rq.max_post = qp->rq.wqe_cnt; } else { @@ -703,12 +710,8 @@ static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi) static int num_med_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi) { - int n; - - n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs - - NUM_NON_BLUE_FLAME_BFREGS; - - return n >= 0 ? n : 0; + return max(0, max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs - + NUM_NON_BLUE_FLAME_BFREGS); } static int first_med_bfreg(struct mlx5_ib_dev *dev, @@ -890,7 +893,7 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (!ucmd->buf_addr) return -EINVAL; - rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0); + rwq->umem = ib_umem_get_va(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0); if (IS_ERR(rwq->umem)) { mlx5_ib_dbg(dev, "umem_get failed\n"); err = PTR_ERR(rwq->umem); @@ -918,7 +921,7 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas, offset); - err = mlx5_ib_db_map_user(ucontext, ucmd->db_addr, &rwq->db); + err = mlx5_ib_db_map_user(ucontext, NULL, 0, ucmd->db_addr, &rwq->db); if (err) { mlx5_ib_dbg(dev, "map failed\n"); goto err_umem; @@ -938,8 +941,17 @@ static int adjust_bfregn(struct mlx5_ib_dev *dev, bfregn % MLX5_NON_FP_BFREGS_PER_UAR; } +static u16 mlx5_qp_buf_attr(struct mlx5_ib_qp *qp) +{ + if (qp->type == IB_QPT_RAW_PACKET || + qp->flags & IB_QP_CREATE_SOURCE_QPN) + return UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM; + return UVERBS_ATTR_CREATE_QP_BUF_UMEM; +} + static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, struct mlx5_ib_qp *qp, struct ib_udata *udata, + struct uverbs_attr_bundle *attrs, struct ib_qp_init_attr *attr, u32 **in, struct mlx5_ib_create_qp_resp *resp, int *inlen, struct mlx5_ib_qp_base *base, @@ -998,14 +1010,20 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (err) goto err_bfreg; - if (ucmd->buf_addr && ubuffer->buf_size) { + ubuffer->umem = NULL; + if (ubuffer->buf_size) { ubuffer->buf_addr = ucmd->buf_addr; - ubuffer->umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr, - ubuffer->buf_size, 0); + ubuffer->umem = ib_umem_get_attr_or_va(&dev->ib_dev, attrs, + mlx5_qp_buf_attr(qp), + ubuffer->buf_addr, + ubuffer->buf_size, 0); if (IS_ERR(ubuffer->umem)) { err = PTR_ERR(ubuffer->umem); + ubuffer->umem = NULL; goto err_bfreg; } + } + if (ubuffer->umem) { page_size = mlx5_umem_find_best_quantized_pgoff( ubuffer->umem, qpc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64, @@ -1015,8 +1033,6 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, goto err_umem; } ncont = ib_umem_num_dma_blocks(ubuffer->umem, page_size); - } else { - ubuffer->umem = NULL; } *inlen = MLX5_ST_SZ_BYTES(create_qp_in) + @@ -1044,7 +1060,9 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, resp->bfreg_index = MLX5_IB_INVALID_BFREG; qp->bfregn = bfregn; - err = mlx5_ib_db_map_user(context, ucmd->db_addr, &qp->db); + err = mlx5_ib_db_map_user(context, attrs, + MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM, + ucmd->db_addr, &qp->db); if (err) { mlx5_ib_dbg(dev, "map failed\n"); goto err_free; @@ -1333,6 +1351,7 @@ static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq, static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, struct ib_udata *udata, + struct uverbs_attr_bundle *attrs, struct mlx5_ib_sq *sq, void *qpin, struct ib_pd *pd, struct mlx5_ib_cq *cq) { @@ -1352,8 +1371,10 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev, if (ts_format < 0) return ts_format; - sq->ubuffer.umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr, - ubuffer->buf_size, 0); + sq->ubuffer.umem = ib_umem_get_attr_or_va(&dev->ib_dev, attrs, + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + ubuffer->buf_addr, + ubuffer->buf_size, 0); if (IS_ERR(sq->ubuffer.umem)) return PTR_ERR(sq->ubuffer.umem); page_size = mlx5_umem_find_best_quantized_pgoff( @@ -1566,6 +1587,7 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev, static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, u32 *in, size_t inlen, struct ib_pd *pd, struct ib_udata *udata, + struct uverbs_attr_bundle *attrs, struct mlx5_ib_create_qp_resp *resp, struct ib_qp_init_attr *init_attr) { @@ -1586,7 +1608,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, if (err) return err; - err = create_raw_packet_qp_sq(dev, udata, sq, in, pd, + err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd, to_mcq(init_attr->send_cq)); if (err) goto err_destroy_tis; @@ -1704,6 +1726,7 @@ static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *q struct mlx5_create_qp_params { struct ib_udata *udata; + struct uverbs_attr_bundle *attrs; size_t inlen; size_t outlen; size_t ucmd_size; @@ -2125,8 +2148,8 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (ts_format < 0) return ts_format; - err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, ¶ms->resp, - &inlen, base, ucmd); + err = _create_user_qp(dev, pd, qp, udata, params->attrs, init_attr, + &in, ¶ms->resp, &inlen, base, ucmd); if (err) return err; @@ -2293,8 +2316,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, return ts_format; } - err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, ¶ms->resp, - &inlen, base, ucmd); + err = _create_user_qp(dev, pd, qp, udata, params->attrs, init_attr, + &in, ¶ms->resp, &inlen, base, ucmd); if (err) return err; @@ -2398,7 +2421,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr; raw_packet_qp_copy_info(qp, &qp->raw_packet_qp); err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata, - ¶ms->resp, init_attr); + params->attrs, ¶ms->resp, + init_attr); } else err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); @@ -2727,6 +2751,10 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, if (err) mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n", base->mqp.qpn); + if (qp->rl.rate) { + mlx5_rl_remove_rate(dev->mdev, &qp->rl); + memset(&qp->rl, 0, sizeof(qp->rl)); + } } destroy_qp(dev, qp, base, udata); @@ -3109,12 +3137,14 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, int err; if (params->is_rss_raw) { + rdma_restrack_no_track(&qp->ibqp.res); err = create_rss_raw_qp_tir(dev, pd, qp, params); goto out; } switch (qp->type) { case MLX5_IB_QPT_DCT: + rdma_restrack_no_track(&qp->ibqp.res); err = create_dct(dev, pd, qp, params); break; case MLX5_IB_QPT_DCI: @@ -3278,6 +3308,7 @@ int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, return err; params.udata = udata; + params.attrs = udata ? rdma_udata_to_uverbs_attr_bundle(udata) : NULL; params.uidx = MLX5_IB_DEFAULT_UIDX; params.attr = attr; params.is_rss_raw = !!attr->rwq_ind_tbl; @@ -3332,7 +3363,7 @@ int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, * including MLX5_IB_QPT_DCT, which doesn't need it. * In that case, resp will be filled with zeros. */ - err = ib_copy_to_udata(udata, ¶ms.resp, params.outlen); + err = ib_respond_udata(udata, params.resp); if (err) goto destroy_qp; @@ -3675,8 +3706,10 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q MLX5_QP_OPTPAR_RNR_TIMEOUT, [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | MLX5_QP_OPTPAR_RWE | - MLX5_QP_OPTPAR_PM_STATE, - [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY, + MLX5_QP_OPTPAR_PM_STATE | + MLX5_QP_OPTPAR_PP_INDEX, + [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY | + MLX5_QP_OPTPAR_PP_INDEX, [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE | @@ -3695,10 +3728,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q MLX5_QP_OPTPAR_ALT_ADDR_PATH, [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_PM_STATE | - MLX5_QP_OPTPAR_ALT_ADDR_PATH, + MLX5_QP_OPTPAR_ALT_ADDR_PATH | + MLX5_QP_OPTPAR_PP_INDEX, [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY | MLX5_QP_OPTPAR_SRQN | - MLX5_QP_OPTPAR_CQN_RCV, + MLX5_QP_OPTPAR_CQN_RCV | + MLX5_QP_OPTPAR_PP_INDEX, [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE | MLX5_QP_OPTPAR_RWE | @@ -3842,15 +3877,117 @@ static int modify_raw_packet_qp_rq( return err; } +static bool qp_rate_limit_supported(struct mlx5_ib_dev *dev, + struct mlx5_ib_qp *qp) +{ + if (qp->type == IB_QPT_RAW_PACKET || + qp->flags & IB_QP_CREATE_SOURCE_QPN) + return true; + + if (qp->type == IB_QPT_UD) + return MLX5_CAP_QOS(dev->mdev, packet_pacing_req_ud); + + if (qp->type == IB_QPT_UC) + return MLX5_CAP_QOS(dev->mdev, packet_pacing_req_uc); + + return false; +} + +static int qp_rl_parse(struct mlx5_ib_dev *dev, + struct mlx5_ib_qp *qp, + const struct ib_qp_attr *attr, + const struct mlx5_ib_modify_qp *ucmd, + struct mlx5_rate_limit *rl_desired) +{ + if (!qp_rate_limit_supported(dev, qp)) + return -EOPNOTSUPP; + + rl_desired->rate = attr->rate_limit; + + if (ucmd->burst_info.max_burst_sz) { + if (!attr->rate_limit || + !MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) + return -EINVAL; + rl_desired->max_burst_sz = ucmd->burst_info.max_burst_sz; + } + + if (ucmd->burst_info.typical_pkt_sz) { + if (!attr->rate_limit || + !MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) + return -EINVAL; + rl_desired->typical_pkt_sz = ucmd->burst_info.typical_pkt_sz; + } + + return 0; +} + +static int qp_rl_prepare(struct mlx5_ib_dev *dev, + struct mlx5_ib_qp *qp, u16 op, + struct mlx5_rate_limit_ctx *ctx) +{ + int err; + + ctx->rl_old = qp->rl; + + if (!qp->sq.wqe_cnt) + return 0; + + if (op != MLX5_CMD_OP_RTR2RTS_QP && + op != MLX5_CMD_OP_RTS2RTS_QP) + return 0; + + ctx->rl_changed = true; + + if (ctx->rl_desired.rate) { + err = mlx5_rl_add_rate(dev->mdev, &ctx->rl_desired_index, + &ctx->rl_desired); + if (err) { + pr_err("Failed configuring rate limit(err %d): rate %u, max_burst_sz %u, typical_pkt_sz %u\n", + err, ctx->rl_desired.rate, + ctx->rl_desired.max_burst_sz, + ctx->rl_desired.typical_pkt_sz); + return err; + } + } + + return 0; +} + +static void qp_rl_rollback(struct mlx5_core_dev *dev, + struct mlx5_rate_limit_ctx *ctx) +{ + if (ctx->rl_desired_index) + mlx5_rl_remove_rate(dev, &ctx->rl_desired); +} + +static void qp_rl_commit(struct mlx5_core_dev *dev, + struct mlx5_ib_qp *qp, + struct mlx5_rate_limit_ctx *ctx, + enum ib_qp_state new_state) +{ + if (ctx->rl_changed) { + if (ctx->rl_old.rate) + mlx5_rl_remove_rate(dev, &ctx->rl_old); + qp->rl = ctx->rl_desired; + } + + if (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR) { + if (qp->rl.rate) + mlx5_rl_remove_rate(dev, &qp->rl); + memset(&qp->rl, 0, sizeof(qp->rl)); + memset(&qp->rl_desired, 0, sizeof(qp->rl_desired)); + return; + } + + qp->rl_desired = ctx->rl_desired; +} + static int modify_raw_packet_qp_sq( struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state, const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd) { + const struct mlx5_rate_limit_ctx *rl_ctx = &raw_qp_param->rl_ctx; struct mlx5_ib_qp *ibqp = sq->base.container_mibqp; - struct mlx5_rate_limit old_rl = ibqp->rl; - struct mlx5_rate_limit new_rl = old_rl; - bool new_rate_added = false; - u16 rl_index = 0; void *in; void *sqc; int inlen; @@ -3868,49 +4005,27 @@ static int modify_raw_packet_qp_sq( MLX5_SET(sqc, sqc, state, new_state); if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) { - if (new_state != MLX5_SQC_STATE_RDY) + if (new_state != MLX5_SQC_STATE_RDY) { pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n", __func__); - else - new_rl = raw_qp_param->rl; - } - - if (!mlx5_rl_are_equal(&old_rl, &new_rl)) { - if (new_rl.rate) { - err = mlx5_rl_add_rate(dev, &rl_index, &new_rl); - if (err) { - pr_err("Failed configuring rate limit(err %d): \ - rate %u, max_burst_sz %u, typical_pkt_sz %u\n", - err, new_rl.rate, new_rl.max_burst_sz, - new_rl.typical_pkt_sz); - - goto out; - } - new_rate_added = true; + } else if (rl_ctx->rl_changed) { + MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); + /* index 0 means no limit */ + MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, + rl_ctx->rl_desired_index); } - - MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); - /* index 0 means no limit */ - MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index); } err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in); - if (err) { - /* Remove new rate from table if failed */ - if (new_rate_added) - mlx5_rl_remove_rate(dev, &new_rl); + if (err) goto out; + + if (new_state != MLX5_SQC_STATE_RDY) { + mlx5_rl_remove_rate(dev, &ibqp->rl); + memset(&ibqp->rl, 0, sizeof(ibqp->rl)); + memset(&ibqp->rl_desired, 0, sizeof(ibqp->rl_desired)); } - /* Only remove the old rate after new rate was set */ - if ((old_rl.rate && !mlx5_rl_are_equal(&old_rl, &new_rl)) || - (new_state != MLX5_SQC_STATE_RDY)) { - mlx5_rl_remove_rate(dev, &old_rl); - if (new_state != MLX5_SQC_STATE_RDY) - memset(&new_rl, 0, sizeof(new_rl)); - } - - ibqp->rl = new_rl; sq->state = new_state; out: @@ -4172,6 +4287,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, struct mlx5_ib_qp *qp = to_mqp(ibqp); struct mlx5_ib_qp_base *base = &qp->trans_qp.base; struct mlx5_ib_cq *send_cq, *recv_cq; + struct mlx5_rate_limit_ctx rl_ctx = {}; struct mlx5_ib_pd *pd; enum mlx5_qp_state mlx5_cur, mlx5_new; void *qpc, *pri_path, *alt_path; @@ -4362,20 +4478,31 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, goto out; } - op = optab[mlx5_cur][mlx5_new]; - optpar |= ib_mask_to_mlx5_opt(attr_mask); - optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st]; - - if (attr_mask & IB_QP_RATE_LIMIT && qp->type != IB_QPT_RAW_PACKET) { - err = -EOPNOTSUPP; - goto out; + if (attr_mask & IB_QP_RATE_LIMIT) { + err = qp_rl_parse(dev, qp, attr, ucmd, &rl_ctx.rl_desired); + if (err) + goto out; + } else { + rl_ctx.rl_desired = qp->rl_desired; } + op = optab[mlx5_cur][mlx5_new]; + if (!mlx5_rl_are_equal(&rl_ctx.rl_desired, &qp->rl)) { + err = qp_rl_prepare(dev, qp, op, &rl_ctx); + if (err) + goto out; + } + optpar |= ib_mask_to_mlx5_opt(attr_mask); + if (rl_ctx.rl_changed) + optpar |= MLX5_QP_OPTPAR_PP_INDEX; + optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st]; + if (qp->type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { struct mlx5_modify_raw_qp_param raw_qp_param = {}; raw_qp_param.operation = op; + raw_qp_param.rl_ctx = rl_ctx; if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { raw_qp_param.rq_q_ctr_id = set_id; raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID; @@ -4384,36 +4511,22 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, if (attr_mask & IB_QP_PORT) raw_qp_param.port = attr->port_num; - if (attr_mask & IB_QP_RATE_LIMIT) { - raw_qp_param.rl.rate = attr->rate_limit; - - if (ucmd->burst_info.max_burst_sz) { - if (attr->rate_limit && - MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) { - raw_qp_param.rl.max_burst_sz = - ucmd->burst_info.max_burst_sz; - } else { - err = -EINVAL; - goto out; - } - } - - if (ucmd->burst_info.typical_pkt_sz) { - if (attr->rate_limit && - MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) { - raw_qp_param.rl.typical_pkt_sz = - ucmd->burst_info.typical_pkt_sz; - } else { - err = -EINVAL; - goto out; - } - } - + if (rl_ctx.rl_changed) raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT; - } err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity); + if (err) { + qp_rl_rollback(dev->mdev, &raw_qp_param.rl_ctx); + goto out; + } + + qp_rl_commit(dev->mdev, qp, &raw_qp_param.rl_ctx, new_state); } else { + if (rl_ctx.rl_changed) { + MLX5_SET(qpc, qpc, packet_pacing_rate_limit_index, + rl_ctx.rl_desired_index); + } + if (udata) { /* For the kernel flows, the resp will stay zero */ resp->ece_options = @@ -4423,6 +4536,13 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, } err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp, &resp->ece_options); + + if (err) { + qp_rl_rollback(dev->mdev, &rl_ctx); + goto out; + } + + qp_rl_commit(dev->mdev, qp, &rl_ctx, new_state); } if (err) @@ -4631,7 +4751,7 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr, resp.dctn = qp->dct.mdct.mqp.qpn; if (MLX5_CAP_GEN(dev->mdev, ece_support)) resp.ece_options = MLX5_GET(create_dct_out, out, ece); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) { mlx5_core_destroy_dct(dev, &qp->dct.mdct); return err; @@ -4747,6 +4867,13 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, attr_mask); goto out; } + } else if (attr_mask == IB_QP_RATE_LIMIT && cur_state != IB_QPS_RTS) { + struct mlx5_rate_limit rl_desired = {}; + + err = qp_rl_parse(dev, qp, attr, &ucmd, &rl_desired); + if (!err) + qp->rl_desired = rl_desired; + goto out; } else if (qp_type != MLX5_IB_QPT_REG_UMR && qp_type != MLX5_IB_QPT_DCI && !ib_modify_qp_is_ok(cur_state, new_state, qp_type, @@ -4790,7 +4917,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (!err && resp.response_length && udata->outlen >= resp.response_length) /* Return -EFAULT to the user and expect him to destroy QP. */ - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); out: mutex_unlock(&qp->mutex); @@ -5490,7 +5617,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, if (udata->outlen) { resp.response_length = offsetofend( struct mlx5_ib_create_wq_resp, response_length); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto err_copy; } @@ -5538,10 +5665,9 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, u32 *in; void *rqtc; - if (udata->inlen > 0 && - !ib_is_udata_cleared(udata, 0, - udata->inlen)) - return -EOPNOTSUPP; + err = ib_is_udata_in_empty(udata); + if (err) + return err; if (init_attr->log_ind_tbl_size > MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) { @@ -5582,7 +5708,7 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, resp.response_length = offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, response_length); - err = ib_copy_to_udata(udata, &resp, resp.response_length); + err = ib_respond_udata(udata, resp); if (err) goto err_copy; } @@ -5862,3 +5988,15 @@ void mlx5_ib_qp_event_cleanup(void) { destroy_workqueue(mlx5_ib_qp_event_wq); } + +ADD_UVERBS_ATTRIBUTES_SIMPLE( + mlx5_ib_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_UMEM(MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM, + UA_OPTIONAL)); + +const struct uapi_definition mlx5_ib_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &mlx5_ib_qp_create), + {}, +}; diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c index 67841922c7b8..00a9bcb2603f 100644 --- a/drivers/infiniband/hw/mlx5/restrack.c +++ b/drivers/infiniband/hw/mlx5/restrack.c @@ -178,9 +178,6 @@ static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "REG_UMR"); break; - case MLX5_IB_QPT_DCT: - ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT"); - break; case MLX5_IB_QPT_DCI: ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI"); break; diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 852f6f502d14..5bc48fef3744 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -66,7 +66,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); - srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0); + srq->umem = ib_umem_get_va(pd->device, ucmd.buf_addr, buf_size, 0); if (IS_ERR(srq->umem)) { mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); err = PTR_ERR(srq->umem); @@ -74,7 +74,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, } in->umem = srq->umem; - err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db); + err = mlx5_ib_db_map_user(ucontext, NULL, 0, ucmd.db_addr, &srq->db); if (err) { mlx5_ib_dbg(dev, "map doorbell failed\n"); goto err_umem; @@ -292,12 +292,9 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, .srqn = srq->msrq.srqn, }; - if (ib_copy_to_udata(udata, &resp, min(udata->outlen, - sizeof(resp)))) { - mlx5_ib_dbg(dev, "copy to user failed\n"); - err = -EFAULT; + err = ib_respond_udata(udata, resp); + if (err) goto err_core; - } } init_attr->attr.max_wr = srq->msrq.max - 1; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index f2139474be37..48cae5cc1c1b 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -603,11 +603,11 @@ mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg, static void mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev *dev, struct mlx5_mkey_seg *mkey_seg, - struct mlx5_ib_mr *mr, + struct mlx5_ib_mr *mr, u32 pdn, unsigned int page_shift) { mlx5r_umr_set_access_flags(dev, mkey_seg, mr->access_flags); - MLX5_SET(mkc, mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); + MLX5_SET(mkc, mkey_seg, pd, pdn); MLX5_SET64(mkc, mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, mkey_seg, len, mr->ibmr.length); MLX5_SET(mkc, mkey_seg, log_page_size, page_shift); @@ -670,23 +670,22 @@ static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev, wqe->data_seg.byte_count = cpu_to_be32(sg->length); } -static void -_mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, struct mlx5r_umr_wqe *wqe, - struct ib_sge *sg, unsigned int flags, - unsigned int page_shift, bool dd) +static void _mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, + struct mlx5r_umr_wqe *wqe, struct ib_sge *sg, + unsigned int flags, u32 pdn, + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); mlx5r_umr_set_update_xlt_ctrl_seg(&wqe->ctrl_seg, flags, sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, page_shift); - if (dd) /* Use the data direct internal kernel PD */ - MLX5_SET(mkc, &wqe->mkey_seg, pd, dev->ddr.pdn); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe->data_seg, sg); } -static int -_mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, - size_t start_block, size_t nblocks) +static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, + u32 pdn, bool dd, size_t start_block, + size_t nblocks) { size_t ent_size = dd ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -720,7 +719,7 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, orig_sg_length = sg.length; - _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, mr->page_shift, dd); + _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, pdn, mr->page_shift); /* Set initial translation offset to start_block */ offset = (u64)start_block * ent_size; @@ -811,7 +810,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, true, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + true, start_block, nblocks); } int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, @@ -821,12 +821,13 @@ int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, } int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks) + u32 pdn, size_t start_block, size_t nblocks) { if (WARN_ON(mr->umem->is_odp)) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, false, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, pdn, false, start_block, + nblocks); } /* @@ -834,9 +835,9 @@ int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP * flag may be used. */ -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn) { - return mlx5r_umr_update_mr_pas_range(mr, flags, 0, 0); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, 0, 0); } static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) @@ -845,7 +846,7 @@ static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) } int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, - int page_shift, int flags) + int page_shift, int flags, u32 pdn) { int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) ? sizeof(struct mlx5_klm) @@ -895,7 +896,8 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, } mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, page_shift); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg); for (pages_mapped = 0; @@ -915,7 +917,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, */ err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); if (err) - return err; + break; dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT); @@ -937,8 +939,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, * pinned and the HW can switch from 4K to huge-page alignment). */ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd) + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); struct mlx5r_umr_wqe wqe = {}; @@ -953,16 +954,8 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, /* Fill mkey segment with the new page size, keep the rest unchanged */ MLX5_SET(mkc, &wqe.mkey_seg, log_page_size, page_shift); - if (dd) - MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn); - else - MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); - MLX5_SET64(mkc, &wqe.mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, &wqe.mkey_seg, len, mr->ibmr.length); - MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); - MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, - mlx5_mkey_variant(mr->mmkey.key)); err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); if (!err) @@ -971,17 +964,18 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, return err; } -static inline int -_mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks, bool dd) +static inline int _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, + unsigned int flags, u32 pdn, + size_t start_block, + size_t nblocks, bool dd) { if (dd) return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, start_block, nblocks); else - return mlx5r_umr_update_mr_pas_range(mr, flags, start_block, - nblocks); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, + start_block, nblocks); } /** @@ -995,11 +989,9 @@ _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, * Return: On success, returns the number of entries that were zapped. * On error, returns a negative error code. */ -static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, - unsigned int flags, - unsigned int page_shift, - size_t *nblocks, - bool dd) +static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, unsigned int flags, + unsigned int page_shift, size_t *nblocks, + u32 pdn, bool dd) { unsigned int old_page_shift = mr->page_shift; struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -1039,7 +1031,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, */ if (*nblocks) mr->page_shift = max_page_shift; - err = _mlx5r_dmabuf_umr_update_pas(mr, flags, 0, *nblocks, dd); + err = _mlx5r_dmabuf_umr_update_pas(mr, flags, pdn, 0, *nblocks, dd); if (err) { mr->page_shift = old_page_shift; return err; @@ -1049,7 +1041,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * non-present. */ if (*nblocks) { - err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift, dd); + err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift); if (err) { mr->page_shift = old_page_shift; return err; @@ -1064,6 +1056,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * entries accordingly * @mr: The memory region to update * @xlt_flags: Translation table update flags + * @pdn: Protection domain number * @page_shift: The new (optimized) page shift to use * * This function updates the page size and mkey translation entries for a DMABUF @@ -1083,7 +1076,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * * Returns 0 on success or a negative error code on failure. */ -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift) { unsigned int old_page_shift = mr->page_shift; @@ -1092,7 +1085,7 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, int err; err = _mlx5r_umr_zap_mkey(mr, xlt_flags, page_shift, &zapped_blocks, - mr->data_direct); + pdn, mr->data_direct); if (err) return err; @@ -1105,20 +1098,17 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, * the page size in the mkey yet. */ err = _mlx5r_dmabuf_umr_update_pas( - mr, - xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, - zapped_blocks, - total_blocks - zapped_blocks, + mr, xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, pdn, + zapped_blocks, total_blocks - zapped_blocks, mr->data_direct); if (err) goto err; } - err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift, - mr->data_direct); + err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift); if (err) goto err; - err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks, + err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, pdn, 0, zapped_blocks, mr->data_direct); if (err) goto err; diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index 7eeaf6a94c97..bda7123781a9 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -101,14 +101,13 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, size_t nblocks); int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int flags); int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks); -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags); + u32 pdn, size_t start_block, size_t nblocks); +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn); int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, - int page_shift, int flags); + int page_shift, int flags, u32 pdn); int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd); -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, + unsigned int page_shift); +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift); #endif /* _MLX5_IB_UMR_H */ diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index e8d5d865c1f1..f90f67afc8fa 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -55,16 +55,19 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr { struct ib_smp *in_mad; struct ib_smp *out_mad; - int err = -ENOMEM; + int err; struct mthca_dev *mdev = to_mdev(ibdev); - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; in_mad = kzalloc_obj(*in_mad); out_mad = kmalloc_obj(*out_mad); - if (!in_mad || !out_mad) + if (!in_mad || !out_mad) { + err = -ENOMEM; goto out; + } memset(props, 0, sizeof *props); @@ -111,7 +114,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * props->max_mcast_grp; - err = 0; + err = ib_respond_empty_udata(uhw); out: kfree(in_mad); kfree(out_mad); @@ -311,10 +314,11 @@ static int mthca_alloc_ucontext(struct ib_ucontext *uctx, return err; } - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { + err = ib_respond_udata(udata, uresp); + if (err) { mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab); mthca_uar_free(to_mdev(ibdev), &context->uar); - return -EFAULT; + return err; } context->reg_mr_warned = 0; @@ -356,9 +360,12 @@ static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return err; if (udata) { - if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) { + struct mthca_alloc_pd_resp uresp = { .pdn = pd->pd_num }; + + err = ib_respond_udata(udata, uresp); + if (err) { mthca_pd_free(to_mdev(ibdev), pd); - return -EFAULT; + return err; } } @@ -427,11 +434,17 @@ static int mthca_create_srq(struct ib_srq *ibsrq, if (err) return err; - if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) { - mthca_free_srq(to_mdev(ibsrq->device), srq); - mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, - context->db_tab, ucmd.db_index); - return -EFAULT; + if (context) { + struct mthca_create_srq_resp uresp = { .srqn = srq->srqn }; + + err = ib_respond_udata(udata, uresp); + if (err) { + mthca_free_srq(to_mdev(ibsrq->device), srq); + mthca_unmap_user_db(to_mdev(ibsrq->device), + &context->uar, context->db_tab, + ucmd.db_index); + return err; + } } return 0; @@ -630,10 +643,14 @@ static int mthca_create_cq(struct ib_cq *ibcq, if (err) goto err_unmap_arm; - if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) { - mthca_free_cq(to_mdev(ibdev), cq); - err = -EFAULT; - goto err_unmap_arm; + if (udata) { + struct mthca_create_cq_resp uresp = { .cqn = cq->cqn }; + + err = ib_respond_udata(udata, uresp); + if (err) { + mthca_free_cq(to_mdev(ibdev), cq); + goto err_unmap_arm; + } } cq->resize_buf = NULL; @@ -864,7 +881,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (!mr) return ERR_PTR(-ENOMEM); - mr->umem = ib_umem_get(pd->device, start, length, acc); + mr->umem = ib_umem_get_va(pd->device, start, length, acc); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); goto err; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index a88cc5d84af8..53ba32d168a1 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -68,9 +68,11 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *uhw) { struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); + int err; - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; memset(attr, 0, sizeof *attr); memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], @@ -110,7 +112,7 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; attr->max_pkeys = 1; - return 0; + return ib_respond_empty_udata(uhw); } static inline void get_link_speed_and_width(struct ocrdma_dev *dev, @@ -502,7 +504,7 @@ int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) resp.dpp_wqe_size = dev->attr.wqe_size; memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); - status = ib_copy_to_udata(udata, &resp, sizeof(resp)); + status = ib_respond_udata(udata, resp); if (status) goto cpy_err; return 0; @@ -586,11 +588,10 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, u64 db_page_addr; u64 dpp_page_addr = 0; u32 db_page_size; - struct ocrdma_alloc_pd_uresp rsp; + struct ocrdma_alloc_pd_uresp rsp = {}; struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct ocrdma_ucontext, ibucontext); - memset(&rsp, 0, sizeof(rsp)); rsp.id = pd->id; rsp.dpp_enabled = pd->dpp_enabled; db_page_addr = ocrdma_get_db_addr(dev, pd->id); @@ -611,7 +612,7 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, rsp.dpp_page_addr_lo = dpp_page_addr; } - status = ib_copy_to_udata(udata, &rsp, sizeof(rsp)); + status = ib_respond_udata(udata, rsp); if (status) goto ucopy_err; @@ -865,7 +866,7 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr = kzalloc_obj(*mr); if (!mr) return ERR_PTR(status); - mr->umem = ib_umem_get(ibpd->device, start, len, acc); + mr->umem = ib_umem_get_va(ibpd->device, start, len, acc); if (IS_ERR(mr->umem)) { status = -EFAULT; goto umem_err; @@ -930,13 +931,12 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, int status; struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct ocrdma_ucontext, ibucontext); - struct ocrdma_create_cq_uresp uresp; + struct ocrdma_create_cq_uresp uresp = {}; /* this must be user flow! */ if (!udata) return -EINVAL; - memset(&uresp, 0, sizeof(uresp)); uresp.cq_id = cq->id; uresp.page_size = PAGE_ALIGN(cq->len); uresp.num_pages = 1; @@ -945,12 +945,9 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); uresp.db_page_size = dev->nic_info.db_page_size; uresp.phase_change = cq->phase_change ? 1 : 0; - status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (status) { - pr_err("%s(%d) copy error cqid=0x%x.\n", - __func__, dev->id, cq->id); + status = ib_respond_udata(udata, uresp); + if (status) goto err; - } status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); if (status) goto err; @@ -1176,11 +1173,10 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, { int status; u64 usr_db; - struct ocrdma_create_qp_uresp uresp; + struct ocrdma_create_qp_uresp uresp = {}; struct ocrdma_pd *pd = qp->pd; struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); - memset(&uresp, 0, sizeof(uresp)); usr_db = dev->nic_info.unmapped_db + (pd->id * dev->nic_info.db_page_size); uresp.qp_id = qp->id; @@ -1206,11 +1202,9 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, uresp.dpp_credit = dpp_credit_lmt; uresp.dpp_offset = dpp_offset; } - status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (status) { - pr_err("%s(%d) user copy error.\n", __func__, dev->id); + status = ib_respond_udata(udata, uresp); + if (status) goto err; - } status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); if (status) @@ -1735,9 +1729,8 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, struct ib_udata *udata) { int status; - struct ocrdma_create_srq_uresp uresp; + struct ocrdma_create_srq_uresp uresp = {}; - memset(&uresp, 0, sizeof(uresp)); uresp.rq_dbid = srq->rq.dbid; uresp.num_rq_pages = 1; uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); @@ -1754,7 +1747,7 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, uresp.db_shift = 16; } - status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + status = ib_respond_udata(udata, uresp); if (status) return status; status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 679aa6f3a63b..c90a1b5c8ee7 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -64,14 +64,6 @@ enum { QEDR_USER_MMAP_PHYS_PAGE, }; -static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, - size_t len) -{ - size_t min_len = min_t(size_t, len, udata->outlen); - - return ib_copy_to_udata(udata, src, min_len); -} - int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { if (index >= QEDR_ROCE_PKEY_TABLE_LEN) @@ -113,6 +105,7 @@ int qedr_query_device(struct ib_device *ibdev, { struct qedr_dev *dev = get_qedr_dev(ibdev); struct qedr_device_attr *qattr = &dev->attr; + int rc; if (!dev->rdma_ctx) { DP_ERR(dev, @@ -121,6 +114,10 @@ int qedr_query_device(struct ib_device *ibdev, return -EINVAL; } + rc = ib_is_udata_in_empty(udata); + if (rc) + return rc; + memset(attr, 0, sizeof(*attr)); attr->fw_ver = qattr->fw_ver; @@ -163,7 +160,7 @@ int qedr_query_device(struct ib_device *ibdev, attr->max_pkeys = qattr->max_pkey; attr->max_ah = qattr->max_ah; - return 0; + return ib_respond_empty_udata(udata); } static inline void get_link_speed_and_width(int speed, u16 *ib_speed, @@ -340,7 +337,7 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) uresp.sges_per_srq_wr = dev->attr.max_srq_sge; uresp.max_cqes = QEDR_MAX_CQES; - rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rc = ib_respond_udata(udata, uresp); if (rc) goto err; @@ -459,9 +456,8 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct qedr_ucontext *context = rdma_udata_to_drv_context( udata, struct qedr_ucontext, ibucontext); - rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rc = ib_respond_udata(udata, uresp); if (rc) { - DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); return rc; } @@ -696,14 +692,10 @@ static void qedr_db_recovery_del(struct qedr_dev *dev, dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); } -static int qedr_copy_cq_uresp(struct qedr_dev *dev, - struct qedr_cq *cq, struct ib_udata *udata, +static int qedr_copy_cq_uresp(struct qedr_cq *cq, struct ib_udata *udata, u32 db_offset) { - struct qedr_create_cq_uresp uresp; - int rc; - - memset(&uresp, 0, sizeof(uresp)); + struct qedr_create_cq_uresp uresp = {}; uresp.db_offset = db_offset; uresp.icid = cq->icid; @@ -711,11 +703,7 @@ static int qedr_copy_cq_uresp(struct qedr_dev *dev, uresp.db_rec_addr = rdma_user_mmap_get_offset(cq->q.db_mmap_entry); - rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (rc) - DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); - - return rc; + return ib_respond_udata(udata, uresp); } static void consume_cqe(struct qedr_cq *cq) @@ -796,9 +784,9 @@ static inline int qedr_init_user_queue(struct ib_udata *udata, q->buf_addr = buf_addr; q->buf_len = buf_len; - q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); + q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access); if (IS_ERR(q->umem)) { - DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", + DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n", PTR_ERR(q->umem)); return PTR_ERR(q->umem); } @@ -994,7 +982,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, spin_lock_init(&cq->cq_lock); if (udata) { - rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); + rc = qedr_copy_cq_uresp(cq, udata, db_offset); if (rc) goto err2; @@ -1251,15 +1239,10 @@ static int qedr_copy_srq_uresp(struct qedr_dev *dev, struct qedr_srq *srq, struct ib_udata *udata) { struct qedr_create_srq_uresp uresp = {}; - int rc; uresp.srq_id = srq->srq_id; - rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); - if (rc) - DP_ERR(dev, "create srq: problem copying data to user space\n"); - - return rc; + return ib_respond_udata(udata, uresp); } static void qedr_copy_rq_uresp(struct qedr_dev *dev, @@ -1303,10 +1286,6 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_udata *udata, struct qedr_create_qp_uresp *uresp) { - int rc; - - memset(uresp, 0, sizeof(*uresp)); - if (qedr_qp_has_sq(qp)) qedr_copy_sq_uresp(dev, uresp, qp); @@ -1316,13 +1295,7 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev, uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; uresp->qp_id = qp->qp_id; - rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); - if (rc) - DP_ERR(dev, - "create qp: failed a copy to user space with qp icid=0x%x.\n", - qp->icid); - - return rc; + return ib_respond_udata(udata, *uresp); } static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) @@ -1471,13 +1444,14 @@ static int qedr_init_srq_user_params(struct ib_udata *udata, if (rc) return rc; - srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, - sizeof(struct rdma_srq_producers), access); + srq->prod_umem = ib_umem_get_va(srq->ibsrq.device, ureq->prod_pair_addr, + sizeof(struct rdma_srq_producers), + access); if (IS_ERR(srq->prod_umem)) { qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); ib_umem_release(srq->usrq.umem); DP_ERR(srq->dev, - "create srq: failed ib_umem_get for producer, got %ld\n", + "create srq: failed ib_umem_get_va for producer, got %ld\n", PTR_ERR(srq->prod_umem)); return PTR_ERR(srq->prod_umem); } @@ -2964,7 +2938,7 @@ struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->type = QEDR_MR_USER; - mr->umem = ib_umem_get(ibpd->device, start, len, acc); + mr->umem = ib_umem_get_va(ibpd->device, start, len, acc); if (IS_ERR(mr->umem)) { rc = -EFAULT; goto err0; diff --git a/drivers/infiniband/hw/usnic/usnic_fwd.c b/drivers/infiniband/hw/usnic/usnic_fwd.c index 39cdd72eabf6..59c363db0355 100644 --- a/drivers/infiniband/hw/usnic/usnic_fwd.c +++ b/drivers/infiniband/hw/usnic/usnic_fwd.c @@ -93,7 +93,7 @@ struct usnic_fwd_dev *usnic_fwd_dev_alloc(struct pci_dev *pdev) ufdev->netdev = pci_get_drvdata(pdev); spin_lock_init(&ufdev->lock); BUILD_BUG_ON(sizeof(ufdev->name) != sizeof(ufdev->netdev->name)); - strcpy(ufdev->name, ufdev->netdev->name); + strscpy(ufdev->name, ufdev->netdev->name); return ufdev; } diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 615de9c4209b..dc355b00f61c 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -82,17 +82,13 @@ static void usnic_ib_fw_string_to_u64(char *fw_ver_str, u64 *fw_ver) static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, struct ib_udata *udata) { - struct usnic_ib_dev *us_ibdev; - struct usnic_ib_create_qp_resp resp; + struct usnic_ib_create_qp_resp resp = {}; struct pci_dev *pdev; struct vnic_dev_bar *bar; struct usnic_vnic_res_chunk *chunk; struct usnic_ib_qp_grp_flow *default_flow; int i, err; - memset(&resp, 0, sizeof(resp)); - - us_ibdev = qp_grp->vf->pf; pdev = usnic_vnic_get_pdev(qp_grp->vf->vnic); if (!pdev) { usnic_err("Failed to get pdev of qp_grp %d\n", @@ -157,12 +153,9 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, struct usnic_ib_qp_grp_flow, link); resp.transport = default_flow->trans_type; - err = ib_copy_to_udata(udata, &resp, sizeof(resp)); - if (err) { - usnic_err("Failed to copy udata for %s", - dev_name(&us_ibdev->ib_dev.dev)); + err = ib_respond_udata(udata, resp); + if (err) return err; - } return 0; } @@ -282,10 +275,12 @@ int usnic_ib_query_device(struct ib_device *ibdev, union ib_gid gid; struct ethtool_drvinfo info; int qp_per_vf; + int err; usnic_dbg("\n"); - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; mutex_lock(&us_ibdev->usdev_lock); us_ibdev->netdev->ethtool_ops->get_drvinfo(us_ibdev->netdev, &info); @@ -329,7 +324,7 @@ int usnic_ib_query_device(struct ib_device *ibdev, * max_qp_wr, max_sge, max_sge_rd, max_cqe */ mutex_unlock(&us_ibdev->usdev_lock); - return 0; + return ib_respond_empty_udata(uhw); } int usnic_ib_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index bc3adcc1ae67..0bdb4452d6f6 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -138,8 +138,8 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, if (ret) goto err_cq; - cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, ucmd.buf_size, - IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_va(ibdev, ucmd.buf_addr, ucmd.buf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { ret = PTR_ERR(cq->umem); goto err_cq; @@ -203,11 +203,10 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->uar = &context->uar; /* Copy udata back. */ - if (ib_copy_to_udata(udata, &cq_resp, sizeof(cq_resp))) { - dev_warn(&dev->pdev->dev, - "failed to copy back udata\n"); + ret = ib_respond_udata(udata, cq_resp); + if (ret) { pvrdma_destroy_cq(&cq->ibcq, udata); - return -EINVAL; + return ret; } } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c index 05a6bd991502..942381ab0367 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c @@ -131,7 +131,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, return ERR_PTR(-EINVAL); } - umem = ib_umem_get(pd->device, start, length, access_flags); + umem = ib_umem_get_va(pd->device, start, length, access_flags); if (IS_ERR(umem)) { dev_warn(&dev->pdev->dev, "could not get umem for mem region\n"); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 16aab967a203..e939cd5ce40b 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -268,9 +268,9 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, if (!is_srq) { /* set qp->sq.wqe_cnt, shift, buf_size.. */ - qp->rumem = ib_umem_get(ibqp->device, - ucmd.rbuf_addr, - ucmd.rbuf_size, 0); + qp->rumem = ib_umem_get_va(ibqp->device, + ucmd.rbuf_addr, + ucmd.rbuf_size, 0); if (IS_ERR(qp->rumem)) { ret = PTR_ERR(qp->rumem); goto err_qp; @@ -281,8 +281,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, qp->srq = to_vsrq(init_attr->srq); } - qp->sumem = ib_umem_get(ibqp->device, ucmd.sbuf_addr, - ucmd.sbuf_size, 0); + qp->sumem = ib_umem_get_va(ibqp->device, ucmd.sbuf_addr, + ucmd.sbuf_size, 0); if (IS_ERR(qp->sumem)) { if (!is_srq) ib_umem_release(qp->rumem); @@ -406,12 +406,10 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, qp_resp.qpn = qp->ibqp.qp_num; qp_resp.qp_handle = qp->qp_handle; - if (ib_copy_to_udata(udata, &qp_resp, - min(udata->outlen, sizeof(qp_resp)))) { - dev_warn(&dev->pdev->dev, - "failed to copy back udata\n"); + ret = ib_respond_udata(udata, qp_resp); + if (ret) { __pvrdma_destroy_qp(dev, qp); - return -EINVAL; + return ret; } } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c index d31fb692fcaa..345ec486a223 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c @@ -146,7 +146,7 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, if (ret) goto err_srq; - srq->umem = ib_umem_get(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); + srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); if (IS_ERR(srq->umem)) { ret = PTR_ERR(srq->umem); goto err_srq; @@ -195,10 +195,10 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, spin_unlock_irqrestore(&dev->srq_tbl_lock, flags); /* Copy udata back. */ - if (ib_copy_to_udata(udata, &srq_resp, sizeof(srq_resp))) { - dev_warn(&dev->pdev->dev, "failed to copy back udata\n"); + ret = ib_respond_udata(udata, srq_resp); + if (ret) { pvrdma_destroy_srq(&srq->ibsrq, udata); - return -EINVAL; + return ret; } return 0; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index c7c2b41060e5..1d29a535f76a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -67,9 +67,11 @@ int pvrdma_query_device(struct ib_device *ibdev, struct ib_udata *uhw) { struct pvrdma_dev *dev = to_vdev(ibdev); + int err; - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; props->fw_ver = dev->dsr->caps.fw_ver; props->sys_image_guid = dev->dsr->caps.sys_image_guid; @@ -114,7 +116,7 @@ int pvrdma_query_device(struct ib_device *ibdev, props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN; - return 0; + return ib_respond_empty_udata(uhw); } /** @@ -320,11 +322,11 @@ int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) /* copy back to user */ uresp.qp_tab_size = vdev->dsr->caps.max_qp; - ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + ret = ib_respond_udata(udata, uresp); if (ret) { /* pvrdma_dealloc_ucontext() also frees the UAR */ pvrdma_dealloc_ucontext(&context->ibucontext); - return -EFAULT; + return ret; } return 0; @@ -430,11 +432,10 @@ int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) pd_resp.pdn = resp->pd_handle; if (udata) { - if (ib_copy_to_udata(udata, &pd_resp, sizeof(pd_resp))) { - dev_warn(&dev->pdev->dev, - "failed to copy back protection domain\n"); + ret = ib_respond_udata(udata, pd_resp); + if (ret) { pvrdma_dealloc_pd(&pd->ibpd, udata); - return -EFAULT; + return ret; } } diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c index 30904c6ae852..45404611c9ce 100644 --- a/drivers/infiniband/sw/rdmavt/cq.c +++ b/drivers/infiniband/sw/rdmavt/cq.c @@ -372,7 +372,7 @@ int rvt_resize_cq(struct ib_cq *ibcq, unsigned int cqe, struct ib_udata *udata) if (udata && udata->outlen >= sizeof(__u64)) { __u64 offset = 0; - ret = ib_copy_to_udata(udata, &offset, sizeof(offset)); + ret = ib_respond_udata(udata, offset); if (ret) goto bail_free; } diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c index 15f1ff917d6c..3266129e63e7 100644 --- a/drivers/infiniband/sw/rdmavt/mr.c +++ b/drivers/infiniband/sw/rdmavt/mr.c @@ -351,7 +351,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (length == 0) return ERR_PTR(-EINVAL); - umem = ib_umem_get(pd->device, start, length, mr_access_flags); + umem = ib_umem_get_va(pd->device, start, length, mr_access_flags); if (IS_ERR(umem)) return ERR_CAST(umem); diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 816624e0991a..70e7d08fdce6 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -1192,8 +1192,7 @@ int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, if (!qp->r_rq.wq) { __u64 offset = 0; - ret = ib_copy_to_udata(udata, &offset, - sizeof(offset)); + ret = ib_respond_udata(udata, offset); if (ret) goto bail_qpn; } else { diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c index fe125bf85b27..d022aa56c5bf 100644 --- a/drivers/infiniband/sw/rdmavt/srq.c +++ b/drivers/infiniband/sw/rdmavt/srq.c @@ -128,6 +128,7 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); struct rvt_rq tmp_rq = {}; + __u64 offset_addr; int ret = 0; if (attr_mask & IB_SRQ_MAX_WR) { @@ -149,19 +150,17 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, return -ENOMEM; /* Check that we can write the offset to mmap. */ if (udata && udata->inlen >= sizeof(__u64)) { - __u64 offset_addr; __u64 offset = 0; ret = ib_copy_from_udata(&offset_addr, udata, sizeof(offset_addr)); if (ret) goto bail_free; - udata->outbuf = (void __user *) - (unsigned long)offset_addr; - ret = ib_copy_to_udata(udata, &offset, - sizeof(offset)); - if (ret) + if (copy_to_user(u64_to_user_ptr(offset_addr), &offset, + sizeof(offset))) { + ret = -EFAULT; goto bail_free; + } } spin_lock_irq(&srq->rq.kwq->c_lock); @@ -236,10 +235,10 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, * See rvt_mmap() for details. */ if (udata && udata->inlen >= sizeof(__u64)) { - ret = ib_copy_to_udata(udata, &ip->offset, - sizeof(ip->offset)); - if (ret) - return ret; + if (copy_to_user(u64_to_user_ptr(offset_addr), + &ip->offset, + sizeof(ip->offset))) + return -EFAULT; } /* diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index 40aa64208364..5fa3a1f33326 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "vt.h" #include "cq.h" #include "trace.h" @@ -79,14 +80,16 @@ static int rvt_query_device(struct ib_device *ibdev, struct ib_udata *uhw) { struct rvt_dev_info *rdi = ib_to_rvt(ibdev); + int err; - if (uhw->inlen || uhw->outlen) - return -EINVAL; + err = ib_is_udata_in_empty(uhw); + if (err) + return err; /* * Return rvt_dev_info.dparms.props contents */ *props = rdi->dparms.props; - return 0; + return ib_respond_empty_udata(uhw); } static int rvt_get_numa_node(struct ib_device *ibdev) diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile index 3977f4f13258..e097c1ca1874 100644 --- a/drivers/infiniband/sw/rxe/Makefile +++ b/drivers/infiniband/sw/rxe/Makefile @@ -23,6 +23,7 @@ rdma_rxe-y := \ rxe_task.o \ rxe_net.o \ rxe_hw_counters.o \ + rxe_mad.o \ rxe_ns.o rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c index b0714f9abe3d..af39209d0fcf 100644 --- a/drivers/infiniband/sw/rxe/rxe.c +++ b/drivers/infiniband/sw/rxe/rxe.c @@ -81,7 +81,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe, struct net_device *ndev) } else { /* * This device does not have a HW address, but - * connection mangagement requires a unique gid. + * connection management requires a unique gid. */ eth_random_addr(rxe->raw_gid); } diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c index 437917a7d8f2..17edaa9a9b9b 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c @@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_LINK_DOWNED].name = "link_downed", [RXE_CNT_RDMA_SEND].name = "rdma_sends", [RXE_CNT_RDMA_RECV].name = "rdma_recvs", + [RXE_CNT_SENT_BYTES].name = "sent_bytes", + [RXE_CNT_RCVD_BYTES].name = "rcvd_bytes", }; int rxe_ib_get_hw_stats(struct ib_device *ibdev, diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h index 051f9e1c3852..01b355103cbc 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h @@ -26,6 +26,8 @@ enum rxe_counters { RXE_CNT_LINK_DOWNED, RXE_CNT_RDMA_SEND, RXE_CNT_RDMA_RECV, + RXE_CNT_SENT_BYTES, + RXE_CNT_RCVD_BYTES, RXE_NUM_OF_COUNTERS }; diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index e095c12699cb..64d636bf80fd 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -242,4 +242,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd, #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +/* rxe-mad.c */ +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, + const struct ib_wc *in_wc, const struct ib_grh *in_grh, + const struct ib_mad *in, struct ib_mad *out, + size_t *out_mad_size, u16 *out_mad_pkey_index); + #endif /* RXE_LOC_H */ diff --git a/drivers/infiniband/sw/rxe/rxe_mad.c b/drivers/infiniband/sw/rxe/rxe_mad.c new file mode 100644 index 000000000000..cb2d55830f42 --- /dev/null +++ b/drivers/infiniband/sw/rxe/rxe_mad.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2026 zhenwei pi + */ + +#include +#include "rxe.h" +#include "rxe_hw_counters.h" + +static int rxe_get_pma_info(struct ib_mad *out) +{ + struct ib_class_port_info cpi = {}; + + cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH; + memcpy((out->data + 40), &cpi, sizeof(cpi)); + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_get_pma_counters(struct rxe_dev *rxe, struct ib_mad *out) +{ + struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data + 40); + s64 val; + + /* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop + * at all ones. + */ + val = atomic64_read(&rxe->stats_counters[RXE_CNT_LINK_DOWNED]); + pma_cnt->link_downed_counter = clamp(val, 0, U8_MAX); + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_get_pma_counters_ext(struct rxe_dev *rxe, struct ib_mad *out) +{ + struct ib_pma_portcounters_ext *pma_cnt_ext = + (struct ib_pma_portcounters_ext *)(out->data + 40); + s64 val; + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_SENT_BYTES]); + pma_cnt_ext->port_xmit_data = cpu_to_be64(val >> 2); + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_RCVD_BYTES]); + pma_cnt_ext->port_rcv_data = cpu_to_be64(val >> 2); + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_SENT_PKTS]); + pma_cnt_ext->port_xmit_packets = cpu_to_be64(val); + + val = atomic64_read(&rxe->stats_counters[RXE_CNT_RCVD_PKTS]); + pma_cnt_ext->port_rcv_packets = cpu_to_be64(val); + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + +static int rxe_get_perf_mgmt(struct rxe_dev *rxe, const struct ib_mad *in, struct ib_mad *out) +{ + switch (in->mad_hdr.attr_id) { + case IB_PMA_CLASS_PORT_INFO: + return rxe_get_pma_info(out); + + case IB_PMA_PORT_COUNTERS: + return rxe_get_pma_counters(rxe, out); + + case IB_PMA_PORT_COUNTERS_EXT: + return rxe_get_pma_counters_ext(rxe, out); + + default: + out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB); + return IB_MAD_RESULT_SUCCESS; + } +} + +int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, + const struct ib_wc *in_wc, const struct ib_grh *in_grh, + const struct ib_mad *in, struct ib_mad *out, + size_t *out_mad_size, u16 *out_mad_pkey_index) +{ + struct rxe_dev *rxe = to_rdev(ibdev); + u8 mgmt_class = in->mad_hdr.mgmt_class; + u8 method = in->mad_hdr.method; + + if (port_num != 1) + return IB_MAD_RESULT_FAILURE; + + memset(out, 0, sizeof(*out)); + switch (mgmt_class) { + case IB_MGMT_CLASS_PERF_MGMT: + if (method == IB_MGMT_METHOD_GET) + return rxe_get_perf_mgmt(rxe, in, out); + break; + + default: + out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD); + return IB_MAD_RESULT_SUCCESS; + } + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c index 5cad72073eca..acd03bd87794 100644 --- a/drivers/infiniband/sw/rxe/rxe_mcast.c +++ b/drivers/infiniband/sw/rxe/rxe_mcast.c @@ -34,7 +34,7 @@ static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid) struct net_device *ndev; int ret; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return -ENODEV; @@ -59,7 +59,7 @@ static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid) struct net_device *ndev; int ret; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return -ENODEV; diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c index db380302149e..7f723a2f3700 100644 --- a/drivers/infiniband/sw/rxe/rxe_mmap.c +++ b/drivers/infiniband/sw/rxe/rxe_mmap.c @@ -93,18 +93,31 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) goto done; found_it: + /* + * Increment refcount and check whether it is being freed atm while + * holding lock to prevent UAF + */ + if (!kref_get_unless_zero(&ip->ref)) { + spin_unlock_bh(&rxe->pending_lock); + ret = -ENXIO; + goto done; + } + list_del_init(&ip->pending_mmaps); spin_unlock_bh(&rxe->pending_lock); + vma->vm_ops = &rxe_vm_ops; + vma->vm_private_data = ip; + ret = remap_vmalloc_range(vma, ip->obj, 0); if (ret) { + vma->vm_private_data = NULL; + vma->vm_ops = NULL; + kref_put(&ip->ref, rxe_mmap_release); rxe_dbg_dev(rxe, "err %d from remap_vmalloc_range\n", ret); goto done; } - vma->vm_ops = &rxe_vm_ops; - vma->vm_private_data = ip; - rxe_vma_open(vma); done: return ret; } diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index c696ff874980..875eceb55fdf 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -197,7 +197,7 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access, mr); - umem = ib_umem_get(&rxe->ib_dev, start, length, access); + umem = ib_umem_get_va(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", (int)PTR_ERR(umem)); diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 082ff387d081..3741b2c4b0bb 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -501,6 +501,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, int err; int is_request = pkt->mask & RXE_REQ_MASK; struct rxe_dev *rxe = to_rdev(qp->ibqp.device); + unsigned int skblen = skb->len; unsigned long flags; spin_lock_irqsave(&qp->state_lock, flags); @@ -524,6 +525,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, } rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); + rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen); goto done; drop: @@ -600,7 +602,7 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num) struct net_device *ndev; char *ndev_name; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return NULL; ndev_name = ndev->name; @@ -644,12 +646,11 @@ static void rxe_sock_put(struct sock *sk, void rxe_net_del(struct ib_device *dev) { - struct rxe_dev *rxe = container_of(dev, struct rxe_dev, ib_dev); struct net_device *ndev; struct sock *sk; struct net *net; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(dev, 1); if (!ndev) return; @@ -697,7 +698,7 @@ void rxe_set_port_state(struct rxe_dev *rxe) { struct net_device *ndev; - ndev = rxe_ib_device_get_netdev(&rxe->ib_dev); + ndev = ib_device_get_netdev(&rxe->ib_dev, 1); if (!ndev) return; diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h index 767870568372..1cc77c46bbcb 100644 --- a/drivers/infiniband/sw/rxe/rxe_param.h +++ b/drivers/infiniband/sw/rxe/rxe_param.h @@ -109,7 +109,7 @@ enum rxe_device_param { RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, - /* Max number of interations of each work item + /* Max number of iterations of each work item * before yielding the cpu to let other * work make progress */ diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 2d5e701ff961..2c90387aa126 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb) int err; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); struct rxe_dev *rxe = pkt->rxe; + unsigned int skblen = skb->len - skb_network_offset(skb); if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop; @@ -353,6 +354,7 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(err)) goto drop; + rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen); rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS); if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN)) diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 9cb2f6fbf2dd..d8cbdfa70cdb 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -264,6 +264,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) struct rxe_recv_wqe *wqe; struct ib_event ev; unsigned int count; + unsigned int num_sge; size_t size; unsigned long flags; @@ -279,12 +280,13 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) } /* don't trust user space data */ - if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) { + num_sge = wqe->dma.num_sge; + if (unlikely(num_sge > srq->rq.max_sge)) { spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n"); return RESPST_ERR_MALFORMED_WQE; } - size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge); + size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); qp->resp.wqe = &qp->resp.srq_wqe.wqe; @@ -308,6 +310,29 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) return RESPST_CHK_LENGTH; } +static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp) +{ + struct rxe_queue *q = qp->rq.queue; + struct rxe_recv_wqe *wqe; + unsigned int num_sge; + size_t size; + + wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT); + if (!wqe) + return RESPST_ERR_RNR; + + num_sge = wqe->dma.num_sge; + if (unlikely(num_sge > qp->rq.max_sge)) { + rxe_dbg_qp(qp, "invalid num_sge in recv WQE\n"); + return RESPST_ERR_MALFORMED_WQE; + } + size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); + memcpy(&qp->resp.srq_wqe, wqe, size); + + qp->resp.wqe = &qp->resp.srq_wqe.wqe; + return RESPST_CHK_LENGTH; +} + static enum resp_states check_resource(struct rxe_qp *qp, struct rxe_pkt_info *pkt) { @@ -328,9 +353,7 @@ static enum resp_states check_resource(struct rxe_qp *qp, if (srq) return get_srq_wqe(qp); - qp->resp.wqe = queue_head(qp->rq.queue, - QUEUE_TYPE_FROM_CLIENT); - return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR; + return rxe_get_recv_wqe(qp); } return RESPST_CHK_LENGTH; @@ -1505,7 +1528,7 @@ static int flush_recv_wqe(struct rxe_qp *qp, struct rxe_recv_wqe *wqe) return err; } -/* drain and optionally complete the recive queue +/* drain and optionally complete the receive queue * if unable to complete a wqe stop completing and * just flush the remaining wqes */ diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 4cf04a44189c..1ec130fee8ea 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -22,19 +22,13 @@ static int rxe_query_device(struct ib_device *ibdev, struct rxe_dev *rxe = to_rdev(ibdev); int err; - if (udata->inlen || udata->outlen) { - rxe_dbg_dev(rxe, "malformed udata\n"); - err = -EINVAL; - goto err_out; - } + err = ib_is_udata_in_empty(udata); + if (err) + return err; memcpy(attr, &rxe->attr, sizeof(*attr)); - return 0; - -err_out: - rxe_err_dev(rxe, "returned err = %d\n", err); - return err; + return ib_respond_empty_udata(udata); } static int rxe_query_port(struct ib_device *ibdev, @@ -50,7 +44,7 @@ static int rxe_query_port(struct ib_device *ibdev, goto err_out; } - ndev = rxe_ib_device_get_netdev(ibdev); + ndev = ib_device_get_netdev(ibdev, 1); if (!ndev) { err = -ENODEV; goto err_out; @@ -1446,7 +1440,7 @@ static int rxe_enable_driver(struct ib_device *ib_dev) struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev); struct net_device *ndev; - ndev = rxe_ib_device_get_netdev(ib_dev); + ndev = ib_device_get_netdev(ib_dev, 1); if (!ndev) return -ENODEV; @@ -1501,6 +1495,7 @@ static const struct ib_device_ops rxe_dev_ops = { .post_recv = rxe_post_recv, .post_send = rxe_post_send, .post_srq_recv = rxe_post_srq_recv, + .process_mad = rxe_process_mad, .query_ah = rxe_query_ah, .query_device = rxe_query_device, .query_pkey = rxe_query_pkey, diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index d92f80d16f78..0f5ffd94643f 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -415,7 +415,6 @@ struct rxe_port { u32 qp_gsi_index; }; -#define RXE_PORT 1 struct rxe_dev { struct ib_device ib_dev; struct ib_device_attr attr; @@ -451,16 +450,17 @@ struct rxe_dev { struct rxe_port port; }; -static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev) -{ - return ib_device_get_netdev(dev, RXE_PORT); -} - static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) { atomic64_inc(&rxe->stats_counters[index]); } +static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index, + s64 val) +{ + atomic64_add(val, &rxe->stats_counters[index]); +} + static inline struct rxe_dev *to_rdev(struct ib_device *dev) { return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL; diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h index f5fd71717b80..dbc998248b1e 100644 --- a/drivers/infiniband/sw/siw/siw.h +++ b/drivers/infiniband/sw/siw/siw.h @@ -119,9 +119,10 @@ struct siw_page_chunk { struct siw_umem { struct ib_umem *base_mem; - struct siw_page_chunk *page_chunk; - int num_pages; + unsigned int num_pages; + unsigned int num_chunks; u64 fp_addr; /* First page base address */ + struct siw_page_chunk page_chunk[] __counted_by(num_chunks); }; struct siw_pble { diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index f7ac81c0f267..87c79527ac09 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -138,6 +138,7 @@ static void siw_socket_disassoc(struct socket *s) cep = sk_to_cep(sk); if (cep) { siw_sk_restore_upcalls(sk, cep); + cep->sock = NULL; siw_cep_put(cep); } else { pr_warn("siw: cannot restore sk callbacks: no ep\n"); @@ -418,10 +419,11 @@ static void siw_free_cm_id(struct siw_cep *cep) static void siw_destroy_cep_sock(struct siw_cep *cep) { - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + struct socket *s = cep->sock; + + if (s) { + siw_socket_disassoc(s); + sock_release(s); } } @@ -1050,7 +1052,6 @@ static void siw_accept_newconn(struct siw_cep *cep) if (new_s) { siw_socket_disassoc(new_s); sock_release(new_s); - new_cep->sock = NULL; } siw_dbg_cep(cep, "error %d\n", rv); } @@ -1202,6 +1203,8 @@ static void siw_cm_work_handler(struct work_struct *w) WARN(1, "Undefined CM work type: %d\n", work->type); } if (release_cep) { + struct socket *s = cep->sock; + siw_dbg_cep(cep, "release: timer=%s, QP[%u]\n", cep->mpa_timer ? "y" : "n", @@ -1227,10 +1230,9 @@ static void siw_cm_work_handler(struct work_struct *w) cep->qp = NULL; siw_qp_put(qp); } - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + if (s) { + siw_socket_disassoc(s); + sock_release(s); } if (cep->cm_id) { siw_free_cm_id(cep); @@ -1561,7 +1563,6 @@ int siw_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) if (cep) { siw_socket_disassoc(s); sock_release(s); - cep->sock = NULL; cep->qp = NULL; @@ -1937,7 +1938,6 @@ int siw_create_listen(struct iw_cm_id *id, int backlog) siw_cep_set_inuse(cep); siw_free_cm_id(cep); - cep->sock = NULL; siw_socket_disassoc(s); cep->state = SIW_EPSTATE_CLOSED; @@ -1959,6 +1959,7 @@ static void siw_drop_listeners(struct iw_cm_id *id) */ list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) { struct siw_cep *cep = list_entry(p, struct siw_cep, listenq); + struct socket *s = cep->sock; list_del(p); @@ -1967,10 +1968,9 @@ static void siw_drop_listeners(struct iw_cm_id *id) siw_cep_set_inuse(cep); siw_free_cm_id(cep); - if (cep->sock) { - siw_socket_disassoc(cep->sock); - sock_release(cep->sock); - cep->sock = NULL; + if (s) { + siw_socket_disassoc(s); + sock_release(s); } cep->state = SIW_EPSTATE_CLOSED; siw_cep_set_free_and_put(cep); diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c index 98c802b3ed72..2a08817d3cce 100644 --- a/drivers/infiniband/sw/siw/siw_mem.c +++ b/drivers/infiniband/sw/siw/siw_mem.c @@ -41,16 +41,14 @@ struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index) void siw_umem_release(struct siw_umem *umem) { - int i, num_pages = umem->num_pages; + unsigned int i, num_chunks = umem->num_chunks; if (umem->base_mem) ib_umem_release(umem->base_mem); - for (i = 0; num_pages > 0; i++) { + for (i = 0; i < num_chunks; i++) kfree(umem->page_chunk[i].plist); - num_pages -= PAGES_PER_CHUNK; - } - kfree(umem->page_chunk); + kfree(umem); } @@ -188,7 +186,7 @@ int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr, * lookup is being done and mem is not released it check fails. */ int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge, struct siw_mem *mem[], - enum ib_access_flags perms, u32 off, int len) + enum ib_access_flags perms, u32 off, u32 len) { struct siw_device *sdev = to_siw_dev(pd->device); struct siw_mem *new = NULL; @@ -338,26 +336,21 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start, struct sg_page_iter sg_iter; struct sg_table *sgt; u64 first_page_va; - int num_pages, num_chunks, i, rv = 0; + unsigned int num_pages, num_chunks, i; + int rv = 0; if (!len) return ERR_PTR(-EINVAL); first_page_va = start & PAGE_MASK; num_pages = PAGE_ALIGN(start + len - first_page_va) >> PAGE_SHIFT; - num_chunks = (num_pages >> CHUNK_SHIFT) + 1; + num_chunks = ((num_pages - 1) >> CHUNK_SHIFT) + 1; - umem = kzalloc_obj(*umem); + umem = kzalloc_flex(*umem, page_chunk, num_chunks); if (!umem) return ERR_PTR(-ENOMEM); - umem->page_chunk = - kzalloc_objs(struct siw_page_chunk, num_chunks); - if (!umem->page_chunk) { - rv = -ENOMEM; - goto err_out; - } - base_mem = ib_umem_get(base_dev, start, len, rights); + base_mem = ib_umem_get_va(base_dev, start, len, rights); if (IS_ERR(base_mem)) { rv = PTR_ERR(base_mem); siw_dbg(base_dev, "Cannot pin user memory: %d\n", rv); @@ -365,33 +358,40 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start, } umem->fp_addr = first_page_va; umem->base_mem = base_mem; + umem->num_pages = num_pages; + umem->num_chunks = num_chunks; sgt = &base_mem->sgt_append.sgt; __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); - if (!__sg_page_iter_next(&sg_iter)) { - rv = -EINVAL; - goto err_out; - } - for (i = 0; num_pages > 0; i++) { - int nents = min_t(int, num_pages, PAGES_PER_CHUNK); - struct page **plist = - kzalloc_objs(struct page *, nents); + for (i = 0; i < num_chunks; i++) { + struct page **plist; + unsigned int pix, nents = min(num_pages, PAGES_PER_CHUNK); + plist = kzalloc_objs(struct page *, nents); if (!plist) { rv = -ENOMEM; goto err_out; } umem->page_chunk[i].plist = plist; - while (nents--) { - *plist = sg_page_iter_page(&sg_iter); - umem->num_pages++; - num_pages--; - plist++; + + for (pix = 0; pix < nents; pix++) { if (!__sg_page_iter_next(&sg_iter)) break; + plist[pix] = sg_page_iter_page(&sg_iter); + num_pages--; } } + + if (num_pages) { + /* + * Unexpected size of sg list provided by ib_umem_get_va() + */ + siw_dbg(base_dev, "Short SG list, missing %u pages\n", + num_pages); + rv = -EINVAL; + goto err_out; + } return umem; err_out: siw_umem_release(umem); diff --git a/drivers/infiniband/sw/siw/siw_mem.h b/drivers/infiniband/sw/siw/siw_mem.h index 8e769d30e2ac..e4f3a5a4f81d 100644 --- a/drivers/infiniband/sw/siw/siw_mem.h +++ b/drivers/infiniband/sw/siw/siw_mem.h @@ -17,7 +17,7 @@ int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr, enum ib_access_flags perms, int len); int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge, struct siw_mem *mem[], enum ib_access_flags perms, - u32 off, int len); + u32 off, u32 len); void siw_wqe_put_mem(struct siw_wqe *wqe, enum siw_opcode op); int siw_mr_add_mem(struct siw_mr *mr, struct ib_pd *pd, void *mem_obj, u64 start, u64 len, int rights); @@ -45,7 +45,6 @@ static inline void siw_unref_mem_sgl(struct siw_mem **mem, unsigned int num_sge) #define CHUNK_SHIFT 9 /* sets number of pages per chunk */ #define PAGES_PER_CHUNK (_AC(1, UL) << CHUNK_SHIFT) #define CHUNK_MASK (~(PAGES_PER_CHUNK - 1)) -#define PAGE_CHUNK_SIZE (PAGES_PER_CHUNK * sizeof(struct page *)) /* * siw_get_upage() @@ -61,7 +60,7 @@ static inline struct page *siw_get_upage(struct siw_umem *umem, u64 addr) chunk_idx = page_idx >> CHUNK_SHIFT, page_in_chunk = page_idx & ~CHUNK_MASK; - if (likely(page_idx < umem->num_pages)) + if (page_idx < umem->num_pages) return umem->page_chunk[chunk_idx].plist[page_in_chunk]; return NULL; diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index 34d03584160c..b566d163c5aa 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -844,6 +844,15 @@ int siw_proc_rresp(struct siw_qp *qp) } mem_p = *mem; + if (unlikely(wqe->processed + srx->fpdu_part_rem > wqe->bytes)) { + siw_dbg_qp(qp, "rresp len: %d + %d > %d\n", + wqe->processed, srx->fpdu_part_rem, wqe->bytes); + wqe->wc_status = SIW_WC_LOC_LEN_ERR; + siw_init_terminate(qp, TERM_ERROR_LAYER_DDP, + DDP_ETYPE_TAGGED_BUF, + DDP_ECODE_T_BASE_BOUNDS, 0); + return -EINVAL; + } bytes = min(srx->fpdu_part_rem, srx->skb_new); rv = siw_rx_data(mem_p, srx, &frx->pbl_idx, sge->laddr + wqe->processed, bytes); diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 1e1d262a4ae2..b74ac85c1b8b 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -102,7 +102,7 @@ int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata) rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; @@ -130,9 +130,11 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, struct ib_udata *udata) { struct siw_device *sdev = to_siw_dev(base_dev); + int rv; - if (udata->inlen || udata->outlen) - return -EINVAL; + rv = ib_is_udata_in_empty(udata); + if (rv) + return rv; memset(attr, 0, sizeof(*attr)); @@ -165,7 +167,7 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, addrconf_addr_eui48((u8 *)&attr->sys_image_guid, sdev->raw_gid); - return 0; + return ib_respond_empty_udata(udata); } int siw_query_port(struct ib_device *base_dev, u32 port, @@ -472,7 +474,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, rv = -EINVAL; goto err_out_xa; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out_xa; } @@ -1205,7 +1207,7 @@ int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr, rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; } @@ -1386,7 +1388,7 @@ struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; } @@ -1646,7 +1648,7 @@ int siw_create_srq(struct ib_srq *base_srq, rv = -EINVAL; goto err_out; } - rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + rv = ib_respond_udata(udata, uresp); if (rv) goto err_out; } diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c index 287e0ea43287..f8b833bd81ad 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c @@ -37,8 +37,6 @@ static void rtrs_clt_path_stats_release(struct kobject *kobj) stats = container_of(kobj, struct rtrs_clt_stats, kobj_stats); free_percpu(stats->pcpu_stats); - - kfree(stats); } static struct kobj_type ktype_stats = { diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index e351552733df..d34d7e5f34d6 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1536,7 +1536,7 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, int cpu; size_t total_con; - clt_path = kzalloc_obj(*clt_path); + clt_path = kzalloc_flex(*clt_path, stats, 1); if (!clt_path) goto err; @@ -1552,10 +1552,6 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, clt_path->s.con_num = total_con; clt_path->s.irq_con_num = con_num + 1; - clt_path->stats = kzalloc_obj(*clt_path->stats); - if (!clt_path->stats) - goto err_free_con; - mutex_init(&clt_path->init_mutex); uuid_gen(&clt_path->s.uuid); memcpy(&clt_path->s.dst_addr, path->dst, @@ -1583,7 +1579,7 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, clt_path->mp_skip_entry = alloc_percpu(typeof(*clt_path->mp_skip_entry)); if (!clt_path->mp_skip_entry) - goto err_free_stats; + goto err_free_con; for_each_possible_cpu(cpu) INIT_LIST_HEAD(per_cpu_ptr(clt_path->mp_skip_entry, cpu)); @@ -1596,8 +1592,6 @@ static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, err_free_percpu: free_percpu(clt_path->mp_skip_entry); -err_free_stats: - kfree(clt_path->stats); err_free_con: kfree(clt_path->s.con); err_free_path: @@ -2863,7 +2857,6 @@ struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, list_del_rcu(&clt_path->s.entry); rtrs_clt_close_conns(clt_path, true); free_percpu(clt_path->stats->pcpu_stats); - kfree(clt_path->stats); free_path(clt_path); goto close_all_path; } @@ -2873,7 +2866,6 @@ struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, list_del_rcu(&clt_path->s.entry); rtrs_clt_close_conns(clt_path, true); free_percpu(clt_path->stats->pcpu_stats); - kfree(clt_path->stats); free_path(clt_path); goto close_all_path; } @@ -3166,7 +3158,6 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt, rtrs_clt_remove_path_from_arr(clt_path); rtrs_clt_close_conns(clt_path, true); free_percpu(clt_path->stats->pcpu_stats); - kfree(clt_path->stats); free_path(clt_path); return err; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h index 986239ed2d3b..1305601a6251 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h @@ -142,12 +142,12 @@ struct rtrs_clt_path { u32 flags; struct kobject kobj; u8 for_new_clt; - struct rtrs_clt_stats *stats; /* cache hca_port and hca_name to display in sysfs */ u8 hca_port; char hca_name[IB_DEVICE_NAME_MAX]; struct list_head __percpu *mp_skip_entry; + struct rtrs_clt_stats stats[]; }; struct rtrs_clt_sess { diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 6482ad859bd1..7d8e4422cc57 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -225,8 +225,9 @@ static int rdma_write_sg(struct rtrs_srv_op *id) /* WR will fail with length error * if this is 0 */ - if (plist->length == 0) { - rtrs_err(s, "Invalid RDMA-Write sg list length 0\n"); + if (plist->length == 0 || plist->length > max_chunk_size) { + rtrs_err(s, "Invalid RDMA-Write sg list length %u\n", + plist->length); return -EINVAL; } @@ -1059,6 +1060,11 @@ static void process_read(struct rtrs_srv_con *con, "Processing read request failed, invalid message\n"); return; } + usr_len = le16_to_cpu(msg->usr_len); + if (usr_len > off) { + pr_debug("rtrs-srv: Invalid usr_len %zu > off %u\n", usr_len, off); + return; + } rtrs_srv_get_ops_ids(srv_path); rtrs_srv_update_rdma_stats(srv_path->stats, off, READ); id = srv_path->ops_ids[buf_id]; @@ -1066,7 +1072,6 @@ static void process_read(struct rtrs_srv_con *con, id->dir = READ; id->msg_id = buf_id; id->rd_msg = msg; - usr_len = le16_to_cpu(msg->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, @@ -1112,6 +1117,11 @@ static void process_write(struct rtrs_srv_con *con, rtrs_srv_state_str(srv_path->state)); return; } + usr_len = le16_to_cpu(req->usr_len); + if (usr_len > off) { + pr_debug("rtrs-srv: Invalid usr_len %zu > off %u\n", usr_len, off); + return; + } rtrs_srv_get_ops_ids(srv_path); rtrs_srv_update_rdma_stats(srv_path->stats, off, WRITE); id = srv_path->ops_ids[buf_id]; @@ -1119,7 +1129,6 @@ static void process_write(struct rtrs_srv_con *con, id->dir = WRITE; id->msg_id = buf_id; - usr_len = le16_to_cpu(req->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 9aec5d80117f..f66cfd70c263 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1129,9 +1129,10 @@ static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx, struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd); void *data = (void *)srp_cmd + imm_data_offset; uint32_t len = be32_to_cpu(imm_buf->len); - uint32_t req_size = imm_data_offset + len; + uint32_t req_size; - if (req_size > srp_max_req_size) { + if (check_add_overflow((uint32_t)imm_data_offset, len, &req_size) || + req_size > srp_max_req_size) { pr_err("Immediate data (length %d + %d) exceeds request size %d\n", imm_data_offset, len, srp_max_req_size); return -EINVAL; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 4f59b7e8a3d5..695c86ee6d7a 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1116,7 +1116,10 @@ struct mlx5_ifc_qos_cap_bits { u8 log_esw_max_sched_depth[0x4]; u8 reserved_at_10[0x10]; - u8 reserved_at_20[0x9]; + u8 reserved_at_20[0x2]; + u8 packet_pacing_req_ud[0x1]; + u8 packet_pacing_req_uc[0x1]; + u8 reserved_at_24[0x5]; u8 esw_cross_esw_sched[0x1]; u8 reserved_at_2a[0x1]; u8 log_max_qos_nic_queue_group[0x5]; @@ -3709,7 +3712,8 @@ struct mlx5_ifc_qpc_bits { u8 cur_retry_count[0x3]; u8 reserved_at_39b[0x5]; - u8 reserved_at_3a0[0x20]; + u8 reserved_at_3a0[0x10]; + u8 packet_pacing_rate_limit_index[0x10]; u8 reserved_at_3c0[0x8]; u8 next_send_psn[0x18]; diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index d67aedc6ea68..40f889403b07 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -72,6 +72,7 @@ enum mlx5_qp_optpar { MLX5_QP_OPTPAR_CQN_RCV = 1 << 19, MLX5_QP_OPTPAR_DC_HS = 1 << 20, MLX5_QP_OPTPAR_DC_KEY = 1 << 21, + MLX5_QP_OPTPAR_PP_INDEX = 1 << 22, MLX5_QP_OPTPAR_COUNTER_SET_ID = 1 << 25, }; diff --git a/include/rdma/frmr_pools.h b/include/rdma/frmr_pools.h index af1b88801fa4..aed4d69d3841 100644 --- a/include/rdma/frmr_pools.h +++ b/include/rdma/frmr_pools.h @@ -34,6 +34,7 @@ int ib_frmr_pools_init(struct ib_device *device, const struct ib_frmr_pool_ops *pool_ops); void ib_frmr_pools_cleanup(struct ib_device *device); int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr); -int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); +void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr); +void ib_frmr_pool_drop(struct ib_mr *mr); #endif /* FRMR_POOLS_H */ diff --git a/include/rdma/ib_ucaps.h b/include/rdma/ib_ucaps.h index d9f96be3a553..b629c99117d8 100644 --- a/include/rdma/ib_ucaps.h +++ b/include/rdma/ib_ucaps.h @@ -14,7 +14,6 @@ enum rdma_user_cap { RDMA_UCAP_MAX }; -void ib_cleanup_ucaps(void); int ib_get_ucaps(int *fds, int fd_count, uint64_t *idx_mask); #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int ib_create_ucap(enum rdma_user_cap type); diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 49172098a8de..31b3a86fe73a 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -73,16 +73,43 @@ static inline size_t ib_umem_num_pages(struct ib_umem *umem) { return ib_umem_num_dma_blocks(umem, PAGE_SIZE); } + +struct ib_udata; +struct ib_uverbs_buffer_desc; +struct uverbs_attr_bundle; + #ifdef CONFIG_INFINIBAND_USER_MEM -struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, - size_t size, int access); +struct ib_umem *ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access); +struct ib_umem *ib_umem_get_attr(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, size_t size, int access); +struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u16 attr_id, u64 addr, size_t size, + int access); +struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + size_t size, int access); +struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, + u64 addr, size_t size, int access); + +static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, + unsigned long addr, size_t size, + int access) +{ + return ib_umem_get_attr_or_va(device, NULL, 0, addr, size, access); +} + void ib_umem_release(struct ib_umem *umem); int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length); unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt); + u64 virt); /** * ib_umem_find_best_pgoff - Find best HW page size @@ -118,16 +145,11 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, static inline bool ib_umem_is_contiguous(struct ib_umem *umem) { - dma_addr_t dma_addr; unsigned long pgsz; - /* - * Select the smallest aligned page that can contain the whole umem if - * it was contiguous. - */ - dma_addr = ib_umem_start_dma_addr(umem); - pgsz = roundup_pow_of_two((dma_addr ^ (umem->length - 1 + dma_addr)) + 1); - return !!ib_umem_find_best_pgoff(umem, pgsz, U64_MAX); + pgsz = ib_umem_find_best_pgsz(umem, ULONG_MAX, + ib_umem_start_dma_addr(umem)); + return pgsz && ib_umem_num_dma_blocks(umem, pgsz) == 1; } struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, @@ -162,9 +184,43 @@ int ib_umem_check_rereg(struct ib_umem *umem, int flags, int new_access_flags); #include -static inline struct ib_umem *ib_umem_get(struct ib_device *device, - unsigned long addr, size_t size, - int access) +static inline struct ib_umem * +ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, + unsigned long addr, size_t size, + int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * +ib_umem_get_attr(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + size_t size, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * +ib_umem_get_attr_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, u16 attr_id, + u64 addr, size_t size, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * +ib_umem_get_cq_buf(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, size_t size, + int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct ib_umem * +ib_umem_get_cq_buf_or_va(struct ib_device *device, + const struct uverbs_attr_bundle *attrs, u64 addr, + size_t size, int access) { return ERR_PTR(-EOPNOTSUPP); } @@ -175,7 +231,7 @@ static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offs } static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt) + u64 virt) { return 0; } @@ -185,6 +241,10 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, { return 0; } +static inline bool ib_umem_is_contiguous(struct ib_umem *umem) +{ + return false; +} static inline struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, unsigned long offset, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 9dd76f489a0b..794746de8db0 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -275,6 +275,7 @@ enum ib_device_cap_flags { IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL, IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT, IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE, + IB_DEVICE_CC_DMA_BOUNCE = IB_UVERBS_DEVICE_CC_DMA_BOUNCE, }; enum ib_kernel_cap_flags { @@ -1738,7 +1739,6 @@ struct ib_cq { u8 interrupt:1; u8 shared:1; unsigned int comp_vector; - struct ib_umem *umem; /* * Implementation details of the RDMA core, don't use in drivers: @@ -1977,6 +1977,11 @@ struct ib_dmah { struct ib_mr { struct ib_device *device; + /* + * Due to IB_MR_REREG_PD pd is not a fixed pointer and can change. For a + * user MR, this value should only be read from a system call that holds + * the uobject lock, or the driver should disable in-place REREG_PD. + */ struct ib_pd *pd; u32 lkey; u32 rkey; @@ -2951,6 +2956,8 @@ struct ib_device { u16 kverbs_provider:1; /* CQ adaptive moderation (RDMA DIM) */ u16 use_cq_dim:1; + /* CoCo guest with DMA bounce buffering required */ + u16 cc_dma_bounce:1; u8 node_type; u32 phys_port_cnt; struct ib_device_attr attrs; @@ -3101,6 +3108,7 @@ void ib_set_client_data(struct ib_device *device, struct ib_client *client, void ib_set_device_ops(struct ib_device *device, const struct ib_device_ops *ops); +#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma, unsigned long pfn, unsigned long size, pgprot_t prot, struct rdma_user_mmap_entry *entry); @@ -3112,13 +3120,7 @@ int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext, size_t length, u32 min_pgoff, u32 max_pgoff); -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) void rdma_user_mmap_disassociate(struct ib_device *device); -#else -static inline void rdma_user_mmap_disassociate(struct ib_device *device) -{ -} -#endif static inline int rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext, @@ -3138,6 +3140,66 @@ rdma_user_mmap_entry_get(struct ib_ucontext *ucontext, void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry); void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry); +#else +static inline int rdma_user_mmap_io(struct ib_ucontext *ucontext, + struct vm_area_struct *vma, + unsigned long pfn, unsigned long size, + pgprot_t prot, + struct rdma_user_mmap_entry *entry) +{ + return -EINVAL; +} + +static inline int +rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext, + struct rdma_user_mmap_entry *entry, size_t length) +{ + return -EINVAL; +} + +static inline int +rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext, + struct rdma_user_mmap_entry *entry, + size_t length, u32 min_pgoff, u32 max_pgoff) +{ + return -EINVAL; +} + +static inline void rdma_user_mmap_disassociate(struct ib_device *device) +{ +} + +static inline int +rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext, + struct rdma_user_mmap_entry *entry, + size_t length, u32 pgoff) +{ + return -EINVAL; +} + +static inline struct rdma_user_mmap_entry * +rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext, + unsigned long pgoff) +{ + return NULL; +} + +static inline struct rdma_user_mmap_entry * +rdma_user_mmap_entry_get(struct ib_ucontext *ucontext, + struct vm_area_struct *vma) +{ + return NULL; +} + +static inline void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry) +{ +} + +static inline void +rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) +{ +} +#endif static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len) { diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h index 7d8de561f71b..7ffc83262a01 100644 --- a/include/rdma/rdma_vt.h +++ b/include/rdma/rdma_vt.h @@ -438,26 +438,6 @@ struct rvt_dev_info { struct rvt_wss *wss; }; -/** - * rvt_set_ibdev_name - Craft an IB device name from client info - * @rdi: pointer to the client rvt_dev_info structure - * @name: client specific name - * @unit: client specific unit number. - */ -static inline void rvt_set_ibdev_name(struct rvt_dev_info *rdi, - const char *fmt, const char *name, - const int unit) -{ - /* - * FIXME: rvt and its users want to touch the ibdev before - * registration and have things like the name work. We don't have the - * infrastructure in the core to support this directly today, hack it - * to work by setting the name manually here. - */ - dev_set_name(&rdi->ibdev.dev, fmt, name, unit); - strscpy(rdi->ibdev.name, dev_name(&rdi->ibdev.dev), IB_DEVICE_NAME_MAX); -} - /** * rvt_get_ibdev_name - return the IB name * @rdi: rdmavt device diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index c89428030d61..24fd36213023 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -590,6 +590,28 @@ struct uapi_definition { UA_OPTIONAL, \ .is_udata = 1) +/* + * Per-attribute UMEM descriptor. The payload is a single + * struct ib_uverbs_buffer_desc identifying a memory region backed by + * dma-buf or user virtual address. _access selects UA_OPTIONAL or + * UA_MANDATORY. Drivers obtain a umem from the attribute via the + * ib_umem_get_*() wrapper helpers. + */ +#define UVERBS_ATTR_UMEM(_attr_id, _access) \ + UVERBS_ATTR_PTR_IN(_attr_id, \ + UVERBS_ATTR_TYPE(struct ib_uverbs_buffer_desc), \ + _access) + +/* + * Bit masks of the @flags / @optional_flags fields of struct + * ib_uverbs_buffer_desc that the kernel understands. @flags is strict: + * any bit outside the known mask makes the call fail with -EINVAL. + * @optional_flags is advisory: bits outside the known mask are silently + * dropped. Both masks are extended as new bits are introduced. + */ +#define IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK 0U +#define IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK 0U + /* ================================================= * Parsing infrastructure * ================================================= @@ -668,8 +690,6 @@ rdma_udata_to_uverbs_attr_bundle(struct ib_udata *udata) (udata ? container_of(rdma_udata_to_uverbs_attr_bundle(udata)->context, \ drv_dev_struct, member) : (drv_dev_struct *)NULL) -struct ib_device *rdma_udata_to_dev(struct ib_udata *udata); - #define IS_UVERBS_COPY_ERR(_ret) ((_ret) && (_ret) != -ENOENT) static inline const struct uverbs_attr *uverbs_attr_get(const struct uverbs_attr_bundle *attrs_bundle, @@ -864,6 +884,8 @@ int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, size_t idx, u64 allowed_bits); int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx, const void *from, size_t size); +int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle, + u16 attr_id, struct ib_uverbs_buffer_desc *desc); __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, gfp_t flags); @@ -902,6 +924,8 @@ int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req, size_t kernel_size, size_t minimum_size); int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len); +int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm, + u64 valid_cm); #else static inline int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, @@ -920,6 +944,12 @@ static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, { return -EINVAL; } +static inline int +uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle, + u16 attr_id, struct ib_uverbs_buffer_desc *desc) +{ + return -EINVAL; +} static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size) { @@ -971,6 +1001,12 @@ static inline int _ib_respond_udata(struct ib_udata *udata, const void *src, { return -EINVAL; } + +static inline int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, + u64 req_cm, u64 valid_cm) +{ + return -EINVAL; +} #endif #define uverbs_get_const_signed(_to, _attrs_bundle, _idx) \ @@ -1051,9 +1087,6 @@ uverbs_get_raw_fd(int *to, const struct uverbs_attr_bundle *attrs_bundle, _ib_copy_validate_udata_in(_udata, &(_req), sizeof(_req), \ offsetofend(typeof(_req), _end_member)) -int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm, - u64 valid_cm); - /** * ib_copy_validate_udata_in_cm - Copy the req structure and check the comp_mask * @_udata: The system calls ib_udata struct diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index 6a253b7dc5ea..5a07f9a6dcd1 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -147,6 +147,7 @@ struct uverbs_obj_fd_type { struct uverbs_obj_type type; void (*destroy_object)(struct ib_uobject *uobj, enum rdma_remove_reason why); + void (*release_cleanup)(struct ib_uobject *uobj); const struct file_operations *fops; const char *name; int flags; @@ -190,7 +191,8 @@ int uverbs_uobject_release(struct ib_uobject *uobj); #define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) - \ sizeof(char)) -#define UVERBS_TYPE_ALLOC_FD(_obj_size, _destroy_object, _fops, _name, _flags) \ +#define UVERBS_TYPE_ALLOC_FD_RELEASE(_obj_size, _destroy_object, \ + _release_cleanup, _fops, _name, _flags) \ ((&((const struct uverbs_obj_fd_type) \ {.type = { \ .type_class = &uverbs_fd_class, \ @@ -199,9 +201,13 @@ int uverbs_uobject_release(struct ib_uobject *uobj); sizeof(struct ib_uobject)), \ }, \ .destroy_object = _destroy_object, \ + .release_cleanup = _release_cleanup, \ .fops = _fops, \ .name = _name, \ .flags = _flags}))->type) +#define UVERBS_TYPE_ALLOC_FD(_obj_size, _destroy_object, _fops, _name, _flags) \ + UVERBS_TYPE_ALLOC_FD_RELEASE(_obj_size, _destroy_object, NULL, \ + _fops, _name, _flags) #define UVERBS_TYPE_ALLOC_IDR_SZ(_size, _destroy_object) \ ((&((const struct uverbs_obj_idr_type) \ {.type = { \ diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index 40955eaba32e..a4599d7b736a 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -126,7 +126,7 @@ struct bnxt_re_resize_cq_req { }; enum bnxt_re_qp_mask { - BNXT_RE_QP_REQ_MASK_VAR_WQE_SQ_SLOTS = 0x1, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR = 0x1, }; struct bnxt_re_qp_req { @@ -135,6 +135,11 @@ struct bnxt_re_qp_req { __aligned_u64 qp_handle; __aligned_u64 comp_mask; __u32 sq_slots; + __u32 sq_npsn; +}; + +enum bnxt_re_create_qp_attrs { + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE = UVERBS_ID_DRIVER_NS_WITH_UHW, }; struct bnxt_re_qp_resp { diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 72041c1b0ea5..839835bd4b23 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -117,6 +117,7 @@ enum uverbs_attrs_create_cq_cmd_attr_ids { UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH, UVERBS_ATTR_CREATE_CQ_BUFFER_FD, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET, + UVERBS_ATTR_CREATE_CQ_BUF_UMEM, }; enum uverbs_attrs_destroy_cq_cmd_attr_ids { @@ -158,6 +159,9 @@ enum uverbs_attrs_create_qp_cmd_attr_ids { UVERBS_ATTR_CREATE_QP_EVENT_FD, UVERBS_ATTR_CREATE_QP_RESP_CAP, UVERBS_ATTR_CREATE_QP_RESP_QP_NUM, + UVERBS_ATTR_CREATE_QP_BUF_UMEM, + UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, }; enum uverbs_attrs_destroy_qp_cmd_attr_ids { diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h index 90c5cd8e7753..51030c27d479 100644 --- a/include/uapi/rdma/ib_user_ioctl_verbs.h +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h @@ -273,4 +273,31 @@ struct ib_uverbs_gid_entry { __u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */ }; +enum ib_uverbs_buffer_type { + IB_UVERBS_BUFFER_TYPE_DMABUF, + IB_UVERBS_BUFFER_TYPE_VA, +}; + +/* + * Describes a single buffer backed by dma-buf or user virtual address. + * Used as the payload of a per-attribute UVERBS_ATTR_UMEM-typed attribute. + * + * @type: buffer type from enum ib_uverbs_buffer_type + * @fd: dma-buf file descriptor (valid for IB_UVERBS_BUFFER_TYPE_DMABUF) + * @flags: required flags; the kernel rejects the call with -EINVAL if any + * bit is not understood. No bits are defined yet. + * @optional_flags: advisory flags; bits the kernel does not understand are + * silently ignored. No bits are defined yet. + * @addr: offset within dma-buf, or user virtual address for VA + * @length: buffer length in bytes + */ +struct ib_uverbs_buffer_desc { + __u32 type; + __s32 fd; + __u32 flags; + __u32 optional_flags; + __aligned_u64 addr; + __aligned_u64 length; +}; + #endif diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h index 3b7bd99813e9..d2aeadb6d2f9 100644 --- a/include/uapi/rdma/ib_user_verbs.h +++ b/include/uapi/rdma/ib_user_verbs.h @@ -1368,6 +1368,8 @@ enum ib_uverbs_device_cap_flags { IB_UVERBS_DEVICE_FLUSH_PERSISTENT = 1ULL << 39, /* Atomic write attributes */ IB_UVERBS_DEVICE_ATOMIC_WRITE = 1ULL << 40, + /* CoCo guest with DMA bounce buffering required */ + IB_UVERBS_DEVICE_CC_DMA_BOUNCE = 1ULL << 41, }; enum ib_uverbs_raw_packet_caps { diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h index 01a2a050e468..ddb898afd813 100644 --- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h +++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h @@ -274,6 +274,11 @@ enum mlx5_ib_device_query_context_attrs { enum mlx5_ib_create_cq_attrs { MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX = UVERBS_ID_DRIVER_NS_WITH_UHW, + MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM, +}; + +enum mlx5_ib_create_qp_attrs { + MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM = UVERBS_ID_DRIVER_NS_WITH_UHW, }; enum mlx5_ib_reg_dmabuf_mr_attrs { diff --git a/tools/testing/selftests/rdma/Makefile b/tools/testing/selftests/rdma/Makefile index 7dd7cba7a73c..07af7f15c1bf 100644 --- a/tools/testing/selftests/rdma/Makefile +++ b/tools/testing/selftests/rdma/Makefile @@ -2,6 +2,7 @@ TEST_PROGS := rxe_rping_between_netns.sh \ rxe_ipv6.sh \ rxe_socket_with_netns.sh \ - rxe_test_NETDEV_UNREGISTER.sh + rxe_test_NETDEV_UNREGISTER.sh \ + rxe_sent_rcvd_bytes.sh include ../lib.mk diff --git a/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh new file mode 100755 index 000000000000..0e4fbfeebd22 --- /dev/null +++ b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Configuration +PORT=4791 +MODS=("tun" "rdma_rxe") + +exec > /dev/null + +# --- Helper: Cleanup Routine --- +cleanup() { + echo "Cleaning up resources..." + rdma link del rxe0 2>/dev/null + ip link del tun0 2>/dev/null + for m in "${MODS[@]}"; do modprobe -r "$m" 2>/dev/null; done +} + +# Ensure cleanup runs on script exit or interrupt +trap cleanup EXIT + +# --- Phase 1: Environment Check --- +if [[ $EUID -ne 0 ]]; then + echo "Error: This script must be run as root." + exit 1 +fi + +for m in "${MODS[@]}"; do + modprobe "$m" || { echo "Error: Failed to load $m"; exit 1; } +done + +# --- Phase 2: Create Interfaces & RXE Links --- +echo "Creating tun0 (1.1.1.1) and rxe0..." +ip tuntap add mode tun tun0 +ip addr add 1.1.1.1/24 dev tun0 +ip link set tun0 up +rdma link add rxe0 type rxe netdev tun0 + +# Verify port 4791 is listening +if ! ss -Huln sport = :$PORT | grep -q ":$PORT"; then + echo "Error: UDP port $PORT not found after rxe0 creation" + exit 1 +fi + +orig_s=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_xmit_data` +orig_r=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_rcv_data` + +rping -s -a 1.1.1.1 -C 3 -v & +sleep 1 +rping -c -a 1.1.1.1 -C 3 -d -v + +new_s=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_xmit_data` +new_r=`cat /sys/class/infiniband/rxe0/ports/1/counters/port_rcv_data` + +echo sent $new_s $orig_s +echo rcvd $new_r $orig_r + +result0=$((new_s - orig_s)) +result1=$((new_r - orig_r)) + +if [ $result0 != $result1 ]; then + echo "Error: sent and rcvd bytes different" + echo $result0 + echo $result1 + exit 1 +fi + +echo "Deleting rxe0..." +rdma link del rxe0 + +# Port should now be gone +if ss -Huln sport = :$PORT | grep -q ":$PORT"; then + echo "Error: UDP port $PORT still exists after all links deleted" + exit 1 +fi + +echo "Test passed successfully."