mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-22 03:27:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Browse Source 
Pull rdma updates from Jason Gunthorpe:
 "Many AI driven bug fixes, and several big driver API cleanups

   - Driver bug fixes and minor cleanups in mlx5, hns, rxe, efa, siw,
     rtrs, mana, irdma, mlx4. Commonly error path flows, integer
     arithmetic overflows on unsafe data, out of bounds access, and use
     after free issues under races.

   - Second half of the new udata API for drivers focusing on uAPI
     response

   - bnxt_re supports more options for QP creation that will allow a dv
     path in rdma-core

   - Untangle the module dependencies so drivers don't link to
     ib_uverbs.ko as was originall intended

   - Provide a new way to handle umems with a consistent simplified uAPI
     and update several drivers to use it. This brings dmabuf support to
     more places and more drivers

   - Support for mlx5 rate limit and packet pacing for UD and UC

   - A batch of fixes for the new shared FRMR pools infrastructure"

* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (148 commits)
  RDMA/irdma: Replace waitqueue and flag with completion
  RDMA/hns: Fix memory leak of bonding resources
  RDMA/rtrs-srv: Bound RDMA-Write length to chunk size in rdma_write_sg
  docs: infiniband: correct name of option to enable the ib_uverbs module
  RDMA/bnxt_re: Reject GET_TOGGLE_MEM when toggle page was not allocated
  RDMA/bnxt_re: Fail DBR related page allocation UAPIs if the feature is disabled
  RDMA/bnxt_re: Avoid repeated requests to allocate WC pages
  RDMA/bnxt_re: Proper rollback if the ioremap fails
  RDMA/bnxt_re: Add a max slot check for SQ
  RDMA/bnxt_re: Avoid displaying the kernel pointer
  RDMA/bnxt_re: Free CQ toggle page after firmware teardown
  RDMA/bnxt_re: Free SRQ toggle page after firmware teardown
  RDMA/bnxt_re: Initialize dpi variable to zero
  ABI: sysfs-class-infiniband: minor cleanup
  RDMA/mlx5: Release the HW‑provided UAR index rather than the SW one
  RDMA/mlx5: Fix undefined shift of user RQ WQE size
  RDMA/mlx5: Remove raw RSS QP restrack tracking
  RDMA/mlx5: Remove DCT restrack tracking
  RDMA/mlx5: Drop FRMR pool handle on UMR revoke failure
  RDMA/core: Add ib_frmr_pool_drop for unrecoverable handles
  ...
This commit is contained in:
Linus Torvalds
2026-06-18 08:16:21 -07:00
parent 00d07402df d9c8c45e6d
commit 9e7e663345
163 changed files with 3782 additions and 1840 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Documentation/ABI/stable/sysfs-class-infiniband
View File

@@ -148,17 +148,17 @@ Description:
**Data info**:
port_xmit_data: (RO) Total number of data octets, divided by 4
(lanes), transmitted on all VLs. This is 64 bit counter
(lanes), transmitted on all VLs. This is a 64-bit counter
port_rcv_data: (RO) Total number of data octets, divided by 4
(lanes), received on all VLs. This is 64 bit counter.
(lanes), received on all VLs. This is a 64-bit counter.
port_xmit_packets: (RO) Total number of packets transmitted on
all VLs from this port. This may include packets with errors.
This is 64 bit counter.
This is a 64-bit counter.
port_rcv_packets: (RO) Total number of packets (this may include
packets containing Errors. This is 64 bit counter.
packets containing Errors). This is a 64-bit counter.
link_downed: (RO) Total number of times the Port Training state
machine has failed the link error recovery process and downed

2
Documentation/infiniband/user_verbs.rst
View File

@@ -2,7 +2,7 @@
Userspace verbs access
======================
The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_VERBS,
The ib_uverbs module, built by enabling CONFIG_INFINIBAND_USER_ACCESS,
enables direct userspace access to IB hardware via "verbs," as
described in chapter 11 of the InfiniBand Architecture Specification.

2
Documentation/translations/zh_CN/infiniband/user_verbs.rst
View File

@@ -17,7 +17,7 @@
用户空间verbs访问
=================
ib_uverbs模块，通过启用CONFIG_INFINIBAND_USER_VERBS构建，使用户空间
ib_uverbs模块，通过启用CONFIG_INFINIBAND_USER_ACCESS构建，使用户空间
通过“verbs”直接访问IB硬件，如InfiniBand架构规范第11章所述。
要使用verbs，需要libibverbs库，可从https://github.com/linux-rdma/rdma-core。

1
MAINTAINERS
View File

@@ -12937,7 +12937,6 @@ F: include/linux/avf/virtchnl.h
F: include/linux/net/intel/*/
INTEL ETHERNET PROTOCOL DRIVER FOR RDMA
M: Krzysztof Czurylo <krzysztof.czurylo@intel.com>
M: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
L: linux-rdma@vger.kernel.org
S: Supported

4
drivers/infiniband/Kconfig
View File

@@ -37,6 +37,10 @@ config INFINIBAND_USER_ACCESS
libibverbs, libibcm and a hardware driver library from
rdma-core <https://github.com/linux-rdma/rdma-core>.
config INFINIBAND_USER_ACCESS_CORE
bool
default y if INFINIBAND_USER_ACCESS != n
config INFINIBAND_USER_MEM
bool
depends on INFINIBAND_USER_ACCESS != n

16
drivers/infiniband/core/Makefile
View File

@@ -5,19 +5,22 @@ user_access-$(CONFIG_INFINIBAND_ADDR_TRANS) := rdma_ucm.o
obj-$(CONFIG_INFINIBAND) += ib_core.o ib_cm.o iw_cm.o \
$(infiniband-y)
obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o
obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y)
obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o \
$(user_access-y) \
ib_uverbs_support.o
ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
device.o cache.o netlink.o \
roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
multicast.o mad.o smi.o agent.o mad_rmpp.o \
nldev.o restrack.o counters.o ib_core_uverbs.o \
nldev.o restrack.o counters.o \
trace.o lag.o iter.o frmr_pools.o
ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o
ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o
ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o
ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
ib_core-$(CONFIG_INFINIBAND_USER_ACCESS_CORE) += ib_core_uverbs.o
ib_cm-y := cm.o cm_trace.o
@@ -33,7 +36,7 @@ rdma_ucm-y := ucma.o
ib_umad-y := user_mad.o
ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
rdma_core.o uverbs_std_types.o uverbs_ioctl.o \
uverbs_std_types.o uverbs_ioctl.o \
uverbs_std_types_cq.o \
uverbs_std_types_dmabuf.o \
uverbs_std_types_dmah.o \
@@ -43,5 +46,8 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
uverbs_std_types_async_fd.o \
uverbs_std_types_srq.o \
uverbs_std_types_wq.o \
uverbs_std_types_qp.o \
ucaps.o
uverbs_std_types_qp.o
ib_uverbs_support-y := rdma_core.o \
ucaps.o \
uverbs_flow.o

4
drivers/infiniband/core/addr.c
View File

@@ -438,7 +438,7 @@ static int addr6_resolve(struct sockaddr *src_sock,
static bool is_dst_local(const struct dst_entry *dst)
{
if (dst->ops->family == AF_INET)
return !!(dst_rtable(dst)->rt_type & RTN_LOCAL);
return dst_rtable(dst)->rt_type == RTN_LOCAL;
else if (dst->ops->family == AF_INET6)
return !!(dst_rt6_info(dst)->rt6i_flags & RTF_LOCAL);
else
@@ -850,7 +850,7 @@ static struct notifier_block nb = {
int addr_init(void)
{
addr_wq = alloc_ordered_workqueue("ib_addr", 0);
addr_wq = alloc_workqueue("ib_addr", WQ_UNBOUND, 0);
if (!addr_wq)
return -ENOMEM;

13
drivers/infiniband/core/cm.c
View File

@@ -530,6 +530,7 @@ static int cm_init_av_by_path(struct sa_path_rec *path,
struct rdma_ah_attr new_ah_attr;
struct cm_device *cm_dev;
struct cm_port *port;
u16 pkey_index;
int ret;
port = get_cm_port_from_path(path, sgid_attr);
@@ -538,12 +539,10 @@ static int cm_init_av_by_path(struct sa_path_rec *path,
cm_dev = port->cm_dev;
ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
be16_to_cpu(path->pkey), &av->pkey_index);
be16_to_cpu(path->pkey), &pkey_index);
if (ret)
return ret;
cm_set_av_port(av, port);
/*
* av->ah_attr might be initialized based on wc or during
* request processing time which might have reference to sgid_attr.
@@ -558,6 +557,8 @@ static int cm_init_av_by_path(struct sa_path_rec *path,
if (ret)
return ret;
av->pkey_index = pkey_index;
cm_set_av_port(av, port);
av->timeout = path->packet_life_time + 1;
rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
return 0;
@@ -2184,8 +2185,10 @@ static int cm_req_handler(struct cm_work *work)
cm_id_priv->av.ah_attr.roce.dmac);
work->path[0].hop_limit = grh->hop_limit;
/* This destroy call is needed to pair with cm_init_av_for_response */
cm_destroy_av(&cm_id_priv->av);
/*
* cm_init_av_by_path() will internally pair with the above
* cm_init_av_for_response() if it succeeds.
*/
ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
if (ret) {
int err;

6
drivers/infiniband/core/cma_configfs.c
View File

@@ -255,7 +255,7 @@ static void release_cma_ports_group(struct config_item *item)
cma_dev_group->ports = NULL;
};
static struct configfs_item_operations cma_ports_item_ops = {
static const struct configfs_item_operations cma_ports_item_ops = {
.release = release_cma_ports_group
};
@@ -264,7 +264,7 @@ static const struct config_item_type cma_ports_group_type = {
.ct_owner = THIS_MODULE
};
static struct configfs_item_operations cma_device_item_ops = {
static const struct configfs_item_operations cma_device_item_ops = {
.release = release_cma_dev
};
@@ -327,7 +327,7 @@ static void drop_cma_dev(struct config_group *cgroup, struct config_item *item)
config_item_put(item);
}
static struct configfs_group_operations cma_subsys_group_ops = {
static const struct configfs_group_operations cma_subsys_group_ops = {
.make_group = make_cma_dev,
.drop_item = drop_cma_dev,
};

2
drivers/infiniband/core/core_priv.h
View File

@@ -321,7 +321,7 @@ void nldev_exit(void);
struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
struct ib_qp_init_attr *attr,
struct ib_udata *udata,
struct uverbs_attr_bundle *uattrs,
struct ib_uqp_object *uobj, const char *caller);
void ib_qp_usecnt_inc(struct ib_qp *qp);

12
drivers/infiniband/core/counters.c
View File

@@ -198,12 +198,20 @@ static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port,
ret = __rdma_counter_bind_qp(counter, qp, port);
if (ret)
goto err_mode;
goto err_bind;
rdma_restrack_parent_name(&counter->res, &qp->res);
rdma_restrack_add(&counter->res);
return counter;
err_bind:
mutex_lock(&port_counter->lock);
port_counter->num_counters--;
if (!port_counter->num_counters &&
port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL)
__counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0,
false);
mutex_unlock(&port_counter->lock);
err_mode:
rdma_free_hw_stats_struct(counter->stats);
err_stats:
@@ -661,7 +669,7 @@ void rdma_counter_init(struct ib_device *dev)
fail:
for (i = port; i >= rdma_start_port(dev); i--) {
port_counter = &dev->port_data[port].port_counter;
port_counter = &dev->port_data[i].port_counter;
rdma_free_hw_stats_struct(port_counter->hstats);
port_counter->hstats = NULL;
mutex_destroy(&port_counter->lock);

12
drivers/infiniband/core/device.c
View File

@@ -42,6 +42,7 @@
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/hashtable.h>
#include <linux/cc_platform.h>
#include <rdma/rdma_netlink.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_cache.h>
@@ -1245,7 +1246,6 @@ static int assign_name(struct ib_device *device, const char *name)
*/
static int setup_device(struct ib_device *device)
{
struct ib_udata uhw = {.outlen = 0, .inlen = 0};
int ret;
ib_device_check_mandatory(device);
@@ -1257,7 +1257,7 @@ static int setup_device(struct ib_device *device)
}
memset(&device->attrs, 0, sizeof(device->attrs));
ret = device->ops.query_device(device, &device->attrs, &uhw);
ret = device->ops.query_device(device, &device->attrs, NULL);
if (ret) {
dev_warn(&device->dev,
"Couldn't query the device attributes\n");
@@ -1419,6 +1419,14 @@ int ib_register_device(struct ib_device *device, const char *name,
*/
WARN_ON(dma_device && !dma_device->dma_parms);
device->dma_device = dma_device;
/*
* In a CoCo guest every device is currently assumed to be untrusted
* (T=0) and therefore subject to DMA bouncing. Once trusted (T=1)
* device detection is wired up, narrow this check to exclude such
* devices.
*/
if (dma_device && cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
device->cc_dma_bounce = 1;
ret = setup_device(device);
if (ret)

106
drivers/infiniband/core/frmr_pools.c
View File

@@ -97,13 +97,44 @@ static void destroy_all_handles_in_queue(struct ib_device *device,
}
}
/*
* Bulk-move all handles from @src into @dst without allocating new pages.
* If @dst has a partial tail page, fill it handle-by-handle from @src first
* to preserve the invariant that only the tail page is partial, then splice
* the remaining @src pages onto @dst. On return @src is empty.
*
* Caller must hold the lock protecting both queues.
*/
static void splice_frmr_queue_locked(struct frmr_queue *dst,
struct frmr_queue *src)
{
u32 free_in_tail = dst->ci % NUM_HANDLES_PER_PAGE;
u32 handle;
if (free_in_tail) {
free_in_tail = NUM_HANDLES_PER_PAGE - free_in_tail;
while (free_in_tail && src->ci) {
handle = pop_handle_from_queue_locked(src);
push_handle_to_queue_locked(dst, handle);
free_in_tail--;
}
}
if (src->ci > 0) {
list_splice_tail_init(&src->pages_list, &dst->pages_list);
dst->num_pages += src->num_pages;
dst->ci += src->ci;
src->num_pages = 0;
src->ci = 0;
}
}
static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool)
{
struct ib_frmr_pools *pools = device->frmr_pools;
u32 total, to_destroy, destroyed = 0;
bool has_work = false;
u32 *handles;
u32 handle;
spin_lock(&pool->lock);
total = pool->queue.ci + pool->inactive_queue.ci + pool->in_use;
@@ -112,7 +143,7 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool)
return false;
}
to_destroy = total - pool->pinned_handles;
to_destroy = min(total - pool->pinned_handles, pool->inactive_queue.ci);
handles = kcalloc(to_destroy, sizeof(*handles), GFP_ATOMIC);
if (!handles) {
@@ -121,15 +152,13 @@ static bool age_pinned_pool(struct ib_device *device, struct ib_frmr_pool *pool)
}
/* Destroy all excess handles in the inactive queue */
while (pool->inactive_queue.ci && destroyed < to_destroy) {
handles[destroyed++] = pop_handle_from_queue_locked(
for (; destroyed < to_destroy; destroyed++)
handles[destroyed] = pop_handle_from_queue_locked(
&pool->inactive_queue);
}
/* Move all handles from regular queue to inactive queue */
while (pool->queue.ci) {
handle = pop_handle_from_queue_locked(&pool->queue);
push_handle_to_queue_locked(&pool->inactive_queue, handle);
if (pool->queue.ci > 0) {
splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue);
has_work = true;
}
@@ -158,13 +187,7 @@ static void pool_aging_work(struct work_struct *work)
/* Move all pages from regular queue to inactive queue */
spin_lock(&pool->lock);
if (pool->queue.ci > 0) {
list_splice_tail_init(&pool->queue.pages_list,
&pool->inactive_queue.pages_list);
pool->inactive_queue.num_pages = pool->queue.num_pages;
pool->inactive_queue.ci = pool->queue.ci;
pool->queue.num_pages = 0;
pool->queue.ci = 0;
splice_frmr_queue_locked(&pool->inactive_queue, &pool->queue);
has_work = true;
}
spin_unlock(&pool->lock);
@@ -426,7 +449,7 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key,
if (!handles)
return -ENOMEM;
ret = pools->pool_ops->create_frmrs(device, key, handles,
ret = pools->pool_ops->create_frmrs(device, &driver_key, handles,
needed_handles);
if (ret) {
kfree(handles);
@@ -438,11 +461,16 @@ int ib_frmr_pools_set_pinned(struct ib_device *device, struct ib_frmr_key *key,
ret = push_handle_to_queue_locked(&pool->queue,
handles[i]);
if (ret)
goto end;
break;
}
spin_unlock(&pool->lock);
if (ret) {
/* Destroy handles created but never pushed to the pool. */
pools->pool_ops->destroy_frmrs(device, &handles[i],
needed_handles - i);
}
end:
spin_unlock(&pool->lock);
kfree(handles);
schedule_aging:
@@ -501,7 +529,9 @@ int ib_frmr_pool_pop(struct ib_device *device, struct ib_mr *mr)
struct ib_frmr_pools *pools = device->frmr_pools;
struct ib_frmr_pool *pool;
WARN_ON_ONCE(!device->frmr_pools);
if (WARN_ON_ONCE(!pools))
return -EINVAL;
pool = ib_frmr_pool_find(pools, &mr->frmr.key);
if (!pool) {
pool = create_frmr_pool(device, &mr->frmr.key);
@@ -519,9 +549,8 @@ EXPORT_SYMBOL(ib_frmr_pool_pop);
* @device: The device to push the FRMR handle to.
* @mr: The MR containing the FRMR handle to push back to the pool.
*
* Returns 0 on success, negative error code on failure.
*/
int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
void ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
{
struct ib_frmr_pool *pool = mr->frmr.pool;
struct ib_frmr_pools *pools = device->frmr_pools;
@@ -529,19 +558,38 @@ int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
int ret;
spin_lock(&pool->lock);
/* Schedule aging every time an empty pool becomes non-empty */
if (pool->queue.ci == 0)
schedule_aging = true;
pool->in_use--;
ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle);
if (ret == 0)
pool->in_use--;
/* Schedule aging every time an empty pool becomes non-empty */
if (!ret && pool->queue.ci == 1)
schedule_aging = true;
spin_unlock(&pool->lock);
if (ret == 0 && schedule_aging)
if (ret) {
pools->pool_ops->destroy_frmrs(device, &mr->frmr.handle, 1);
return;
}
if (schedule_aging)
queue_delayed_work(pools->aging_wq, &pool->aging_work,
secs_to_jiffies(READ_ONCE(pools->aging_period_sec)));
return ret;
}
EXPORT_SYMBOL(ib_frmr_pool_push);
/*
* Drop a handle previously popped from the pool without returning it for
* reuse. The caller is responsible for destroying the underlying hardware
* resource.
*/
void ib_frmr_pool_drop(struct ib_mr *mr)
{
struct ib_frmr_pool *pool = mr->frmr.pool;
spin_lock(&pool->lock);
pool->in_use--;
spin_unlock(&pool->lock);
}
EXPORT_SYMBOL(ib_frmr_pool_drop);

263
drivers/infiniband/core/ib_core_uverbs.c
View File

@@ -398,7 +398,7 @@ EXPORT_SYMBOL(rdma_user_mmap_entry_insert);
* The struct ib_device that is handling the uverbs call. Must not be called if
* udata is NULL. The result can be NULL.
*/
struct ib_device *rdma_udata_to_dev(struct ib_udata *udata)
static struct ib_device *rdma_udata_to_dev(struct ib_udata *udata)
{
struct uverbs_attr_bundle *bundle =
rdma_udata_to_uverbs_attr_bundle(udata);
@@ -415,10 +415,9 @@ struct ib_device *rdma_udata_to_dev(struct ib_udata *udata)
return srcu_dereference(bundle->ufile->device->ib_dev,
&bundle->ufile->device->disassociate_srcu);
}
EXPORT_SYMBOL(rdma_udata_to_dev);
#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs);
static uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
{
struct uverbs_attr_bundle *bundle =
rdma_udata_to_uverbs_attr_bundle(udata);
@@ -502,4 +501,258 @@ int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len)
return -EFAULT;
}
EXPORT_SYMBOL(_ib_respond_udata);
#endif
/*
* Must be called with the ufile->device->disassociate_srcu held, and the lock
* must be held until use of the ucontext is finished.
*/
struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile)
{
/*
* We do not hold the hw_destroy_rwsem lock for this flow, instead
* srcu is used. It does not matter if someone races this with
* get_context, we get NULL or valid ucontext.
*/
struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext);
if (!srcu_dereference(ufile->device->ib_dev,
&ufile->device->disassociate_srcu))
return ERR_PTR(-EIO);
if (!ucontext)
return ERR_PTR(-EINVAL);
return ucontext;
}
EXPORT_SYMBOL(ib_uverbs_get_ucontext_file);
int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
{
return 0;
}
EXPORT_SYMBOL(uverbs_destroy_def_handler);
/*
* When calling a destroy function during an error unwind we need to pass in
* the udata that is sanitized of all user arguments. Ie from the driver
* perspective it looks like no udata was passed.
*/
struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
{
attrs->driver_udata = (struct ib_udata){};
return &attrs->driver_udata;
}
EXPORT_SYMBOL_NS_GPL(uverbs_get_cleared_udata, "rdma_core");
/**
* _uverbs_alloc() - Quickly allocate memory for use with a bundle
* @bundle: The bundle
* @size: Number of bytes to allocate
* @flags: Allocator flags
*
* The bundle allocator is intended for allocations that are connected with
* processing the system call related to the bundle. The allocated memory is
* always freed once the system call completes, and cannot be freed any other
* way.
*
* This tries to use a small pool of pre-allocated memory for performance.
*/
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
gfp_t flags)
{
struct bundle_priv *pbundle =
container_of(&bundle->hdr, struct bundle_priv, bundle);
size_t new_used;
void *res;
if (check_add_overflow(size, pbundle->internal_used, &new_used))
return ERR_PTR(-EOVERFLOW);
if (new_used > pbundle->internal_avail) {
struct bundle_alloc_head *buf;
buf = kvmalloc_flex(*buf, data, size, flags);
if (!buf)
return ERR_PTR(-ENOMEM);
buf->next = pbundle->allocated_mem;
pbundle->allocated_mem = buf;
return buf->data;
}
res = (void *)pbundle->internal_buffer + pbundle->internal_used;
pbundle->internal_used =
ALIGN(new_used, sizeof(*pbundle->internal_buffer));
if (want_init_on_alloc(flags))
memset(res, 0, size);
return res;
}
EXPORT_SYMBOL(_uverbs_alloc);
int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
const void *from, size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
size_t min_size;
if (IS_ERR(attr))
return PTR_ERR(attr);
min_size = min_t(size_t, attr->ptr_attr.len, size);
if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
return -EFAULT;
return uverbs_set_output(bundle, attr);
}
EXPORT_SYMBOL(uverbs_copy_to);
int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
size_t idx, const void *from, size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
if (size < attr->ptr_attr.len) {
if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
attr->ptr_attr.len - size))
return -EFAULT;
}
return uverbs_copy_to(bundle, idx, from, size);
}
EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
int _uverbs_get_const_unsigned(u64 *to,
const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 upper_bound, u64 *def_val)
{
const struct uverbs_attr *attr;
attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr)) {
if ((PTR_ERR(attr) != -ENOENT) || !def_val)
return PTR_ERR(attr);
*to = *def_val;
} else {
*to = attr->ptr_attr.data;
}
if (*to > upper_bound)
return -EINVAL;
return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_unsigned);
int _uverbs_get_const_signed(s64 *to,
const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, s64 lower_bound, u64 upper_bound,
s64 *def_val)
{
const struct uverbs_attr *attr;
attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr)) {
if ((PTR_ERR(attr) != -ENOENT) || !def_val)
return PTR_ERR(attr);
*to = *def_val;
} else {
*to = attr->ptr_attr.data;
}
if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
return -EINVAL;
return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_signed);
int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
const struct uverbs_attr *attr;
u64 flags;
attr = uverbs_attr_get(attrs_bundle, idx);
/* Missing attribute means 0 flags */
if (IS_ERR(attr)) {
*to = 0;
return 0;
}
/*
* New userspace code should use 8 bytes to pass flags, but we
* transparently support old userspaces that were using 4 bytes as
* well.
*/
if (attr->ptr_attr.len == 8)
flags = attr->ptr_attr.data;
else if (attr->ptr_attr.len == 4)
flags = *(u32 *)&attr->ptr_attr.data;
else
return -EINVAL;
if (flags & ~allowed_bits)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags64);
int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
u64 flags;
int ret;
ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
if (ret)
return ret;
if (flags > U32_MAX)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags32);
/**
* uverbs_get_buffer_desc - Read a buffer descriptor from a uverbs attr.
* @attrs_bundle: uverbs attribute bundle.
* @attr_id: id of an UVERBS_ATTR_UMEM-typed attribute.
* @desc: descriptor to fill.
*
* Return: 0 on success, -ENOENT if @attr_id is not set, -EINVAL on a
* malformed descriptor, or any other negative errno propagated from
* uverbs_copy_from() (notably -EFAULT on copy_from_user() failure).
*/
int uverbs_get_buffer_desc(const struct uverbs_attr_bundle *attrs_bundle,
u16 attr_id, struct ib_uverbs_buffer_desc *desc)
{
int ret;
ret = uverbs_copy_from(desc, attrs_bundle, attr_id);
if (ret)
return ret;
if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK)
return -EINVAL;
desc->optional_flags &= IB_UVERBS_BUFFER_DESC_OPTIONAL_FLAGS_KNOWN_MASK;
return 0;
}
EXPORT_SYMBOL(uverbs_get_buffer_desc);
/* Once called an abort will call through to the type's destroy_hw() */
void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
u16 idx)
{
struct bundle_priv *pbundle =
container_of(&bundle->hdr, struct bundle_priv, bundle);
__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
pbundle->uobj_hw_obj_valid);
}
EXPORT_SYMBOL(uverbs_finalize_uobj_create);

4
drivers/infiniband/core/iwcm.c
View File

@@ -518,8 +518,8 @@ static int iw_cm_map(struct iw_cm_id *cm_id, bool active)
cm_id->m_local_addr = cm_id->local_addr;
cm_id->m_remote_addr = cm_id->remote_addr;
strcpy(pm_reg_msg.dev_name, devname);
strcpy(pm_reg_msg.if_name, ifname);
strscpy(pm_reg_msg.dev_name, devname);
strscpy(pm_reg_msg.if_name, ifname);
if (iwpm_register_pid(&pm_reg_msg, RDMA_NL_IWCM) ||
!iwpm_valid_pid())

15
drivers/infiniband/core/nldev.c
View File

@@ -695,7 +695,7 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
struct rdma_restrack_entry *res, uint32_t port)
{
struct ib_mr *mr = container_of(res, struct ib_mr, res);
struct ib_device *dev = mr->pd->device;
struct ib_device *dev = mr->device;
if (has_cap_net_admin) {
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
@@ -711,9 +711,12 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
return -EMSGSIZE;
if (!rdma_is_kernel_res(res) &&
nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id))
return -EMSGSIZE;
if (!rdma_is_kernel_res(res)) {
struct ib_pd *pd = READ_ONCE(mr->pd);
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, pd->res.id))
return -EMSGSIZE;
}
if (fill_res_name_pid(msg, res))
return -EMSGSIZE;
@@ -727,7 +730,7 @@ static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
struct rdma_restrack_entry *res, uint32_t port)
{
struct ib_mr *mr = container_of(res, struct ib_mr, res);
struct ib_device *dev = mr->pd->device;
struct ib_device *dev = mr->device;
if (!dev->ops.fill_res_mr_entry_raw)
return -EINVAL;
@@ -1017,7 +1020,7 @@ static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
struct rdma_restrack_entry *res, uint32_t port)
{
struct ib_mr *mr = container_of(res, struct ib_mr, res);
struct ib_device *dev = mr->pd->device;
struct ib_device *dev = mr->device;
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
goto err;

152
drivers/infiniband/core/rdma_core.c
View File

@@ -42,6 +42,40 @@
#include "core_priv.h"
#include "rdma_core.h"
static void release_ufile_idr_uobject(struct ib_uverbs_file *ufile);
void ib_uverbs_release_file(struct kref *ref)
{
struct ib_uverbs_file *file =
container_of(ref, struct ib_uverbs_file, ref);
struct ib_device *ib_dev;
int srcu_key;
release_ufile_idr_uobject(file);
srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
ib_dev = srcu_dereference(file->device->ib_dev,
&file->device->disassociate_srcu);
if (ib_dev && !ib_dev->ops.disassociate_ucontext)
module_put(ib_dev->ops.owner);
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
if (refcount_dec_and_test(&file->device->refcount))
ib_uverbs_comp_dev(file->device);
if (file->default_async_file)
uverbs_uobject_put(&file->default_async_file->uobj);
put_device(&file->device->dev);
if (file->disassociate_page)
__free_pages(file->disassociate_page, 0);
mutex_destroy(&file->disassociation_lock);
mutex_destroy(&file->umap_lock);
mutex_destroy(&file->ucontext_lock);
kfree(file);
}
EXPORT_SYMBOL_NS_GPL(ib_uverbs_release_file, "rdma_core");
static void uverbs_uobject_free(struct kref *ref)
{
kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu);
@@ -214,6 +248,7 @@ int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)
up_read(&ufile->hw_destroy_rwsem);
return ret;
}
EXPORT_SYMBOL_NS_GPL(uobj_destroy, "rdma_core");
/*
* uobj_get_destroy destroys the HW object and returns a handle to the uobj
@@ -239,6 +274,7 @@ struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
return uobj;
}
EXPORT_SYMBOL_NS_GPL(__uobj_get_destroy, "rdma_core");
/*
* Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success
@@ -255,6 +291,7 @@ int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
uobj_put_destroy(uobj);
return 0;
}
EXPORT_SYMBOL_NS_GPL(__uobj_perform_destroy, "rdma_core");
/* alloc_uobj must be undone by uverbs_destroy_uobject() */
static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs,
@@ -420,6 +457,7 @@ struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
uverbs_uobject_put(uobj);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_NS_GPL(rdma_lookup_get_uobject, "rdma_core");
static struct ib_uobject *
alloc_begin_idr_uobject(const struct uverbs_api_object *obj,
@@ -465,8 +503,8 @@ alloc_begin_fd_uobject(const struct uverbs_api_object *obj,
fd_type =
container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
if (WARN_ON(fd_type->fops && fd_type->fops->release != &uverbs_uobject_fd_release &&
fd_type->fops->release != &uverbs_async_event_release)) {
if (WARN_ON(fd_type->fops &&
fd_type->fops->release != &uverbs_uobject_fd_release)) {
ret = ERR_PTR(-EINVAL);
goto err_fd;
}
@@ -522,6 +560,7 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
}
return ret;
}
EXPORT_SYMBOL_NS_GPL(rdma_alloc_begin_uobject, "rdma_core");
static void alloc_abort_idr_uobject(struct ib_uobject *uobj)
{
@@ -668,6 +707,7 @@ void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
/* Matches the down_read in rdma_alloc_begin_uobject */
up_read(&ufile->hw_destroy_rwsem);
}
EXPORT_SYMBOL_NS_GPL(rdma_alloc_commit_uobject, "rdma_core");
/*
* new_uobj will be assigned to the handle currently used by to_uobj, and
@@ -697,6 +737,7 @@ void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj
*/
uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs);
}
EXPORT_SYMBOL_NS_GPL(rdma_assign_uobject, "rdma_core");
/*
* This consumes the kref for uobj. It is up to the caller to unwind the HW
@@ -727,6 +768,7 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
/* Matches the down_read in rdma_alloc_begin_uobject */
up_read(&ufile->hw_destroy_rwsem);
}
EXPORT_SYMBOL_NS_GPL(rdma_alloc_abort_uobject, "rdma_core");
static void lookup_put_idr_uobject(struct ib_uobject *uobj,
enum rdma_lookup_mode mode)
@@ -770,13 +812,15 @@ void rdma_lookup_put_uobject(struct ib_uobject *uobj,
/* Pairs with the kref obtained by type->lookup_get */
uverbs_uobject_put(uobj);
}
EXPORT_SYMBOL_NS_GPL(rdma_lookup_put_uobject, "rdma_core");
void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile)
{
xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC);
}
EXPORT_SYMBOL_NS_GPL(setup_ufile_idr_uobject, "rdma_core");
void release_ufile_idr_uobject(struct ib_uverbs_file *ufile)
static void release_ufile_idr_uobject(struct ib_uverbs_file *ufile)
{
struct ib_uobject *entry;
unsigned long id;
@@ -839,6 +883,7 @@ int uverbs_uobject_release(struct ib_uobject *uobj)
uverbs_uobject_put(uobj);
return 0;
}
EXPORT_SYMBOL_NS_GPL(uverbs_uobject_release, "rdma_core");
/*
* Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct
@@ -846,51 +891,42 @@ int uverbs_uobject_release(struct ib_uobject *uobj)
*/
int uverbs_uobject_fd_release(struct inode *inode, struct file *filp)
{
void (*release_cleanup)(struct ib_uobject *uobj) = NULL;
struct ib_uobject *uobj = filp->private_data;
const struct uverbs_obj_type *type_attrs;
int ret;
/*
* This can only happen if the fput came from alloc_abort_fd_uobject()
*/
if (!filp->private_data)
if (!uobj)
return 0;
return uverbs_uobject_release(filp->private_data);
/*
* uverbs_disassociate_api() can NULL type_attrs after disassociate, but
* it won't if release_cleanup is used.
*/
type_attrs = READ_ONCE(uobj->uapi_object->type_attrs);
if (type_attrs)
release_cleanup = container_of(type_attrs,
struct uverbs_obj_fd_type, type)
->release_cleanup;
if (release_cleanup)
uverbs_uobject_get(uobj);
ret = uverbs_uobject_release(uobj);
if (release_cleanup) {
release_cleanup(uobj);
uverbs_uobject_put(uobj);
}
return ret;
}
EXPORT_SYMBOL(uverbs_uobject_fd_release);
/*
* Drop the ucontext off the ufile and completely disconnect it from the
* ib_device
*/
static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
struct ib_ucontext *ucontext = ufile->ucontext;
struct ib_device *ib_dev = ucontext->device;
/*
* If we are closing the FD then the user mmap VMAs must have
* already been destroyed as they hold on to the filep, otherwise
* they need to be zap'd.
*/
if (reason == RDMA_REMOVE_DRIVER_REMOVE) {
uverbs_user_mmap_disassociate(ufile);
if (ib_dev->ops.disassociate_ucontext)
ib_dev->ops.disassociate_ucontext(ucontext);
}
ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev,
RDMACG_RESOURCE_HCA_HANDLE);
rdma_restrack_del(&ucontext->res);
ib_dev->ops.dealloc_ucontext(ucontext);
WARN_ON(!xa_empty(&ucontext->mmap_xa));
kfree(ucontext);
ufile->ucontext = NULL;
}
static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
struct uverbs_attr_bundle attrs = { .ufile = ufile };
struct ib_ucontext *ucontext = ufile->ucontext;
@@ -931,36 +967,7 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
}
return ret;
}
/*
* Destroy the ucontext and every uobject associated with it.
*
* This is internally locked and can be called in parallel from multiple
* contexts.
*/
void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
down_write(&ufile->hw_destroy_rwsem);
/*
* If a ucontext was never created then we can't have any uobjects to
* cleanup, nothing to do.
*/
if (!ufile->ucontext)
goto done;
while (!list_empty(&ufile->uobjects) &&
!__uverbs_cleanup_ufile(ufile, reason)) {
}
if (WARN_ON(!list_empty(&ufile->uobjects)))
__uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE);
ufile_destroy_ucontext(ufile, reason);
done:
up_write(&ufile->hw_destroy_rwsem);
}
EXPORT_SYMBOL_NS_GPL(__uverbs_cleanup_ufile, "rdma_core");
const struct uverbs_obj_type_class uverbs_fd_class = {
.alloc_begin = alloc_begin_fd_uobject,
@@ -998,6 +1005,7 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
return ERR_PTR(-EOPNOTSUPP);
}
}
EXPORT_SYMBOL_NS_GPL(uverbs_get_uobject_from_file, "rdma_core");
void uverbs_finalize_object(struct ib_uobject *uobj,
enum uverbs_obj_access access, bool hw_obj_valid,
@@ -1030,6 +1038,7 @@ void uverbs_finalize_object(struct ib_uobject *uobj,
WARN_ON(true);
}
}
EXPORT_SYMBOL_NS_GPL(uverbs_finalize_object, "rdma_core");
/**
* rdma_uattrs_has_raw_cap() - Returns whether a rdma device linked to the
@@ -1059,3 +1068,6 @@ bool rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs)
return has_cap;
}
EXPORT_SYMBOL(rdma_uattrs_has_raw_cap);
MODULE_DESCRIPTION("InfiniBand uverbs objects");
MODULE_LICENSE("Dual BSD/GPL");

11
drivers/infiniband/core/rdma_core.h
View File

@@ -70,9 +70,15 @@ void uverbs_finalize_object(struct ib_uobject *uobj,
int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);
void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile);
void release_ufile_idr_uobject(struct ib_uverbs_file *ufile);
#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs);
#else
static inline struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
{
return NULL;
}
#endif
/*
* This is the runtime description of the uverbs API, used by the syscall
@@ -151,9 +157,6 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
unsigned int num_attrs);
void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile);
typedef int (*uverbs_api_ioctl_handler_fn)(struct uverbs_attr_bundle *attrs);
uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata);
extern const struct uapi_definition uverbs_def_obj_async_fd[];
extern const struct uapi_definition uverbs_def_obj_counters[];
extern const struct uapi_definition uverbs_def_obj_cq[];

49
drivers/infiniband/core/restrack.c
View File

@@ -71,6 +71,8 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,
xa_lock(&rt->xa);
xas_for_each(&xas, e, U32_MAX) {
if (xa_is_zero(e))
continue;
if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details)
continue;
cnt++;
@@ -276,6 +278,53 @@ int rdma_restrack_put(struct rdma_restrack_entry *res)
}
EXPORT_SYMBOL(rdma_restrack_put);
/**
* rdma_restrack_sync() - Fence concurrent netlink dumps on an entry
* @res: resource entry
*
* After this returns any concurrent netlink dump threads will see the current
* value of the object. This is useful if the object has to be changed and there
* is not locking to protect the nl side. Eg for mr->pd. This effectively
* destroys the object from a kref/xarray perspective and then immediately
* restores it. The kref is acting like a lock to barrier concurrent nl threads.
* Callers must ensure rdma_restrack_del() is not concurrently called.
*/
void rdma_restrack_sync(struct rdma_restrack_entry *res)
{
struct rdma_restrack_entry *old;
struct rdma_restrack_root *rt;
struct task_struct *task;
struct ib_device *dev;
if (!res->valid || res->no_track)
return;
dev = res_to_dev(res);
if (WARN_ON(!dev))
return;
rt = &dev->res[res->type];
if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD)))
return;
old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL);
if (WARN_ON(old != res))
return;
task = res->task;
if (task)
get_task_struct(task);
rdma_restrack_put(res);
wait_for_completion(&res->comp);
reinit_completion(&res->comp);
if (task)
res->task = task;
kref_init(&res->kref);
xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL);
}
EXPORT_SYMBOL(rdma_restrack_sync);
/**
* rdma_restrack_del() - delete object from the resource tracking database
* @res: resource entry

1
drivers/infiniband/core/restrack.h
View File

@@ -27,6 +27,7 @@ int rdma_restrack_init(struct ib_device *dev);
void rdma_restrack_clean(struct ib_device *dev);
void rdma_restrack_add(struct rdma_restrack_entry *res);
void rdma_restrack_del(struct rdma_restrack_entry *res);
void rdma_restrack_sync(struct rdma_restrack_entry *res);
void rdma_restrack_new(struct rdma_restrack_entry *res,
enum rdma_restrack_type type);
void rdma_restrack_set_name(struct rdma_restrack_entry *res,

10
drivers/infiniband/core/ucaps.c
View File

@@ -46,12 +46,7 @@ static const struct file_operations ucaps_cdev_fops = {
.open = simple_open,
};
/**
* ib_cleanup_ucaps - cleanup all API resources and class.
*
* This is called once, when removing the ib_uverbs module.
*/
void ib_cleanup_ucaps(void)
static __exit void ib_cleanup_ucaps(void)
{
mutex_lock(&ucaps_mutex);
if (!ucaps_class_is_registered) {
@@ -263,3 +258,6 @@ int ib_get_ucaps(int *fds, int fd_count, uint64_t *idx_mask)
mutex_unlock(&ucaps_mutex);
return ret;
}
EXPORT_SYMBOL_NS_GPL(ib_get_ucaps, "rdma_core");
module_exit(ib_cleanup_ucaps);

370
drivers/infiniband/core/umem.c
View File

@@ -84,14 +84,17 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
*/
unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
unsigned long pgsz_bitmap,
unsigned long virt)
u64 virt)
{
unsigned long curr_len = 0;
dma_addr_t curr_base = ~0;
unsigned long va, pgoff;
unsigned long pgoff;
struct scatterlist *sg;
dma_addr_t mask;
unsigned long mask = 0;
unsigned int bits;
dma_addr_t end;
u64 last_va;
u64 va;
int i;
umem->iova = va = virt;
@@ -109,9 +112,12 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
* number of required pages. Compute the largest page size that could
* work based on VA address bits that don't change.
*/
mask = pgsz_bitmap &
GENMASK(BITS_PER_LONG - 1,
bits_per((umem->length - 1 + virt) ^ virt));
if (check_add_overflow(umem->length - 1, virt, &last_va))
return 0;
bits = bits_per(virt ^ last_va);
if (bits < BITS_PER_LONG)
mask = pgsz_bitmap & GENMASK(BITS_PER_LONG - 1, bits);
/* offset into first SGL */
pgoff = umem->address & ~PAGE_MASK;
@@ -153,16 +159,9 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
}
EXPORT_SYMBOL(ib_umem_find_best_pgsz);
/**
* ib_umem_get - Pin and DMA map userspace memory.
*
* @device: IB device to connect UMEM
* @addr: userspace virtual address to start at
* @size: length of region to pin
* @access: IB_ACCESS_xxx flags for memory being pinned
*/
struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
size_t size, int access)
static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
unsigned long addr, size_t size,
int access)
{
struct ib_umem *umem;
struct page **page_list;
@@ -174,6 +173,9 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
int pinned, ret;
unsigned int gup_flags = FOLL_LONGTERM;
if (device->cc_dma_bounce)
return ERR_PTR(-EOPNOTSUPP);
/*
* If the combination of the addr and size requested for this memory
* region causes an integer overflow, return error.
@@ -275,10 +277,342 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
}
return ret ? ERR_PTR(ret) : umem;
}
EXPORT_SYMBOL(ib_umem_get);
/**
* ib_umem_release - release memory pinned with ib_umem_get
* ib_umem_get_desc - Pin a umem from a buffer descriptor.
* @device: IB device.
* @desc: buffer descriptor (VA or DMABUF).
* @access: IB access flags.
*
* Return: caller-owned umem on success, ERR_PTR(...) on error.
*/
struct ib_umem *ib_umem_get_desc(struct ib_device *device,
const struct ib_uverbs_buffer_desc *desc,
int access)
{
struct ib_umem_dmabuf *umem_dmabuf;
if (desc->flags & ~IB_UVERBS_BUFFER_DESC_FLAGS_KNOWN_MASK)
return ERR_PTR(-EINVAL);
if (overflows_type(desc->addr, unsigned long) ||
overflows_type(desc->length, size_t))
return ERR_PTR(-EOVERFLOW);
switch (desc->type) {
case IB_UVERBS_BUFFER_TYPE_DMABUF:
umem_dmabuf = ib_umem_dmabuf_get_pinned(device, desc->addr,
desc->length, desc->fd,
access);
if (IS_ERR(umem_dmabuf))
return ERR_CAST(umem_dmabuf);
return &umem_dmabuf->umem;
case IB_UVERBS_BUFFER_TYPE_VA:
return __ib_umem_get_va(device, desc->addr, desc->length,
access);
default:
return ERR_PTR(-EINVAL);
}
}
EXPORT_SYMBOL(ib_umem_get_desc);
/*
* Per-command legacy buffer-desc filler.
* Returns 0 on success (desc filled), -ENODATA if no legacy attrs apply,
* negative errno on validation failure.
*/
typedef int (*ib_umem_buf_desc_filler_t)(const struct uverbs_attr_bundle *attrs,
struct ib_uverbs_buffer_desc *desc);
/*
* ib_umem_resolve_desc - Resolve a buffer descriptor from a per-command UMEM
* attribute and/or a legacy attr filler.
*
* Return:
* 0 @desc filled.
* -ENOENT no source produced a buffer.
* -EINVAL both the UMEM attribute and the legacy filler produced a buffer.
* -errno propagated from attr read / filler validation.
*/
static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs,
u16 attr_id,
ib_umem_buf_desc_filler_t legacy_filler,
struct ib_uverbs_buffer_desc *desc)
{
bool have_desc = false;
int ret;
if (!attrs)
return -ENOENT;
ret = uverbs_get_buffer_desc(attrs, attr_id, desc);
if (!ret)
have_desc = true;
else if (ret != -ENOENT)
return ret;
if (legacy_filler) {
struct ib_uverbs_buffer_desc legacy_desc = {};
ret = legacy_filler(attrs, &legacy_desc);
if (!ret) {
if (have_desc)
return -EINVAL;
*desc = legacy_desc;
have_desc = true;
} else if (ret != -ENODATA) {
return ret;
}
}
return have_desc ? 0 : -ENOENT;
}
/*
* ib_umem_get_desc_check - Pin a umem from @desc and verify it meets
* @min_size.
*/
static struct ib_umem *
ib_umem_get_desc_check(struct ib_device *device,
const struct ib_uverbs_buffer_desc *desc,
size_t min_size, int access)
{
struct ib_umem *umem;
umem = ib_umem_get_desc(device, desc, access);
if (IS_ERR(umem))
return umem;
if (umem->length < min_size) {
ib_umem_release(umem);
return ERR_PTR(-EINVAL);
}
return umem;
}
/*
* ib_umem_get_from_attrs - Pin a umem from a per-command UMEM attribute
* and/or a legacy attr filler.
*
* Return: caller-owned umem on success; NULL when no source supplied a
* buffer; ERR_PTR(...) on error.
*/
static struct ib_umem *
ib_umem_get_from_attrs(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler,
size_t size, int access)
{
struct ib_uverbs_buffer_desc desc = {};
int ret;
ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc);
if (ret == -ENOENT)
return NULL;
if (ret)
return ERR_PTR(ret);
return ib_umem_get_desc_check(device, &desc, size, access);
}
/*
* ib_umem_get_from_attrs_or_va - Pin a umem from a per-command UMEM
* attribute and/or a legacy attr filler,
* falling back to a UHW VA when no source
* matched.
*
* @size is always consumed: it is the length to pin on the VA fallback
* path AND the post-pin minimum-length check on the attr / legacy paths.
* Callers must always pass a meaningful, validated value.
*
* Return: caller-owned umem on success, ERR_PTR(...) on error.
*/
static struct ib_umem *
ib_umem_get_from_attrs_or_va(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u16 attr_id,
ib_umem_buf_desc_filler_t legacy_filler,
u64 addr, size_t size, int access)
{
struct ib_uverbs_buffer_desc desc = {};
int ret;
ret = ib_umem_resolve_desc(attrs, attr_id, legacy_filler, &desc);
if (ret == -ENOENT)
desc = (struct ib_uverbs_buffer_desc){
.type = IB_UVERBS_BUFFER_TYPE_VA,
.addr = addr,
.length = size,
};
else if (ret)
return ERR_PTR(ret);
return ib_umem_get_desc_check(device, &desc, size, access);
}
/**
* ib_umem_get_attr - Pin a umem from a per-command UMEM attribute.
* @device: IB device.
* @attrs: uverbs attribute bundle (may be NULL).
* @attr_id: per-command UMEM attribute id.
* @size: minimum required umem length.
* @access: IB access flags.
*
* Return: caller-owned umem on success; NULL when no source supplied
* a buffer; ERR_PTR(...) on error.
*/
struct ib_umem *ib_umem_get_attr(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u16 attr_id, size_t size, int access)
{
return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size,
access);
}
EXPORT_SYMBOL(ib_umem_get_attr);
/**
* ib_umem_get_attr_or_va - Pin a umem from a per-command UMEM attribute,
* falling back to a UHW VA.
* @device: IB device.
* @attrs: uverbs attribute bundle (may be NULL).
* @attr_id: per-command UMEM attribute id.
* @addr: UHW user VA used when no per-command attribute matched.
* @size: on the attr / legacy paths, the minimum required umem length
* validated post-pin; on the VA fallback path, the length to pin.
* @access: IB access flags.
*
* Like ib_umem_get_attr(), but pins @addr/@size when no per-command
* UMEM attribute is supplied.
*
* IMPORTANT: @size is always consumed. On the attr / legacy paths it is
* used as the post-pin minimum-length check; on the VA fallback path it
* is the length to pin. Callers MUST pass a meaningful, validated value
* even when they expect an attribute-supplied buffer to be used.
*
* Every in-tree caller passes the same value for the two roles of @size
* because no driver today distinguishes a user-passed buffer length from
* a driver-computed minimum. Drivers that currently accept a user-supplied
* length without cross-checking it against a driver minimum (vmw_pvrdma
* CQ/QP/SRQ, qedr CQ/QP/SRQ, mana WQ/QP, ionic CQ/QP), once tightened to
* compute and check a real minimum, will want to introduce a separate
* helper that passes these as distinct values.
*
* Return: caller-owned umem on success, ERR_PTR(...) on error.
*/
struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u16 attr_id, u64 addr, size_t size,
int access)
{
return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr,
size, access);
}
EXPORT_SYMBOL(ib_umem_get_attr_or_va);
static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *attrs,
struct ib_uverbs_buffer_desc *desc)
{
struct ib_device *ib_dev = attrs->context->device;
int ret;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) {
ret = uverbs_copy_from(&desc->addr, attrs,
UVERBS_ATTR_CREATE_CQ_BUFFER_VA);
if (ret)
return ret;
ret = uverbs_copy_from(&desc->length, attrs,
UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);
if (ret)
return ret;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) ||
uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
!ib_dev->ops.create_user_cq)
return -EINVAL;
desc->type = IB_UVERBS_BUFFER_TYPE_VA;
return 0;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) {
ret = uverbs_get_raw_fd(&desc->fd, attrs,
UVERBS_ATTR_CREATE_CQ_BUFFER_FD);
if (ret)
return ret;
ret = uverbs_copy_from(&desc->addr, attrs,
UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET);
if (ret)
return ret;
ret = uverbs_copy_from(&desc->length, attrs,
UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);
if (ret)
return ret;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) ||
!ib_dev->ops.create_user_cq)
return -EINVAL;
desc->type = IB_UVERBS_BUFFER_TYPE_DMABUF;
return 0;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH))
return -EINVAL;
return -ENODATA;
}
/**
* ib_umem_get_cq_buf - Pin a CQ buffer umem from per-command attributes.
* @device: IB device.
* @attrs: uverbs attribute bundle (may be NULL).
* @size: minimum required CQ buffer length.
* @access: IB access flags.
*
* Resolves the CQ buffer from the new UMEM attribute or the legacy
* CQ buffer attributes. There is no UHW VA fallback, so the caller
* must arrange its own backing (typically an in-kernel allocation)
* when no source is available.
*
* Return: caller-owned umem on success; NULL when no source supplied
* a buffer; ERR_PTR(...) on error.
*/
struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
size_t size, int access)
{
return ib_umem_get_from_attrs(device, attrs,
UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
uverbs_create_cq_get_buffer_desc,
size, access);
}
EXPORT_SYMBOL(ib_umem_get_cq_buf);
/**
* ib_umem_get_cq_buf_or_va - Pin a CQ buffer umem with UHW VA fallback.
* @device: IB device.
* @attrs: uverbs attribute bundle (may be NULL).
* @addr: UHW user VA used when no per-command attribute matched.
* @size: on the attr / legacy paths, the minimum required umem length
* validated post-pin; on the VA fallback path, the length to pin.
* @access: IB access flags.
*
* Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the
* UMEM attribute nor the legacy CQ buffer attributes are supplied.
*
* See ib_umem_get_attr_or_va() for the note on @size's dual role and
* the migration path for drivers that would distinguish a user-supplied
* length from a driver-computed minimum.
*
* Return: caller-owned umem on success, ERR_PTR(...) on error.
*/
struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u64 addr, size_t size, int access)
{
return ib_umem_get_from_attrs_or_va(device, attrs,
UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
uverbs_create_cq_get_buffer_desc,
addr, size, access);
}
EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va);
/**
* ib_umem_release - release pinned memory
* @umem: umem struct to release
*/
void ib_umem_release(struct ib_umem *umem)

25
drivers/infiniband/core/uverbs.h
View File

@@ -203,7 +203,6 @@ void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue);
void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file);
void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue);
void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res);
int uverbs_async_event_release(struct inode *inode, struct file *filp);
int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs);
int ib_init_ucontext(struct uverbs_attr_bundle *attrs);
@@ -263,6 +262,21 @@ struct bundle_priv {
u64 internal_buffer[32];
};
static inline int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
const struct uverbs_attr *attr)
{
struct bundle_priv *pbundle =
container_of(&bundle->hdr, struct bundle_priv, bundle);
u16 flags;
flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
UVERBS_ATTR_F_VALID_OUTPUT;
if (put_user(flags,
&pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
return -EFAULT;
return 0;
}
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
struct ib_uverbs_flow_spec {
@@ -345,4 +359,13 @@ static inline void ib_uverbs_dmabuf_done(struct kref *kref)
complete(&priv->comp);
}
int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason);
static inline void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
{
complete(&dev->comp);
}
#endif /* UVERBS_H */

103
drivers/infiniband/core/uverbs_cmd.c
View File

@@ -47,6 +47,7 @@
#include "uverbs.h"
#include "core_priv.h"
#include "restrack.h"
/*
* Copy a response to userspace. If the provided 'resp' is larger than the
@@ -163,17 +164,6 @@ static int uverbs_request_finish(struct uverbs_req_iter *iter)
return 0;
}
/*
* When calling a destroy function during an error unwind we need to pass in
* the udata that is sanitized of all user arguments. Ie from the driver
* perspective it looks like no udata was passed.
*/
struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
{
attrs->driver_udata = (struct ib_udata){};
return &attrs->driver_udata;
}
static struct ib_uverbs_completion_event_file *
_ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
{
@@ -819,6 +809,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
ret = PTR_ERR(new_pd);
goto put_uobjs;
}
if (new_pd == orig_pd) {
uobj_put_obj_read(new_pd);
cmd.flags &= ~IB_MR_REREG_PD;
}
} else {
new_pd = mr->pd;
}
@@ -866,9 +860,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
mr = new_mr;
} else {
if (cmd.flags & IB_MR_REREG_PD) {
atomic_dec(&orig_pd->usecnt);
mr->pd = new_pd;
atomic_inc(&new_pd->usecnt);
WRITE_ONCE(mr->pd, new_pd);
rdma_restrack_sync(&mr->res);
atomic_dec(&orig_pd->usecnt);
}
if (cmd.flags & IB_MR_REREG_TRANS) {
mr->iova = cmd.hca_va;
@@ -1084,7 +1079,6 @@ static int create_cq(struct uverbs_attr_bundle *attrs,
return uverbs_response(attrs, &resp, sizeof(resp));
err_free:
ib_umem_release(cq->umem);
rdma_restrack_put(&cq->res);
kfree(cq);
err_file:
@@ -1462,8 +1456,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
attr.source_qpn = cmd->source_qpn;
}
qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj,
KBUILD_MODNAME);
qp = ib_create_qp_user(device, pd, &attr, attrs, obj, KBUILD_MODNAME);
if (IS_ERR(qp)) {
ret = PTR_ERR(qp);
goto err_put;
@@ -2594,82 +2587,6 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
return ret;
}
struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
{
struct ib_uflow_resources *resources;
resources = kzalloc_obj(*resources);
if (!resources)
return NULL;
if (!num_specs)
goto out;
resources->counters =
kzalloc_objs(*resources->counters, num_specs);
resources->collection =
kzalloc_objs(*resources->collection, num_specs);
if (!resources->counters || !resources->collection)
goto err;
out:
resources->max = num_specs;
return resources;
err:
kfree(resources->counters);
kfree(resources);
return NULL;
}
EXPORT_SYMBOL(flow_resources_alloc);
void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
{
unsigned int i;
if (!uflow_res)
return;
for (i = 0; i < uflow_res->collection_num; i++)
atomic_dec(&uflow_res->collection[i]->usecnt);
for (i = 0; i < uflow_res->counters_num; i++)
atomic_dec(&uflow_res->counters[i]->usecnt);
kfree(uflow_res->collection);
kfree(uflow_res->counters);
kfree(uflow_res);
}
EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
void flow_resources_add(struct ib_uflow_resources *uflow_res,
enum ib_flow_spec_type type,
void *ibobj)
{
WARN_ON(uflow_res->num >= uflow_res->max);
switch (type) {
case IB_FLOW_SPEC_ACTION_HANDLE:
atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
uflow_res->collection[uflow_res->collection_num++] =
(struct ib_flow_action *)ibobj;
break;
case IB_FLOW_SPEC_ACTION_COUNT:
atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
uflow_res->counters[uflow_res->counters_num++] =
(struct ib_counters *)ibobj;
break;
default:
WARN_ON(1);
}
uflow_res->num++;
}
EXPORT_SYMBOL(flow_resources_add);
static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
struct ib_uverbs_flow_spec *kern_spec,
union ib_flow_spec *ib_spec,
@@ -3661,6 +3578,8 @@ static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
resp.timestamp_mask = attr.timestamp_mask;
resp.hca_core_clock = attr.hca_core_clock;
resp.device_cap_flags_ex = attr.device_cap_flags;
if (ib_dev->cc_dma_bounce)
resp.device_cap_flags_ex |= IB_UVERBS_DEVICE_CC_DMA_BOUNCE;
resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
resp.rss_caps.max_rwq_indirection_tables =
attr.rss_caps.max_rwq_indirection_tables;

78
drivers/infiniband/core/uverbs_flow.c Normal file
View File

@@ -0,0 +1,78 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
#include "uverbs.h"
struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
{
struct ib_uflow_resources *resources;
resources = kzalloc_obj(*resources);
if (!resources)
return NULL;
if (!num_specs)
goto out;
resources->counters =
kzalloc_objs(*resources->counters, num_specs);
resources->collection =
kzalloc_objs(*resources->collection, num_specs);
if (!resources->counters || !resources->collection)
goto err;
out:
resources->max = num_specs;
return resources;
err:
kfree(resources->counters);
kfree(resources);
return NULL;
}
EXPORT_SYMBOL(flow_resources_alloc);
void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
{
unsigned int i;
if (!uflow_res)
return;
for (i = 0; i < uflow_res->collection_num; i++)
atomic_dec(&uflow_res->collection[i]->usecnt);
for (i = 0; i < uflow_res->counters_num; i++)
atomic_dec(&uflow_res->counters[i]->usecnt);
kfree(uflow_res->collection);
kfree(uflow_res->counters);
kfree(uflow_res);
}
EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
void flow_resources_add(struct ib_uflow_resources *uflow_res,
enum ib_flow_spec_type type,
void *ibobj)
{
WARN_ON(uflow_res->num >= uflow_res->max);
switch (type) {
case IB_FLOW_SPEC_ACTION_HANDLE:
atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
uflow_res->collection[uflow_res->collection_num++] =
(struct ib_flow_action *)ibobj;
break;
case IB_FLOW_SPEC_ACTION_COUNT:
atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
uflow_res->counters[uflow_res->counters_num++] =
(struct ib_counters *)ibobj;
break;
default:
WARN_ON(1);
}
uflow_res->num++;
}
EXPORT_SYMBOL(flow_resources_add);

204
drivers/infiniband/core/uverbs_ioctl.c
View File

@@ -58,50 +58,6 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
}
/**
* _uverbs_alloc() - Quickly allocate memory for use with a bundle
* @bundle: The bundle
* @size: Number of bytes to allocate
* @flags: Allocator flags
*
* The bundle allocator is intended for allocations that are connected with
* processing the system call related to the bundle. The allocated memory is
* always freed once the system call completes, and cannot be freed any other
* way.
*
* This tries to use a small pool of pre-allocated memory for performance.
*/
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
gfp_t flags)
{
struct bundle_priv *pbundle =
container_of(&bundle->hdr, struct bundle_priv, bundle);
size_t new_used;
void *res;
if (check_add_overflow(size, pbundle->internal_used, &new_used))
return ERR_PTR(-EOVERFLOW);
if (new_used > pbundle->internal_avail) {
struct bundle_alloc_head *buf;
buf = kvmalloc_flex(*buf, data, size, flags);
if (!buf)
return ERR_PTR(-ENOMEM);
buf->next = pbundle->allocated_mem;
pbundle->allocated_mem = buf;
return buf->data;
}
res = (void *)pbundle->internal_buffer + pbundle->internal_used;
pbundle->internal_used =
ALIGN(new_used, sizeof(*pbundle->internal_buffer));
if (want_init_on_alloc(flags))
memset(res, 0, size);
return res;
}
EXPORT_SYMBOL(_uverbs_alloc);
static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
u16 len)
{
@@ -113,21 +69,6 @@ static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
0, uattr->len - len);
}
static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
const struct uverbs_attr *attr)
{
struct bundle_priv *pbundle =
container_of(&bundle->hdr, struct bundle_priv, bundle);
u16 flags;
flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
UVERBS_ATTR_F_VALID_OUTPUT;
if (put_user(flags,
&pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
return -EFAULT;
return 0;
}
static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
const struct uverbs_api_attr *attr_uapi,
struct uverbs_objs_arr_attr *attr,
@@ -616,57 +557,6 @@ long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
return err;
}
int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
const struct uverbs_attr *attr;
u64 flags;
attr = uverbs_attr_get(attrs_bundle, idx);
/* Missing attribute means 0 flags */
if (IS_ERR(attr)) {
*to = 0;
return 0;
}
/*
* New userspace code should use 8 bytes to pass flags, but we
* transparently support old userspaces that were using 4 bytes as
* well.
*/
if (attr->ptr_attr.len == 8)
flags = attr->ptr_attr.data;
else if (attr->ptr_attr.len == 4)
flags = *(u32 *)&attr->ptr_attr.data;
else
return -EINVAL;
if (flags & ~allowed_bits)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags64);
int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
u64 flags;
int ret;
ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
if (ret)
return ret;
if (flags > U32_MAX)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags32);
/*
* Fill a ib_udata struct (core or uhw) using the given attribute IDs.
* This is primarily used to convert the UVERBS_ATTR_UHW() into the
@@ -707,24 +597,6 @@ void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
}
}
int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
const void *from, size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
size_t min_size;
if (IS_ERR(attr))
return PTR_ERR(attr);
min_size = min_t(size_t, attr->ptr_attr.len, size);
if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
return -EFAULT;
return uverbs_set_output(bundle, attr);
}
EXPORT_SYMBOL(uverbs_copy_to);
/*
* This is only used if the caller has directly used copy_to_use to write the
* data. It signals to user space that the buffer is filled in.
@@ -738,79 +610,3 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
return uverbs_set_output(bundle, attr);
}
int _uverbs_get_const_signed(s64 *to,
const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, s64 lower_bound, u64 upper_bound,
s64 *def_val)
{
const struct uverbs_attr *attr;
attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr)) {
if ((PTR_ERR(attr) != -ENOENT) || !def_val)
return PTR_ERR(attr);
*to = *def_val;
} else {
*to = attr->ptr_attr.data;
}
if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
return -EINVAL;
return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_signed);
int _uverbs_get_const_unsigned(u64 *to,
const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 upper_bound, u64 *def_val)
{
const struct uverbs_attr *attr;
attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr)) {
if ((PTR_ERR(attr) != -ENOENT) || !def_val)
return PTR_ERR(attr);
*to = *def_val;
} else {
*to = attr->ptr_attr.data;
}
if (*to > upper_bound)
return -EINVAL;
return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_unsigned);
int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
size_t idx, const void *from, size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
if (size < attr->ptr_attr.len) {
if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
attr->ptr_attr.len - size))
return -EFAULT;
}
return uverbs_copy_to(bundle, idx, from, size);
}
EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
/* Once called an abort will call through to the type's destroy_hw() */
void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
u16 idx)
{
struct bundle_priv *pbundle =
container_of(&bundle->hdr, struct bundle_priv, bundle);
__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
pbundle->uobj_hw_obj_valid);
}
EXPORT_SYMBOL(uverbs_finalize_uobj_create);

127
drivers/infiniband/core/uverbs_main.c
View File

@@ -61,6 +61,7 @@
MODULE_AUTHOR("Roland Dreier");
MODULE_DESCRIPTION("InfiniBand userspace verbs access");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_IMPORT_NS("rdma_core");
enum {
IB_UVERBS_MAJOR = 231,
@@ -91,30 +92,6 @@ static const struct class uverbs_class = {
.devnode = uverbs_devnode,
};
/*
* Must be called with the ufile->device->disassociate_srcu held, and the lock
* must be held until use of the ucontext is finished.
*/
struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile)
{
/*
* We do not hold the hw_destroy_rwsem lock for this flow, instead
* srcu is used. It does not matter if someone races this with
* get_context, we get NULL or valid ucontext.
*/
struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext);
if (!srcu_dereference(ufile->device->ib_dev,
&ufile->device->disassociate_srcu))
return ERR_PTR(-EIO);
if (!ucontext)
return ERR_PTR(-EINVAL);
return ucontext;
}
EXPORT_SYMBOL(ib_uverbs_get_ucontext_file);
int uverbs_dealloc_mw(struct ib_mw *mw)
{
struct ib_pd *pd = mw->pd;
@@ -189,42 +166,6 @@ void ib_uverbs_detach_umcast(struct ib_qp *qp,
}
}
static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
{
complete(&dev->comp);
}
void ib_uverbs_release_file(struct kref *ref)
{
struct ib_uverbs_file *file =
container_of(ref, struct ib_uverbs_file, ref);
struct ib_device *ib_dev;
int srcu_key;
release_ufile_idr_uobject(file);
srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
ib_dev = srcu_dereference(file->device->ib_dev,
&file->device->disassociate_srcu);
if (ib_dev && !ib_dev->ops.disassociate_ucontext)
module_put(ib_dev->ops.owner);
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
if (refcount_dec_and_test(&file->device->refcount))
ib_uverbs_comp_dev(file->device);
if (file->default_async_file)
uverbs_uobject_put(&file->default_async_file->uobj);
put_device(&file->device->dev);
if (file->disassociate_page)
__free_pages(file->disassociate_page, 0);
mutex_destroy(&file->disassociation_lock);
mutex_destroy(&file->umap_lock);
mutex_destroy(&file->ucontext_lock);
kfree(file);
}
static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue,
struct file *filp, char __user *buf,
size_t count, loff_t *pos,
@@ -362,7 +303,7 @@ const struct file_operations uverbs_async_event_fops = {
.owner = THIS_MODULE,
.read = ib_uverbs_async_event_read,
.poll = ib_uverbs_async_event_poll,
.release = uverbs_async_event_release,
.release = uverbs_uobject_fd_release,
.fasync = ib_uverbs_async_event_fasync,
};
@@ -1009,6 +950,69 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)
return ret;
}
/*
* Drop the ucontext off the ufile and completely disconnect it from the
* ib_device
*/
static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
struct ib_ucontext *ucontext = ufile->ucontext;
struct ib_device *ib_dev = ucontext->device;
/*
* If we are closing the FD then the user mmap VMAs must have
* already been destroyed as they hold on to the filep, otherwise
* they need to be zap'd.
*/
if (reason == RDMA_REMOVE_DRIVER_REMOVE) {
uverbs_user_mmap_disassociate(ufile);
if (ib_dev->ops.disassociate_ucontext)
ib_dev->ops.disassociate_ucontext(ucontext);
}
ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev,
RDMACG_RESOURCE_HCA_HANDLE);
rdma_restrack_del(&ucontext->res);
ib_dev->ops.dealloc_ucontext(ucontext);
WARN_ON(!xa_empty(&ucontext->mmap_xa));
kfree(ucontext);
ufile->ucontext = NULL;
}
/*
* Destroy the ucontext and every uobject associated with it.
*
* This is internally locked and can be called in parallel from multiple
* contexts.
*/
void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
down_write(&ufile->hw_destroy_rwsem);
/*
* If a ucontext was never created then we can't have any uobjects to
* cleanup, nothing to do.
*/
if (!ufile->ucontext)
goto done;
while (!list_empty(&ufile->uobjects) &&
!__uverbs_cleanup_ufile(ufile, reason)) {
}
if (WARN_ON(!list_empty(&ufile->uobjects)))
__uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE);
ufile_destroy_ucontext(ufile, reason);
done:
up_write(&ufile->hw_destroy_rwsem);
}
static int ib_uverbs_close(struct inode *inode, struct file *filp)
{
struct ib_uverbs_file *file = filp->private_data;
@@ -1346,7 +1350,6 @@ static void __exit ib_uverbs_cleanup(void)
IB_UVERBS_NUM_FIXED_MINOR);
unregister_chrdev_region(dynamic_uverbs_dev,
IB_UVERBS_NUM_DYNAMIC_MINOR);
ib_cleanup_ucaps();
mmu_notifier_synchronize();
}

6
drivers/infiniband/core/uverbs_std_types.c
View File

@@ -165,12 +165,6 @@ uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj,
ib_uverbs_free_event_queue(&file->ev_queue);
}
int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
{
return 0;
}
EXPORT_SYMBOL(uverbs_destroy_def_handler);
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_COMP_CHANNEL,
UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file),

22
drivers/infiniband/core/uverbs_std_types_async_fd.c
View File

@@ -32,14 +32,9 @@ static void uverbs_async_event_destroy_uobj(struct ib_uobject *uobj,
NULL, NULL);
}
int uverbs_async_event_release(struct inode *inode, struct file *filp)
static void uverbs_async_event_free_event_queue(struct ib_uobject *uobj)
{
struct ib_uverbs_async_event_file *event_file;
struct ib_uobject *uobj = filp->private_data;
int ret;
if (!uobj)
return uverbs_uobject_fd_release(inode, filp);
event_file =
container_of(uobj, struct ib_uverbs_async_event_file, uobj);
@@ -50,11 +45,7 @@ int uverbs_async_event_release(struct inode *inode, struct file *filp)
* release. The user knows it has reached the end of the event stream
* when it sees IB_EVENT_DEVICE_FATAL.
*/
uverbs_uobject_get(uobj);
ret = uverbs_uobject_fd_release(inode, filp);
ib_uverbs_free_event_queue(&event_file->ev_queue);
uverbs_uobject_put(uobj);
return ret;
}
DECLARE_UVERBS_NAMED_METHOD(
@@ -66,11 +57,12 @@ DECLARE_UVERBS_NAMED_METHOD(
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_ASYNC_EVENT,
UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file),
uverbs_async_event_destroy_uobj,
&uverbs_async_event_fops,
"[infinibandevent]",
O_RDONLY),
UVERBS_TYPE_ALLOC_FD_RELEASE(sizeof(struct ib_uverbs_async_event_file),
uverbs_async_event_destroy_uobj,
uverbs_async_event_free_event_queue,
&uverbs_async_event_fops,
"[infinibandevent]",
O_RDONLY),
&UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC));
const struct uapi_definition uverbs_def_obj_async_fd[] = {

73
drivers/infiniband/core/uverbs_std_types_cq.c
View File

@@ -66,16 +66,10 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
typeof(*obj), uevent.uobject);
struct ib_uverbs_completion_event_file *ev_file = NULL;
struct ib_device *ib_dev = attrs->context->device;
struct ib_umem_dmabuf *umem_dmabuf;
struct ib_cq_init_attr attr = {};
struct ib_uobject *ev_file_uobj;
struct ib_umem *umem = NULL;
u64 buffer_length;
u64 buffer_offset;
struct ib_cq *cq;
u64 user_handle;
u64 buffer_va;
int buffer_fd;
int ret;
if ((!ib_dev->ops.create_cq && !ib_dev->ops.create_user_cq) ||
@@ -122,65 +116,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
INIT_LIST_HEAD(&obj->comp_list);
INIT_LIST_HEAD(&obj->uevent.event_list);
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) {
ret = uverbs_copy_from(&buffer_va, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA);
if (ret)
goto err_event_file;
ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);
if (ret)
goto err_event_file;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) ||
uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
!ib_dev->ops.create_user_cq) {
ret = -EINVAL;
goto err_event_file;
}
umem = ib_umem_get(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
ret = PTR_ERR(umem);
goto err_event_file;
}
} else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) {
ret = uverbs_get_raw_fd(&buffer_fd, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD);
if (ret)
goto err_event_file;
ret = uverbs_copy_from(&buffer_offset, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET);
if (ret)
goto err_event_file;
ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);
if (ret)
goto err_event_file;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) ||
!ib_dev->ops.create_user_cq) {
ret = -EINVAL;
goto err_event_file;
}
umem_dmabuf = ib_umem_dmabuf_get_pinned(ib_dev, buffer_offset, buffer_length,
buffer_fd, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem_dmabuf)) {
ret = PTR_ERR(umem_dmabuf);
goto err_event_file;
}
umem = &umem_dmabuf->umem;
} else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH)) {
ret = -EINVAL;
goto err_event_file;
}
cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
if (!cq) {
ret = -ENOMEM;
ib_umem_release(umem);
goto err_event_file;
}
@@ -189,11 +127,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
cq->comp_handler = ib_uverbs_comp_handler;
cq->event_handler = ib_uverbs_cq_event_handler;
cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
/*
* If UMEM is not provided here, legacy drivers will set it during
* CQ creation based on their internal udata.
*/
cq->umem = umem;
atomic_set(&cq->usecnt, 0);
rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
@@ -206,9 +139,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
if (ret)
goto err_free;
/* Check that driver didn't overrun existing umem */
WARN_ON(umem && cq->umem != umem);
obj->uevent.uobject.object = cq;
obj->uevent.uobject.user_handle = user_handle;
rdma_restrack_add(&cq->res);
@@ -219,7 +149,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
return ret;
err_free:
ib_umem_release(cq->umem);
rdma_restrack_put(&cq->res);
kfree(cq);
err_event_file:
@@ -269,6 +198,8 @@ DECLARE_UVERBS_NAMED_METHOD(
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET,
UVERBS_ATTR_TYPE(u64),
UA_OPTIONAL),
UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
UA_OPTIONAL),
UVERBS_ATTR_UHW());
static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(

9
drivers/infiniband/core/uverbs_std_types_qp.c
View File

@@ -248,8 +248,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)(
set_caps(&attr, &cap, true);
mutex_init(&obj->mcast_lock);
qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj,
KBUILD_MODNAME);
qp = ib_create_qp_user(device, pd, &attr, attrs, obj, KBUILD_MODNAME);
if (IS_ERR(qp)) {
ret = PTR_ERR(qp);
goto err_put;
@@ -340,6 +339,12 @@ DECLARE_UVERBS_NAMED_METHOD(
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_QP_NUM,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_BUF_UMEM,
UA_OPTIONAL),
UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM,
UA_OPTIONAL),
UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM,
UA_OPTIONAL),
UVERBS_ATTR_UHW());
static int UVERBS_HANDLER(UVERBS_METHOD_QP_DESTROY)(

13
drivers/infiniband/core/uverbs_uapi.c
View File

@@ -718,12 +718,25 @@ void uverbs_disassociate_api(struct uverbs_api *uapi)
if (uapi_key_is_object(iter.index)) {
struct uverbs_api_object *object_elm =
rcu_dereference_protected(*slot, true);
const struct uverbs_obj_type *type_attrs =
object_elm->type_attrs;
/*
* Some type_attrs are in the driver module. We don't
* bother to keep track of which since there should be
* no use of this after disassociate.
*
* release_cleanup is the exception because
* uverbs_uobject_fd_release() needs it. In this case
* the module reference held by the fops will guarentee
* the type_class remains valid too.
*/
if (type_attrs &&
type_attrs->type_class == &uverbs_fd_class &&
container_of(type_attrs, struct uverbs_obj_fd_type,
type)->release_cleanup)
continue;
object_elm->type_attrs = NULL;
} else if (uapi_key_is_attr(iter.index)) {
struct uverbs_api_attr *elm =

40
drivers/infiniband/core/verbs.c
View File

@@ -53,6 +53,7 @@
#include <rdma/rw.h>
#include <rdma/lag.h>
#include "rdma_core.h"
#include "core_priv.h"
#include <trace/events/rdma_core.h>
@@ -1265,10 +1266,9 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
struct ib_qp_init_attr *attr,
struct ib_udata *udata,
struct uverbs_attr_bundle *uattrs,
struct ib_uqp_object *uobj, const char *caller)
{
struct ib_udata dummy = {};
struct ib_qp *qp;
int ret;
@@ -1301,9 +1301,10 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
qp->recv_cq = attr->recv_cq;
rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
rdma_restrack_set_name(&qp->res, udata ? NULL : caller);
ret = dev->ops.create_qp(qp, attr, udata);
WARN_ONCE(!uattrs && !caller, "Missing kernel QP owner");
rdma_restrack_set_name(&qp->res, uattrs ? NULL : caller);
ret = dev->ops.create_qp(qp, attr,
uattrs ? &uattrs->driver_udata : NULL);
if (ret)
goto err_create;
@@ -1322,7 +1323,8 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
return qp;
err_security:
qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL);
qp->device->ops.destroy_qp(
qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL);
err_create:
rdma_restrack_put(&qp->res);
kfree(qp);
@@ -1338,13 +1340,13 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
* @attr: A list of initial attributes required to create the
* QP. If QP creation succeeds, then the attributes are updated to
* the actual capabilities of the created QP.
* @udata: User data
* @uattrs: User ioctl attributes and udata
* @uobj: uverbs obect
* @caller: caller's build-time module name
*/
struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
struct ib_qp_init_attr *attr,
struct ib_udata *udata,
struct uverbs_attr_bundle *uattrs,
struct ib_uqp_object *uobj, const char *caller)
{
struct ib_qp *qp, *xrc_qp;
@@ -1352,7 +1354,7 @@ struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
if (attr->qp_type == IB_QPT_XRC_TGT)
qp = create_qp(dev, pd, attr, NULL, NULL, caller);
else
qp = create_qp(dev, pd, attr, udata, uobj, NULL);
qp = create_qp(dev, pd, attr, uattrs, uobj, NULL);
if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(qp))
return qp;
@@ -1538,8 +1540,7 @@ static const struct {
IB_QP_PKEY_INDEX),
[IB_QPT_RC] = (IB_QP_ALT_PATH |
IB_QP_ACCESS_FLAGS |
IB_QP_PKEY_INDEX |
IB_QP_RATE_LIMIT),
IB_QP_PKEY_INDEX),
[IB_QPT_XRC_INI] = (IB_QP_ALT_PATH |
IB_QP_ACCESS_FLAGS |
IB_QP_PKEY_INDEX),
@@ -1587,8 +1588,7 @@ static const struct {
IB_QP_ALT_PATH |
IB_QP_ACCESS_FLAGS |
IB_QP_MIN_RNR_TIMER |
IB_QP_PATH_MIG_STATE |
IB_QP_RATE_LIMIT),
IB_QP_PATH_MIG_STATE),
[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
IB_QP_ALT_PATH |
IB_QP_ACCESS_FLAGS |
@@ -1602,7 +1602,6 @@ static const struct {
IB_QP_QKEY),
[IB_QPT_GSI] = (IB_QP_CUR_STATE |
IB_QP_QKEY),
[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
}
}
},
@@ -1622,8 +1621,7 @@ static const struct {
IB_QP_ACCESS_FLAGS |
IB_QP_ALT_PATH |
IB_QP_PATH_MIG_STATE |
IB_QP_MIN_RNR_TIMER |
IB_QP_RATE_LIMIT),
IB_QP_MIN_RNR_TIMER),
[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
IB_QP_ACCESS_FLAGS |
IB_QP_ALT_PATH |
@@ -1637,7 +1635,6 @@ static const struct {
IB_QP_QKEY),
[IB_QPT_GSI] = (IB_QP_CUR_STATE |
IB_QP_QKEY),
[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
}
},
[IB_QPS_SQD] = {
@@ -1775,7 +1772,7 @@ bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
if ((mask & req_param) != req_param)
return false;
if (mask & ~(req_param | opt_param | IB_QP_STATE))
if (mask & ~(req_param | opt_param | IB_QP_STATE | IB_QP_RATE_LIMIT))
return false;
return true;
@@ -2221,12 +2218,6 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
kfree(cq);
return ERR_PTR(ret);
}
/*
* We are in kernel verbs flow and drivers are not allowed
* to set umem pointer, it needs to stay NULL.
*/
WARN_ON_ONCE(cq->umem);
rdma_restrack_add(&cq->res);
return cq;
}
@@ -2257,7 +2248,6 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
if (ret)
return ret;
ib_umem_release(cq->umem);
rdma_restrack_del(&cq->res);
kfree(cq);
return ret;

5
drivers/infiniband/hw/bng_re/bng_fw.c
View File

@@ -123,7 +123,6 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,
bool is_waiter_alive;
struct pci_dev *pdev;
u32 wait_cmds = 0;
int rc = 0;
pdev = rcfw->pdev;
switch (qp_event->event) {
@@ -152,7 +151,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,
"rcfw timedout: cookie = %#x, free_slots = %d",
cookie, crsqe->free_slots);
spin_unlock(&hwq->lock);
return rc;
return 0;
}
if (crsqe->is_waiter_alive) {
@@ -182,7 +181,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,
spin_unlock(&hwq->lock);
}
*num_wait += wait_cmds;
return rc;
return 0;
}
/* function events */

2
drivers/infiniband/hw/bnxt_re/debugfs.c
View File

@@ -498,6 +498,8 @@ void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev)
bnxt_re_debugfs_add_info(rdev);
rdev->cc_config_params = kzalloc_obj(*cc_params);
if (!rdev->cc_config_params)
return;
for (i = 0; i < BNXT_RE_CC_PARAM_GEN0; i++) {
struct bnxt_re_cc_param *tmp_params = &rdev->cc_config_params->gen0_parms[i];

379
drivers/infiniband/hw/bnxt_re/ib_verbs.c
View File

@@ -265,7 +265,10 @@ int bnxt_re_query_device(struct ib_device *ibdev,
resp.packet_pacing_caps.supported_qpts =
1 << IB_QPT_RC;
}
return ib_respond_udata(udata, resp);
if (udata)
return ib_respond_udata(udata, resp);
return 0;
}
int bnxt_re_modify_device(struct ib_device *ibdev,
@@ -1024,6 +1027,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
if (rc)
ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
if (qp->dbr_obj)
kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release);
if (rdma_is_kernel_res(&qp->ib_qp.res)) {
flags = bnxt_re_lock_cqs(qp);
bnxt_qplib_clean_qp(&qp->qplib_qp);
@@ -1136,23 +1142,42 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
return 0;
}
static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
struct bnxt_re_qp_req *ureq)
static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev,
struct ib_umem *umem,
struct bnxt_qplib_sg_info *sginfo)
{
struct bnxt_qplib_qp *qplib_qp;
int bytes = 0, psn_sz;
struct ib_umem *umem;
int psn_nume;
unsigned long page_size;
qplib_qp = &qp->qplib_qp;
if (!umem)
return -EINVAL;
bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
/* Consider mapping PSN search memory only for RC QPs. */
if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0);
if (!page_size || page_size != SZ_4K)
return -EINVAL;
sginfo->umem = umem;
sginfo->npages = ib_umem_num_dma_blocks(umem, page_size);
sginfo->pgsize = page_size;
sginfo->pgshft = __builtin_ctz(page_size);
return 0;
}
static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev,
struct bnxt_re_ucontext *cntx,
struct bnxt_qplib_qp *qplib_qp,
struct bnxt_re_qp_req *ureq,
bool fixed_que_attr)
{
int psn_sz, psn_nume;
if (rdev->dev_attr &&
_is_host_msn_table(rdev->dev_attr->dev_cap_flags2))
psn_sz = sizeof(struct sq_msn_search);
else
psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
sizeof(struct sq_psn_search_ext) :
sizeof(struct sq_psn_search);
sizeof(struct sq_psn_search_ext) :
sizeof(struct sq_psn_search);
if (!fixed_que_attr) {
if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) {
psn_nume = ureq->sq_slots;
} else {
@@ -1162,43 +1187,80 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
}
if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
psn_nume = roundup_pow_of_two(psn_nume);
bytes += (psn_nume * psn_sz);
} else {
psn_nume = ureq->sq_npsn;
}
return psn_nume * psn_sz;
}
static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
struct bnxt_re_qp_req *ureq,
bool fixed_que_attr,
struct bnxt_re_dbr_obj *dbr_obj,
struct uverbs_attr_bundle *attrs)
{
struct bnxt_qplib_qp *qplib_qp;
struct ib_umem *umem;
int bytes;
int rc;
qplib_qp = &qp->qplib_qp;
bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
/* Consider mapping PSN search memory only for RC QPs. */
if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC)
bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr);
bytes = PAGE_ALIGN(bytes);
umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes,
IB_ACCESS_LOCAL_WRITE);
umem = ib_umem_get_attr_or_va(&rdev->ibdev, attrs,
UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM,
ureq->qpsva, bytes, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem))
return PTR_ERR(umem);
qp->sumem = umem;
qplib_qp->sq.sg_info.umem = umem;
qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
qplib_qp->qp_handle = ureq->qp_handle;
rc = bnxt_re_setup_sginfo(rdev, qp->sumem, &qplib_qp->sq.sg_info);
if (rc)
goto fail;
if (!qp->qplib_qp.srq) {
bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
bytes = PAGE_ALIGN(bytes);
umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem))
goto rqfail;
qp->rumem = umem;
qplib_qp->rq.sg_info.umem = umem;
qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
if (qp->qplib_qp.srq)
goto done;
bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
bytes = PAGE_ALIGN(bytes);
umem = ib_umem_get_attr_or_va(&rdev->ibdev, attrs,
UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM,
ureq->qprva, bytes, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
rc = PTR_ERR(umem);
goto fail;
}
qplib_qp->dpi = &cntx->dpi;
qp->rumem = umem;
rc = bnxt_re_setup_sginfo(rdev, qp->rumem, &qplib_qp->rq.sg_info);
if (rc)
goto rqfail;
done:
if (dbr_obj)
qplib_qp->dpi = &dbr_obj->dpi;
else
qplib_qp->dpi = &cntx->dpi;
qplib_qp->qp_handle = ureq->qp_handle;
qplib_qp->is_user = true;
return 0;
rqfail:
ib_umem_release(qp->rumem);
qp->rumem = NULL;
memset(&qplib_qp->rq.sg_info, 0, sizeof(qplib_qp->rq.sg_info));
fail:
ib_umem_release(qp->sumem);
qp->sumem = NULL;
memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
return PTR_ERR(umem);
return rc;
}
static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
@@ -1297,7 +1359,7 @@ static int bnxt_re_qp_alloc_init_xrrq(struct bnxt_re_qp *qp)
return rc;
}
static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp)
static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp, bool fixed_que_attr)
{
struct bnxt_qplib_res *res = &qp->rdev->qplib_res;
struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
@@ -1311,12 +1373,17 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp)
hwq_attr.res = res;
hwq_attr.sginfo = &sq->sg_info;
hwq_attr.stride = bnxt_qplib_get_stride();
hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true);
hwq_attr.aux_stride = qplib_qp->psn_sz;
hwq_attr.aux_depth = (qplib_qp->psn_sz) ?
bnxt_qplib_set_sq_size(sq, wqe_mode) : 0;
if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz)
if (!fixed_que_attr) {
hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true);
hwq_attr.aux_depth = (qplib_qp->psn_sz) ?
bnxt_qplib_set_sq_size(sq, wqe_mode) : 0;
if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz)
hwq_attr.aux_depth = qplib_qp->msn_tbl_sz;
} else {
hwq_attr.depth = sq->max_wqe;
hwq_attr.aux_depth = qplib_qp->msn_tbl_sz;
}
hwq_attr.type = HWQ_TYPE_QUEUE;
rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
if (rc)
@@ -1327,6 +1394,9 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp)
CMDQ_CREATE_QP_SQ_LVL_SFT);
sq->hwq.pg_sz_lvl = pg_sz_lvl;
if (qplib_qp->srq)
goto done;
hwq_attr.res = res;
hwq_attr.sginfo = &rq->sg_info;
hwq_attr.stride = bnxt_qplib_get_stride();
@@ -1343,6 +1413,7 @@ static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp)
CMDQ_CREATE_QP_RQ_LVL_SFT);
rq->hwq.pg_sz_lvl = pg_sz_lvl;
done:
if (qplib_qp->psn_sz) {
rc = bnxt_re_qp_alloc_init_xrrq(qp);
if (rc)
@@ -1411,7 +1482,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
qp->qplib_qp.dpi = &rdev->dpi_privileged;
rc = bnxt_re_setup_qp_hwqs(qp);
rc = bnxt_re_setup_qp_hwqs(qp, false);
if (rc)
goto fail;
@@ -1436,7 +1507,8 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
struct ib_qp_init_attr *init_attr,
struct bnxt_re_ucontext *uctx)
struct bnxt_re_ucontext *uctx,
bool fixed_que_attr)
{
struct bnxt_qplib_dev_attr *dev_attr;
struct bnxt_qplib_qp *qplqp;
@@ -1461,12 +1533,16 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
init_attr->cap.max_recv_sge = rq->max_sge;
rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
dev_attr->max_qp_sges);
/* Allocate 1 more than what's provided so posting max doesn't
* mean empty.
*/
rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1,
dev_attr->max_qp_wqes + 1,
uctx);
if (!fixed_que_attr) {
/* Allocate 1 more than what's provided so posting max doesn't
* mean empty.
*/
rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1,
dev_attr->max_qp_wqes + 1,
uctx);
} else {
rq->max_wqe = init_attr->cap.max_recv_wr;
}
rq->max_sw_wqe = rq->max_wqe;
rq->q_full_delta = 0;
rq->sg_info.pgsize = PAGE_SIZE;
@@ -1497,7 +1573,8 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
struct ib_qp_init_attr *init_attr,
struct bnxt_re_ucontext *uctx,
struct bnxt_re_qp_req *ureq)
struct bnxt_re_qp_req *ureq,
bool fixed_que_attr)
{
struct bnxt_qplib_dev_attr *dev_attr;
struct bnxt_qplib_qp *qplqp;
@@ -1538,13 +1615,18 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
sq->max_sw_wqe = sq->max_wqe;
}
sq->q_full_delta = diff + 1;
/*
* Reserving one slot for Phantom WQE. Application can
* post one extra entry in this case. But allowing this to avoid
* unexpected Queue full condition
*/
qplqp->sq.q_full_delta -= 1;
if (!fixed_que_attr) {
sq->q_full_delta = diff + 1;
/*
* Reserving one slot for Phantom WQE. Application can
* post one extra entry in this case. But allowing this to avoid
* unexpected Queue full condition
*/
qplqp->sq.q_full_delta -= 1;
} else {
sq->q_full_delta = 0;
}
qplqp->sq.sg_info.pgsize = PAGE_SIZE;
qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
@@ -1597,7 +1679,9 @@ static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
return qptype;
}
static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp)
static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp,
bool fixed_que_attr,
struct bnxt_re_qp_req *req)
{
struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
struct bnxt_qplib_q *sq = &qplib_qp->sq;
@@ -1620,12 +1704,17 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp)
/* Update msn tbl size */
if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) {
if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
qplib_qp->msn_tbl_sz =
roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode));
else
qplib_qp->msn_tbl_sz =
roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2;
if (!fixed_que_attr) {
if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
qplib_qp->msn_tbl_sz =
roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode));
else
qplib_qp->msn_tbl_sz =
roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode))
/ 2;
} else {
qplib_qp->msn_tbl_sz = req->sq_npsn;
}
qplib_qp->msn = 0;
}
}
@@ -1633,7 +1722,10 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp)
static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
struct ib_qp_init_attr *init_attr,
struct bnxt_re_ucontext *uctx,
struct bnxt_re_qp_req *ureq)
struct bnxt_re_qp_req *ureq,
struct bnxt_re_dbr_obj *dbr_obj,
bool fixed_que_attr,
struct uverbs_attr_bundle *attrs)
{
struct bnxt_qplib_dev_attr *dev_attr;
struct bnxt_qplib_qp *qplqp;
@@ -1656,6 +1748,16 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
return qptype;
qplqp->type = (u8)qptype;
qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);
if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE &&
(!ureq->sq_slots || ureq->sq_slots > BNXT_RE_MAX_SQ_SLOTS))
return -EINVAL;
if (fixed_que_attr) {
if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE)
return -EOPNOTSUPP;
if (!ureq->sq_npsn ||
ureq->sq_npsn > roundup_pow_of_two(ureq->sq_slots / 2))
return -EINVAL;
}
qplqp->dev_cap_flags = dev_attr->dev_cap_flags;
qplqp->cctx = rdev->chip_ctx;
if (init_attr->qp_type == IB_QPT_RC) {
@@ -1685,28 +1787,29 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
}
/* Setup RQ/SRQ */
rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
rc = bnxt_re_init_rq_attr(qp, init_attr, uctx, fixed_que_attr);
if (rc)
return rc;
if (init_attr->qp_type == IB_QPT_GSI)
bnxt_re_adjust_gsi_rq_attr(qp);
/* Setup SQ */
rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);
rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq, fixed_que_attr);
if (rc)
return rc;
if (init_attr->qp_type == IB_QPT_GSI)
bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
if (uctx) { /* This will update DPI and qp_handle */
rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq);
rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr,
dbr_obj, attrs);
if (rc)
return rc;
}
bnxt_re_qp_calculate_msn_psn_size(qp);
bnxt_re_qp_calculate_msn_psn_size(qp, fixed_que_attr, ureq);
rc = bnxt_re_setup_qp_hwqs(qp);
rc = bnxt_re_setup_qp_hwqs(qp, fixed_que_attr);
if (rc)
goto free_umem;
@@ -1835,8 +1938,11 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev)
int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
struct ib_udata *udata)
{
struct uverbs_attr_bundle *attrs = NULL;
struct bnxt_re_dbr_obj *dbr_obj = NULL;
struct bnxt_qplib_dev_attr *dev_attr;
struct bnxt_re_ucontext *uctx;
bool fixed_que_attr = false;
struct bnxt_re_qp_req ureq;
struct bnxt_re_dev *rdev;
struct bnxt_re_pd *pd;
@@ -1853,9 +1959,23 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
if (udata) {
rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0);
rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle,
BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR);
if (rc)
return rc;
attrs = rdma_udata_to_uverbs_attr_bundle(udata);
if (uverbs_attr_is_valid(attrs,
BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) {
dbr_obj = uverbs_attr_get_obj(attrs,
BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE);
if (IS_ERR(dbr_obj))
return PTR_ERR(dbr_obj);
kref_get(&dbr_obj->usecnt);
qp->dbr_obj = dbr_obj;
}
if (ureq.comp_mask & BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR)
fixed_que_attr = true;
}
rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
@@ -1865,7 +1985,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
}
qp->rdev = rdev;
rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq);
rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq,
dbr_obj, fixed_que_attr, attrs);
if (rc)
goto fail;
@@ -1884,7 +2005,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
}
if (udata) {
struct bnxt_re_qp_resp resp;
struct bnxt_re_qp_resp resp = {};
resp.qpid = qp->qplib_qp.id;
resp.rsvd = 0;
@@ -1935,6 +2056,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
bnxt_re_qp_free_umem(qp);
fail:
if (dbr_obj)
kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release);
return rc;
}
@@ -2029,11 +2152,11 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
if (ret)
return ret;
if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
free_page((unsigned long)srq->uctx_srq_page);
if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT)
hash_del(&srq->hash_entry);
}
bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT)
free_page((unsigned long)srq->uctx_srq_page);
ib_umem_release(srq->umem);
atomic_dec(&rdev->stats.res.srq_count);
return ib_respond_empty_udata(udata);
@@ -2058,8 +2181,8 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
bytes = PAGE_ALIGN(bytes);
umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
IB_ACCESS_LOCAL_WRITE);
umem = ib_umem_get_va(&rdev->ibdev, ureq.srqva, bytes,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem))
return PTR_ERR(umem);
@@ -2286,6 +2409,23 @@ static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
return rc;
}
static bool bnxt_re_is_modify_ok(enum ib_qp_attr_mask ext_mask,
enum ib_qp_type type, enum ib_qp_state cur,
enum ib_qp_state next)
{
if (!ext_mask)
return true;
if (ext_mask & ~IB_QP_RATE_LIMIT)
return false;
/* Rate limit is only supported for RC QPs during specific transitions */
return type == IB_QPT_RC &&
((cur == IB_QPS_INIT && next == IB_QPS_RTR) ||
(cur == IB_QPS_RTR && next == IB_QPS_RTS) ||
(cur == IB_QPS_RTS && next == IB_QPS_RTS));
}
int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
int qp_attr_mask, struct ib_udata *udata)
{
@@ -2310,7 +2450,10 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
new_qp_state = qp_attr->qp_state;
if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
ib_qp->qp_type, qp_attr_mask)) {
ib_qp->qp_type, qp_attr_mask) ||
!bnxt_re_is_modify_ok(qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS,
ib_qp->qp_type, curr_qp_state,
new_qp_state)) {
ibdev_err(&rdev->ibdev,
"Invalid attribute mask: %#x specified ",
qp_attr_mask);
@@ -3332,39 +3475,20 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
if (ret)
return ret;
if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
free_page((unsigned long)cq->uctx_cq_page);
if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT)
hash_del(&cq->hash_entry);
}
bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT)
free_page((unsigned long)cq->uctx_cq_page);
bnxt_re_put_nq(rdev, nq);
atomic_dec(&rdev->stats.res.cq_count);
kfree(cq->cql);
ib_umem_release(cq->umem);
return ib_respond_empty_udata(udata);
}
static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev,
struct ib_umem *umem,
struct bnxt_qplib_sg_info *sginfo)
{
unsigned long page_size;
if (!umem)
return -EINVAL;
page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0);
if (!page_size || page_size != SZ_4K)
return -EINVAL;
sginfo->umem = umem;
sginfo->npages = ib_umem_num_dma_blocks(umem, page_size);
sginfo->pgsize = page_size;
sginfo->pgshft = __builtin_ctz(page_size);
return 0;
}
int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct uverbs_attr_bundle *attrs)
{
@@ -3402,17 +3526,15 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att
entries = bnxt_re_init_depth(attr->cqe + 1,
dev_attr->max_cq_wqes + 1, uctx);
if (!ibcq->umem) {
ibcq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(ibcq->umem))
return PTR_ERR(ibcq->umem);
}
cq->umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, attrs, req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem))
return PTR_ERR(cq->umem);
rc = bnxt_re_setup_sginfo(rdev, ibcq->umem, &cq->qplib_cq.sg_info);
rc = bnxt_re_setup_sginfo(rdev, cq->umem, &cq->qplib_cq.sg_info);
if (rc)
return rc;
goto free_umem;
cq->qplib_cq.dpi = &uctx->dpi;
cq->qplib_cq.max_wqe = entries;
@@ -3422,7 +3544,7 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att
rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
if (rc)
return rc;
goto free_umem;
cq->ib_cq.cqe = entries;
cq->cq_period = cq->qplib_cq.period;
@@ -3435,8 +3557,10 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att
hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
/* Allocate a page */
cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
if (!cq->uctx_cq_page)
return -ENOMEM;
if (!cq->uctx_cq_page) {
rc = -ENOMEM;
goto destroy_cq;
}
resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
}
@@ -3444,15 +3568,17 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att
resp.tail = cq->qplib_cq.hwq.cons;
resp.phase = cq->qplib_cq.period;
rc = ib_respond_udata(udata, resp);
if (rc) {
bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
if (rc)
goto free_mem;
}
return 0;
free_mem:
free_page((unsigned long)cq->uctx_cq_page);
destroy_cq:
bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
free_umem:
ib_umem_release(cq->umem);
return rc;
}
@@ -3516,8 +3642,8 @@ static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
cq->qplib_cq.max_wqe = cq->resize_cqe;
if (cq->resize_umem) {
ib_umem_release(cq->ib_cq.umem);
cq->ib_cq.umem = cq->resize_umem;
ib_umem_release(cq->umem);
cq->umem = cq->resize_umem;
cq->resize_umem = NULL;
cq->resize_cqe = 0;
}
@@ -3562,12 +3688,12 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe,
if (rc)
goto fail;
cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->resize_umem)) {
rc = PTR_ERR(cq->resize_umem);
ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %pe\n",
ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n",
__func__, cq->resize_umem);
cq->resize_umem = NULL;
goto fail;
@@ -4113,7 +4239,7 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
/* User CQ; the only processing we do is to
* complete any pending CQ resize operation.
*/
if (cq->ib_cq.umem) {
if (cq->umem) {
if (cq->resize_umem)
bnxt_re_resize_cq_complete(cq);
return 0;
@@ -4577,7 +4703,7 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
umem = ib_umem_get_va(&rdev->ibdev, start, length, mr_access_flags);
if (IS_ERR(umem))
return ERR_CAST(umem);
@@ -4644,6 +4770,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
goto fail;
}
spin_lock_init(&uctx->sh_lock);
mutex_init(&uctx->wcdpi_lock);
resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
chip_met_rev_num = rdev->chip_ctx->chip_num;
@@ -4886,6 +5013,10 @@ void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
rdma_entry);
if (bnxt_entry->dpi_valid)
bnxt_qplib_free_uc_dpi(&bnxt_entry->uctx->rdev->qplib_res,
&bnxt_entry->dpi);
kfree(bnxt_entry);
}

8
drivers/infiniband/hw/bnxt_re/ib_verbs.h
View File

@@ -96,6 +96,7 @@ struct bnxt_re_qp {
struct bnxt_re_cq *scq;
struct bnxt_re_cq *rcq;
struct dentry *dentry;
struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */
};
struct bnxt_re_cq {
@@ -108,6 +109,7 @@ struct bnxt_re_cq {
struct bnxt_qplib_cqe *cql;
#define MAX_CQL_PER_POLL 1024
u32 max_cql;
struct ib_umem *umem;
struct ib_umem *resize_umem;
int resize_cqe;
void *uctx_cq_page;
@@ -141,6 +143,7 @@ struct bnxt_re_ucontext {
struct bnxt_re_dev *rdev;
struct bnxt_qplib_dpi dpi;
struct bnxt_qplib_dpi wcdpi;
struct mutex wcdpi_lock; /* serialises WC DPI alloc/free */
void *shpg;
spinlock_t sh_lock; /* protect shpg */
struct rdma_user_mmap_entry *shpage_mmap;
@@ -161,13 +164,15 @@ struct bnxt_re_user_mmap_entry {
struct bnxt_re_ucontext *uctx;
u64 mem_offset;
u8 mmap_flag;
bool dpi_valid;
struct bnxt_qplib_dpi dpi;
};
struct bnxt_re_dbr_obj {
struct bnxt_re_dev *rdev;
struct bnxt_qplib_dpi dpi;
struct bnxt_re_user_mmap_entry *entry;
atomic_t usecnt; /* QPs using this dbr */
struct kref usecnt; /* 1 (uobject) + n (QPs using this dbr) */
};
struct bnxt_re_flow {
@@ -308,4 +313,5 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags);
struct bnxt_re_user_mmap_entry*
bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
enum bnxt_re_mmap_flag mmap_flag, u64 *offset);
void bnxt_re_dbr_kref_release(struct kref *ref);
#endif /* __BNXT_RE_IB_VERBS_H__ */

2
drivers/infiniband/hw/bnxt_re/main.c
View File

@@ -1093,8 +1093,6 @@ static int bnxt_re_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)
goto err;
if (rdma_nl_put_driver_u32(msg, "element_size", mr_hwq->element_size))
goto err;
if (rdma_nl_put_driver_u64_hex(msg, "hwq", (unsigned long)mr_hwq))
goto err;
if (rdma_nl_put_driver_u64_hex(msg, "va", mr->qplib_mr.va))
goto err;

11
drivers/infiniband/hw/bnxt_re/qplib_res.c
View File

@@ -764,9 +764,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
break;
case BNXT_QPLIB_DPI_TYPE_WC:
dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE);
if (!dpi->dbr)
goto fail_ioremap;
break;
default:
dpi->dbr = ioremap(umaddr, PAGE_SIZE);
if (!dpi->dbr)
goto fail_ioremap;
break;
}
@@ -774,6 +778,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
mutex_unlock(&res->dpi_tbl_lock);
return 0;
fail_ioremap:
/* Roll back the bit we just claimed. */
set_bit(bit_num, dpit->tbl);
dpit->app_tbl[bit_num] = NULL;
mutex_unlock(&res->dpi_tbl_lock);
return -ENOMEM;
}
int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,

1
drivers/infiniband/hw/bnxt_re/qplib_sp.h
View File

@@ -369,6 +369,7 @@ int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res);
#define BNXT_VAR_MAX_SLOT_ALIGN 256
#define BNXT_VAR_MAX_SGE 13
#define BNXT_RE_MAX_RQ_WQES 65536
#define BNXT_RE_MAX_SQ_SLOTS 65536
#define BNXT_STATIC_MAX_SGE 6

92
drivers/infiniband/hw/bnxt_re/uapi.c
View File

@@ -76,8 +76,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *
struct ib_ucontext *ib_uctx;
struct bnxt_re_dev *rdev;
u64 mmap_offset;
u32 dpi = 0;
u32 length;
u32 dpi;
u64 addr;
int err;
@@ -98,26 +98,39 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *
switch (alloc_type) {
case BNXT_RE_ALLOC_WC_PAGE:
if (cctx->modes.db_push) {
if (cctx->modes.db_push) {
mutex_lock(&uctx->wcdpi_lock);
/* already allocated — one WC page per context */
if (uctx->wcdpi.dbr) {
mutex_unlock(&uctx->wcdpi_lock);
return -EEXIST;
}
if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi,
uctx, BNXT_QPLIB_DPI_TYPE_WC))
uctx, BNXT_QPLIB_DPI_TYPE_WC)) {
mutex_unlock(&uctx->wcdpi_lock);
return -ENOMEM;
}
length = PAGE_SIZE;
dpi = uctx->wcdpi.dpi;
addr = (u64)uctx->wcdpi.umdbr;
mmap_flag = BNXT_RE_MMAP_WC_DB;
mutex_unlock(&uctx->wcdpi_lock);
} else {
return -EINVAL;
}
break;
case BNXT_RE_ALLOC_DBR_BAR_PAGE:
if (!rdev->pacing.dbr_pacing)
return -EOPNOTSUPP;
length = PAGE_SIZE;
addr = (u64)rdev->pacing.dbr_bar_addr;
mmap_flag = BNXT_RE_MMAP_DBR_BAR;
break;
case BNXT_RE_ALLOC_DBR_PAGE:
if (!rdev->pacing.dbr_pacing)
return -EOPNOTSUPP;
length = PAGE_SIZE;
addr = (u64)rdev->pacing.dbr_page;
mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
@@ -128,8 +141,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *
}
entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset);
if (!entry)
if (!entry) {
if (mmap_flag == BNXT_RE_MMAP_WC_DB) {
mutex_lock(&uctx->wcdpi_lock);
bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
uctx->wcdpi.dbr = NULL;
mutex_unlock(&uctx->wcdpi_lock);
}
return -ENOMEM;
}
uobj->object = entry;
uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
@@ -160,11 +180,16 @@ static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
switch (entry->mmap_flag) {
case BNXT_RE_MMAP_WC_DB:
if (uctx && uctx->wcdpi.dbr) {
if (uctx) {
struct bnxt_re_dev *rdev = uctx->rdev;
bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
uctx->wcdpi.dbr = NULL;
mutex_lock(&uctx->wcdpi_lock);
if (uctx->wcdpi.dbr) {
bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
&uctx->wcdpi);
uctx->wcdpi.dbr = NULL;
}
mutex_unlock(&uctx->wcdpi_lock);
}
break;
case BNXT_RE_MMAP_DBR_BAR:
@@ -252,6 +277,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
return -EINVAL;
addr = (u64)cq->uctx_cq_page;
if (!addr)
return -EOPNOTSUPP;
break;
case BNXT_RE_SRQ_TOGGLE_MEM:
srq = bnxt_re_search_for_srq(rdev, res_id);
@@ -259,6 +286,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
return -EINVAL;
addr = (u64)srq->uctx_srq_page;
if (!addr)
return -EOPNOTSUPP;
break;
default:
@@ -368,7 +397,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a
goto free_dpi;
}
/* Save DPI info to the mmap entry so that bnxt_re_mmap_free()
* can free the DPI slot only after the last reference to the
* mmap entry is released.
*/
obj->entry->dpi = *dpi;
obj->entry->dpi_valid = true;
obj->rdev = rdev;
kref_init(&obj->usecnt);
uobj->object = obj;
uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE);
@@ -391,15 +428,35 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a
return ret;
}
void bnxt_re_dbr_kref_release(struct kref *ref)
{
struct bnxt_re_dbr_obj *obj =
container_of(ref, struct bnxt_re_dbr_obj, usecnt);
/* Drop the driver's reference to the mmap entry (_remove()).
* The DPI slot gets freed from bnxt_re_mmap_free() only
* when there's no VMA mapping reference to it.
*/
rdma_user_mmap_entry_remove(&obj->entry->rdma_entry);
kfree(obj);
}
static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct bnxt_re_dbr_obj *obj = uobject->object;
struct bnxt_re_dev *rdev = obj->rdev;
rdma_user_mmap_entry_remove(&obj->entry->rdma_entry);
bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi);
/* If it is being destroyed explicitly while QPs still hold a
* reference (> 1), reject it with EBUSY. If no QP references
* or implicit teardown (process exit, driver removal), drop
* the uobject reference unconditionally. The object gets freed
* (bnxt_re_dbr_kref_release) when the usecnt goes to zero.
*/
if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1)
return -EBUSY;
kref_put(&obj->usecnt, bnxt_re_dbr_kref_release);
return 0;
}
@@ -459,11 +516,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR,
DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR,
&UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR));
ADD_UVERBS_ATTRIBUTES_SIMPLE(
bnxt_re_qp_create,
UVERBS_OBJECT_QP,
UVERBS_METHOD_QP_CREATE,
UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE,
BNXT_RE_OBJECT_DBR,
UVERBS_ACCESS_READ,
UA_OPTIONAL));
const struct uapi_definition bnxt_re_create_qp_defs[] = {
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create),
{},
};
const struct uapi_definition bnxt_re_uapi_defs[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR),
UAPI_DEF_CHAIN(bnxt_re_create_qp_defs),
{}
};

11
drivers/infiniband/hw/cxgb4/cq.c
View File

@@ -1004,7 +1004,7 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device);
struct c4iw_cq *chp = to_c4iw_cq(ibcq);
struct c4iw_create_cq ucmd;
struct c4iw_create_cq_resp uresp;
struct c4iw_create_cq_resp uresp = {};
int ret, wr_len;
size_t memsize, hwentries;
struct c4iw_mm_entry *mm, *mm2;
@@ -1102,7 +1102,6 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
if (!mm2)
goto err_free_mm;
memset(&uresp, 0, sizeof(uresp));
uresp.qid_mask = rhp->rdev.cqmask;
uresp.cqid = chp->cq.cqid;
uresp.size = chp->cq.size;
@@ -1115,13 +1114,11 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
/* communicate to the userspace that
* kernel driver supports 64B CQE
*/
uresp.flags |= C4IW_64B_CQE;
if (!ucontext->is_32b_cqe)
uresp.flags |= C4IW_64B_CQE;
spin_unlock(&ucontext->mmap_lock);
ret = ib_copy_to_udata(udata, &uresp,
ucontext->is_32b_cqe ?
sizeof(uresp) - sizeof(uresp.flags) :
sizeof(uresp));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_free_mm2;

2
drivers/infiniband/hw/cxgb4/mem.c
View File

@@ -530,7 +530,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
mhp->rhp = rhp;
mhp->umem = ib_umem_get(pd->device, start, length, acc);
mhp->umem = ib_umem_get_va(pd->device, start, length, acc);
if (IS_ERR(mhp->umem))
goto err_free_skb;

22
drivers/infiniband/hw/cxgb4/provider.c
View File

@@ -52,6 +52,7 @@
#include <rdma/ib_smi.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/uverbs_ioctl.h>
#include "iw_cxgb4.h"
@@ -79,7 +80,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
struct ib_device *ibdev = ucontext->device;
struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext);
struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
struct c4iw_alloc_ucontext_resp uresp;
struct c4iw_alloc_ucontext_resp uresp = {};
int ret = 0;
struct c4iw_mm_entry *mm = NULL;
@@ -105,8 +106,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
context->key += PAGE_SIZE;
spin_unlock(&context->mmap_lock);
ret = ib_copy_to_udata(udata, &uresp,
sizeof(uresp) - sizeof(uresp.reserved));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_mm;
@@ -209,8 +209,9 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
{
struct c4iw_pd *php = to_c4iw_pd(pd);
struct ib_device *ibdev = pd->device;
u32 pdid;
struct c4iw_dev *rhp;
u32 pdid;
int ret;
pr_debug("ibdev %p\n", ibdev);
rhp = (struct c4iw_dev *) ibdev;
@@ -223,9 +224,10 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
if (udata) {
struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid};
if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
ret = ib_respond_udata(udata, uresp);
if (ret) {
c4iw_deallocate_pd(&php->ibpd, udata);
return -EFAULT;
return ret;
}
}
mutex_lock(&rhp->rdev.stats.lock);
@@ -257,11 +259,13 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
{
struct c4iw_dev *dev;
int err;
pr_debug("ibdev %p\n", ibdev);
if (uhw->inlen || uhw->outlen)
return -EINVAL;
err = ib_is_udata_in_empty(uhw);
if (err)
return err;
dev = to_c4iw_dev(ibdev);
addrconf_addr_eui48((u8 *)&props->sys_image_guid,
@@ -296,7 +300,7 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro
props->max_fast_reg_page_list_len =
t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl);
return 0;
return ib_respond_empty_udata(uhw);
}
static int c4iw_query_port(struct ib_device *ibdev, u32 port,

10
drivers/infiniband/hw/cxgb4/qp.c
View File

@@ -2120,7 +2120,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
struct c4iw_pd *php;
struct c4iw_cq *schp;
struct c4iw_cq *rchp;
struct c4iw_create_qp_resp uresp;
struct c4iw_create_qp_resp uresp = {};
unsigned int sqsize, rqsize = 0;
struct c4iw_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct c4iw_ucontext, ibucontext);
@@ -2242,7 +2242,6 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
goto err_free_sq_db_key;
}
}
memset(&uresp, 0, sizeof(uresp));
if (t4_sq_onchip(&qhp->wq.sq)) {
ma_sync_key_mm = kmalloc_obj(*ma_sync_key_mm);
if (!ma_sync_key_mm) {
@@ -2280,7 +2279,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
ucontext->key += PAGE_SIZE;
}
spin_unlock(&ucontext->mmap_lock);
ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_free_ma_sync_key;
sq_key_mm->key = uresp.sq_key;
@@ -2686,7 +2685,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
struct c4iw_dev *rhp;
struct c4iw_srq *srq = to_c4iw_srq(ib_srq);
struct c4iw_pd *php;
struct c4iw_create_srq_resp uresp;
struct c4iw_create_srq_resp uresp = {};
struct c4iw_ucontext *ucontext;
struct c4iw_mm_entry *srq_key_mm, *srq_db_key_mm;
int rqsize;
@@ -2764,7 +2763,6 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
ret = -ENOMEM;
goto err_free_srq_key_mm;
}
memset(&uresp, 0, sizeof(uresp));
uresp.flags = srq->flags;
uresp.qid_mask = rhp->rdev.qpmask;
uresp.srqid = srq->wq.qid;
@@ -2777,7 +2775,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
uresp.srq_db_gts_key = ucontext->key;
ucontext->key += PAGE_SIZE;
spin_unlock(&ucontext->mmap_lock);
ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_free_srq_db_key_mm;
srq_key_mm->key = uresp.srq_key;

3
drivers/infiniband/hw/efa/Kconfig
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
#
# Amazon fabric device configuration
#
@@ -8,6 +8,7 @@ config INFINIBAND_EFA
tristate "Amazon Elastic Fabric Adapter (EFA) support"
depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN
depends on INFINIBAND_USER_ACCESS
select CRC16
help
This driver supports Amazon Elastic Fabric Adapter (EFA).

1
drivers/infiniband/hw/efa/efa.h
View File

@@ -148,6 +148,7 @@ int efa_query_device(struct ib_device *ibdev,
struct ib_udata *udata);
int efa_query_port(struct ib_device *ibdev, u32 port,
struct ib_port_attr *props);
int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed);
int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_qp_init_attr *qp_init_attr);

3
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
View File

@@ -6,9 +6,6 @@
#ifndef _EFA_ADMIN_CMDS_H_
#define _EFA_ADMIN_CMDS_H_
#define EFA_ADMIN_API_VERSION_MAJOR 0
#define EFA_ADMIN_API_VERSION_MINOR 1
/* EFA admin queue opcodes */
enum efa_admin_aq_opcode {
EFA_ADMIN_CREATE_QP = 1,

15
drivers/infiniband/hw/efa/efa_admin_defs.h
View File

@@ -1,18 +1,20 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_ADMIN_H_
#define _EFA_ADMIN_H_
#define EFA_ADMIN_API_VERSION_MAJOR 0
#define EFA_ADMIN_API_VERSION_MINOR 2
enum efa_admin_aq_completion_status {
EFA_ADMIN_SUCCESS = 0,
EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1,
EFA_ADMIN_BAD_OPCODE = 2,
EFA_ADMIN_UNSUPPORTED_OPCODE = 3,
EFA_ADMIN_MALFORMED_REQUEST = 4,
/* Additional status is provided in ACQ entry extended_status */
EFA_ADMIN_ILLEGAL_PARAMETER = 5,
EFA_ADMIN_UNKNOWN_ERROR = 6,
EFA_ADMIN_RESOURCE_BUSY = 7,
@@ -78,13 +80,10 @@ struct efa_admin_acq_common_desc {
*/
u8 flags;
u16 extended_status;
/* Poly 0x8005 CRC16 with initial value 0xFFFF and final XOR of 0xFFFF */
u16 checksum;
/*
* indicates to the driver which AQ entry has been consumed by the
* device and could be reused
*/
u16 sq_head_indx;
u16 reserved;
};
struct efa_admin_acq_entry {

50
drivers/infiniband/hw/efa/efa_com.c
View File

@@ -3,6 +3,7 @@
* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#include <linux/crc16.h>
#include <linux/log2.h>
#include "efa_com.h"
@@ -22,6 +23,14 @@
#define EFA_CTRL_MINOR 0
#define EFA_CTRL_SUB_MINOR 1
#define EFA_CRC16_INIT_VAL 0xffff
#define EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR 0
#define EFA_CRC_MIN_ADMIN_API_VERSION_MINOR 2
#define EFA_MIN_ADMIN_API_VERSION_MAJOR 0
#define EFA_MIN_ADMIN_API_VERSION_MINOR 1
enum efa_cmd_status {
EFA_CMD_UNUSED,
EFA_CMD_ALLOCATED,
@@ -167,9 +176,8 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)
struct efa_com_admin_queue *aq = &edev->aq;
struct efa_com_admin_cq *cq = &aq->cq;
u16 size = aq->depth * sizeof(*cq->entries);
u32 acq_caps = 0;
u32 addr_high;
u32 addr_low;
u32 acq_caps = 0, crc_min_ver = 0;
u32 addr_high, addr_low;
cq->entries =
dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);
@@ -178,6 +186,11 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)
spin_lock_init(&cq->lock);
EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR);
EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MINOR);
if (edev->dev_api_ver >= crc_min_ver)
cq->validate_checksum = true;
cq->cc = 0;
cq->phase = 1;
@@ -409,12 +422,35 @@ static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
return 0;
}
static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq,
struct efa_admin_acq_entry *cqe)
{
u16 cqe_checksum = cqe->acq_common_descriptor.checksum;
u16 calc_checksum;
cqe->acq_common_descriptor.checksum = 0;
calc_checksum = crc16(EFA_CRC16_INIT_VAL, (u8 *)cqe, sizeof(*cqe)) ^ EFA_CRC16_INIT_VAL;
if (calc_checksum != cqe_checksum) {
ibdev_err(aq->efa_dev,
"Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n",
cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc,
aq->cq.cc);
return false;
}
return true;
}
static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,
struct efa_admin_acq_entry *cqe)
{
struct efa_comp_ctx *comp_ctx;
u16 cmd_id;
if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe))
return -EINVAL;
cmd_id = EFA_GET(&cqe->acq_common_descriptor.command,
EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
@@ -954,16 +990,16 @@ int efa_com_validate_version(struct efa_com_dev *edev)
EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION),
EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION));
EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION,
EFA_ADMIN_API_VERSION_MAJOR);
EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION,
EFA_ADMIN_API_VERSION_MINOR);
EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MAJOR);
EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MINOR);
if (ver < min_ver) {
ibdev_err(edev->efa_dev,
"EFA version is lower than the minimal version the driver supports\n");
return -EOPNOTSUPP;
}
edev->dev_api_ver = ver;
ibdev_dbg(
edev->efa_dev,
"efa controller version: %d.%d.%d implementation version %d\n",

4
drivers/infiniband/hw/efa/efa_com.h
View File

@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_COM_H_
@@ -25,6 +25,7 @@ struct efa_com_admin_cq {
struct efa_admin_acq_entry *entries;
dma_addr_t dma_addr;
spinlock_t lock; /* Protects ACQ */
bool validate_checksum;
u16 cc; /* consumer counter */
u8 phase;
@@ -112,6 +113,7 @@ struct efa_com_dev {
u32 supported_features;
u32 dma_addr_bits;
u32 dev_api_ver;
struct efa_com_mmio_read mmio_read;
};

4
drivers/infiniband/hw/efa/efa_com_cmd.c
View File

@@ -6,6 +6,8 @@
#include "efa_com.h"
#include "efa_com_cmd.h"
#define EFA_DEFAULT_LINK_SPEED_GBPS 100
int efa_com_create_qp(struct efa_com_dev *edev,
struct efa_com_create_qp_params *params,
struct efa_com_create_qp_result *res)
@@ -468,6 +470,8 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,
result->device_caps = resp.u.device_attr.device_caps;
result->guid = resp.u.device_attr.guid;
result->max_link_speed_gbps = resp.u.device_attr.max_link_speed_gbps;
if (!result->max_link_speed_gbps)
result->max_link_speed_gbps = EFA_DEFAULT_LINK_SPEED_GBPS;
if (result->admin_api_version < 1) {
ibdev_err_ratelimited(

1
drivers/infiniband/hw/efa/efa_main.c
View File

@@ -390,6 +390,7 @@ static const struct ib_device_ops efa_dev_ops = {
.query_gid = efa_query_gid,
.query_pkey = efa_query_pkey,
.query_port = efa_query_port,
.query_port_speed = efa_query_port_speed,
.query_qp = efa_query_qp,
.reg_user_mr = efa_reg_mr,
.reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,

137
drivers/infiniband/hw/efa/efa_verbs.c
View File

@@ -90,8 +90,6 @@ static const struct rdma_stat_desc efa_port_stats_descs[] = {
EFA_DEFINE_PORT_STATS(EFA_STATS_STR)
};
#define EFA_DEFAULT_LINK_SPEED_GBPS 100
#define EFA_CHUNK_PAYLOAD_SHIFT 12
#define EFA_CHUNK_PAYLOAD_SIZE BIT(EFA_CHUNK_PAYLOAD_SHIFT)
#define EFA_CHUNK_PAYLOAD_PTR_SIZE 8
@@ -218,12 +216,9 @@ int efa_query_device(struct ib_device *ibdev,
struct efa_dev *dev = to_edev(ibdev);
int err;
if (udata && udata->inlen &&
!ib_is_udata_cleared(udata, 0, udata->inlen)) {
ibdev_dbg(ibdev,
"Incompatible ABI params, udata not cleared\n");
return -EINVAL;
}
err = ib_is_udata_in_empty(udata);
if (err)
return err;
dev_attr = &dev->dev_attr;
@@ -273,13 +268,9 @@ int efa_query_device(struct ib_device *ibdev,
if (dev->neqs)
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS;
err = ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(ibdev,
"Failed to copy udata for query_device\n");
err = ib_respond_udata(udata, resp);
if (err)
return err;
}
}
return 0;
@@ -289,7 +280,13 @@ static void efa_link_gbps_to_speed_and_width(u16 gbps,
enum ib_port_speed *speed,
enum ib_port_width *width)
{
if (gbps >= 400) {
if (gbps >= 1600) {
*width = IB_WIDTH_8X;
*speed = IB_SPEED_XDR;
} else if (gbps >= 800) {
*width = IB_WIDTH_8X;
*speed = IB_SPEED_NDR;
} else if (gbps >= 400) {
*width = IB_WIDTH_8X;
*speed = IB_SPEED_HDR;
} else if (gbps >= 200) {
@@ -333,7 +330,7 @@ int efa_query_port(struct ib_device *ibdev, u32 port,
props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
props->gid_tbl_len = 1;
props->pkey_tbl_len = 1;
link_gbps = dev->dev_attr.max_link_speed_gbps ?: EFA_DEFAULT_LINK_SPEED_GBPS;
link_gbps = dev->dev_attr.max_link_speed_gbps;
efa_link_gbps_to_speed_and_width(link_gbps, &link_speed, &link_width);
props->active_speed = link_speed;
props->active_width = link_width;
@@ -345,6 +342,15 @@ int efa_query_port(struct ib_device *ibdev, u32 port,
return 0;
}
int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed)
{
struct efa_dev *dev = to_edev(ibdev);
*speed = dev->dev_attr.max_link_speed_gbps * 10;
return 0;
}
int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_qp_init_attr *qp_init_attr)
@@ -433,13 +439,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
struct efa_pd *pd = to_epd(ibpd);
int err;
if (udata->inlen &&
!ib_is_udata_cleared(udata, 0, udata->inlen)) {
ibdev_dbg(&dev->ibdev,
"Incompatible ABI params, udata not cleared\n");
err = -EINVAL;
err = ib_is_udata_in_empty(udata);
if (err)
goto err_out;
}
err = efa_com_alloc_pd(&dev->edev, &result);
if (err)
@@ -449,13 +451,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
resp.pdn = result.pdn;
if (udata->outlen) {
err = ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(&dev->ibdev,
"Failed to copy udata for alloc_pd\n");
err = ib_respond_udata(udata, resp);
if (err)
goto err_dealloc_pd;
}
}
ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn);
@@ -798,14 +796,9 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
qp->max_inline_data = init_attr->cap.max_inline_data;
if (udata->outlen) {
err = ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(&dev->ibdev,
"Failed to copy udata for qp[%u]\n",
create_qp_resp.qp_num);
err = ib_respond_udata(udata, resp);
if (err)
goto err_remove_mmap_entries;
}
}
ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num);
@@ -991,12 +984,9 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
return -EOPNOTSUPP;
if (udata->inlen &&
!ib_is_udata_cleared(udata, 0, udata->inlen)) {
ibdev_dbg(&dev->ibdev,
"Incompatible ABI params, udata not cleared\n");
return -EINVAL;
}
err = ib_is_udata_in_empty(udata);
if (err)
return err;
cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state :
qp->state;
@@ -1081,6 +1071,7 @@ int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
if (cq->cpu_addr)
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE);
ib_umem_release(cq->umem);
return 0;
}
@@ -1133,6 +1124,7 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct efa_ibv_create_cq cmd;
struct efa_cq *cq = to_ecq(ibcq);
int entries = attr->cqe;
struct ib_umem *umem;
bool set_src_addr;
int err;
@@ -1181,26 +1173,29 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
cq->ucontext = ucontext;
cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs);
if (ibcq->umem) {
if (ibcq->umem->length < cq->size) {
ibdev_dbg(&dev->ibdev, "External memory too small\n");
err = -EINVAL;
goto err_out;
}
umem = ib_umem_get_cq_buf(ibcq->device, attrs, cq->size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
err = PTR_ERR(umem);
goto err_out;
}
if (!ib_umem_is_contiguous(ibcq->umem)) {
cq->umem = umem;
if (umem) {
if (!ib_umem_is_contiguous(umem)) {
ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n");
err = -EINVAL;
goto err_out;
goto err_release_umem;
}
cq->dma_addr = ib_umem_start_dma_addr(ibcq->umem);
cq->dma_addr = ib_umem_start_dma_addr(umem);
} else {
cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
if (!cq->cpu_addr) {
err = -ENOMEM;
goto err_out;
goto err_release_umem;
}
}
@@ -1245,13 +1240,9 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
}
if (udata->outlen) {
err = ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(ibdev,
"Failed to copy udata for create_cq\n");
err = ib_respond_udata(udata, resp);
if (err)
goto err_xa_erase;
}
}
ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",
@@ -1271,6 +1262,8 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
if (cq->cpu_addr)
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
err_release_umem:
ib_umem_release(cq->umem);
err_out:
atomic64_inc(&dev->stats.create_cq_err);
return err;
@@ -1621,13 +1614,11 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,
struct efa_dev *dev = to_edev(ibpd->device);
int supp_access_flags;
struct efa_mr *mr;
int ret;
if (udata && udata->inlen &&
!ib_is_udata_cleared(udata, 0, udata->inlen)) {
ibdev_dbg(&dev->ibdev,
"Incompatible ABI params, udata not cleared\n");
return ERR_PTR(-EINVAL);
}
ret = ib_is_udata_in_empty(udata);
if (ret)
return ERR_PTR(ret);
supp_access_flags =
IB_ACCESS_LOCAL_WRITE |
@@ -1783,7 +1774,7 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
goto err_out;
}
mr->umem = ib_umem_get(ibpd->device, start, length, access_flags);
mr->umem = ib_umem_get_va(ibpd->device, start, length, access_flags);
if (IS_ERR(mr->umem)) {
err = PTR_ERR(mr->umem);
ibdev_dbg(&dev->ibdev,
@@ -1956,8 +1947,7 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
resp.max_tx_batch = dev->dev_attr.max_tx_batch;
resp.min_sq_wr = dev->dev_attr.min_sq_depth;
err = ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen));
err = ib_respond_udata(udata, resp);
if (err)
goto err_dealloc_uar;
@@ -2091,12 +2081,9 @@ int efa_create_ah(struct ib_ah *ibah,
goto err_out;
}
if (udata->inlen &&
!ib_is_udata_cleared(udata, 0, udata->inlen)) {
ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
err = -EINVAL;
err = ib_is_udata_in_empty(udata);
if (err)
goto err_out;
}
memcpy(params.dest_addr, ah_attr->grh.dgid.raw,
sizeof(params.dest_addr));
@@ -2111,13 +2098,9 @@ int efa_create_ah(struct ib_ah *ibah,
resp.efa_address_handle = result.ah;
if (udata->outlen) {
err = ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(&dev->ibdev,
"Failed to copy udata for create_ah response\n");
err = ib_respond_udata(udata, resp);
if (err)
goto err_destroy_ah;
}
}
ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah);

28
drivers/infiniband/hw/erdma/erdma_verbs.c
View File

@@ -315,9 +315,14 @@ erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
}
int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
struct ib_udata *unused)
struct ib_udata *udata)
{
struct erdma_dev *dev = to_edev(ibdev);
int err;
err = ib_is_udata_in_empty(udata);
if (err)
return err;
memset(attr, 0, sizeof(*attr));
@@ -358,7 +363,7 @@ int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
dev->netdev->dev_addr);
return 0;
return ib_respond_empty_udata(udata);
}
int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
@@ -829,7 +834,7 @@ static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
{
int ret = 0;
mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
if (IS_ERR(mem->umem)) {
ret = PTR_ERR(mem->umem);
mem->umem = NULL;
@@ -896,8 +901,8 @@ static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
page->va = (dbrecords_va & PAGE_MASK);
page->refcnt = 0;
page->umem = ib_umem_get(ctx->ibucontext.device,
dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
page->umem = ib_umem_get_va(ctx->ibucontext.device,
dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
if (IS_ERR(page->umem)) {
rv = PTR_ERR(page->umem);
kfree(page);
@@ -996,7 +1001,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
udata, struct erdma_ucontext, ibucontext);
struct erdma_ureq_create_qp ureq;
struct erdma_uresp_create_qp uresp;
struct erdma_uresp_create_qp uresp = {};
void *old_entry;
int ret = 0;
@@ -1048,14 +1053,12 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
if (ret)
goto err_out_xa;
memset(&uresp, 0, sizeof(uresp));
uresp.num_sqe = qp->attrs.sq_size;
uresp.num_rqe = qp->attrs.rq_size;
uresp.qp_id = QP_ID(qp);
uresp.rq_offset = qp->user_qp.rq_offset;
ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_out_cmd;
} else {
@@ -1571,7 +1574,7 @@ int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
uresp.dev_id = dev->pdev->device;
ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_put_mmap_entries;
@@ -1977,7 +1980,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
if (!rdma_is_kernel_res(&ibcq->res)) {
struct erdma_ureq_create_cq ureq;
struct erdma_uresp_create_cq uresp;
struct erdma_uresp_create_cq uresp = {};
ret = ib_copy_validate_udata_in(udata, ureq, rsvd0);
if (ret)
@@ -1990,8 +1993,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
uresp.cq_id = cq->cqn;
uresp.num_cqe = depth;
ret = ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen));
ret = ib_respond_udata(udata, uresp);
if (ret)
goto err_free_res;
} else {

13
drivers/infiniband/hw/hfi1/init.c
View File

@@ -1206,6 +1206,7 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
size_t extra)
{
struct hfi1_devdata *dd;
struct ib_device *ibdev;
int ret, nports;
/* extra is * number of ports */
@@ -1227,7 +1228,17 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
"Could not allocate unit ID: error %d\n", -ret);
goto bail;
}
rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit);
/*
* FIXME: rvt and its users want to touch the ibdev before
* registration and have things like the name work. We don't have the
* infrastructure in the core to support this directly today, hack it
* to work by setting the name manually here.
*/
ibdev = &dd->verbs_dev.rdi.ibdev;
dev_set_name(&ibdev->dev, "%s_%d", class_name(), dd->unit);
strscpy(ibdev->name, dev_name(&ibdev->dev), IB_DEVICE_NAME_MAX);
/*
* If the BIOS does not have the NUMA node information set, select
* NUMA 0 so we get consistent performance.

4
drivers/infiniband/hw/hns/hns_roce_ah.c
View File

@@ -32,6 +32,7 @@
#include <rdma/ib_addr.h>
#include <rdma/ib_cache.h>
#include <rdma/uverbs_ioctl.h>
#include "hns_roce_device.h"
#include "hns_roce_hw_v2.h"
@@ -112,8 +113,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
resp.priority = ah->av.sl;
resp.tc_mode = tc_mode;
memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN);
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
ret = ib_respond_udata(udata, resp);
}
err_out:

9
drivers/infiniband/hw/hns/hns_roce_cq.c
View File

@@ -174,9 +174,9 @@ static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev,
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC,
hr_cq->cqn);
if (ret)
ibdev_err(ibdev,
"failed to send create cmd for CQ(0x%lx), ret = %d.\n",
hr_cq->cqn, ret);
ibdev_err_ratelimited(ibdev,
"failed to send create cmd for CQ(0x%lx), ret = %d.\n",
hr_cq->cqn, ret);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
@@ -452,8 +452,7 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
if (udata) {
resp.cqn = hr_cq->cqn;
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
ret = ib_respond_udata(udata, resp);
if (ret)
goto err_cqc;
}

4
drivers/infiniband/hw/hns/hns_roce_db.c
View File

@@ -29,8 +29,8 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
refcount_set(&page->refcount, 1);
page->user_virt = page_addr;
page->umem = ib_umem_get(context->ibucontext.device, page_addr,
PAGE_SIZE, 0);
page->umem = ib_umem_get_va(context->ibucontext.device, page_addr,
PAGE_SIZE, 0);
if (IS_ERR(page->umem)) {
ret = PTR_ERR(page->umem);
kfree(page);

287
drivers/infiniband/hw/hns/hns_roce_debugfs.c
View File

@@ -3,11 +3,13 @@
* Copyright (c) 2023 Hisilicon Limited.
*/
#include <linux/cleanup.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/pci.h>
#include "hns_roce_device.h"
#include "hns_roce_hw_v2.h"
static struct dentry *hns_roce_dbgfs_root;
@@ -18,25 +20,34 @@ static int hns_debugfs_seqfile_open(struct inode *inode, struct file *f)
return single_open(f, seqfile->read, seqfile->data);
}
static ssize_t hns_debugfs_seqfile_write(struct file *file,
const char __user *buffer,
size_t count, loff_t *ppos)
{
struct hns_debugfs_seqfile *seqfile = file_inode(file)->i_private;
char buf[16] = {};
if (!seqfile->write)
return -EOPNOTSUPP;
if (count >= sizeof(buf))
return -EINVAL;
if (copy_from_user(buf, buffer, count))
return -EFAULT;
return seqfile->write(buf, count, seqfile->data);
}
static const struct file_operations hns_debugfs_seqfile_fops = {
.owner = THIS_MODULE,
.open = hns_debugfs_seqfile_open,
.release = single_release,
.read = seq_read,
.write = hns_debugfs_seqfile_write,
.llseek = seq_lseek
};
static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq,
const char *name, struct dentry *parent,
int (*read_fn)(struct seq_file *, void *),
void *data)
{
debugfs_create_file(name, 0400, parent, seq, &hns_debugfs_seqfile_fops);
seq->read = read_fn;
seq->data = data;
}
static const char * const sw_stat_info[] = {
[HNS_ROCE_DFX_AEQE_CNT] = "aeqe",
[HNS_ROCE_DFX_CEQE_CNT] = "ceqe",
@@ -76,10 +87,256 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev,
{
struct hns_sw_stat_debugfs *dbgfs = &hr_dev->dbgfs.sw_stat_root;
dbgfs->root = debugfs_create_dir("sw_stat", parent);
dbgfs->sw_stat.read = sw_stat_debugfs_show;
dbgfs->sw_stat.data = hr_dev;
init_debugfs_seqfile(&dbgfs->sw_stat, "sw_stat", dbgfs->root,
sw_stat_debugfs_show, hr_dev);
dbgfs->root = debugfs_create_dir("sw_stat", parent);
debugfs_create_file("sw_stat", 0400, dbgfs->root, &dbgfs->sw_stat,
&hns_debugfs_seqfile_fops);
}
#define __HNS_SCC_ATTR(_name, _type, _offset, _size, _min, _max) { \
.name = _name, \
.algo_type = _type, \
.offset = _offset, \
.size = _size, \
.min = _min, \
.max = _max, \
}
#define HNS_DCQCN_CC_ATTR_RW(_name, NAME) \
__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DCQCN, \
HNS_ROCE_DCQCN_##NAME##_OFS, \
HNS_ROCE_DCQCN_##NAME##_SZ, \
0, HNS_ROCE_DCQCN_##NAME##_MAX)
#define HNS_LDCP_CC_ATTR_RW(_name, NAME) \
__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_LDCP, \
HNS_ROCE_LDCP_##NAME##_OFS, \
HNS_ROCE_LDCP_##NAME##_SZ, \
0, HNS_ROCE_LDCP_##NAME##_MAX)
#define HNS_HC3_CC_ATTR_RW(_name, NAME) \
__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_HC3, \
HNS_ROCE_HC3_##NAME##_OFS, \
HNS_ROCE_HC3_##NAME##_SZ, \
HNS_ROCE_HC3_##NAME##_MIN, \
HNS_ROCE_HC3_##NAME##_MAX)
#define HNS_DIP_CC_ATTR_RW(_name, NAME) \
__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DIP, \
HNS_ROCE_DIP_##NAME##_OFS, \
HNS_ROCE_DIP_##NAME##_SZ, \
0, HNS_ROCE_DIP_##NAME##_MAX)
static const struct hns_roce_cong_attr {
enum hns_roce_cong_type cong_type;
const char *name;
struct hns_roce_cc_param_attr params[HNS_ROCE_CC_PARAM_MAX_NUM];
} cong_attrs[] = {
{ CONG_TYPE_DCQCN, "dcqcn_cc_param",
{
HNS_DCQCN_CC_ATTR_RW("ai", AI),
HNS_DCQCN_CC_ATTR_RW("f", F),
HNS_DCQCN_CC_ATTR_RW("tkp", TKP),
HNS_DCQCN_CC_ATTR_RW("tmp", TMP),
HNS_DCQCN_CC_ATTR_RW("alp", ALP),
HNS_DCQCN_CC_ATTR_RW("max_speed", MAX_SPEED),
HNS_DCQCN_CC_ATTR_RW("g", G),
HNS_DCQCN_CC_ATTR_RW("al", AL),
HNS_DCQCN_CC_ATTR_RW("cnp_time", CNP_TIME),
HNS_DCQCN_CC_ATTR_RW("ashift", ASHIFT),
}
},
{ CONG_TYPE_LDCP, "ldcp_cc_param",
{
HNS_LDCP_CC_ATTR_RW("cwd0", CWD0),
HNS_LDCP_CC_ATTR_RW("alpha", ALPHA),
HNS_LDCP_CC_ATTR_RW("gamma", GAMMA),
HNS_LDCP_CC_ATTR_RW("beta", BETA),
HNS_LDCP_CC_ATTR_RW("eta", ETA),
}
},
{ CONG_TYPE_HC3, "hc3_cc_param",
{
HNS_HC3_CC_ATTR_RW("initial_window", INITIAL_WINDOW),
HNS_HC3_CC_ATTR_RW("bandwidth", BANDWIDTH),
HNS_HC3_CC_ATTR_RW("qlen_shift", QLEN_SHIFT),
HNS_HC3_CC_ATTR_RW("port_usage_shift", PORT_USAGE_SHIFT),
HNS_HC3_CC_ATTR_RW("over_period", OVER_PERIOD),
HNS_HC3_CC_ATTR_RW("max_stage", MAX_STAGE),
HNS_HC3_CC_ATTR_RW("gamma_shift", GAMMA_SHIFT),
}
},
{ CONG_TYPE_DIP, "dip_cc_param",
{
HNS_DIP_CC_ATTR_RW("ai", AI),
HNS_DIP_CC_ATTR_RW("f", F),
HNS_DIP_CC_ATTR_RW("tkp", TKP),
HNS_DIP_CC_ATTR_RW("tmp", TMP),
HNS_DIP_CC_ATTR_RW("alp", ALP),
HNS_DIP_CC_ATTR_RW("max_speed", MAX_SPEED),
HNS_DIP_CC_ATTR_RW("g", G),
HNS_DIP_CC_ATTR_RW("al", AL),
HNS_DIP_CC_ATTR_RW("cnp_time", CNP_TIME),
HNS_DIP_CC_ATTR_RW("ashift", ASHIFT),
}
}
};
static int cc_param_debugfs_show(struct seq_file *file, void *offset)
{
struct hns_cc_param_seqfile *param_seqfile = file->private;
const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr;
int algo_type = param_attr->algo_type;
int index = param_seqfile->index;
struct hns_roce_dev *hr_dev =
container_of(param_seqfile, struct hns_roce_dev,
dbgfs.cc_param_root[algo_type].params[index]);
struct hns_roce_scc_param *scc_param;
__le32 val = 0;
scc_param = &hr_dev->scc_param[algo_type];
scoped_guard(mutex, &scc_param->scc_mutex) {
memcpy(&val,
(void *)scc_param->param + param_attr->offset,
param_attr->size);
}
seq_printf(file, "%u\n", le32_to_cpu(val));
return 0;
}
static ssize_t cc_param_debugfs_store(char *buf, size_t count, void *data)
{
struct hns_cc_param_seqfile *param_seqfile = data;
const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr;
int algo_type = param_attr->algo_type;
int index = param_seqfile->index;
struct hns_roce_dev *hr_dev =
container_of(param_seqfile, struct hns_roce_dev,
dbgfs.cc_param_root[algo_type].params[index]);
struct hns_roce_scc_param *scc_param;
__le32 attr_val, old_val = 0;
u32 val;
int ret;
if (kstrtou32(buf, 0, &val))
return -EINVAL;
if (val > param_attr->max || val < param_attr->min)
return -EINVAL;
attr_val = cpu_to_le32(val);
scc_param = &hr_dev->scc_param[algo_type];
guard(mutex)(&scc_param->scc_mutex);
memcpy(&old_val, (void *)scc_param->param + param_attr->offset,
param_attr->size);
memcpy((void *)scc_param->param + param_attr->offset, &attr_val,
param_attr->size);
ret = hr_dev->hw->config_scc_param(hr_dev, algo_type);
if (ret) {
memcpy((void *)scc_param->param + param_attr->offset, &old_val,
param_attr->size);
return ret;
}
return count;
}
static void get_default_scc_param(struct hns_roce_dev *hr_dev)
{
int ret;
int i;
for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) {
ret = hr_dev->hw->query_scc_param(hr_dev, i);
if (ret && ret != -EOPNOTSUPP)
ibdev_warn_ratelimited(&hr_dev->ib_dev,
"failed to get default parameters of scc algo %d, ret = %d.\n",
i, ret);
}
}
static int hns_roce_alloc_scc_param(struct hns_roce_dev *hr_dev)
{
struct hns_roce_scc_param *scc_param;
int i;
scc_param = kvcalloc(HNS_ROCE_SCC_ALGO_TOTAL, sizeof(*scc_param),
GFP_KERNEL);
if (!scc_param)
return -ENOMEM;
for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) {
scc_param[i].algo_type = i;
scc_param[i].hr_dev = hr_dev;
mutex_init(&scc_param[i].scc_mutex);
}
hr_dev->scc_param = scc_param;
get_default_scc_param(hr_dev);
return 0;
}
static void hns_roce_dealloc_scc_param(struct hns_roce_dev *hr_dev)
{
int i;
if (!hr_dev->scc_param)
return;
for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++)
mutex_destroy(&hr_dev->scc_param[i].scc_mutex);
kvfree(hr_dev->scc_param);
}
static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev,
struct dentry *parent)
{
const struct hns_roce_cong_attr *cong_attr;
struct hns_cc_param_debugfs *dbgfs;
int i, j;
int ret;
if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 ||
!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) ||
hr_dev->is_vf)
return;
ret = hns_roce_alloc_scc_param(hr_dev);
if (ret)
return;
for (i = 0; i < CONG_TYPE_MAX_NUM; i++) {
cong_attr = &cong_attrs[i];
if (!test_bit(cong_attr->cong_type,
(unsigned long *)&hr_dev->caps.cong_cap))
continue;
dbgfs = &hr_dev->dbgfs.cc_param_root[i];
dbgfs->root = debugfs_create_dir(cong_attr->name, parent);
for (j = 0; j < HNS_ROCE_CC_PARAM_MAX_NUM; j++) {
if (!cong_attr->params[j].name)
break;
dbgfs->params[j].param_attr = &cong_attr->params[j];
dbgfs->params[j].index = j;
dbgfs->params[j].seqfile.read = cc_param_debugfs_show;
dbgfs->params[j].seqfile.write = cc_param_debugfs_store;
dbgfs->params[j].seqfile.data = &dbgfs->params[j];
debugfs_create_file(cong_attr->params[j].name, 0600,
dbgfs->root,
&dbgfs->params[j].seqfile,
&hns_debugfs_seqfile_fops);
}
}
}
/* debugfs for device */
@@ -91,11 +348,13 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
hns_roce_dbgfs_root);
create_sw_stat_debugfs(hr_dev, dbgfs->root);
create_cc_param_debugfs(hr_dev, dbgfs->root);
}
void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev)
{
debugfs_remove_recursive(hr_dev->dbgfs.root);
hns_roce_dealloc_scc_param(hr_dev);
}
/* debugfs for hns module */

26
drivers/infiniband/hw/hns/hns_roce_debugfs.h
View File

@@ -9,6 +9,7 @@
/* debugfs seqfile */
struct hns_debugfs_seqfile {
int (*read)(struct seq_file *seq, void *data);
ssize_t (*write)(char *buf, size_t count, void *data);
void *data;
};
@@ -17,10 +18,35 @@ struct hns_sw_stat_debugfs {
struct hns_debugfs_seqfile sw_stat;
};
struct hns_roce_cc_param_attr {
const char *name;
int algo_type;
u32 offset;
u32 size;
u32 max;
u32 min;
};
struct hns_cc_param_seqfile {
struct hns_debugfs_seqfile seqfile;
const struct hns_roce_cc_param_attr *param_attr;
int index;
};
#define HNS_ROCE_CC_PARAM_MAX_NUM 11
struct hns_cc_param_debugfs {
struct dentry *root;
struct hns_cc_param_seqfile params[HNS_ROCE_CC_PARAM_MAX_NUM];
};
#define CONG_TYPE_MAX_NUM 4
/* Debugfs for device */
struct hns_roce_dev_debugfs {
struct dentry *root;
struct hns_sw_stat_debugfs sw_stat_root;
struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM];
};
struct hns_roce_dev;

21
drivers/infiniband/hw/hns/hns_roce_device.h
View File

@@ -726,6 +726,14 @@ struct hns_roce_eq_table {
struct hns_roce_eq *eq;
};
enum hns_roce_scc_algo {
HNS_ROCE_SCC_ALGO_DCQCN = 0,
HNS_ROCE_SCC_ALGO_LDCP,
HNS_ROCE_SCC_ALGO_HC3,
HNS_ROCE_SCC_ALGO_DIP,
HNS_ROCE_SCC_ALGO_TOTAL,
};
struct hns_roce_caps {
u64 fw_ver;
u8 num_ports;
@@ -963,6 +971,18 @@ struct hns_roce_hw {
u8 *tc_mode, u8 *priority);
const struct ib_device_ops *hns_roce_dev_ops;
const struct ib_device_ops *hns_roce_dev_srq_ops;
int (*config_scc_param)(struct hns_roce_dev *hr_dev,
enum hns_roce_scc_algo algo);
int (*query_scc_param)(struct hns_roce_dev *hr_dev,
enum hns_roce_scc_algo alog);
};
#define HNS_ROCE_SCC_PARAM_SIZE 4
struct hns_roce_scc_param {
__le32 param[HNS_ROCE_SCC_PARAM_SIZE];
enum hns_roce_scc_algo algo_type;
struct hns_roce_dev *hr_dev;
struct mutex scc_mutex; /* protect @param */
};
struct hns_roce_dev {
@@ -1026,6 +1046,7 @@ struct hns_roce_dev {
u64 dwqe_page;
struct hns_roce_dev_debugfs dbgfs;
atomic64_t *dfx_cnt;
struct hns_roce_scc_param *scc_param;
};
enum hns_roce_trace_type {

10
drivers/infiniband/hw/hns/hns_roce_hem.c
View File

@@ -314,14 +314,14 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev,
bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
switch (bt_num) {
case 3:
index->l1 = l0_idx * chunk_ba_num + l1_idx;
index->l1 = (u64)l0_idx * chunk_ba_num + l1_idx;
index->l0 = l0_idx;
index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
l1_idx * chunk_ba_num + l2_idx;
index->buf = (u64)l0_idx * chunk_ba_num * chunk_ba_num +
(u64)l1_idx * chunk_ba_num + l2_idx;
break;
case 2:
index->l0 = l0_idx;
index->buf = l0_idx * chunk_ba_num + l1_idx;
index->buf = (u64)l0_idx * chunk_ba_num + l1_idx;
break;
case 1:
index->buf = l0_idx;
@@ -1269,8 +1269,6 @@ setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,
root_hem = list_first_entry(&head->root,
struct hns_roce_hem_item, list);
if (!root_hem)
return -ENOMEM;
total = 0;
for (i = 0; i < region_cnt && total <= max_ba_num; i++) {

106
drivers/infiniband/hw/hns/hns_roce_hw_v2.c
View File

@@ -31,6 +31,7 @@
*/
#include <linux/acpi.h>
#include <linux/cleanup.h>
#include <linux/etherdevice.h>
#include <linux/interrupt.h>
#include <linux/iopoll.h>
@@ -6192,9 +6193,9 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
if (ret)
ibdev_err(&hr_dev->ib_dev,
"failed to handle cmd of modifying SRQ, ret = %d.\n",
ret);
ibdev_err_ratelimited(&hr_dev->ib_dev,
"failed to handle cmd of modifying SRQ, ret = %d.\n",
ret);
}
out:
@@ -6220,9 +6221,9 @@ static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
HNS_ROCE_CMD_QUERY_SRQC, srq->srqn);
if (ret) {
ibdev_err(&hr_dev->ib_dev,
"failed to process cmd of querying SRQ, ret = %d.\n",
ret);
ibdev_err_ratelimited(&hr_dev->ib_dev,
"failed to process cmd of querying SRQ, ret = %d.\n",
ret);
goto out;
}
@@ -6328,9 +6329,9 @@ static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key,
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,
key_to_hw_index(key));
if (ret) {
ibdev_err(&hr_dev->ib_dev,
"failed to process cmd when querying MPT, ret = %d.\n",
ret);
ibdev_err_ratelimited(&hr_dev->ib_dev,
"failed to process cmd when querying MPT, ret = %d.\n",
ret);
goto err_mailbox;
}
@@ -7196,6 +7197,57 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
kfree(eq_table->eq);
}
static const enum hns_roce_opcode_type scc_opcode[] = {
HNS_ROCE_OPC_CFG_DCQCN_PARAM,
HNS_ROCE_OPC_CFG_LDCP_PARAM,
HNS_ROCE_OPC_CFG_HC3_PARAM,
HNS_ROCE_OPC_CFG_DIP_PARAM,
};
static int hns_roce_v2_config_scc_param(struct hns_roce_dev *hr_dev,
enum hns_roce_scc_algo algo)
{
struct hns_roce_scc_param *scc_param = &hr_dev->scc_param[algo];
struct hns_roce_cmq_desc desc;
int ret;
lockdep_assert_held(&scc_param->scc_mutex);
hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], false);
memcpy(&desc.data, scc_param->param, sizeof(scc_param->param));
ret = hns_roce_cmq_send(hr_dev, &desc, 1);
if (ret)
ibdev_err_ratelimited(&hr_dev->ib_dev,
"failed to configure scc param, opcode: 0x%x, ret = %d.\n",
le16_to_cpu(desc.opcode), ret);
return ret;
}
static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev,
enum hns_roce_scc_algo algo)
{
struct hns_roce_scc_param *scc_param;
struct hns_roce_cmq_desc desc;
int ret;
hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], true);
ret = hns_roce_cmq_send(hr_dev, &desc, 1);
if (ret) {
ibdev_err_ratelimited(&hr_dev->ib_dev,
"failed to query scc param, opcode: 0x%x, ret = %d.\n",
le16_to_cpu(desc.opcode), ret);
return ret;
}
scc_param = &hr_dev->scc_param[algo];
scoped_guard(mutex, &scc_param->scc_mutex)
memcpy(scc_param->param, &desc.data, sizeof(scc_param->param));
return 0;
}
static const struct ib_device_ops hns_roce_v2_dev_ops = {
.destroy_qp = hns_roce_v2_destroy_qp,
.modify_cq = hns_roce_v2_modify_cq,
@@ -7246,19 +7298,35 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
.get_dscp = hns_roce_hw_v2_get_dscp,
.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
.config_scc_param = hns_roce_v2_config_scc_param,
.query_scc_param = hns_roce_v2_query_scc_param,
};
static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
{
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
.driver_data = 0,
}, {
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
.driver_data = 0,
}, {
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
.driver_data = 0,
}, {
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
.driver_data = 0,
}, {
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
.driver_data = 0,
}, {
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
.driver_data = 0,
}, {
PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
},
/* required last entry */
{0, }
{ }
};
MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
@@ -7586,8 +7654,8 @@ static int __init hns_roce_hw_v2_init(void)
static void __exit hns_roce_hw_v2_exit(void)
{
hns_roce_dealloc_bond_grp();
hnae3_unregister_client(&hns_roce_hw_v2_client);
hns_roce_dealloc_bond_grp();
hns_roce_cleanup_debugfs();
}

125
drivers/infiniband/hw/hns/hns_roce_hw_v2.h
View File

@@ -202,6 +202,10 @@ enum {
/* CMQ command */
enum hns_roce_opcode_type {
HNS_QUERY_FW_VER = 0x0001,
HNS_ROCE_OPC_CFG_DCQCN_PARAM = 0x1A80,
HNS_ROCE_OPC_CFG_LDCP_PARAM = 0x1A81,
HNS_ROCE_OPC_CFG_HC3_PARAM = 0x1A82,
HNS_ROCE_OPC_CFG_DIP_PARAM = 0x1A83,
HNS_ROCE_OPC_QUERY_HW_VER = 0x8000,
HNS_ROCE_OPC_CFG_GLOBAL_PARAM = 0x8001,
HNS_ROCE_OPC_ALLOC_PF_RES = 0x8004,
@@ -1459,6 +1463,127 @@ struct hns_roce_wqe_atomic_seg {
__le64 cmp_data;
};
#define HNS_ROCE_DCQCN_AI_OFS 0
#define HNS_ROCE_DCQCN_AI_SZ sizeof(u16)
#define HNS_ROCE_DCQCN_AI_MAX ((u16)(~0U))
#define HNS_ROCE_DCQCN_F_OFS (HNS_ROCE_DCQCN_AI_OFS + HNS_ROCE_DCQCN_AI_SZ)
#define HNS_ROCE_DCQCN_F_SZ sizeof(u8)
#define HNS_ROCE_DCQCN_F_MAX ((u8)(~0U))
#define HNS_ROCE_DCQCN_TKP_OFS (HNS_ROCE_DCQCN_F_OFS + HNS_ROCE_DCQCN_F_SZ)
#define HNS_ROCE_DCQCN_TKP_SZ sizeof(u8)
#define HNS_ROCE_DCQCN_TKP_MAX 10
#define HNS_ROCE_DCQCN_TMP_OFS (HNS_ROCE_DCQCN_TKP_OFS + HNS_ROCE_DCQCN_TKP_SZ)
#define HNS_ROCE_DCQCN_TMP_SZ sizeof(u16)
#define HNS_ROCE_DCQCN_TMP_MAX 15
#define HNS_ROCE_DCQCN_ALP_OFS (HNS_ROCE_DCQCN_TMP_OFS + HNS_ROCE_DCQCN_TMP_SZ)
#define HNS_ROCE_DCQCN_ALP_SZ sizeof(u16)
#define HNS_ROCE_DCQCN_ALP_MAX ((u16)(~0U))
#define HNS_ROCE_DCQCN_MAX_SPEED_OFS (HNS_ROCE_DCQCN_ALP_OFS + \
HNS_ROCE_DCQCN_ALP_SZ)
#define HNS_ROCE_DCQCN_MAX_SPEED_SZ sizeof(u32)
#define HNS_ROCE_DCQCN_MAX_SPEED_MAX ((u32)(~0U))
#define HNS_ROCE_DCQCN_G_OFS (HNS_ROCE_DCQCN_MAX_SPEED_OFS + \
HNS_ROCE_DCQCN_MAX_SPEED_SZ)
#define HNS_ROCE_DCQCN_G_SZ sizeof(u8)
#define HNS_ROCE_DCQCN_G_MAX 15
#define HNS_ROCE_DCQCN_AL_OFS (HNS_ROCE_DCQCN_G_OFS + HNS_ROCE_DCQCN_G_SZ)
#define HNS_ROCE_DCQCN_AL_SZ sizeof(u8)
#define HNS_ROCE_DCQCN_AL_MAX ((u8)(~0U))
#define HNS_ROCE_DCQCN_CNP_TIME_OFS (HNS_ROCE_DCQCN_AL_OFS + \
HNS_ROCE_DCQCN_AL_SZ)
#define HNS_ROCE_DCQCN_CNP_TIME_SZ sizeof(u8)
#define HNS_ROCE_DCQCN_CNP_TIME_MAX ((u8)(~0U))
#define HNS_ROCE_DCQCN_ASHIFT_OFS (HNS_ROCE_DCQCN_CNP_TIME_OFS + \
HNS_ROCE_DCQCN_CNP_TIME_SZ)
#define HNS_ROCE_DCQCN_ASHIFT_SZ sizeof(u8)
#define HNS_ROCE_DCQCN_ASHIFT_MAX 15
#define HNS_ROCE_LDCP_CWD0_OFS 0
#define HNS_ROCE_LDCP_CWD0_SZ sizeof(u32)
#define HNS_ROCE_LDCP_CWD0_MAX ((u32)(~0U))
#define HNS_ROCE_LDCP_ALPHA_OFS (HNS_ROCE_LDCP_CWD0_OFS + HNS_ROCE_LDCP_CWD0_SZ)
#define HNS_ROCE_LDCP_ALPHA_SZ sizeof(u8)
#define HNS_ROCE_LDCP_ALPHA_MAX ((u8)(~0U))
#define HNS_ROCE_LDCP_GAMMA_OFS (HNS_ROCE_LDCP_ALPHA_OFS + \
HNS_ROCE_LDCP_ALPHA_SZ)
#define HNS_ROCE_LDCP_GAMMA_SZ sizeof(u8)
#define HNS_ROCE_LDCP_GAMMA_MAX 7
#define HNS_ROCE_LDCP_BETA_OFS (HNS_ROCE_LDCP_GAMMA_OFS + \
HNS_ROCE_LDCP_GAMMA_SZ)
#define HNS_ROCE_LDCP_BETA_SZ sizeof(u8)
#define HNS_ROCE_LDCP_BETA_MAX 7
#define HNS_ROCE_LDCP_ETA_OFS (HNS_ROCE_LDCP_BETA_OFS + HNS_ROCE_LDCP_BETA_SZ)
#define HNS_ROCE_LDCP_ETA_SZ sizeof(u8)
#define HNS_ROCE_LDCP_ETA_MAX 7
#define HNS_ROCE_HC3_INITIAL_WINDOW_OFS 0
#define HNS_ROCE_HC3_INITIAL_WINDOW_SZ sizeof(u32)
#define HNS_ROCE_HC3_INITIAL_WINDOW_MIN 0
#define HNS_ROCE_HC3_INITIAL_WINDOW_MAX ((u32)(~0U))
#define HNS_ROCE_HC3_BANDWIDTH_OFS (HNS_ROCE_HC3_INITIAL_WINDOW_OFS + \
HNS_ROCE_HC3_INITIAL_WINDOW_SZ)
#define HNS_ROCE_HC3_BANDWIDTH_SZ sizeof(u32)
#define HNS_ROCE_HC3_BANDWIDTH_MIN 1000
#define HNS_ROCE_HC3_BANDWIDTH_MAX ((u32)(~0U))
#define HNS_ROCE_HC3_QLEN_SHIFT_OFS (HNS_ROCE_HC3_BANDWIDTH_OFS + \
HNS_ROCE_HC3_BANDWIDTH_SZ)
#define HNS_ROCE_HC3_QLEN_SHIFT_SZ sizeof(u8)
#define HNS_ROCE_HC3_QLEN_SHIFT_MIN 0
#define HNS_ROCE_HC3_QLEN_SHIFT_MAX 31
#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS (HNS_ROCE_HC3_QLEN_SHIFT_OFS + \
HNS_ROCE_HC3_QLEN_SHIFT_SZ)
#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ sizeof(u8)
#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MIN 0
#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MAX 100
#define HNS_ROCE_HC3_OVER_PERIOD_OFS (HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS + \
HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ)
#define HNS_ROCE_HC3_OVER_PERIOD_SZ sizeof(u8)
#define HNS_ROCE_HC3_OVER_PERIOD_MIN 0
#define HNS_ROCE_HC3_OVER_PERIOD_MAX ((u8)(~0U))
#define HNS_ROCE_HC3_MAX_STAGE_OFS (HNS_ROCE_HC3_OVER_PERIOD_OFS + \
HNS_ROCE_HC3_OVER_PERIOD_SZ)
#define HNS_ROCE_HC3_MAX_STAGE_SZ sizeof(u8)
#define HNS_ROCE_HC3_MAX_STAGE_MIN 0
#define HNS_ROCE_HC3_MAX_STAGE_MAX ((u8)(~0U))
#define HNS_ROCE_HC3_GAMMA_SHIFT_OFS (HNS_ROCE_HC3_MAX_STAGE_OFS + \
HNS_ROCE_HC3_MAX_STAGE_SZ)
#define HNS_ROCE_HC3_GAMMA_SHIFT_SZ sizeof(u8)
#define HNS_ROCE_HC3_GAMMA_SHIFT_MIN 0
#define HNS_ROCE_HC3_GAMMA_SHIFT_MAX 15
#define HNS_ROCE_DIP_AI_OFS 0
#define HNS_ROCE_DIP_AI_SZ sizeof(u16)
#define HNS_ROCE_DIP_AI_MAX ((u16)(~0U))
#define HNS_ROCE_DIP_F_OFS (HNS_ROCE_DIP_AI_OFS + HNS_ROCE_DIP_AI_SZ)
#define HNS_ROCE_DIP_F_SZ sizeof(u8)
#define HNS_ROCE_DIP_F_MAX ((u8)(~0U))
#define HNS_ROCE_DIP_TKP_OFS (HNS_ROCE_DIP_F_OFS + HNS_ROCE_DIP_F_SZ)
#define HNS_ROCE_DIP_TKP_SZ sizeof(u8)
#define HNS_ROCE_DIP_TKP_MAX 10
#define HNS_ROCE_DIP_TMP_OFS (HNS_ROCE_DIP_TKP_OFS + HNS_ROCE_DIP_TKP_SZ)
#define HNS_ROCE_DIP_TMP_SZ sizeof(u16)
#define HNS_ROCE_DIP_TMP_MAX 15
#define HNS_ROCE_DIP_ALP_OFS (HNS_ROCE_DIP_TMP_OFS + HNS_ROCE_DIP_TMP_SZ)
#define HNS_ROCE_DIP_ALP_SZ sizeof(u16)
#define HNS_ROCE_DIP_ALP_MAX ((u16)(~0U))
#define HNS_ROCE_DIP_MAX_SPEED_OFS (HNS_ROCE_DIP_ALP_OFS + HNS_ROCE_DIP_ALP_SZ)
#define HNS_ROCE_DIP_MAX_SPEED_SZ sizeof(u32)
#define HNS_ROCE_DIP_MAX_SPEED_MAX ((u32)(~0U))
#define HNS_ROCE_DIP_G_OFS (HNS_ROCE_DIP_MAX_SPEED_OFS + \
HNS_ROCE_DIP_MAX_SPEED_SZ)
#define HNS_ROCE_DIP_G_SZ sizeof(u8)
#define HNS_ROCE_DIP_G_MAX 15
#define HNS_ROCE_DIP_AL_OFS (HNS_ROCE_DIP_G_OFS + HNS_ROCE_DIP_G_SZ)
#define HNS_ROCE_DIP_AL_SZ sizeof(u8)
#define HNS_ROCE_DIP_AL_MAX ((u8)(~0U))
#define HNS_ROCE_DIP_CNP_TIME_OFS (HNS_ROCE_DIP_AL_OFS + HNS_ROCE_DIP_AL_SZ)
#define HNS_ROCE_DIP_CNP_TIME_SZ sizeof(u8)
#define HNS_ROCE_DIP_CNP_TIME_MAX ((u8)(~0U))
#define HNS_ROCE_DIP_ASHIFT_OFS (HNS_ROCE_DIP_CNP_TIME_OFS + \
HNS_ROCE_DIP_CNP_TIME_SZ)
#define HNS_ROCE_DIP_ASHIFT_SZ sizeof(u8)
#define HNS_ROCE_DIP_ASHIFT_MAX 15
struct hns_roce_sccc_clr {
__le32 qpn;
__le32 rsv[5];

12
drivers/infiniband/hw/hns/hns_roce_main.c
View File

@@ -221,6 +221,11 @@ static int hns_roce_query_device(struct ib_device *ib_dev,
struct ib_udata *uhw)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
int ret;
ret = ib_is_udata_in_empty(uhw);
if (ret)
return ret;
memset(props, 0, sizeof(*props));
@@ -274,7 +279,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev,
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)
props->device_cap_flags |= IB_DEVICE_XRC;
return 0;
return ib_respond_empty_udata(uhw);
}
static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num,
@@ -477,8 +482,7 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
resp.cqe_size = hr_dev->caps.cqe_sz;
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
ret = ib_respond_udata(udata, resp);
if (ret)
goto error_fail_copy_to_udata;
@@ -1113,7 +1117,7 @@ static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq)
unsigned long flags;
spin_lock_irqsave(&hr_cq->lock, flags);
if (cq->comp_handler) {
if (cq->comp_handler && hr_cq->ib_cq.poll_ctx != IB_POLL_DIRECT) {
if (!hr_cq->is_armed) {
hr_cq->is_armed = 1;
list_add_tail(&hr_cq->node, cq_list);

10
drivers/infiniband/hw/hns/hns_roce_mr.c
View File

@@ -173,7 +173,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,
mtpt_idx & (hr_dev->caps.num_mtpts - 1));
if (ret) {
dev_err(dev, "failed to create mpt, ret = %d.\n", ret);
dev_err_ratelimited(dev, "failed to create mpt, ret = %d.\n", ret);
goto err_page;
}
@@ -319,7 +319,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start,
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,
mtpt_idx);
if (ret)
ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret);
ibdev_warn_ratelimited(ib_dev, "failed to destroy MPT, ret = %d.\n", ret);
mr->enabled = 0;
mr->iova = virt_addr;
@@ -350,7 +350,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start,
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,
mtpt_idx);
if (ret) {
ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret);
ibdev_err_ratelimited(ib_dev, "failed to create MPT, ret = %d.\n", ret);
goto free_cmd_mbox;
}
@@ -595,8 +595,8 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
if (udata) {
mtr->kmem = NULL;
mtr->umem = ib_umem_get(ibdev, user_addr, total_size,
buf_attr->user_access);
mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
buf_attr->user_access);
if (IS_ERR(mtr->umem)) {
ibdev_err(ibdev, "failed to get umem, ret = %pe.\n",
mtr->umem);

8
drivers/infiniband/hw/hns/hns_roce_pd.c
View File

@@ -30,6 +30,7 @@
* SOFTWARE.
*/
#include <rdma/uverbs_ioctl.h>
#include "hns_roce_device.h"
void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev)
@@ -61,12 +62,9 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
if (udata) {
struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd->pdn};
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
if (ret) {
ret = ib_respond_udata(udata, resp);
if (ret)
ida_free(&pd_ida->ida, id);
ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret);
}
}
return ret;

13
drivers/infiniband/hw/hns/hns_roce_qp.c
View File

@@ -1236,12 +1236,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
if (udata) {
resp.cap_flags = hr_qp->en_flags;
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
if (ret) {
ibdev_err(ibdev, "copy qp resp failed!\n");
ret = ib_respond_udata(udata, resp);
if (ret)
goto err_flow_ctrl;
}
}
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
@@ -1494,11 +1491,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
if (udata && udata->outlen) {
resp.tc_mode = hr_qp->tc_mode;
resp.priority = hr_qp->sl;
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
if (ret)
ibdev_err_ratelimited(&hr_dev->ib_dev,
"failed to copy modify qp resp.\n");
ret = ib_respond_udata(udata, resp);
}
out:

8
drivers/infiniband/hw/hns/hns_roce_srq.c
View File

@@ -103,7 +103,7 @@ static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev,
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ,
srq->srqn);
if (ret)
ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret);
ibdev_err_ratelimited(ibdev, "failed to config SRQC, ret = %d.\n", ret);
err_mbox:
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
@@ -477,11 +477,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
if (udata) {
resp.cap_flags = srq->cap_flags;
resp.srqn = srq->srqn;
if (ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)))) {
ret = -EFAULT;
ret = ib_respond_udata(udata, resp);
if (ret)
goto err_srqc;
}
}
return 0;

39
drivers/infiniband/hw/ionic/ionic_controlpath.c
View File

@@ -110,8 +110,8 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
if (rc)
goto err_qdesc;
cq->umem = ib_umem_get(&dev->ibdev, req_cq->addr, req_cq->size,
IB_ACCESS_LOCAL_WRITE);
cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr,
req_cq->size, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
rc = PTR_ERR(cq->umem);
goto err_qdesc;
@@ -414,7 +414,7 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
if (dev->lif_cfg.rq_expdb)
resp.expdb_qtypes |= IONIC_EXPDB_RQ;
rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
rc = ib_respond_udata(udata, resp);
if (rc)
goto err_resp;
@@ -752,7 +752,7 @@ int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
if (udata) {
resp.ahid = ah->ahid;
rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
rc = ib_respond_udata(udata, resp);
if (rc)
goto err_resp;
}
@@ -895,7 +895,7 @@ struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
mr->umem = ib_umem_get(&dev->ibdev, start, length, access);
mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access);
if (IS_ERR(mr->umem)) {
rc = PTR_ERR(mr->umem);
goto err_umem;
@@ -1263,7 +1263,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
if (udata) {
resp.udma_mask = vcq->udma_mask;
rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
rc = ib_respond_udata(udata, resp);
if (rc)
goto err_resp;
}
@@ -1837,7 +1837,7 @@ static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
qp->sq_meta = NULL;
qp->sq_msn_idx = NULL;
qp->sq_umem = ib_umem_get(&dev->ibdev, sq->addr, sq->size, 0);
qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0);
if (IS_ERR(qp->sq_umem))
return PTR_ERR(qp->sq_umem);
} else {
@@ -2050,7 +2050,7 @@ static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
qp->rq_meta = NULL;
qp->rq_umem = ib_umem_get(&dev->ibdev, rq->addr, rq->size, 0);
qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0);
if (IS_ERR(qp->rq_umem))
return PTR_ERR(qp->rq_umem);
} else {
@@ -2315,7 +2315,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
resp.rq_cmb = qp->rq_cmb;
}
rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
rc = ib_respond_udata(udata, resp);
if (rc)
goto err_resp;
}
@@ -2535,6 +2535,23 @@ static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state,
return false;
}
static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask,
enum ib_qp_type type, enum ib_qp_state cur,
enum ib_qp_state next)
{
if (!ext_mask)
return true;
if (ext_mask & ~IB_QP_RATE_LIMIT)
return false;
/* Rate limit is only supported for RC QPs during specific transitions */
return type == IB_QPT_RC &&
((cur == IB_QPS_INIT && next == IB_QPS_RTR) ||
(cur == IB_QPS_RTR && next == IB_QPS_RTS) ||
(cur == IB_QPS_RTS && next == IB_QPS_RTS));
}
static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr,
int mask)
{
@@ -2547,7 +2564,9 @@ static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr,
!ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state))
return -EINVAL;
if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask))
if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) ||
!ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS,
qp->ibqp.qp_type, cur_state, next_state))
return -EINVAL;
/* unprivileged qp not allowed privileged qkey */

7
drivers/infiniband/hw/ionic/ionic_ibdev.c
View File

@@ -25,6 +25,11 @@ static int ionic_query_device(struct ib_device *ibdev,
{
struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);
struct net_device *ndev;
int err;
err = ib_is_udata_in_empty(udata);
if (err)
return err;
ndev = ib_device_get_netdev(ibdev, 1);
addrconf_ifid_eui48((u8 *)&attr->sys_image_guid, ndev);
@@ -69,7 +74,7 @@ static int ionic_query_device(struct ib_device *ibdev,
attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2;
attr->max_pkeys = IONIC_PKEY_TBL_LEN;
return 0;
return ib_respond_empty_udata(udata);
}
static int ionic_query_port(struct ib_device *ibdev, u32 port,

7
drivers/infiniband/hw/irdma/hw.c
View File

@@ -235,8 +235,7 @@ static void irdma_complete_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request)
{
if (cqp_request->waiting) {
WRITE_ONCE(cqp_request->request_done, true);
wake_up(&cqp_request->waitq);
complete_all(&cqp_request->comp);
} else if (cqp_request->callback_fcn) {
cqp_request->callback_fcn(cqp_request);
}
@@ -1107,9 +1106,9 @@ static int irdma_create_cqp(struct irdma_pci_f *rf)
INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
/* init the waitqueue of the cqp_requests and add them to the list */
/* init the completion of the cqp_requests and add them to the list */
for (i = 0; i < sqsize; i++) {
init_waitqueue_head(&cqp->cqp_requests[i].waitq);
init_completion(&cqp->cqp_requests[i].comp);
list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
}
init_waitqueue_head(&cqp->remove_wq);

3
drivers/infiniband/hw/irdma/main.h
View File

@@ -161,13 +161,12 @@ struct irdma_cqp_compl_info {
struct irdma_cqp_request {
struct cqp_cmds_info info;
wait_queue_head_t waitq;
struct completion comp;
struct list_head list;
refcount_t refcnt;
void (*callback_fcn)(struct irdma_cqp_request *cqp_request);
void *param;
struct irdma_cqp_compl_info compl_info;
bool request_done; /* READ/WRITE_ONCE macros operate on it */
bool waiting:1;
bool dynamic:1;
bool pending:1;

9
drivers/infiniband/hw/irdma/uk.c
View File

@@ -1568,15 +1568,12 @@ static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = {
* irdma_setup_connection_wqes - setup WQEs necessary to complete
* connection.
* @qp: hw qp (user and kernel)
* @info: qp initialization info
*/
static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info)
static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp)
{
u16 move_cnt = 1;
if (!info->legacy_mode &&
(qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE))
if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
qp->conn_wqes = move_cnt;
@@ -1727,7 +1724,7 @@ int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
sq_ring_size = qp->sq_size << info->sq_shift;
IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
if (info->first_sq_wq) {
irdma_setup_connection_wqes(qp, info);
irdma_setup_connection_wqes(qp);
qp->swqe_polarity = 1;
qp->first_sq_wq = true;
} else {

1
drivers/infiniband/hw/irdma/user.h
View File

@@ -563,7 +563,6 @@ struct irdma_qp_uk_init_info {
u8 sq_shift;
u8 rq_shift;
int abi_ver;
bool legacy_mode;
struct irdma_srq_uk *srq_uk;
};

14
drivers/infiniband/hw/irdma/utils.c
View File

@@ -442,7 +442,7 @@ struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
if (cqp_request) {
cqp_request->dynamic = true;
if (wait)
init_waitqueue_head(&cqp_request->waitq);
init_completion(&cqp_request->comp);
}
}
if (!cqp_request) {
@@ -480,7 +480,7 @@ void irdma_free_cqp_request(struct irdma_cqp *cqp,
if (cqp_request->dynamic) {
kfree(cqp_request);
} else {
WRITE_ONCE(cqp_request->request_done, false);
reinit_completion(&cqp_request->comp);
cqp_request->callback_fcn = NULL;
cqp_request->waiting = false;
cqp_request->pending = false;
@@ -515,8 +515,7 @@ irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
{
if (cqp_request->waiting) {
cqp_request->compl_info.error = true;
WRITE_ONCE(cqp_request->request_done, true);
wake_up(&cqp_request->waitq);
complete_all(&cqp_request->comp);
}
wait_event_timeout(cqp->remove_wq,
refcount_read(&cqp_request->refcnt) == 1, 1000);
@@ -609,9 +608,8 @@ static int irdma_wait_event(struct irdma_pci_f *rf,
cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
if (wait_event_timeout(cqp_request->waitq,
READ_ONCE(cqp_request->request_done),
msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
if (wait_for_completion_timeout(&cqp_request->comp,
msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
break;
if (cqp_request->pending)
@@ -2442,7 +2440,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp)
cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
sw_wqe = qp->sq_base[wqe_idx].elem;
get_64bit_val(sw_wqe, 24, &wqe_qword);
cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
/* remove the SQ WR by moving SQ tail*/
IRDMA_RING_SET_TAIL(*sq_ring,

86
drivers/infiniband/hw/irdma/verbs.c
View File

@@ -16,9 +16,11 @@ static int irdma_query_device(struct ib_device *ibdev,
struct irdma_pci_f *rf = iwdev->rf;
struct pci_dev *pcidev = iwdev->rf->pcidev;
struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs;
int err;
if (udata->inlen || udata->outlen)
return -EINVAL;
err = ib_is_udata_in_empty(udata);
if (err)
return err;
memset(props, 0, sizeof(*props));
addrconf_addr_eui48((u8 *)&props->sys_image_guid,
@@ -74,7 +76,7 @@ static int irdma_query_device(struct ib_device *ibdev,
if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3)
props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B;
return 0;
return ib_respond_empty_udata(udata);
}
/**
@@ -325,9 +327,9 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx,
uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
uresp.kernel_ver = req.userspace_ver;
if (ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen)))
return -EFAULT;
ret = ib_respond_udata(udata, uresp);
if (ret)
return ret;
} else {
u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
@@ -354,10 +356,10 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx,
uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA;
if (ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen))) {
ret = ib_respond_udata(udata, uresp);
if (ret) {
rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
return -EFAULT;
return ret;
}
}
@@ -420,11 +422,9 @@ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
ibucontext);
irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
uresp.pd_id = pd_id;
if (ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen))) {
err = -EFAULT;
err = ib_respond_udata(udata, uresp);
if (err)
goto error;
}
} else {
irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
}
@@ -634,7 +634,6 @@ static int irdma_setup_umode_qp(struct ib_udata *udata,
iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
iwqp->user_mode = 1;
if (req.user_wqe_bufs) {
info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode;
spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,
&ucontext->qp_reg_mem_list);
@@ -1124,10 +1123,8 @@ static int irdma_create_qp(struct ib_qp *ibqp,
uresp.qp_id = qp_num;
uresp.qp_caps = qp->qp_uk.qp_caps;
err_code = ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen));
err_code = ib_respond_udata(udata, uresp);
if (err_code) {
ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n");
irdma_destroy_qp(&iwqp->ibqp, udata);
return err_code;
}
@@ -1612,12 +1609,9 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
uresp.push_valid = 1;
uresp.push_offset = iwqp->sc_qp.push_offset;
}
ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp),
udata->outlen));
ret = ib_respond_udata(udata, uresp);
if (ret) {
irdma_remove_push_mmap_entries(iwqp);
ibdev_dbg(&iwdev->ibdev,
"VERBS: copy_to_udata failed\n");
return ret;
}
}
@@ -1860,12 +1854,9 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
uresp.push_offset = iwqp->sc_qp.push_offset;
}
err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp),
udata->outlen));
err = ib_respond_udata(udata, uresp);
if (err) {
irdma_remove_push_mmap_entries(iwqp);
ibdev_dbg(&iwdev->ibdev,
"VERBS: copy_to_udata failed\n");
return err;
}
}
@@ -2074,10 +2065,6 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries,
rdma_udata_to_drv_context(udata, struct irdma_ucontext,
ibucontext);
/* CQ resize not supported with legacy GEN_1 libi40iw */
if (ucontext->legacy_mode)
return -EOPNOTSUPP;
if (ib_copy_from_udata(&req, udata,
min(sizeof(req), udata->inlen)))
return -EINVAL;
@@ -2418,11 +2405,9 @@ static int irdma_create_srq(struct ib_srq *ibsrq,
resp.srq_id = iwsrq->srq_num;
resp.srq_size = ukinfo->srq_size;
if (ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen))) {
err_code = -EPROTO;
err_code = ib_respond_udata(udata, resp);
if (err_code)
goto srq_destroy;
}
}
return 0;
@@ -2559,7 +2544,7 @@ static int irdma_create_cq(struct ib_cq *ibcq,
cqmr = &iwpbl->cq_mr;
if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) {
IRDMA_FEATURE_CQ_RESIZE) {
spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
iwpbl_shadow = irdma_get_pbl(
(unsigned long)req.user_shadow_area,
@@ -2572,7 +2557,6 @@ static int irdma_create_cq(struct ib_cq *ibcq,
}
cqmr_shadow = &iwpbl_shadow->cq_mr;
info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
cqmr->split = true;
} else {
info.shadow_area_pa = cqmr->shadow;
}
@@ -2664,13 +2648,9 @@ static int irdma_create_cq(struct ib_cq *ibcq,
resp.cq_id = info.cq_uk_init_info.cq_id;
resp.cq_size = info.cq_uk_init_info.cq_size;
if (ib_copy_to_udata(udata, &resp,
min(sizeof(resp), udata->outlen))) {
ibdev_dbg(&iwdev->ibdev,
"VERBS: copy to user data\n");
err_code = -EPROTO;
err_code = ib_respond_udata(udata, resp);
if (err_code)
goto cq_destroy;
}
}
init_completion(&iwcq->free_cq);
@@ -2781,10 +2761,11 @@ static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo,
* irdma_copy_user_pgaddrs - copy user page address to pble's os locally
* @iwmr: iwmr for IB's user page addresses
* @pbl: ple pointer to save 1 level or 0 level pble
* @pbl_len: Max number of PBL entries to populate
* @level: indicated level 0, 1 or 2
*/
static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
enum irdma_pble_level level)
u32 pbl_len, enum irdma_pble_level level)
{
struct ib_umem *region = iwmr->region;
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
@@ -2792,7 +2773,9 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
struct irdma_pble_info *pinfo;
struct ib_block_iter biter;
u32 idx = 0;
u32 pbl_cnt = 0;
if (!pbl_len)
return;
pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf;
@@ -2801,7 +2784,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) {
*pbl = rdma_block_iter_dma_address(&biter);
if (++pbl_cnt == palloc->total_cnt)
if (!--pbl_len)
break;
pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx);
}
@@ -2877,6 +2860,7 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
u64 *pbl;
int status;
enum irdma_pble_level level = PBLE_LEVEL_1;
u32 pbl_len;
if (lvl) {
status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
@@ -2884,16 +2868,18 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
if (status)
return status;
pbl_len = palloc->total_cnt;
iwpbl->pbl_allocated = true;
level = palloc->level;
pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 :
palloc->level2.leaf;
pbl = pinfo->addr;
} else {
pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR;
pbl = iwmr->pgaddrmem;
}
irdma_copy_user_pgaddrs(iwmr, pbl, level);
irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level);
if (lvl)
iwmr->pgaddrmem[0] = *pbl;
@@ -2974,7 +2960,8 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev,
case IRDMA_MEMREG_TYPE_CQ:
hmc_p = &cqmr->cq_pbl;
if (!cqmr->split)
if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
IRDMA_FEATURE_CQ_RESIZE))
cqmr->shadow = (dma_addr_t)arr[req->cq_pages];
if (lvl)
@@ -3319,6 +3306,7 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,
int err;
lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
iwmr->access = access;
err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
if (err)
@@ -3537,7 +3525,7 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
return ERR_PTR(-EINVAL);
region = ib_umem_get(pd->device, start, len, access);
region = ib_umem_get_va(pd->device, start, len, access);
if (IS_ERR(region)) {
ibdev_dbg(&iwdev->ibdev,
@@ -3739,7 +3727,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
struct ib_umem *region;
int err;
region = ib_umem_get(pd->device, start, len, iwmr->access);
region = ib_umem_get_va(pd->device, start, len, iwmr->access);
if (IS_ERR(region))
return PTR_ERR(region);
@@ -5334,7 +5322,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah,
mutex_unlock(&iwdev->rf->ah_tbl_lock);
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
err = ib_respond_udata(udata, uresp);
if (err)
irdma_destroy_ah(ibah, attr->flags);

1
drivers/infiniband/hw/irdma/verbs.h
View File

@@ -65,7 +65,6 @@ struct irdma_hmc_pble {
struct irdma_cq_mr {
struct irdma_hmc_pble cq_pbl;
dma_addr_t shadow;
bool split;
};
struct irdma_srq_mr {

6
drivers/infiniband/hw/mana/cq.c
View File

@@ -79,11 +79,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
if (udata) {
resp.cqid = cq->queue.id;
err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
err = ib_respond_udata(udata, resp);
if (err)
goto err_remove_cq_cb;
}
}
spin_lock_init(&cq->cq_lock);

12
drivers/infiniband/hw/mana/main.c
View File

@@ -262,7 +262,7 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
queue->id = INVALID_QUEUE_ID;
queue->gdma_region = GDMA_INVALID_DMA_REGION;
umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem);
return PTR_ERR(umem);
@@ -573,6 +573,11 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
{
struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)->dev);
int err;
err = ib_is_udata_in_empty(uhw);
if (err)
return err;
memset(props, 0, sizeof(*props));
props->vendor_id = pdev->vendor;
@@ -600,7 +605,7 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
if (!mana_ib_is_rnic(dev))
props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM;
return 0;
return ib_respond_empty_udata(uhw);
}
int mana_ib_query_port(struct ib_device *ibdev, u32 port,
@@ -624,8 +629,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
}
props->active_width = IB_WIDTH_4X;
props->active_speed = IB_SPEED_EDR;
ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width);
props->pkey_tbl_len = 1;
if (mana_ib_is_rnic(dev)) {
props->gid_tbl_len = 16;

2
drivers/infiniband/hw/mana/mr.c
View File

@@ -127,7 +127,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
if (!mr)
return ERR_PTR(-ENOMEM);
mr->umem = ib_umem_get(ibdev, start, length, access_flags);
mr->umem = ib_umem_get_va(ibdev, start, length, access_flags);
if (IS_ERR(mr->umem)) {
err = PTR_ERR(mr->umem);
ibdev_dbg(ibdev,

22
drivers/infiniband/hw/mana/qp.c
View File

@@ -243,13 +243,9 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
if (ret)
goto free_vport;
ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
if (ret) {
ibdev_dbg(&mdev->ib_dev,
"Failed to copy to udata create rss-qp, %d\n",
ret);
ret = ib_respond_udata(udata, resp);
if (ret)
goto err_disable_vport_rx;
}
kfree(mana_ind_table);
@@ -398,13 +394,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
resp.cqid = send_cq->queue.id;
resp.tx_vp_offset = pd->tx_vp_offset;
err = ib_copy_to_udata(udata, &resp, sizeof(resp));
if (err) {
ibdev_dbg(&mdev->ib_dev,
"Failed copy udata for create qp-raw, %d\n",
err);
err = ib_respond_udata(udata, resp);
if (err)
goto err_remove_cq_cb;
}
return 0;
@@ -602,11 +594,9 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
resp.queue_id[j] = qp->rc_qp.queues[i].id;
j++;
}
err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
if (err) {
ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
err = ib_respond_udata(udata, resp);
if (err)
goto destroy_qp;
}
}
err = mana_table_store_qp(mdev, qp);

2
drivers/infiniband/hw/mlx4/alias_GUID.c
View File

@@ -437,7 +437,7 @@ static void aliasguid_query_handler(int status,
queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq,
&dev->sriov.alias_guid.ports_guid[port_index].
alias_guid_work,
msecs_to_jiffies(resched_delay_sec * 1000));
secs_to_jiffies(resched_delay_sec));
}
if (cb_ctx->sa_query) {
list_del(&cb_ctx->list);

63
drivers/infiniband/hw/mlx4/cq.c
View File

@@ -142,6 +142,7 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
{
struct ib_udata *udata = &attrs->driver_udata;
struct ib_device *ibdev = ibcq->device;
struct mlx4_ib_create_cq_resp uresp = {};
int entries = attr->cqe;
int vector = attr->comp_vector;
struct mlx4_ib_dev *dev = to_mdev(ibdev);
@@ -172,32 +173,40 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
if (err)
goto err_cq;
if (ibcq->umem &&
(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT))
return -EOPNOTSUPP;
cq->umem = ib_umem_get_cq_buf(&dev->ib_dev, attrs, entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
err = PTR_ERR(cq->umem);
goto err_cq;
}
if (cq->umem) {
if (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT) {
err = -EOPNOTSUPP;
goto err_umem;
}
} else {
cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
err = PTR_ERR(cq->umem);
goto err_cq;
}
}
buf_addr = (void *)(unsigned long)ucmd.buf_addr;
if (!ibcq->umem)
ibcq->umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(ibcq->umem)) {
err = PTR_ERR(ibcq->umem);
goto err_cq;
}
shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->ibcq.umem, 0, &n);
shift = mlx4_ib_umem_calc_optimal_mtt_size(cq->umem, 0, &n);
if (shift < 0) {
err = shift;
goto err_cq;
goto err_umem;
}
err = mlx4_mtt_init(dev->dev, n, shift, &cq->buf.mtt);
if (err)
goto err_cq;
goto err_umem;
err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->ibcq.umem);
err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);
if (err)
goto err_mtt;
@@ -219,10 +228,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
cq->mcq.event = mlx4_ib_cq_event;
cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
err = -EFAULT;
uresp.cqn = cq->mcq.cqn;
err = ib_respond_udata(udata, uresp);
if (err)
goto err_cq_free;
}
return 0;
@@ -234,7 +243,9 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
err_mtt:
mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
/* UMEM is released by ib_core */
err_umem:
ib_umem_release(cq->umem);
err_cq:
return err;
@@ -343,9 +354,9 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
if (!cq->resize_buf)
return -ENOMEM;
cq->resize_umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->resize_umem)) {
err = PTR_ERR(cq->resize_umem);
goto err_buf;
@@ -471,8 +482,8 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, unsigned int entries,
if (ibcq->uobject) {
cq->buf = cq->resize_buf->buf;
cq->ibcq.cqe = cq->resize_buf->cqe;
ib_umem_release(cq->ibcq.umem);
cq->ibcq.umem = cq->resize_umem;
ib_umem_release(cq->umem);
cq->umem = cq->resize_umem;
kfree(cq->resize_buf);
cq->resize_buf = NULL;
@@ -532,7 +543,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
struct mlx4_ib_ucontext,
ibucontext),
&mcq->db);
/* UMEM is released by ib_core */
ib_umem_release(mcq->umem);
} else {
mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
mlx4_db_free(dev->dev, &mcq->db);

4
drivers/infiniband/hw/mlx4/doorbell.c
View File

@@ -64,8 +64,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt,
page->user_virt = (virt & PAGE_MASK);
page->refcnt = 0;
page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK,
PAGE_SIZE, 0);
page->umem = ib_umem_get_va(context->ibucontext.device,
virt & PAGE_MASK, PAGE_SIZE, 0);
if (IS_ERR(page->umem)) {
err = PTR_ERR(page->umem);
kfree(page);

44
drivers/infiniband/hw/mlx4/main.c
View File

@@ -444,8 +444,9 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
struct mlx4_uverbs_ex_query_device cmd;
struct mlx4_uverbs_ex_query_device_resp resp = {};
struct mlx4_clock_params clock_params;
size_t uhw_outlen = uhw ? uhw->outlen : 0;
if (uhw->inlen) {
if (uhw && uhw->inlen) {
err = ib_copy_validate_udata_in_cm(uhw, cmd, reserved, 0);
if (err)
return err;
@@ -572,7 +573,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
props->cq_caps.max_cq_moderation_count = MLX4_MAX_CQ_COUNT;
props->cq_caps.max_cq_moderation_period = MLX4_MAX_CQ_PERIOD;
if (uhw->outlen >= resp.response_length + sizeof(resp.hca_core_clock_offset)) {
if (uhw_outlen >= resp.response_length + sizeof(resp.hca_core_clock_offset)) {
resp.response_length += sizeof(resp.hca_core_clock_offset);
if (!mlx4_get_internal_clock_params(dev->dev, &clock_params)) {
resp.comp_mask |= MLX4_IB_QUERY_DEV_RESP_MASK_CORE_CLOCK_OFFSET;
@@ -580,14 +581,14 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
}
}
if (uhw->outlen >= resp.response_length +
if (uhw_outlen >= resp.response_length +
sizeof(resp.max_inl_recv_sz)) {
resp.response_length += sizeof(resp.max_inl_recv_sz);
resp.max_inl_recv_sz = dev->dev->caps.max_rq_sg *
sizeof(struct mlx4_wqe_data_seg);
}
if (offsetofend(typeof(resp), rss_caps) <= uhw->outlen) {
if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
if (props->rss_caps.supported_qpts) {
resp.rss_caps.rx_hash_function =
MLX4_IB_RX_HASH_FUNC_TOEPLITZ;
@@ -611,7 +612,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
sizeof(resp.rss_caps);
}
if (offsetofend(typeof(resp), tso_caps) <= uhw->outlen) {
if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
if (dev->dev->caps.max_gso_sz &&
((mlx4_ib_port_link_layer(ibdev, 1) ==
IB_LINK_LAYER_ETHERNET) ||
@@ -625,8 +626,8 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
sizeof(resp.tso_caps);
}
if (uhw->outlen) {
err = ib_copy_to_udata(uhw, &resp, resp.response_length);
if (uhw_outlen) {
err = ib_respond_udata(uhw, resp);
if (err)
goto out;
}
@@ -1090,8 +1091,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_device *ibdev = uctx->device;
struct mlx4_ib_dev *dev = to_mdev(ibdev);
struct mlx4_ib_ucontext *context = to_mucontext(uctx);
struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
struct mlx4_ib_alloc_ucontext_resp resp;
struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3 = {};
struct mlx4_ib_alloc_ucontext_resp resp = {};
int err;
if (!dev->ib_active)
@@ -1121,16 +1122,16 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
mutex_init(&context->wqn_ranges_mutex);
if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
err = ib_respond_udata(udata, resp_v3);
else
err = ib_copy_to_udata(udata, &resp, sizeof(resp));
err = ib_respond_udata(udata, resp);
if (err) {
mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
return -EFAULT;
return err;
}
return err;
return 0;
}
static void mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
@@ -1199,9 +1200,14 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
if (err)
return err;
if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
return -EFAULT;
if (udata) {
struct mlx4_ib_alloc_pd_resp uresp = { .pdn = pd->pdn };
err = ib_respond_udata(udata, uresp);
if (err) {
mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
return err;
}
}
return 0;
}
@@ -1696,9 +1702,9 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
(flow_attr->type != IB_FLOW_ATTR_NORMAL))
return ERR_PTR(-EOPNOTSUPP);
if (udata &&
udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen))
return ERR_PTR(-EOPNOTSUPP);
err = ib_is_udata_in_empty(udata);
if (err)
return ERR_PTR(err);
memset(type, 0, sizeof(type));

2
drivers/infiniband/hw/mlx4/mlx4_ib.h
View File

@@ -121,6 +121,7 @@ struct mlx4_ib_cq {
struct mlx4_db db;
spinlock_t lock;
struct mutex resize_mutex;
struct ib_umem *umem;
struct ib_umem *resize_umem;
/* List of qps that it serves.*/
struct list_head send_qp_list;
@@ -135,6 +136,7 @@ struct mlx4_ib_mr {
dma_addr_t page_map;
u32 npages;
u32 max_pages;
int access_flags;
struct mlx4_mr mmr;
struct ib_umem *umem;
size_t page_map_size;

13
drivers/infiniband/hw/mlx4/mr.c
View File

@@ -110,7 +110,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start,
/*
* Force registering the memory as writable if the underlying pages
* are writable. This is so rereg can change the access permissions
* from readable to writable without having to run through ib_umem_get
* from readable to writable without having to run through ib_umem_get_va
* again
*/
if (!ib_access_writable(access_flags)) {
@@ -135,7 +135,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start,
mmap_read_unlock(current->mm);
}
return ib_umem_get(device, start, length, access_flags);
return ib_umem_get_va(device, start, length, access_flags);
}
struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
@@ -181,6 +181,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
if (err)
goto err_mr;
mr->access_flags = access_flags;
mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;
mr->ibmr.page_size = 1U << shift;
@@ -241,6 +242,8 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
if (err)
goto release_mpt_entry;
} else {
mr_access_flags = mmr->access_flags;
}
if (flags & IB_MR_REREG_TRANS) {
@@ -282,8 +285,10 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
* return a failure. But dereg_mr will free the resources.
*/
err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry);
if (!err && flags & IB_MR_REREG_ACCESS)
mmr->mmr.access = mr_access_flags;
if (!err && flags & IB_MR_REREG_ACCESS) {
mmr->access_flags = mr_access_flags;
mmr->mmr.access = convert_access(mr_access_flags);
}
release_mpt_entry:
mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry);

13
drivers/infiniband/hw/mlx4/qp.c
View File

@@ -897,7 +897,7 @@ static int create_rq(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,
qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
(qp->sq.wqe_cnt << qp->sq.wqe_shift);
qp->umem = ib_umem_get(pd->device, wq.buf_addr, qp->buf_size, 0);
qp->umem = ib_umem_get_va(pd->device, wq.buf_addr, qp->buf_size, 0);
if (IS_ERR(qp->umem)) {
err = PTR_ERR(qp->umem);
goto err;
@@ -1080,7 +1080,7 @@ static int create_qp_common(struct ib_pd *pd, struct ib_qp_init_attr *init_attr,
goto err;
qp->umem =
ib_umem_get(pd->device, ucmd.buf_addr, qp->buf_size, 0);
ib_umem_get_va(pd->device, ucmd.buf_addr, qp->buf_size, 0);
if (IS_ERR(qp->umem)) {
err = PTR_ERR(qp->umem);
goto err;
@@ -4297,10 +4297,9 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
size_t min_resp_len;
int i, err = 0;
if (udata->inlen > 0 &&
!ib_is_udata_cleared(udata, 0,
udata->inlen))
return -EOPNOTSUPP;
err = ib_is_udata_in_empty(udata);
if (err)
return err;
min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
if (udata->outlen && udata->outlen < min_resp_len)
@@ -4332,7 +4331,7 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
if (udata->outlen) {
resp.response_length = offsetof(typeof(resp), response_length) +
sizeof(resp.response_length);
err = ib_copy_to_udata(udata, &resp, resp.response_length);
err = ib_respond_udata(udata, resp);
}
return err;

14
drivers/infiniband/hw/mlx4/srq.c
View File

@@ -116,7 +116,7 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,
return err;
srq->umem =
ib_umem_get(ib_srq->device, ucmd.buf_addr, buf_size, 0);
ib_umem_get_va(ib_srq->device, ucmd.buf_addr, buf_size, 0);
if (IS_ERR(srq->umem))
return PTR_ERR(srq->umem);
@@ -191,11 +191,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,
srq->msrq.event = mlx4_ib_srq_event;
srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
if (udata)
if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
err = -EFAULT;
if (udata) {
struct mlx4_ib_create_srq_resp uresp = {
.srqn = srq->msrq.srqn
};
err = ib_respond_udata(udata, uresp);
if (err)
goto err_srq;
}
}
init_attr->attr.max_wr = srq->msrq.max - 1;

45
drivers/infiniband/hw/mlx4/sysfs.c
View File

@@ -636,12 +636,6 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave)
p->port_num = port_num;
p->slave = slave;
ret = kobject_init_and_add(&p->kobj, &port_type,
kobject_get(dev->dev_ports_parent[slave]),
"%d", port_num);
if (ret)
goto err_alloc;
p->pkey_group.name = "pkey_idx";
p->pkey_group.attrs =
alloc_group_attrs(show_port_pkey,
@@ -649,13 +643,9 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave)
dev->dev->caps.pkey_table_len[port_num]);
if (!p->pkey_group.attrs) {
ret = -ENOMEM;
goto err_alloc;
goto err_free_port;
}
ret = sysfs_create_group(&p->kobj, &p->pkey_group);
if (ret)
goto err_free_pkey;
p->gid_group.name = "gid_idx";
p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1);
if (!p->gid_group.attrs) {
@@ -663,28 +653,47 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave)
goto err_free_pkey;
}
ret = kobject_init_and_add(&p->kobj, &port_type,
kobject_get(dev->dev_ports_parent[slave]),
"%d", port_num);
if (ret)
goto err_put;
ret = sysfs_create_group(&p->kobj, &p->pkey_group);
if (ret)
goto err_del;
ret = sysfs_create_group(&p->kobj, &p->gid_group);
if (ret)
goto err_free_gid;
goto err_remove_pkey;
ret = add_vf_smi_entries(p);
if (ret)
goto err_free_gid;
goto err_remove_gid;
list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]);
return 0;
err_free_gid:
kfree(p->gid_group.attrs[0]);
kfree(p->gid_group.attrs);
err_remove_gid:
sysfs_remove_group(&p->kobj, &p->gid_group);
err_remove_pkey:
sysfs_remove_group(&p->kobj, &p->pkey_group);
err_del:
kobject_del(&p->kobj);
err_put:
kobject_put(dev->dev_ports_parent[slave]);
kobject_put(&p->kobj);
return ret;
err_free_pkey:
for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i)
kfree(p->pkey_group.attrs[i]);
kfree(p->pkey_group.attrs);
err_alloc:
kobject_put(dev->dev_ports_parent[slave]);
err_free_port:
kfree(p);
return ret;
}

2
drivers/infiniband/hw/mlx5/ah.c
View File

@@ -121,7 +121,7 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
resp.response_length = min_resp_len;
memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN);
err = ib_copy_to_udata(udata, &resp, resp.response_length);
err = ib_respond_udata(udata, resp);
if (err)
return err;
}

51
drivers/infiniband/hw/mlx5/cq.c
View File

@@ -169,7 +169,8 @@ enum {
static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
struct mlx5_ib_qp *qp)
{
enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1);
enum rdma_link_layer ll =
rdma_port_get_link_layer(qp->ibqp.device, qp->port);
struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
struct mlx5_ib_srq *srq = NULL;
struct mlx5_ib_wq *wq;
@@ -745,31 +746,33 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
*cqe_size = ucmd.cqe_size;
if (!cq->ibcq.umem)
cq->ibcq.umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
entries * ucmd.cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->ibcq.umem))
return PTR_ERR(cq->ibcq.umem);
cq->buf.umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, attrs,
ucmd.buf_addr,
entries * ucmd.cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->buf.umem))
return PTR_ERR(cq->buf.umem);
page_size = mlx5_umem_find_best_cq_quantized_pgoff(
cq->ibcq.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
cq->buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
page_offset, 64, &page_offset_quantized);
if (!page_size) {
err = -EINVAL;
goto err_umem;
}
err = mlx5_ib_db_map_user(context, ucmd.db_addr, &cq->db);
err = mlx5_ib_db_map_user(context, attrs,
MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM,
ucmd.db_addr, &cq->db);
if (err)
goto err_umem;
ncont = ib_umem_num_dma_blocks(cq->ibcq.umem, page_size);
ncont = ib_umem_num_dma_blocks(cq->buf.umem, page_size);
mlx5_ib_dbg(
dev,
"addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n",
ucmd.buf_addr, entries * ucmd.cqe_size,
ib_umem_num_pages(cq->ibcq.umem), page_size, ncont);
ib_umem_num_pages(cq->buf.umem), page_size, ncont);
*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;
@@ -780,7 +783,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
}
pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
mlx5_ib_populate_pas(cq->ibcq.umem, page_size, pas, 0);
mlx5_ib_populate_pas(cq->buf.umem, page_size, pas, 0);
cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
MLX5_SET(cqc, cqc, log_page_size,
@@ -853,7 +856,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
mlx5_ib_db_unmap_user(context, &cq->db);
err_umem:
/* UMEM is released by ib_core */
ib_umem_release(cq->buf.umem);
return err;
}
@@ -863,6 +866,7 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata)
udata, struct mlx5_ib_ucontext, ibucontext);
mlx5_ib_db_unmap_user(context, &cq->db);
ib_umem_release(cq->buf.umem);
}
static void init_cq_frag_buf(struct mlx5_ib_cq_buf *buf)
@@ -949,6 +953,7 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
{
struct ib_udata *udata = &attrs->driver_udata;
struct ib_device *ibdev = ibcq->device;
struct mlx5_ib_create_cq_resp uresp = {};
int entries = attr->cqe;
int vector = attr->comp_vector;
struct mlx5_ib_dev *dev = to_mdev(ibdev);
@@ -1015,10 +1020,10 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
INIT_LIST_HEAD(&cq->wc_list);
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
err = -EFAULT;
uresp.cqn = cq->mcq.cqn;
err = ib_respond_udata(udata, uresp);
if (err)
goto err_cmd;
}
kvfree(cqb);
return 0;
@@ -1240,9 +1245,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
return -EINVAL;
umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
(size_t)ucmd.cqe_size * entries,
IB_ACCESS_LOCAL_WRITE);
umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
(size_t)ucmd.cqe_size * entries,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
err = PTR_ERR(umem);
return err;
@@ -1434,8 +1439,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, unsigned int entries,
if (udata) {
cq->ibcq.cqe = entries - 1;
ib_umem_release(cq->ibcq.umem);
cq->ibcq.umem = cq->resize_umem;
ib_umem_release(cq->buf.umem);
cq->buf.umem = cq->resize_umem;
cq->resize_umem = NULL;
} else {
struct mlx5_ib_cq_buf tbuf;
@@ -1518,7 +1523,9 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE(
UVERBS_ATTR_PTR_IN(
MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX,
UVERBS_ATTR_TYPE(u32),
UA_OPTIONAL));
UA_OPTIONAL),
UVERBS_ATTR_UMEM(MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM,
UA_OPTIONAL));
const struct uapi_definition mlx5_ib_create_cq_defs[] = {
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_CQ, &mlx5_ib_cq_create),

2
drivers/infiniband/hw/mlx5/data_direct.c
View File

@@ -88,7 +88,7 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)
return -ENOMEM;
reg->ibdev = ibdev;
strcpy(reg->vuid, vuid);
strscpy(reg->vuid, vuid);
mutex_lock(&mlx5_data_direct_mutex);
list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {

68
drivers/infiniband/hw/mlx5/devx.c
View File

@@ -1913,6 +1913,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
return err;
}
static bool devx_key_in_sub_list(struct list_head *list, u32 key_level1)
{
struct devx_event_subscription *s;
list_for_each_entry(s, list, event_list)
if (s->xa_key_level1 == key_level1)
return true;
return false;
}
static void
subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
u32 key_level1,
@@ -2160,10 +2171,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
event_sub = kzalloc_obj(*event_sub);
if (!event_sub) {
if (!devx_key_in_sub_list(&sub_list, key_level1))
subscribe_event_xa_dealloc(devx_event_table,
key_level1,
obj,
obj_id);
err = -ENOMEM;
goto err;
}
event_sub->ev_file = ev_file;
event_sub->xa_key_level1 = key_level1;
list_add_tail(&event_sub->event_list, &sub_list);
uverbs_uobject_get(&ev_file->uobj);
if (use_eventfd) {
@@ -2178,9 +2196,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
}
event_sub->cookie = cookie;
event_sub->ev_file = ev_file;
/* May be needed upon cleanup the devx object/subscription */
event_sub->xa_key_level1 = key_level1;
event_sub->xa_key_level2 = obj_id;
INIT_LIST_HEAD(&event_sub->obj_list);
}
@@ -2225,10 +2240,11 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
list_del(&event_sub->event_list);
subscribe_event_xa_dealloc(devx_event_table,
event_sub->xa_key_level1,
obj,
obj_id);
if (!devx_key_in_sub_list(&sub_list, event_sub->xa_key_level1))
subscribe_event_xa_dealloc(devx_event_table,
event_sub->xa_key_level1,
obj,
obj_id);
if (event_sub->eventfd)
eventfd_ctx_put(event_sub->eventfd);
@@ -2271,7 +2287,7 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
return PTR_ERR(umem_dmabuf);
obj->umem = &umem_dmabuf->umem;
} else {
obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags);
obj->umem = ib_umem_get_va(&dev->ib_dev, addr, size, access_flags);
if (IS_ERR(obj->umem))
return PTR_ERR(obj->umem);
}
@@ -2512,7 +2528,7 @@ static int deliver_event(struct devx_event_subscription *event_sub,
const void *data)
{
struct devx_async_event_file *ev_file;
struct devx_async_event_data *event_data;
struct devx_async_event_data *event_data, *to_free;
unsigned long flags;
ev_file = event_sub->ev_file;
@@ -2543,12 +2559,17 @@ static int deliver_event(struct devx_event_subscription *event_sub,
event_data->hdr.cookie = event_sub->cookie;
memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
to_free = NULL;
spin_lock_irqsave(&ev_file->lock, flags);
if (!ev_file->is_destroyed)
list_add_tail(&event_data->list, &ev_file->event_list);
else
kfree(event_data);
to_free = event_data;
spin_unlock_irqrestore(&ev_file->lock, flags);
kfree(to_free);
wake_up_interruptible(&ev_file->poll_wait);
return 0;
@@ -2942,6 +2963,7 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
uobj);
struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
struct devx_async_data *entry, *tmp;
LIST_HEAD(tmp_list);
spin_lock_irq(&ev_queue->lock);
ev_queue->is_destroyed = 1;
@@ -2951,12 +2973,15 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
spin_lock_irq(&comp_ev_file->ev_queue.lock);
list_for_each_entry_safe(entry, tmp,
&comp_ev_file->ev_queue.event_list, list) {
/* Move all entries to a temporary list and free them outside lock */
list_splice_init(&comp_ev_file->ev_queue.event_list, &tmp_list);
spin_unlock_irq(&comp_ev_file->ev_queue.lock);
/* Free memory outside of critical section */
list_for_each_entry_safe(entry, tmp, &tmp_list, list) {
list_del(&entry->list);
kvfree(entry);
}
spin_unlock_irq(&comp_ev_file->ev_queue.lock);
};
static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
@@ -2966,7 +2991,9 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
container_of(uobj, struct devx_async_event_file,
uobj);
struct devx_event_subscription *event_sub, *event_sub_tmp;
struct devx_async_event_data *entry, *tmp;
struct mlx5_ib_dev *dev = ev_file->dev;
LIST_HEAD(tmp_list);
spin_lock_irq(&ev_file->lock);
ev_file->is_destroyed = 1;
@@ -2980,18 +3007,19 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
list_del_init(&event_sub->event_list);
} else {
struct devx_async_event_data *entry, *tmp;
list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
list) {
list_del(&entry->list);
kfree(entry);
}
/* Move all entries to a temporary list */
list_splice_init(&ev_file->event_list, &tmp_list);
}
spin_unlock_irq(&ev_file->lock);
wake_up_interruptible(&ev_file->poll_wait);
/* Free event data outside of critical section */
list_for_each_entry_safe(entry, tmp, &tmp_list, list) {
list_del(&entry->list);
kfree(entry);
}
mutex_lock(&dev->devx_event_table.event_xa_lock);
/* delete the subscriptions which are related to this FD */
list_for_each_entry_safe(event_sub, event_sub_tmp,

103
drivers/infiniband/hw/mlx5/doorbell.c
View File

@@ -37,50 +37,95 @@
#include "mlx5_ib.h"
#define MLX5_IB_DBR_SIZE (sizeof(__be32) * 2)
struct mlx5_ib_user_db_page {
struct list_head list;
struct ib_umem *umem;
unsigned long user_virt;
struct ib_uverbs_buffer_desc desc;
int refcnt;
struct mm_struct *mm;
};
int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
struct mlx5_db *db)
static int mlx5_ib_db_map_user_desc(struct mlx5_ib_ucontext *context,
const struct ib_uverbs_buffer_desc *desc,
struct mlx5_db *db)
{
struct mlx5_ib_user_db_page *page;
struct ib_umem *umem;
int err = 0;
if (desc->length < MLX5_IB_DBR_SIZE)
return -EINVAL;
/*
* For VA descriptors the umem is normalized to a single PAGE_SIZE
* region, so reject offsets that would place the 8-byte DBR
* straddling the page boundary.
*/
if (desc->type == IB_UVERBS_BUFFER_TYPE_VA &&
(desc->addr & ~PAGE_MASK) > PAGE_SIZE - MLX5_IB_DBR_SIZE)
return -EINVAL;
mutex_lock(&context->db_page_mutex);
list_for_each_entry(page, &context->db_page_list, list)
if ((current->mm == page->mm) &&
(page->user_virt == (virt & PAGE_MASK)))
goto found;
/*
* Only VA-typed descriptors are eligible to share a per-page
* doorbell umem; FD-typed descriptors are pinned individually.
*/
if (desc->type == IB_UVERBS_BUFFER_TYPE_VA) {
list_for_each_entry(page, &context->db_page_list, list) {
if (current->mm != page->mm)
continue;
if (page->desc.addr == (desc->addr & PAGE_MASK))
goto found;
}
}
page = kmalloc_obj(*page);
page = kzalloc_obj(*page);
if (!page) {
err = -ENOMEM;
goto out;
}
page->user_virt = (virt & PAGE_MASK);
page->refcnt = 0;
page->umem = ib_umem_get(context->ibucontext.device, virt & PAGE_MASK,
PAGE_SIZE, 0);
if (IS_ERR(page->umem)) {
err = PTR_ERR(page->umem);
page->desc = *desc;
/*
* Normalize VA descriptors to a page-aligned PAGE_SIZE region so
* multiple DBRs that fall in the same user page share one umem.
*/
if (page->desc.type == IB_UVERBS_BUFFER_TYPE_VA) {
page->desc.addr &= PAGE_MASK;
page->desc.length = PAGE_SIZE;
}
umem = ib_umem_get_desc(context->ibucontext.device, &page->desc, 0);
if (IS_ERR(umem)) {
err = PTR_ERR(umem);
kfree(page);
goto out;
}
mmgrab(current->mm);
page->mm = current->mm;
/*
* The 8-byte DBR is programmed to the device as one DMA address,
* so it must live in a single contiguous DMA segment.
*/
if (!ib_umem_is_contiguous(umem)) {
ib_umem_release(umem);
kfree(page);
err = -EINVAL;
goto out;
}
page->umem = umem;
if (page->desc.type == IB_UVERBS_BUFFER_TYPE_VA) {
mmgrab(current->mm);
page->mm = current->mm;
}
list_add(&page->list, &context->db_page_list);
found:
db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
(virt & ~PAGE_MASK);
(desc->addr & ~PAGE_MASK);
db->u.user_page = page;
++page->refcnt;
@@ -90,13 +135,35 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
return err;
}
int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,
const struct uverbs_attr_bundle *attrs, u16 attr_id,
unsigned long virt, struct mlx5_db *db)
{
struct ib_uverbs_buffer_desc desc = {
.type = IB_UVERBS_BUFFER_TYPE_VA,
.addr = virt,
.length = MLX5_IB_DBR_SIZE,
};
if (attrs) {
int err;
err = uverbs_get_buffer_desc(attrs, attr_id, &desc);
if (err && err != -ENOENT)
return err;
}
return mlx5_ib_db_map_user_desc(context, &desc, db);
}
void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db)
{
mutex_lock(&context->db_page_mutex);
if (!--db->u.user_page->refcnt) {
list_del(&db->u.user_page->list);
mmdrop(db->u.user_page->mm);
if (db->u.user_page->mm)
mmdrop(db->u.user_page->mm);
ib_umem_release(db->u.user_page->umem);
kfree(db->u.user_page);
}

84
drivers/infiniband/hw/mlx5/main.c
View File

@@ -965,8 +965,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
resp.response_length = resp_len;
if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
return -EINVAL;
err = ib_is_udata_in_empty(uhw);
if (err)
return err;
memset(props, 0, sizeof(*props));
err = mlx5_query_system_image_guid(ibdev,
@@ -1213,20 +1214,29 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
}
if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
raw_support) {
if (MLX5_CAP_QOS(mdev, packet_pacing) &&
MLX5_CAP_GEN(mdev, qos)) {
resp.packet_pacing_caps.qp_rate_limit_max =
MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
resp.packet_pacing_caps.qp_rate_limit_min =
MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
resp.packet_pacing_caps.supported_qpts |=
1 << IB_QPT_RAW_PACKET;
if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
resp.packet_pacing_caps.cap_flags |=
MLX5_IB_PP_SUPPORT_BURST;
if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen) {
if (MLX5_CAP_GEN(mdev, qos)) {
if (MLX5_CAP_QOS(mdev, packet_pacing) && raw_support)
resp.packet_pacing_caps.supported_qpts |=
BIT(IB_QPT_RAW_PACKET);
if (MLX5_CAP_QOS(mdev, packet_pacing_req_ud))
resp.packet_pacing_caps.supported_qpts |=
BIT(IB_QPT_UD);
if (MLX5_CAP_QOS(mdev, packet_pacing_req_uc))
resp.packet_pacing_caps.supported_qpts |=
BIT(IB_QPT_UC);
if (resp.packet_pacing_caps.supported_qpts) {
resp.packet_pacing_caps.qp_rate_limit_max =
MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
resp.packet_pacing_caps.qp_rate_limit_min =
MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
resp.packet_pacing_caps.cap_flags |=
MLX5_IB_PP_SUPPORT_BURST;
}
}
resp.response_length += sizeof(resp.packet_pacing_caps);
}
@@ -1355,7 +1365,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
}
if (uhw_outlen) {
err = ib_copy_to_udata(uhw, &resp, resp.response_length);
err = ib_respond_udata(uhw, resp);
if (err)
return err;
@@ -2023,6 +2033,9 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
dev->lb.qps == 1) {
if (!dev->lb.enabled) {
err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
if (err)
goto err_rollback;
dev->lb.enabled = true;
}
}
@@ -2030,6 +2043,14 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
mutex_unlock(&dev->lb.mutex);
return err;
err_rollback:
if (td)
dev->lb.user_td--;
if (qp)
dev->lb.qps--;
mutex_unlock(&dev->lb.mutex);
return err;
}
void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
@@ -2069,9 +2090,13 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
(!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
!MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
return err;
return 0;
return mlx5_ib_enable_lb(dev, true, false);
err = mlx5_ib_enable_lb(dev, true, false);
if (err)
mlx5_cmd_dealloc_transport_domain(dev->mdev, *tdn, uid);
return err;
}
static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
@@ -2280,7 +2305,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
goto out_mdev;
resp.response_length = min(udata->outlen, sizeof(resp));
err = ib_copy_to_udata(udata, &resp, resp.response_length);
err = ib_respond_udata(udata, resp);
if (err)
goto out_mdev;
@@ -2658,7 +2683,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
if (!dyn_uar)
return err;
mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
mlx5_cmd_uar_dealloc(dev->mdev, uar_index, context->devx_uid);
free_bfreg:
mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
@@ -2772,7 +2797,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct mlx5_ib_pd *pd = to_mpd(ibpd);
struct ib_device *ibdev = ibpd->device;
struct mlx5_ib_alloc_pd_resp resp;
struct mlx5_ib_alloc_pd_resp resp = {};
int err;
u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
@@ -2791,9 +2816,10 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
pd->uid = uid;
if (udata) {
resp.pdn = pd->pdn;
if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
err = ib_respond_udata(udata, resp);
if (err) {
mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
return -EFAULT;
return err;
}
}
@@ -4450,6 +4476,7 @@ static const struct uapi_definition mlx5_ib_defs[] = {
UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs),
UAPI_DEF_CHAIN(mlx5_ib_create_qp_defs),
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr),
@@ -4488,6 +4515,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
dev->port[i].roce.last_port_state = IB_PORT_DOWN;
}
mutex_init(&dev->lb.mutex);
err = mlx5r_cmd_query_special_mkeys(dev);
if (err)
return err;
@@ -4788,11 +4817,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
if (err)
return err;
if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
(MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
mutex_init(&dev->lb.mutex);
if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
err = mlx5_ib_init_var_region(dev);
@@ -5112,7 +5136,7 @@ int __mlx5_ib_add(struct mlx5_ib_dev *dev,
if (profile->stage[i].cleanup)
profile->stage[i].cleanup(dev);
}
return -ENOMEM;
return err;
}
static const struct mlx5_ib_profile pf_profile = {

19
drivers/infiniband/hw/mlx5/mlx5_ib.h
View File

@@ -331,6 +331,10 @@ struct mlx5_ib_flow_db {
#define MLX5_IB_QPT_DCT IB_QPT_RESERVED4
#define MLX5_IB_WR_UMR IB_WR_RESERVED1
/*
* A valid pdn is required when flags include MLX5_IB_UPD_XLT_ENABLE,
* MLX5_IB_UPD_XLT_PD or MLX5_IB_UPD_XLT_ACCESS.
*/
#define MLX5_IB_UPD_XLT_ZAP BIT(0)
#define MLX5_IB_UPD_XLT_ENABLE BIT(1)
#define MLX5_IB_UPD_XLT_ATOMIC BIT(2)
@@ -536,6 +540,7 @@ struct mlx5_ib_qp {
struct list_head cq_recv_list;
struct list_head cq_send_list;
struct mlx5_rate_limit rl;
struct mlx5_rate_limit rl_desired;
u32 underlay_qpn;
u32 flags_en;
/*
@@ -1259,8 +1264,9 @@ to_mmmap(struct rdma_user_mmap_entry *rdma_entry)
int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev);
int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev);
int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
struct mlx5_db *db);
int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,
const struct uverbs_attr_bundle *attrs, u16 attr_id,
unsigned long virt, struct mlx5_db *db);
void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
@@ -1413,8 +1419,8 @@ int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
enum ib_uverbs_advise_mr_advice advice,
u32 flags, struct ib_sge *sg_list, u32 num_sge);
int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr);
int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr);
int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd);
int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd);
#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
static inline int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev,
@@ -1442,11 +1448,11 @@ mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
{
return -EOPNOTSUPP;
}
static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr)
static inline int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
{
return -EOPNOTSUPP;
}
static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr)
static inline int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
{
return -EOPNOTSUPP;
}
@@ -1510,6 +1516,7 @@ extern const struct uapi_definition mlx5_ib_flow_defs[];
extern const struct uapi_definition mlx5_ib_qos_defs[];
extern const struct uapi_definition mlx5_ib_std_types_defs[];
extern const struct uapi_definition mlx5_ib_create_cq_defs[];
extern const struct uapi_definition mlx5_ib_create_qp_defs[];
static inline int is_qp1(enum ib_qp_type qp_type)
{

76
drivers/infiniband/hw/mlx5/mr.c
View File

@@ -31,6 +31,7 @@
* SOFTWARE.
*/
#include <linux/bitfield.h>
#include <linux/kref.h>
#include <linux/random.h>
#include <linux/debugfs.h>
@@ -163,9 +164,8 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
#define MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED \
MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK
#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT 16
#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK 0xFF0000
#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK 0xFFFF
#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16)
#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0)
static struct mlx5_ib_mr *
_mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem,
@@ -194,7 +194,8 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem,
ph ^= MLX5_IB_NO_PH;
mr->ibmr.frmr.key.kernel_vendor_key =
st_index | (ph << MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT);
FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) |
FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph);
err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr);
if (err) {
kfree(mr);
@@ -271,9 +272,10 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key,
get_mkc_octo_size(access_mode, key->num_dma_blocks));
MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
st_index = key->kernel_vendor_key &
MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK;
ph = key->kernel_vendor_key & MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK;
st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK,
key->kernel_vendor_key);
ph = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK,
key->kernel_vendor_key);
if (ph) {
/* Normalize ph: swap MLX5_IB_NO_PH for 0 */
if (ph == MLX5_IB_NO_PH)
@@ -294,7 +296,7 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key,
free_in:
kfree(in);
if (err)
for (; i > 0; i--)
for (i--; i >= 0; i--)
mlx5_core_destroy_mkey(dev->mdev, handles[i]);
return err;
}
@@ -781,7 +783,8 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem,
* configured properly but left disabled. It is safe to go ahead
* and configure it again via UMR while enabling it.
*/
err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE);
err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE,
to_mpd(pd)->pdn);
if (err) {
mlx5_ib_dereg_mr(&mr->ibmr, NULL);
return ERR_PTR(err);
@@ -841,7 +844,7 @@ static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length,
if (err)
goto err_dereg_mr;
err = mlx5_ib_init_odp_mr(mr);
err = mlx5_ib_init_odp_mr(mr, pd);
if (err)
goto err_dereg_mr;
return &mr->ibmr;
@@ -874,7 +877,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
if (access_flags & IB_ACCESS_ON_DEMAND)
return create_user_odp_mr(pd, start, length, iova, access_flags,
udata);
umem = ib_umem_get(&dev->ib_dev, start, length, access_flags);
umem = ib_umem_get_va(&dev->ib_dev, start, length, access_flags);
if (IS_ERR(umem))
return ERR_CAST(umem);
return create_real_mr(pd, umem, iova, access_flags, dmah);
@@ -890,7 +893,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach)
if (!umem_dmabuf->sgt || !mr)
return;
mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP);
/* MLX5_IB_UPD_XLT_ZAP does not change the pdn */
mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP, 0);
ib_umem_dmabuf_unmap_pages(umem_dmabuf);
}
@@ -956,6 +960,10 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages);
umem_dmabuf->private = mr;
if (!pinned_mode) {
err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq);
if (err)
goto err_dereg_mr;
err = mlx5r_store_odp_mkey(dev, &mr->mmkey);
if (err)
goto err_dereg_mr;
@@ -963,7 +971,7 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
mr->data_direct = true;
}
err = mlx5_ib_init_dmabuf_mr(mr);
err = mlx5_ib_init_dmabuf_mr(mr, pd);
if (err)
goto err_dereg_mr;
return &mr->ibmr;
@@ -1128,10 +1136,8 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,
if (err)
return err;
if (flags & IB_MR_REREG_PD) {
mr->ibmr.pd = pd;
if (flags & IB_MR_REREG_PD)
upd_flags |= MLX5_IB_UPD_XLT_PD;
}
if (flags & IB_MR_REREG_ACCESS) {
mr->access_flags = access_flags;
upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
@@ -1141,7 +1147,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,
mr->ibmr.length = new_umem->length;
mr->page_shift = order_base_2(page_size);
mr->umem = new_umem;
err = mlx5r_umr_update_mr_pas(mr, upd_flags);
err = mlx5r_umr_update_mr_pas(mr, upd_flags, to_mpd(pd)->pdn);
if (err) {
/*
* The MR is revoked at this point so there is no issue to free
@@ -1188,6 +1194,21 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
if (!(flags & IB_MR_REREG_PD))
new_pd = ib_mr->pd;
if (mr->is_odp_implicit && !(flags & IB_MR_REREG_TRANS)) {
if (!(new_access_flags & IB_ACCESS_ON_DEMAND))
return ERR_PTR(-EOPNOTSUPP);
/*
* Due to all the child mkeys we cannot actually change an
* implicit MR in place. If the user did not specify a new
* translation then force the fixed implicit MR values.
*/
start = 0;
iova = 0;
length = U64_MAX;
flags |= IB_MR_REREG_TRANS;
}
if (!(flags & IB_MR_REREG_TRANS)) {
struct ib_umem *umem;
@@ -1202,7 +1223,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
}
/* DM or ODP MR's don't have a normal umem so we can't re-use it */
if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr))
goto recreate;
return ERR_PTR(-EOPNOTSUPP);
/*
* Only one active MR can refer to a umem at one time, revoke
@@ -1231,8 +1252,8 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
struct ib_umem *new_umem;
unsigned long page_size;
new_umem = ib_umem_get(&dev->ib_dev, start, length,
new_access_flags);
new_umem = ib_umem_get_va(&dev->ib_dev, start, length,
new_access_flags);
if (IS_ERR(new_umem))
return ERR_CAST(new_umem);
@@ -1381,9 +1402,12 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr)
bool is_odp = is_odp_mr(mr);
int ret;
if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr) &&
!ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr))
return 0;
if (mr->ibmr.frmr.pool) {
if (!mlx5_umr_revoke_mr_with_lock(mr)) {
ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr);
return 0;
}
}
if (is_odp)
mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex);
@@ -1404,6 +1428,10 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr)
dma_resv_unlock(
to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv);
}
if (mr->ibmr.frmr.pool && !ret)
ib_frmr_pool_drop(&mr->ibmr);
return ret;
}
@@ -1813,7 +1841,7 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
resp.response_length =
min(offsetofend(typeof(resp), response_length), udata->outlen);
if (resp.response_length) {
err = ib_copy_to_udata(udata, &resp, resp.response_length);
err = ib_respond_udata(udata, resp);
if (err)
goto free_mkey;
}

Some files were not shown because too many files have changed in this diff Show More