diff --git a/net/core/dev.c b/net/core/dev.c index 23c1ad3d393d..26ac8eb9b259 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3993,7 +3993,8 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, struct skb_shared_info *shinfo; struct net_iov *niov; - if (likely(skb_frags_readable(skb))) + if (likely(skb_frags_readable(skb) || + dev->netmem_tx == NETMEM_TX_NO_DMA)) goto out; if (dev->netmem_tx == NETMEM_TX_NONE) diff --git a/net/core/devmem.c b/net/core/devmem.c index 468344739db2..893643909f6a 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -181,7 +181,7 @@ int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, } struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, struct netdev_nl_sock *priv, @@ -212,6 +212,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, } binding->dev = dev; + binding->vdev = vdev; xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); err = percpu_ref_init(&binding->ref, @@ -396,7 +397,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, */ dst_dev = dst_dev_rcu(dst); if (unlikely(!dst_dev) || - unlikely(dst_dev != READ_ONCE(binding->dev))) { + unlikely(dst_dev != READ_ONCE(binding->dev) && + dst_dev != READ_ONCE(binding->vdev))) { err = -ENODEV; goto out_unlock; } diff --git a/net/core/devmem.h b/net/core/devmem.h index 1c5c18581fcb..3852a56036cb 100644 --- a/net/core/devmem.h +++ b/net/core/devmem.h @@ -19,7 +19,13 @@ struct net_devmem_dmabuf_binding { struct dma_buf *dmabuf; struct dma_buf_attachment *attachment; struct sg_table *sgt; + /* Physical NIC that does the actual DMA for this binding. */ struct net_device *dev; + /* Opaque cookie identifying the virtual device (e.g. netkit) the user + * called bind-tx on. Used only for pointer comparison. Never + * dereferenced. + */ + void *vdev; struct gen_pool *chunk_pool; /* Protect dev */ struct mutex lock; @@ -84,7 +90,7 @@ struct dmabuf_genpool_chunk_owner { void __net_devmem_dmabuf_binding_free(struct work_struct *wq); struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, struct netdev_nl_sock *priv, @@ -165,7 +171,7 @@ static inline void net_devmem_put_net_iov(struct net_iov *niov) } static inline struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index 4d2c49371cdb..b4d48f3672a5 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -1077,7 +1077,7 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) goto err_rxq_bitmap; } - binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_FROM_DEVICE, + binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE, dmabuf_fd, priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); @@ -1119,9 +1119,43 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) return err; } +/* Find the DMA-capable device for a netmem TX binding. + * + * For NETMEM_TX_DMA devices, return the device itself. + * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying + * physical device and return it. + */ +static struct net_device * +netdev_find_netmem_tx_dev(struct net_device *dev) +{ + struct netdev_rx_queue *lease_rxq; + struct net_device *phys_dev; + int i; + + if (dev->netmem_tx == NETMEM_TX_DMA) + return dev; + + if (dev->netmem_tx != NETMEM_TX_NO_DMA) + return NULL; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease); + if (!lease_rxq) + continue; + + phys_dev = lease_rxq->dev; + if (netif_device_present(phys_dev) && + phys_dev->netmem_tx == NETMEM_TX_DMA) + return phys_dev; + } + + return NULL; +} + int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) { struct net_devmem_dmabuf_binding *binding; + struct net_device *bind_dev; struct netdev_nl_sock *priv; struct net_device *netdev; struct device *dma_dev; @@ -1171,22 +1205,41 @@ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) goto err_unlock_netdev; } - dma_dev = netdev_queue_get_dma_dev(netdev, 0, NETDEV_QUEUE_TYPE_TX); - binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_TO_DEVICE, - dmabuf_fd, priv, info->extack); + bind_dev = netdev_find_netmem_tx_dev(netdev); + if (!bind_dev) { + err = -EOPNOTSUPP; + NL_SET_ERR_MSG(info->extack, + "No DMA-capable device found for netmem TX"); + goto err_unlock_netdev; + } + + if (bind_dev != netdev) + netdev_lock(bind_dev); + + dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX); + + binding = net_devmem_bind_dmabuf(bind_dev, + bind_dev != netdev ? netdev : NULL, + dma_dev, DMA_TO_DEVICE, dmabuf_fd, + priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); - goto err_unlock_netdev; + goto err_unlock_bind_dev; } nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); genlmsg_end(rsp, hdr); + if (bind_dev != netdev) + netdev_unlock(bind_dev); netdev_unlock(netdev); mutex_unlock(&priv->lock); return genlmsg_reply(rsp, info); +err_unlock_bind_dev: + if (bind_dev != netdev) + netdev_unlock(bind_dev); err_unlock_netdev: netdev_unlock(netdev); err_unlock_sock: