diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index 1c19bb7705df..7b3392553fd6 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -10,7 +10,7 @@ Type Name fastpath_tx_acce =================================== =========================== =================== =================== =================================================================================== unsigned_long:32 priv_flags read_mostly __dev_queue_xmit(tx) unsigned_long:1 lltx read_mostly HARD_TX_LOCK,HARD_TX_TRYLOCK,HARD_TX_UNLOCK(tx) -unsigned long:1 netmem_tx:1; read_mostly +unsigned_long:2 netmem_tx:2; read_mostly char name[16] struct netdev_name_node* name_node struct dev_ifalias* ifalias diff --git a/Documentation/networking/netmem.rst b/Documentation/networking/netmem.rst index b63aded46337..217869d1108d 100644 --- a/Documentation/networking/netmem.rst +++ b/Documentation/networking/netmem.rst @@ -95,4 +95,10 @@ Driver TX Requirements netdev@, or reach out to the maintainers and/or almasrymina@google.com for help adding the netmem API. -2. Driver should declare support by setting `netdev->netmem_tx = true` +2. Driver should declare support by setting `netdev->netmem_tx` to the + appropriate mode: + + - `NETMEM_TX_DMA`: for physical devices that perform DMA. + + - `NETMEM_TX_NO_DMA`: for virtual or passthrough devices that do + not DMA, but still support handling of netmem-backed skbs. diff --git a/Documentation/translations/zh_CN/networking/netmem.rst b/Documentation/translations/zh_CN/networking/netmem.rst index fe351a240f02..320f3eacf51b 100644 --- a/Documentation/translations/zh_CN/networking/netmem.rst +++ b/Documentation/translations/zh_CN/networking/netmem.rst @@ -89,4 +89,9 @@ dma-mapping API 去处理。 使用某个还不存在的 netmem API,你可以自行添加并提交到 netdev@,也可以联系维护 人员或者发送邮件至 almasrymina@google.com 寻求帮助。 -2. 驱动程序应通过设置 netdev->netmem_tx = true 来表明自身支持 netmem 功能。 +2. 驱动程序应将 `netdev->netmem_tx` 设置为适当的模式: + + - `NETMEM_TX_DMA`:适用于执行 DMA 的物理设备。 + + - `NETMEM_TX_NO_DMA`:适用于不执行 DMA 的虚拟或透传设备,但仍支持 + 处理 netmem 支持的 skb。 diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 945a86696f2f..d4f93e62f583 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -17123,7 +17123,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops_unsupp; if (BNXT_SUPPORTS_QUEUE_API(bp)) dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops; - dev->netmem_tx = true; + dev->netmem_tx = NETMEM_TX_DMA; rc = register_netdev(dev); if (rc) diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 00750643e614..e4d78ae52daf 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -2894,7 +2894,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto abort_with_wq; if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) - dev->netmem_tx = true; + dev->netmem_tx = NETMEM_TX_DMA; err = register_netdev(dev); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index a904e468c197..c5d26c6829a0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -5963,7 +5963,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->netmem_tx = true; + netdev->netmem_tx = NETMEM_TX_DMA; netif_set_tso_max_size(netdev, GSO_MAX_SIZE); mlx5e_set_xdp_feature(priv); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 4dea2bb58d2f..f99ca551c1ce 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -752,7 +752,7 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) netdev->netdev_ops = &fbnic_netdev_ops; netdev->stat_ops = &fbnic_stat_ops; netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops; - netdev->netmem_tx = true; + netdev->netmem_tx = NETMEM_TX_DMA; fbnic_set_ethtool_ops(netdev); diff --git a/drivers/net/netkit.c b/drivers/net/netkit.c index 5e2eecc3165d..0ad6a806d7d5 100644 --- a/drivers/net/netkit.c +++ b/drivers/net/netkit.c @@ -466,6 +466,7 @@ static void netkit_setup(struct net_device *dev) dev->priv_flags |= IFF_NO_QUEUE; dev->priv_flags |= IFF_DISABLE_NETPOLL; dev->lltx = true; + dev->netmem_tx = NETMEM_TX_NO_DMA; dev->netdev_ops = &netkit_netdev_ops; dev->ethtool_ops = &netkit_ethtool_ops; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e7af71491a47..bf3dd9b2c1a7 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1794,6 +1794,12 @@ enum netdev_stat_type { NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */ }; +enum netmem_tx_mode { + NETMEM_TX_NONE, /* no netmem TX support */ + NETMEM_TX_DMA, /* DMA-capable netmem TX (real HW) */ + NETMEM_TX_NO_DMA, /* no DMA, e.g. passthrough for virtual devs */ +}; + enum netdev_reg_state { NETREG_UNINITIALIZED = 0, NETREG_REGISTERED, /* completed register_netdevice */ @@ -1815,7 +1821,7 @@ enum netdev_reg_state { * @lltx: device supports lockless Tx. Deprecated for real HW * drivers. Mainly used by logical interfaces, such as * bonding and tunnels - * @netmem_tx: device support netmem_tx. + * @netmem_tx: device netmem TX mode * * @name: This is the first field of the "visible" part of this structure * (i.e. as seen by users in the "Space.c" file). It is the name @@ -2138,7 +2144,7 @@ struct net_device { struct_group(priv_flags_fast, unsigned long priv_flags:32; unsigned long lltx:1; - unsigned long netmem_tx:1; + unsigned long netmem_tx:2; ); const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; diff --git a/net/core/dev.c b/net/core/dev.c index 267cf3854395..26ac8eb9b259 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3993,10 +3993,11 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, struct skb_shared_info *shinfo; struct net_iov *niov; - if (likely(skb_frags_readable(skb))) + if (likely(skb_frags_readable(skb) || + dev->netmem_tx == NETMEM_TX_NO_DMA)) goto out; - if (!dev->netmem_tx) + if (dev->netmem_tx == NETMEM_TX_NONE) goto out_free; shinfo = skb_shinfo(skb); diff --git a/net/core/devmem.c b/net/core/devmem.c index 468344739db2..893643909f6a 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -181,7 +181,7 @@ int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, } struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, struct netdev_nl_sock *priv, @@ -212,6 +212,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, } binding->dev = dev; + binding->vdev = vdev; xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); err = percpu_ref_init(&binding->ref, @@ -396,7 +397,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, */ dst_dev = dst_dev_rcu(dst); if (unlikely(!dst_dev) || - unlikely(dst_dev != READ_ONCE(binding->dev))) { + unlikely(dst_dev != READ_ONCE(binding->dev) && + dst_dev != READ_ONCE(binding->vdev))) { err = -ENODEV; goto out_unlock; } diff --git a/net/core/devmem.h b/net/core/devmem.h index 1c5c18581fcb..3852a56036cb 100644 --- a/net/core/devmem.h +++ b/net/core/devmem.h @@ -19,7 +19,13 @@ struct net_devmem_dmabuf_binding { struct dma_buf *dmabuf; struct dma_buf_attachment *attachment; struct sg_table *sgt; + /* Physical NIC that does the actual DMA for this binding. */ struct net_device *dev; + /* Opaque cookie identifying the virtual device (e.g. netkit) the user + * called bind-tx on. Used only for pointer comparison. Never + * dereferenced. + */ + void *vdev; struct gen_pool *chunk_pool; /* Protect dev */ struct mutex lock; @@ -84,7 +90,7 @@ struct dmabuf_genpool_chunk_owner { void __net_devmem_dmabuf_binding_free(struct work_struct *wq); struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, struct netdev_nl_sock *priv, @@ -165,7 +171,7 @@ static inline void net_devmem_put_net_iov(struct net_iov *niov) } static inline struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index b8f6076d8007..b4d48f3672a5 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -1077,7 +1077,7 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) goto err_rxq_bitmap; } - binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_FROM_DEVICE, + binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE, dmabuf_fd, priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); @@ -1119,9 +1119,43 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) return err; } +/* Find the DMA-capable device for a netmem TX binding. + * + * For NETMEM_TX_DMA devices, return the device itself. + * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying + * physical device and return it. + */ +static struct net_device * +netdev_find_netmem_tx_dev(struct net_device *dev) +{ + struct netdev_rx_queue *lease_rxq; + struct net_device *phys_dev; + int i; + + if (dev->netmem_tx == NETMEM_TX_DMA) + return dev; + + if (dev->netmem_tx != NETMEM_TX_NO_DMA) + return NULL; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease); + if (!lease_rxq) + continue; + + phys_dev = lease_rxq->dev; + if (netif_device_present(phys_dev) && + phys_dev->netmem_tx == NETMEM_TX_DMA) + return phys_dev; + } + + return NULL; +} + int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) { struct net_devmem_dmabuf_binding *binding; + struct net_device *bind_dev; struct netdev_nl_sock *priv; struct net_device *netdev; struct device *dma_dev; @@ -1164,29 +1198,48 @@ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) goto err_unlock_netdev; } - if (!netdev->netmem_tx) { + if (netdev->netmem_tx == NETMEM_TX_NONE) { err = -EOPNOTSUPP; NL_SET_ERR_MSG(info->extack, "Driver does not support netmem TX"); goto err_unlock_netdev; } - dma_dev = netdev_queue_get_dma_dev(netdev, 0, NETDEV_QUEUE_TYPE_TX); - binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_TO_DEVICE, - dmabuf_fd, priv, info->extack); + bind_dev = netdev_find_netmem_tx_dev(netdev); + if (!bind_dev) { + err = -EOPNOTSUPP; + NL_SET_ERR_MSG(info->extack, + "No DMA-capable device found for netmem TX"); + goto err_unlock_netdev; + } + + if (bind_dev != netdev) + netdev_lock(bind_dev); + + dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX); + + binding = net_devmem_bind_dmabuf(bind_dev, + bind_dev != netdev ? netdev : NULL, + dma_dev, DMA_TO_DEVICE, dmabuf_fd, + priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); - goto err_unlock_netdev; + goto err_unlock_bind_dev; } nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); genlmsg_end(rsp, hdr); + if (bind_dev != netdev) + netdev_unlock(bind_dev); netdev_unlock(netdev); mutex_unlock(&priv->lock); return genlmsg_reply(rsp, info); +err_unlock_bind_dev: + if (bind_dev != netdev) + netdev_unlock(bind_dev); err_unlock_netdev: netdev_unlock(netdev); err_unlock_sock: diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile index 82809d5b2478..c7a1206880ea 100644 --- a/tools/testing/selftests/drivers/net/hw/Makefile +++ b/tools/testing/selftests/drivers/net/hw/Makefile @@ -35,6 +35,7 @@ TEST_PROGS = \ irq.py \ loopback.sh \ nic_timestamp.py \ + nk_devmem.py \ nk_netns.py \ nk_qlease.py \ ntuple.py \ @@ -52,6 +53,7 @@ TEST_PROGS = \ # TEST_FILES := \ + devmem_lib.py \ ethtool_lib.sh \ # diff --git a/tools/testing/selftests/drivers/net/hw/devmem.py b/tools/testing/selftests/drivers/net/hw/devmem.py index ee863e90d1e0..031cf9905f65 100755 --- a/tools/testing/selftests/drivers/net/hw/devmem.py +++ b/tools/testing/selftests/drivers/net/hw/devmem.py @@ -2,91 +2,40 @@ # SPDX-License-Identifier: GPL-2.0 from os import path -from lib.py import ksft_run, ksft_exit -from lib.py import ksft_eq, KsftSkipEx +from devmem_lib import setup_test, run_rx, run_tx, run_tx_chunks, run_rx_hds +from lib.py import ksft_run, ksft_exit, ksft_disruptive from lib.py import NetDrvEpEnv -from lib.py import bkg, cmd, rand_port, wait_port_listen -from lib.py import ksft_disruptive - - -def require_devmem(cfg): - if not hasattr(cfg, "_devmem_probed"): - probe_command = f"{cfg.bin_local} -f {cfg.ifname}" - cfg._devmem_supported = cmd(probe_command, fail=False, shell=True).ret == 0 - cfg._devmem_probed = True - - if not cfg._devmem_supported: - raise KsftSkipEx("Test requires devmem support") @ksft_disruptive def check_rx(cfg) -> None: - require_devmem(cfg) - - port = rand_port() - socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" - listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} -c {cfg.remote_addr} -v 7" - - with bkg(listen_cmd, exit_wait=True) as ncdevmem: - wait_port_listen(port) - cmd(f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | \ - head -c 1K | {socat}", host=cfg.remote, shell=True) - - ksft_eq(ncdevmem.ret, 0) + """Run the devmem RX test.""" + run_rx(cfg) @ksft_disruptive def check_tx(cfg) -> None: - require_devmem(cfg) - - port = rand_port() - listen_cmd = f"socat -U - TCP{cfg.addr_ipver}-LISTEN:{port}" - - with bkg(listen_cmd, host=cfg.remote, exit_wait=True) as socat: - wait_port_listen(port, host=cfg.remote) - cmd(f"echo -e \"hello\\nworld\"| {cfg.bin_local} -f {cfg.ifname} -s {cfg.remote_addr} -p {port}", shell=True) - - ksft_eq(socat.stdout.strip(), "hello\nworld") + """Run the devmem TX test.""" + run_tx(cfg) @ksft_disruptive def check_tx_chunks(cfg) -> None: - require_devmem(cfg) - - port = rand_port() - listen_cmd = f"socat -U - TCP{cfg.addr_ipver}-LISTEN:{port}" - - with bkg(listen_cmd, host=cfg.remote, exit_wait=True) as socat: - wait_port_listen(port, host=cfg.remote) - cmd(f"echo -e \"hello\\nworld\"| {cfg.bin_local} -f {cfg.ifname} -s {cfg.remote_addr} -p {port} -z 3", shell=True) - - ksft_eq(socat.stdout.strip(), "hello\nworld") + """Run the devmem TX chunking test.""" + run_tx_chunks(cfg) def check_rx_hds(cfg) -> None: - """Test HDS splitting across payload sizes.""" - require_devmem(cfg) - - for size in [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]: - port = rand_port() - listen_cmd = f"{cfg.bin_local} -L -l -f {cfg.ifname} -s {cfg.addr} -p {port}" - - with bkg(listen_cmd, exit_wait=True) as ncdevmem: - wait_port_listen(port) - cmd(f"dd if=/dev/zero bs={size} count=1 2>/dev/null | " + - f"socat -b {size} -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},nodelay", - host=cfg.remote, shell=True) - - ksft_eq(ncdevmem.ret, 0, f"HDS failed for payload size {size}") + """Run the HDS test.""" + run_rx_hds(cfg) def main() -> None: + """Run the devmem test cases.""" with NetDrvEpEnv(__file__) as cfg: - cfg.bin_local = path.abspath(path.dirname(__file__) + "/ncdevmem") - cfg.bin_remote = cfg.remote.deploy(cfg.bin_local) - + setup_test(cfg, path.abspath(path.dirname(__file__) + "/ncdevmem")) ksft_run([check_rx, check_tx, check_tx_chunks, check_rx_hds], - args=(cfg, )) + args=(cfg,)) ksft_exit() diff --git a/tools/testing/selftests/drivers/net/hw/devmem_lib.py b/tools/testing/selftests/drivers/net/hw/devmem_lib.py new file mode 100644 index 000000000000..0921ff03eb81 --- /dev/null +++ b/tools/testing/selftests/drivers/net/hw/devmem_lib.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-2.0 +"""Shared helpers for devmem TCP selftests.""" + +import re + +from lib.py import (bkg, cmd, defer, ethtool, rand_port, wait_port_listen, + ksft_eq, KsftSkipEx, NetNSEnter, EthtoolFamily, + NetdevFamily) + + +def require_devmem(cfg): + """Probe ncdevmem on cfg.ifname and SKIP the test if devmem isn't supported.""" + if not hasattr(cfg, "devmem_probed"): + probe_command = f"{cfg.bin_local} -f {cfg.ifname}" + cfg.devmem_supported = cmd(probe_command, fail=False, shell=True).ret == 0 + cfg.devmem_probed = True + + if not cfg.devmem_supported: + raise KsftSkipEx("Test requires devmem support") + + +def configure_nic(cfg): + """Channels, rings, RSS, queue lease for netkit devmem.""" + if not hasattr(cfg, 'netns'): + return + + cfg.require_ipver('6') + ethnl = EthtoolFamily() + + channels = ethnl.channels_get({'header': {'dev-index': cfg.ifindex}}) + channels = channels['combined-count'] + if channels < 2: + raise KsftSkipEx( + 'Test requires NETIF with at least 2 combined channels' + ) + + rings = ethnl.rings_get({'header': {'dev-index': cfg.ifindex}}) + orig_rx_rings = rings['rx'] + orig_hds_thresh = rings.get('hds-thresh', 0) + orig_data_split = rings.get('tcp-data-split', 'unknown') + + ethnl.rings_set({'header': {'dev-index': cfg.ifindex}, + 'tcp-data-split': 'enabled', + 'hds-thresh': 0, + 'rx': min(64, orig_rx_rings)}) + defer(ethnl.rings_set, {'header': {'dev-index': cfg.ifindex}, + 'tcp-data-split': orig_data_split, + 'hds-thresh': orig_hds_thresh, + 'rx': orig_rx_rings}) + + cfg.src_queue = channels - 1 + ethtool(f"-X {cfg.ifname} equal {cfg.src_queue}") + defer(ethtool, f"-X {cfg.ifname} default") + + if not hasattr(cfg, 'nk_queue'): + with NetNSEnter(str(cfg.netns)): + netdevnl = NetdevFamily() + lease_result = netdevnl.queue_create({ + "ifindex": cfg.nk_guest_ifindex, + "type": "rx", + "lease": { + "ifindex": cfg.ifindex, + "queue": {"id": cfg.src_queue, "type": "rx"}, + "netns-id": 0, + }, + }) + cfg.nk_queue = lease_result['id'] + + +def set_flow_rule(cfg, port): + """Install a flow rule steering to src_queue and return the flow rule ID.""" + output = ethtool( + f"-N {cfg.ifname} flow-type tcp6 dst-port {port}" + f" action {cfg.src_queue}" + ).stdout + return int(re.search(r'ID (\d+)', output).group(1)) + + +def ncdevmem_rx(cfg, port, verify=True, fail_on_linear=False, flow_steer=False): + """Build the ncdevmem RX listener command.""" + if hasattr(cfg, 'netns'): + flow_rule_id = set_flow_rule(cfg, port) + defer(ethtool, f"-N {cfg.ifname} delete {flow_rule_id}") + + ifname = cfg.nk_guest_ifname + addr = cfg.nk_guest_ipv6 + extras = [f"-t {cfg.nk_queue}", "-q 1", "-n"] + else: + ifname = cfg.ifname + addr = cfg.addr + extras = [] + if flow_steer: + extras.append(f"-c {cfg.remote_addr}") + + if verify: + extras.append("-v 7") + if fail_on_linear: + extras.append("-L") + + parts = [cfg.bin_local, "-l", f"-f {ifname}", f"-s {addr}", + f"-p {port}", *extras] + return " ".join(parts) + + +def ncdevmem_tx(cfg, port, chunk_size=0): + """Build the ncdevmem TX send command.""" + if hasattr(cfg, 'netns'): + ifname = cfg.nk_guest_ifname + addr = cfg.remote_addr_v['6'] + extras = ["-t 0", "-q 1", "-n"] + else: + ifname = cfg.ifname + addr = cfg.remote_addr + extras = [] + + if chunk_size: + extras.append(f"-z {chunk_size}") + + parts = [cfg.bin_local, f"-f {ifname}", f"-s {addr}", + f"-p {port}", *extras] + return " ".join(parts) + + +def socat_send(cfg, port, buf_size=0): + """Socat command for sending to the devmem listener. + + When buf_size > 0, force one TCP segment per write of exactly that size by + setting socat's buffer (-b) and disabling Nagle (TCP_NODELAY). + """ + proto = f"TCP{cfg.addr_ipver}" + + if hasattr(cfg, 'netns'): + addr = f"[{cfg.nk_guest_ipv6}]" + else: + addr = cfg.baddr + + suffix = f",bind={cfg.remote_baddr}:{port}" + + buf = "" + if buf_size: + buf = f"-b {buf_size}" + suffix += ",nodelay" + + return f"socat {buf} -u - {proto}:{addr}:{port}{suffix}" + + +def socat_listen(cfg, port): + """Socat listen command for TX tests.""" + return f"socat -U - TCP{cfg.addr_ipver}-LISTEN:{port}" + + +def setup_test(cfg, bin_local): + """Stash the local ncdevmem path on cfg and deploy it to the remote.""" + cfg.bin_local = bin_local + cfg.bin_remote = cfg.remote.deploy(cfg.bin_local) + + +def run_rx(cfg): + """Run the devmem RX test.""" + require_devmem(cfg) + configure_nic(cfg) + port = rand_port() + socat = socat_send(cfg, port) + data_pipe = (f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | head -c 1K" + f" | {socat}") + netns = getattr(cfg, "netns", None) + + listen_cmd = ncdevmem_rx(cfg, port, flow_steer=not hasattr(cfg, 'netns')) + with bkg(listen_cmd, exit_wait=True, ns=netns) as ncdevmem: + wait_port_listen(port, proto="tcp", ns=netns) + cmd(data_pipe, host=cfg.remote, shell=True) + ksft_eq(ncdevmem.ret, 0) + + +def run_tx(cfg): + """Run the devmem TX test.""" + require_devmem(cfg) + configure_nic(cfg) + netns = getattr(cfg, "netns", None) + port = rand_port() + tx_cmd = ncdevmem_tx(cfg, port) + listen_cmd = socat_listen(cfg, port) + + with bkg(listen_cmd, host=cfg.remote, exit_wait=True) as socat: + wait_port_listen(port, host=cfg.remote) + cmd(f"bash -c 'echo -e \"hello\\nworld\" | {tx_cmd}'", ns=netns, shell=True) + ksft_eq(socat.stdout.strip(), "hello\nworld") + + +def run_tx_chunks(cfg): + """Run the devmem TX chunking test.""" + require_devmem(cfg) + configure_nic(cfg) + netns = getattr(cfg, "netns", None) + port = rand_port() + tx_cmd = ncdevmem_tx(cfg, port, chunk_size=3) + listen_cmd = socat_listen(cfg, port) + + with bkg(listen_cmd, host=cfg.remote, exit_wait=True) as socat: + wait_port_listen(port, host=cfg.remote) + cmd(f"bash -c 'echo -e \"hello\\nworld\" | {tx_cmd}'", ns=netns, shell=True) + ksft_eq(socat.stdout.strip(), "hello\nworld") + + +def run_rx_hds(cfg): + """Run the HDS test by running devmem RX across a segment size sweep.""" + require_devmem(cfg) + configure_nic(cfg) + netns = getattr(cfg, "netns", None) + + for size in [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]: + port = rand_port() + + listen_cmd = ncdevmem_rx(cfg, port, verify=False, + fail_on_linear=True) + socat = socat_send(cfg, port, buf_size=size) + + with bkg(listen_cmd, exit_wait=True, ns=netns) as ncdevmem: + wait_port_listen(port, proto="tcp", ns=netns) + cmd(f"dd if=/dev/zero bs={size} count=1 2>/dev/null | " + f"{socat}", host=cfg.remote, shell=True) + ksft_eq(ncdevmem.ret, 0, f"HDS failed for payload size {size}") diff --git a/tools/testing/selftests/drivers/net/hw/ncdevmem.c b/tools/testing/selftests/drivers/net/hw/ncdevmem.c index e098d6534c3c..d96e8a3b5a65 100644 --- a/tools/testing/selftests/drivers/net/hw/ncdevmem.c +++ b/tools/testing/selftests/drivers/net/hw/ncdevmem.c @@ -93,6 +93,7 @@ static char *port; static size_t do_validation; static int start_queue = -1; static int num_queues = -1; +static int skip_config; static char *ifname; static unsigned int ifindex; static unsigned int dmabuf_id; @@ -828,7 +829,7 @@ static struct netdev_queue_id *create_queues(void) static int do_server(struct memory_buffer *mem) { - struct ethtool_rings_get_rsp *ring_config; + struct ethtool_rings_get_rsp *ring_config = NULL; char ctrl_data[sizeof(int) * 20000]; size_t non_page_aligned_frags = 0; struct sockaddr_in6 client_addr; @@ -851,27 +852,29 @@ static int do_server(struct memory_buffer *mem) return -1; } - ring_config = get_ring_config(); - if (!ring_config) { - pr_err("Failed to get current ring configuration"); - return -1; - } + if (!skip_config) { + ring_config = get_ring_config(); + if (!ring_config) { + pr_err("Failed to get current ring configuration"); + return -1; + } - if (configure_headersplit(ring_config, 1)) { - pr_err("Failed to enable TCP header split"); - goto err_free_ring_config; - } + if (configure_headersplit(ring_config, 1)) { + pr_err("Failed to enable TCP header split"); + goto err_free_ring_config; + } - /* Configure RSS to divert all traffic from our devmem queues */ - if (configure_rss()) { - pr_err("Failed to configure rss"); - goto err_reset_headersplit; - } + /* Configure RSS to divert all traffic from our devmem queues */ + if (configure_rss()) { + pr_err("Failed to configure rss"); + goto err_reset_headersplit; + } - /* Flow steer our devmem flows to start_queue */ - if (configure_flow_steering(&server_sin)) { - pr_err("Failed to configure flow steering"); - goto err_reset_rss; + /* Flow steer our devmem flows to start_queue */ + if (configure_flow_steering(&server_sin)) { + pr_err("Failed to configure flow steering"); + goto err_reset_rss; + } } if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) { @@ -1052,13 +1055,17 @@ static int do_server(struct memory_buffer *mem) err_unbind: ynl_sock_destroy(ys); err_reset_flow_steering: - reset_flow_steering(); + if (!skip_config) + reset_flow_steering(); err_reset_rss: - reset_rss(); + if (!skip_config) + reset_rss(); err_reset_headersplit: - restore_ring_config(ring_config); + if (!skip_config) + restore_ring_config(ring_config); err_free_ring_config: - ethtool_rings_get_rsp_free(ring_config); + if (!skip_config) + ethtool_rings_get_rsp_free(ring_config); return err; } @@ -1404,7 +1411,7 @@ int main(int argc, char *argv[]) int is_server = 0, opt; int ret, err = 1; - while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:")) != -1) { + while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:n")) != -1) { switch (opt) { case 'L': fail_on_linear = true; @@ -1436,6 +1443,9 @@ int main(int argc, char *argv[]) case 'z': max_chunk = atoi(optarg); break; + case 'n': + skip_config = 1; + break; case '?': fprintf(stderr, "unknown option: %c\n", optopt); break; diff --git a/tools/testing/selftests/drivers/net/hw/nk_devmem.py b/tools/testing/selftests/drivers/net/hw/nk_devmem.py new file mode 100755 index 000000000000..300ed2a70ab4 --- /dev/null +++ b/tools/testing/selftests/drivers/net/hw/nk_devmem.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Test devmem TCP with netkit.""" + +import os +from devmem_lib import setup_test, run_rx, run_tx, run_tx_chunks, run_rx_hds +from lib.py import ksft_run, ksft_exit, ksft_disruptive +from lib.py import NetDrvContEnv + + +@ksft_disruptive +def check_nk_rx(cfg) -> None: + """Run the devmem RX test through netkit.""" + run_rx(cfg) + + +@ksft_disruptive +def check_nk_tx(cfg) -> None: + """Run the devmem TX test through netkit.""" + run_tx(cfg) + + +@ksft_disruptive +def check_nk_tx_chunks(cfg) -> None: + """Run the devmem TX chunking test through netkit.""" + run_tx_chunks(cfg) + + +def check_nk_rx_hds(cfg) -> None: + """Run the HDS test through netkit.""" + run_rx_hds(cfg) + + +def main() -> None: + """Run the netkit devmem test cases.""" + with NetDrvContEnv(__file__, rxqueues=2, primary_rx_redirect=True) as cfg: + setup_test(cfg, + os.path.join(os.path.dirname(os.path.abspath(__file__)), + "ncdevmem")) + ksft_run([check_nk_rx, check_nk_tx, check_nk_tx_chunks, + check_nk_rx_hds], args=(cfg,)) + ksft_exit() + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/net/hw/nk_primary_rx_redirect.bpf.c b/tools/testing/selftests/drivers/net/hw/nk_primary_rx_redirect.bpf.c new file mode 100644 index 000000000000..46ff494b23de --- /dev/null +++ b/tools/testing/selftests/drivers/net/hw/nk_primary_rx_redirect.bpf.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#define ctx_ptr(field) ((void *)(long)(field)) + +volatile __u32 phys_ifindex; + +SEC("tc/ingress") +int nk_primary_rx_redirect(struct __sk_buff *skb) +{ + void *data_end = ctx_ptr(skb->data_end); + void *data = ctx_ptr(skb->data); + struct ethhdr *eth; + struct ipv6hdr *ip6h; + + eth = data; + if ((void *)(eth + 1) > data_end) + return TC_ACT_OK; + + if (eth->h_proto != bpf_htons(ETH_P_IPV6)) + return TC_ACT_OK; + + ip6h = data + sizeof(struct ethhdr); + if ((void *)(ip6h + 1) > data_end) + return TC_ACT_OK; + + if (ip6h->nexthdr == IPPROTO_ICMPV6) + return TC_ACT_OK; + + return bpf_redirect_neigh(phys_ifindex, NULL, 0, 0); +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/drivers/net/hw/nk_qlease.py b/tools/testing/selftests/drivers/net/hw/nk_qlease.py index aa83dc321328..139a91ebd229 100755 --- a/tools/testing/selftests/drivers/net/hw/nk_qlease.py +++ b/tools/testing/selftests/drivers/net/hw/nk_qlease.py @@ -71,7 +71,7 @@ def test_iou_zcrx(cfg) -> None: flow_rule_id = set_flow_rule(cfg) defer(ethtool, f"-N {cfg.ifname} delete {flow_rule_id}") - rx_cmd = f"ip netns exec {cfg.netns.name} {cfg.bin_local} -s -p {cfg.port} -i {cfg._nk_guest_ifname} -q {cfg.nk_queue}" + rx_cmd = f"ip netns exec {cfg.netns.name} {cfg.bin_local} -s -p {cfg.port} -i {cfg.nk_guest_ifname} -q {cfg.nk_queue}" tx_cmd = f"{cfg.bin_remote} -c -h {cfg.nk_guest_ipv6} -p {cfg.port} -l 12840" with bkg(rx_cmd, exit_wait=True): wait_port_listen(cfg.port, proto="tcp", ns=cfg.netns) @@ -128,7 +128,7 @@ def test_attach_xdp_with_mp(cfg) -> None: netdevnl = NetdevFamily() - rx_cmd = f"ip netns exec {cfg.netns.name} {cfg.bin_local} -s -p {cfg.port} -i {cfg._nk_guest_ifname} -q {cfg.nk_queue}" + rx_cmd = f"ip netns exec {cfg.netns.name} {cfg.bin_local} -s -p {cfg.port} -i {cfg.nk_guest_ifname} -q {cfg.nk_queue}" with bkg(rx_cmd): wait_port_listen(cfg.port, proto="tcp", ns=cfg.netns) @@ -178,7 +178,7 @@ def test_destroy(cfg) -> None: ethtool(f"-X {cfg.ifname} equal {cfg.src_queue}") defer(ethtool, f"-X {cfg.ifname} default") - rx_cmd = f"ip netns exec {cfg.netns.name} {cfg.bin_local} -s -p {cfg.port} -i {cfg._nk_guest_ifname} -q {cfg.nk_queue}" + rx_cmd = f"ip netns exec {cfg.netns.name} {cfg.bin_local} -s -p {cfg.port} -i {cfg.nk_guest_ifname} -q {cfg.nk_queue}" rx_proc = cmd(rx_cmd, background=True) wait_port_listen(cfg.port, proto="tcp", ns=cfg.netns) @@ -196,7 +196,7 @@ def test_destroy(cfg) -> None: ip(f"link del dev {cfg._nk_host_ifname}") kill_timer.join() cfg._nk_host_ifname = None - cfg._nk_guest_ifname = None + cfg.nk_guest_ifname = None queue_info = netdevnl.queue_get( {"ifindex": cfg.ifindex, "id": cfg.src_queue, "type": "rx"} diff --git a/tools/testing/selftests/drivers/net/lib/py/env.py b/tools/testing/selftests/drivers/net/lib/py/env.py index 24ce122abd9c..ef317aef3a0a 100644 --- a/tools/testing/selftests/drivers/net/lib/py/env.py +++ b/tools/testing/selftests/drivers/net/lib/py/env.py @@ -2,6 +2,7 @@ import ipaddress import os +import sys import time import json from pathlib import Path @@ -336,15 +337,18 @@ class NetDrvContEnv(NetDrvEpEnv): +---------------+ """ - def __init__(self, src_path, rxqueues=1, **kwargs): + def __init__(self, src_path, rxqueues=1, primary_rx_redirect=False, **kwargs): self.netns = None self._nk_host_ifname = None - self._nk_guest_ifname = None + self.nk_guest_ifname = None self._tc_clsact_added = False self._tc_attached = False + self._primary_rx_redirect_attached = False + self._primary_rx_redirect_clsact_added = False self._bpf_prog_pref = None self._bpf_prog_id = None self._init_ns_attached = False + self._remote_route_added = False self._old_fwd = None self._old_accept_ra = None @@ -390,14 +394,24 @@ class NetDrvContEnv(NetDrvEpEnv): netkit_links.sort(key=lambda x: x['ifindex']) self._nk_host_ifname = netkit_links[1]['ifname'] - self._nk_guest_ifname = netkit_links[0]['ifname'] + self.nk_guest_ifname = netkit_links[0]['ifname'] self.nk_host_ifindex = netkit_links[1]['ifindex'] self.nk_guest_ifindex = netkit_links[0]['ifindex'] self._setup_ns() self._attach_bpf() + if primary_rx_redirect: + self._attach_primary_rx_redirect_bpf() def __del__(self): + if self._primary_rx_redirect_attached: + cmd(f"tc filter del dev {self._nk_host_ifname} ingress", fail=False) + self._primary_rx_redirect_attached = False + + if self._primary_rx_redirect_clsact_added: + cmd(f"tc qdisc del dev {self._nk_host_ifname} clsact", fail=False) + self._primary_rx_redirect_clsact_added = False + if self._tc_attached: cmd(f"tc filter del dev {self.ifname} ingress pref {self._bpf_prog_pref}") self._tc_attached = False @@ -406,10 +420,15 @@ class NetDrvContEnv(NetDrvEpEnv): cmd(f"tc qdisc del dev {self.ifname} clsact") self._tc_clsact_added = False + if self._remote_route_added: + cmd(f"ip -6 route del {self.nk_guest_ipv6}/128", + host=self.remote, fail=False) + self._remote_route_added = False + if self._nk_host_ifname: cmd(f"ip link del dev {self._nk_host_ifname}") self._nk_host_ifname = None - self._nk_guest_ifname = None + self.nk_guest_ifname = None if self._init_ns_attached: cmd("ip netns del init", fail=False) @@ -448,24 +467,30 @@ class NetDrvContEnv(NetDrvEpEnv): cmd("ip netns attach init 1") self._init_ns_attached = True ip("netns set init 0", ns=self.netns) - ip(f"link set dev {self._nk_guest_ifname} netns {self.netns.name}") + ip(f"link set dev {self.nk_guest_ifname} netns {self.netns.name}") ip(f"link set dev {self._nk_host_ifname} up") ip(f"-6 addr add fe80::1/64 dev {self._nk_host_ifname} nodad") ip(f"-6 route add {self.nk_guest_ipv6}/128 via fe80::2 dev {self._nk_host_ifname}") ip("link set lo up", ns=self.netns) - ip(f"link set dev {self._nk_guest_ifname} up", ns=self.netns) - ip(f"-6 addr add fe80::2/64 dev {self._nk_guest_ifname}", ns=self.netns) - ip(f"-6 addr add {self.nk_guest_ipv6}/64 dev {self._nk_guest_ifname} nodad", ns=self.netns) - ip(f"-6 route add default via fe80::1 dev {self._nk_guest_ifname}", ns=self.netns) + ip(f"link set dev {self.nk_guest_ifname} up", ns=self.netns) + ip(f"-6 addr add fe80::2/64 dev {self.nk_guest_ifname}", ns=self.netns) + ip(f"-6 addr add {self.nk_guest_ipv6}/64 dev {self.nk_guest_ifname} nodad", ns=self.netns) + ip(f"-6 route add default via fe80::1 dev {self.nk_guest_ifname}", ns=self.netns) - def _tc_ensure_clsact(self): - qdisc = json.loads(cmd(f"tc -j qdisc show dev {self.ifname}").stdout) + def _tc_ensure_clsact(self, ifname=None): + """Ensure a clsact qdisc exists on @ifname. + + Returns True if this call added the qdisc, otherwise returns False. + """ + if ifname is None: + ifname = self.ifname + qdisc = json.loads(cmd(f"tc -j qdisc show dev {ifname}").stdout) for q in qdisc: if q['kind'] == 'clsact': - return - cmd(f"tc qdisc add dev {self.ifname} clsact") - self._tc_clsact_added = True + return False + cmd(f"tc qdisc add dev {ifname} clsact") + return True def _get_bpf_prog_ids(self): filters = json.loads(cmd(f"tc -j filter show dev {self.ifname} ingress").stdout) @@ -476,28 +501,28 @@ class NetDrvContEnv(NetDrvEpEnv): return (bpf['pref'], bpf['options']['prog']['id']) raise Exception("Failed to get BPF prog ID") + def _find_bss_map_id(self, prog_id): + """Find the .bss map ID for a loaded BPF program.""" + prog_info = bpftool(f"prog show id {prog_id}", json=True) + for map_id in prog_info.get("map_ids", []): + map_info = bpftool(f"map show id {map_id}", json=True) + if map_info.get("name", "").endswith("bss"): + return map_id + raise Exception(f"Failed to find .bss map for prog {prog_id}") + def _attach_bpf(self): bpf_obj = self.test_dir / "nk_forward.bpf.o" if not bpf_obj.exists(): raise KsftSkipEx("BPF prog not found") - self._tc_ensure_clsact() + if self._tc_ensure_clsact(): + self._tc_clsact_added = True cmd(f"tc filter add dev {self.ifname} ingress bpf obj {bpf_obj}" " sec tc/ingress direct-action") self._tc_attached = True (self._bpf_prog_pref, self._bpf_prog_id) = self._get_bpf_prog_ids() - prog_info = bpftool(f"prog show id {self._bpf_prog_id}", json=True) - map_ids = prog_info.get("map_ids", []) - - bss_map_id = None - for map_id in map_ids: - map_info = bpftool(f"map show id {map_id}", json=True) - if map_info.get("name").endswith("bss"): - bss_map_id = map_id - - if bss_map_id is None: - raise Exception("Failed to find .bss map") + bss_map_id = self._find_bss_map_id(self._bpf_prog_id) ipv6_addr = ipaddress.IPv6Address(self.ipv6_prefix) ipv6_bytes = ipv6_addr.packed @@ -505,3 +530,36 @@ class NetDrvContEnv(NetDrvEpEnv): value = ipv6_bytes + ifindex_bytes value_hex = ' '.join(f'{b:02x}' for b in value) bpftool(f"map update id {bss_map_id} key hex 00 00 00 00 value hex {value_hex}") + + def _attach_primary_rx_redirect_bpf(self): + """Attach BPF redirect program on the primary netkit ingress.""" + bpf_obj = self.test_dir / "nk_primary_rx_redirect.bpf.o" + if not bpf_obj.exists(): + raise KsftSkipEx("Primary RX redirect BPF prog not found") + + if self._tc_ensure_clsact(self._nk_host_ifname): + self._primary_rx_redirect_clsact_added = True + cmd(f"tc filter add dev {self._nk_host_ifname} ingress" + f" bpf obj {bpf_obj} sec tc/ingress direct-action") + self._primary_rx_redirect_attached = True + + ip(f"-6 route add {self.nk_guest_ipv6}/128 via {self.addr_v['6']}", + host=self.remote) + self._remote_route_added = True + + filters = json.loads( + cmd(f"tc -j filter show dev {self._nk_host_ifname} ingress").stdout) + redirect_prog_id = None + for bpf in filters: + if 'options' not in bpf: + continue + if bpf['options']['bpf_name'].startswith('nk_primary_rx_redirect'): + redirect_prog_id = bpf['options']['prog']['id'] + break + if redirect_prog_id is None: + raise Exception("Failed to get primary RX redirect BPF prog ID") + + bss_map_id = self._find_bss_map_id(redirect_prog_id) + phys_ifindex_bytes = self.ifindex.to_bytes(4, byteorder=sys.byteorder) + value_hex = ' '.join(f'{b:02x}' for b in phys_ifindex_bytes) + bpftool(f"map update id {bss_map_id} key hex 00 00 00 00 value hex {value_hex}")