diff --git a/Documentation/networking/tuntap.rst b/Documentation/networking/tuntap.rst index 4d7087f727be..56c9dc7b96af 100644 --- a/Documentation/networking/tuntap.rst +++ b/Documentation/networking/tuntap.rst @@ -206,6 +206,38 @@ enable is true we enable it, otherwise we disable it:: return ioctl(fd, TUNSETQUEUE, (void *)&ifr); } +3.4 qdisc backpressure +---------------------- + +IFF_BACKPRESSURE can be set to enable qdisc backpressure. Without it, TX +drops occur when the internal ring buffer is full, so any attached qdisc +is effectively bypassed and applications only learn about congestion +through those drops. + +With it, the kernel stops the queue instead, letting the qdisc hold and +schedule packets, so its AQM, shaping and fairness actually apply. This +helps protocols like TCP, which cut throughput in reaction to packet +drops. With IFF_BACKPRESSURE, drops then only occur as a rare race. +Backpressure requires a qdisc to be attached and has no effect with +noqueue. + +The flag is a property of the TUN/TAP device rather than of the file +descriptor it was set on, so it applies to all queues of the device, +regardless of which process opened which queue. + +The flag can only be changed while the device has at most one queue. On a +multiqueue device that already has a second queue attached or detached, a +later TUNSETIFF succeeds but leaves the flag as it is. All the other +TUNSETIFF flags behave the same way. + +The txqueuelen can be reduced alongside this flag to further shift +buffering into the qdisc and reduce bufferbloat, at a possible +performance cost. + +When running multiple network streams in parallel through a single +TUN/TAP queue, the flag may reduce performance due to the extra overhead +of the backpressure mechanism. + Universal TUN/TAP device driver Frequently Asked Question ========================================================= diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 51e80000bd0e..ec90fef4a42f 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -98,7 +98,8 @@ static void tun_default_link_ksettings(struct net_device *dev, #define TUN_FASYNC IFF_ATTACH_QUEUE #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \ - IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS) + IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS | \ + IFF_BACKPRESSURE) #define GOODCOPY_LEN 128 @@ -145,6 +146,8 @@ struct tun_file { struct list_head next; struct tun_struct *detached; struct ptr_ring tx_ring; + /* Protected by tx_ring.consumer_lock */ + int cons_cnt; struct xdp_rxq_info xdp_rxq; }; @@ -585,11 +588,16 @@ static void __tun_detach(struct tun_file *tfile, bool clean) u16 index = tfile->queue_index; BUG_ON(index >= tun->numqueues); + spin_lock(&tfile->tx_ring.consumer_lock); rcu_assign_pointer(tun->tfiles[index], tun->tfiles[tun->numqueues - 1]); + spin_unlock(&tfile->tx_ring.consumer_lock); ntfile = rtnl_dereference(tun->tfiles[index]); + spin_lock(&ntfile->tx_ring.consumer_lock); ntfile->queue_index = index; ntfile->xdp_rxq.queue_index = index; + ntfile->cons_cnt = 0; + spin_unlock(&ntfile->tx_ring.consumer_lock); rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], NULL); @@ -606,6 +614,14 @@ static void __tun_detach(struct tun_file *tfile, bool clean) tun_flow_delete_by_queue(tun, tun->numqueues + 1); /* Drop read queue */ tun_queue_purge(tfile); + spin_lock_bh(&ntfile->tx_ring.consumer_lock); + spin_lock(&ntfile->tx_ring.producer_lock); + ntfile->cons_cnt = 0; + if (netif_running(tun->dev) && + __ptr_ring_empty(&ntfile->tx_ring)) + netif_wake_subqueue(tun->dev, index); + spin_unlock(&ntfile->tx_ring.producer_lock); + spin_unlock_bh(&ntfile->tx_ring.consumer_lock); tun_set_real_num_queues(tun); } else if (tfile->detached && clean) { tun = tun_enable_queue(tfile); @@ -687,6 +703,25 @@ static void tun_detach_all(struct net_device *dev) module_put(THIS_MODULE); } +static void tun_force_wake_queue(struct tun_struct *tun, + struct tun_file *tfile) +{ + /* Ensure that the producer can not stop the + * queue concurrently by taking locks. + */ + spin_lock_bh(&tfile->tx_ring.consumer_lock); + spin_lock(&tfile->tx_ring.producer_lock); + tfile->cons_cnt = 0; + /* Tested under the locks that tun_net_close() takes, so this can not + * undo its stop. tun_net_open() wakes the queues of a device that + * comes back up. + */ + if (netif_running(tun->dev)) + netif_wake_subqueue(tun->dev, tfile->queue_index); + spin_unlock(&tfile->tx_ring.producer_lock); + spin_unlock_bh(&tfile->tx_ring.consumer_lock); +} + static int tun_attach(struct tun_struct *tun, struct file *file, bool skip_filter, bool napi, bool napi_frags, bool publish_tun) @@ -730,8 +765,11 @@ static int tun_attach(struct tun_struct *tun, struct file *file, goto out; } + spin_lock(&tfile->tx_ring.consumer_lock); tfile->queue_index = tun->numqueues; + spin_unlock(&tfile->tx_ring.consumer_lock); tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; + tun_force_wake_queue(tun, tfile); if (tfile->detached) { /* Re-attach detached tfile, updating XDP queue_index */ @@ -964,6 +1002,24 @@ static int tun_net_open(struct net_device *dev) /* Net device close. */ static int tun_net_close(struct net_device *dev) { + struct tun_struct *tun = netdev_priv(dev); + struct tun_file *tfile; + int i; + + /* netif_running() is already false: take both ring locks to keep the + * wake sites out, so the stop below is the last write to + * __QUEUE_STATE_DRV_XOFF. + */ + for (i = 0; i < tun->numqueues; i++) { + tfile = rtnl_dereference(tun->tfiles[i]); + + spin_lock_bh(&tfile->tx_ring.consumer_lock); + spin_lock(&tfile->tx_ring.producer_lock); + tfile->cons_cnt = 0; + spin_unlock(&tfile->tx_ring.producer_lock); + spin_unlock_bh(&tfile->tx_ring.consumer_lock); + } + netif_tx_stop_all_queues(dev); return 0; } @@ -1008,6 +1064,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) struct netdev_queue *queue; struct tun_file *tfile; int len = skb->len; + int ret; rcu_read_lock(); tfile = rcu_dereference(tun->tfiles[txq]); @@ -1062,13 +1119,35 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) nf_reset_ct(skb); - if (ptr_ring_produce(&tfile->tx_ring, skb)) { + queue = netdev_get_tx_queue(dev, txq); + + spin_lock(&tfile->tx_ring.producer_lock); + ret = __ptr_ring_produce(&tfile->tx_ring, skb); + /* Do not touch the queue state of a device that is going down. */ + if ((tun->flags & IFF_BACKPRESSURE) && netif_running(dev) && + !qdisc_txq_has_no_queue(queue) && + __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) { + netif_tx_stop_queue(queue); + /* Paired with smp_mb() in __tun_wake_queue() */ + smp_mb__after_atomic(); + if (!__ptr_ring_check_produce(&tfile->tx_ring)) + netif_tx_wake_queue(queue); + } + spin_unlock(&tfile->tx_ring.producer_lock); + + if (ret) { + /* This should be a rare case if IFF_BACKPRESSURE is enabled and + * a qdisc is present, but can happen due to lltx. + * Since skb_tx_timestamp(), skb_orphan(), + * run_ebpf_filter() and pskb_trim() could have tinkered + * with the SKB, returning NETDEV_TX_BUSY is unsafe and + * we must drop instead. + */ drop_reason = SKB_DROP_REASON_FULL_RING; goto drop; } /* dev->lltx requires to do our own update of trans_start */ - queue = netdev_get_tx_queue(dev, txq); txq_trans_cond_update(queue); /* Notify and wake up reader process */ @@ -2116,13 +2195,61 @@ static ssize_t tun_put_user(struct tun_struct *tun, return total; } -static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) +/* Callers must hold ring.consumer_lock */ +static void __tun_wake_queue(struct tun_struct *tun, + struct tun_file *tfile, int consumed) +{ + u16 queue_index = tfile->queue_index; + struct netdev_queue *txq; + + lockdep_assert_held(&tfile->tx_ring.consumer_lock); + + if (!(tun->flags & IFF_BACKPRESSURE)) + return; + + /* A stop from tun_net_close() is not backpressure, leave it alone. */ + if (unlikely(!netif_running(tun->dev))) + return; + + /* Only the current owner of the slot may wake its subqueue. */ + if (unlikely(rcu_access_pointer(tun->tfiles[queue_index]) != tfile)) + return; + + txq = netdev_get_tx_queue(tun->dev, queue_index); + + /* Paired with smp_mb__after_atomic() in tun_net_xmit() */ + smp_mb(); + if (netif_tx_queue_stopped(txq)) { + tfile->cons_cnt += consumed; + if (tfile->cons_cnt >= tfile->tx_ring.size / 2 || + __ptr_ring_empty(&tfile->tx_ring)) { + netif_tx_wake_queue(txq); + tfile->cons_cnt = 0; + } + } +} + +static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile) +{ + void *ptr; + + spin_lock(&tfile->tx_ring.consumer_lock); + ptr = __ptr_ring_consume(&tfile->tx_ring); + if (ptr) + __tun_wake_queue(tun, tfile, 1); + + spin_unlock(&tfile->tx_ring.consumer_lock); + return ptr; +} + +static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, + int noblock, int *err) { DECLARE_WAITQUEUE(wait, current); void *ptr = NULL; int error = 0; - ptr = ptr_ring_consume(&tfile->tx_ring); + ptr = tun_ring_consume(tun, tfile); if (ptr) goto out; if (noblock) { @@ -2134,7 +2261,7 @@ static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) while (1) { set_current_state(TASK_INTERRUPTIBLE); - ptr = ptr_ring_consume(&tfile->tx_ring); + ptr = tun_ring_consume(tun, tfile); if (ptr) break; if (signal_pending(current)) { @@ -2171,7 +2298,7 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, if (!ptr) { /* Read frames from ring */ - ptr = tun_ring_recv(tfile, noblock, &err); + ptr = tun_ring_recv(tun, tfile, noblock, &err); if (!ptr) return err; } @@ -2703,8 +2830,9 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) { struct tun_struct *tun; struct tun_file *tfile = file->private_data; + struct tun_file *ntfile; struct net_device *dev; - int err; + int err, i; if (tfile->detached) return -EINVAL; @@ -2833,8 +2961,10 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) /* Make sure persistent devices do not get stuck in * xoff state. */ - if (netif_running(tun->dev)) - netif_tx_wake_all_queues(tun->dev); + for (i = 0; i < tun->numqueues; i++) { + ntfile = rtnl_dereference(tun->tfiles[i]); + tun_force_wake_queue(tun, ntfile); + } strscpy(ifr->ifr_name, tun->dev->name); return 0; @@ -3630,6 +3760,13 @@ static int tun_queue_resize(struct tun_struct *tun) dev->tx_queue_len, GFP_KERNEL, tun_ptr_free); + if (!ret) { + for (i = 0; i < tun->numqueues; i++) { + tfile = rtnl_dereference(tun->tfiles[i]); + tun_force_wake_queue(tun, tfile); + } + } + kfree(rings); return ret; } @@ -3738,6 +3875,31 @@ struct ptr_ring *tun_get_tx_ring(struct file *file) } EXPORT_SYMBOL_GPL(tun_get_tx_ring); +/* Callers must hold ring.consumer_lock */ +void tun_wake_queue(struct file *file, int consumed) +{ + struct tun_file *tfile; + struct tun_struct *tun; + + if (file->f_op != &tun_fops) + return; + + tfile = file->private_data; + if (!tfile) + return; + + lockdep_assert_held(&tfile->tx_ring.consumer_lock); + + rcu_read_lock(); + + tun = rcu_dereference(tfile->tun); + if (tun) + __tun_wake_queue(tun, tfile, consumed); + + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(tun_wake_queue); + module_init(tun_init); module_exit(tun_cleanup); MODULE_DESCRIPTION(DRV_DESCRIPTION); diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 6949b704166d..3e72b9c6af0c 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -176,13 +176,21 @@ static void *vhost_net_buf_consume(struct vhost_net_buf *rxq) return ret; } -static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq) +static int vhost_net_buf_produce(struct sock *sk, + struct vhost_net_virtqueue *nvq) { + struct file *file = sk->sk_socket->file; struct vhost_net_buf *rxq = &nvq->rxq; rxq->head = 0; - rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue, - VHOST_NET_BATCH); + spin_lock(&nvq->rx_ring->consumer_lock); + rxq->tail = __ptr_ring_consume_batched(nvq->rx_ring, rxq->queue, + VHOST_NET_BATCH); + + if (rxq->tail) + tun_wake_queue(file, rxq->tail); + + spin_unlock(&nvq->rx_ring->consumer_lock); return rxq->tail; } @@ -209,14 +217,15 @@ static int vhost_net_buf_peek_len(void *ptr) return __skb_array_len_with_tag(ptr); } -static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq) +static int vhost_net_buf_peek(struct sock *sk, + struct vhost_net_virtqueue *nvq) { struct vhost_net_buf *rxq = &nvq->rxq; if (!vhost_net_buf_is_empty(rxq)) goto out; - if (!vhost_net_buf_produce(nvq)) + if (!vhost_net_buf_produce(sk, nvq)) return 0; out: @@ -1004,7 +1013,7 @@ static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk) unsigned long flags; if (rvq->rx_ring) - return vhost_net_buf_peek(rvq); + return vhost_net_buf_peek(sk, rvq); spin_lock_irqsave(&sk->sk_receive_queue.lock, flags); head = skb_peek(&sk->sk_receive_queue); diff --git a/include/linux/if_tun.h b/include/linux/if_tun.h index 80166eb62f41..eeb9ed3c5a23 100644 --- a/include/linux/if_tun.h +++ b/include/linux/if_tun.h @@ -22,6 +22,8 @@ struct tun_msg_ctl { #if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE) struct socket *tun_get_socket(struct file *); struct ptr_ring *tun_get_tx_ring(struct file *file); +/* Callers must hold the consumer_lock of the ring of file */ +void tun_wake_queue(struct file *file, int consumed); static inline bool tun_is_xdp_frame(void *ptr) { @@ -55,6 +57,8 @@ static inline struct ptr_ring *tun_get_tx_ring(struct file *f) return ERR_PTR(-EINVAL); } +static inline void tun_wake_queue(struct file *f, int consumed) {} + static inline bool tun_is_xdp_frame(void *ptr) { return false; diff --git a/include/linux/ptr_ring.h b/include/linux/ptr_ring.h index d2c3629bbe45..631c43fde440 100644 --- a/include/linux/ptr_ring.h +++ b/include/linux/ptr_ring.h @@ -96,6 +96,26 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r) return ret; } +/* Report whether the next __ptr_ring_produce() has room for one entry: + * 0 means the single slot at r->queue[r->producer] is free, -ENOSPC means + * the ring is full, which is transient, and -EINVAL means r->size is 0, + * which is permanent. A caller that stops producing and waits for space + * must therefore do so only for -ENOSPC. + * + * Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must hold producer_lock. + */ +static inline int __ptr_ring_check_produce(struct ptr_ring *r) +{ + if (unlikely(!r->size)) + return -EINVAL; + + if (data_race(r->queue[r->producer])) + return -ENOSPC; + + return 0; +} + /* Note: callers invoking this in a loop must use a compiler barrier, * for example cpu_relax(). Callers must hold producer_lock. * Callers are responsible for making sure pointer that is being queued @@ -103,8 +123,10 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r) */ static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) { - if (unlikely(!r->size) || data_race(r->queue[r->producer])) - return -ENOSPC; + int ret = __ptr_ring_check_produce(r); + + if (ret) + return ret; /* Make sure the pointer we are storing points to a valid data. */ /* Pairs with the dependency ordering in __ptr_ring_consume. */ diff --git a/include/uapi/linux/if_tun.h b/include/uapi/linux/if_tun.h index 79d53c7a1ebd..a0ddc50a7534 100644 --- a/include/uapi/linux/if_tun.h +++ b/include/uapi/linux/if_tun.h @@ -69,6 +69,10 @@ #define IFF_NAPI_FRAGS 0x0020 /* Used in TUNSETIFF to bring up tun/tap without carrier */ #define IFF_NO_CARRIER 0x0040 +/* Stop the queue instead of dropping when the internal ring is full, so an + * attached qdisc applies backpressure instead of being bypassed. + */ +#define IFF_BACKPRESSURE 0x0080 #define IFF_NO_PI 0x1000 /* This flag has no real effect */ #define IFF_ONE_QUEUE 0x2000 diff --git a/tools/include/uapi/linux/if_tun.h b/tools/include/uapi/linux/if_tun.h index 2ec07de1d73b..2c85525704c1 100644 --- a/tools/include/uapi/linux/if_tun.h +++ b/tools/include/uapi/linux/if_tun.h @@ -67,6 +67,10 @@ #define IFF_TAP 0x0002 #define IFF_NAPI 0x0010 #define IFF_NAPI_FRAGS 0x0020 +/* Stop the queue instead of dropping when the internal ring is full, so an + * attached qdisc applies backpressure instead of being bypassed. + */ +#define IFF_BACKPRESSURE 0x0080 #define IFF_NO_PI 0x1000 /* This flag has no real effect */ #define IFF_ONE_QUEUE 0x2000