mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 15:22:21 -04:00
Code Issues Packages Projects Releases Wiki Activity

net/packet: defer vmalloc TX_RING free until skbs finish

Browse Source 
AF_PACKET TX_RING skbs keep a raw pointer to their ring frame. The skb
page references preserve page-backed ring blocks after pg_vec is freed,
but they do not preserve a vmalloc mapping.

tpacket_destruct_skb() currently drops the pending reference before
writing the timestamp and TP_STATUS_AVAILABLE to the frame. Move the
decrement after those stores. The smp_wmb() in __packet_set_status()
orders the frame stores before the decrement.

Also recheck pending TX frames under pg_vec_lock before non-closing
ring replacement, so a racing send cannot add a pending skb between
the initial check and the ring swap.

Ring allocation can produce a mixture of page-backed and vmalloc-backed
blocks. Allocate deferred-work storage during TX ring setup when the
first vmalloc-backed block is encountered, and keep its pointer in the
pg_vec allocation header. If allocation fails, return -ENOMEM from ring
setup. On socket close, a non-NULL pointer identifies a vmalloc-backed
vector without a scan. If TX skbs remain, defer the whole vector to
system_long_wq.

After pg_vec is detached, a late destructor can skip the pending
decrement. Use socket write-memory accounting as the deferred lifetime
gate instead: an skb remains charged through its final sock_wfree(),
after all ring-frame accesses. The delayed work retains a socket
reference and reschedules itself until no TX skbs remain.

Move pending_refcnt release to packet_sock_destruct() so late skb
destructors and deferred cleanup can safely use it after
packet_release(). Page-backed teardown remains synchronous, and no lock
is added to the TX completion hot path.

Fixes: b013840810 ("packet: use percpu mmap tx frame pending refcount")
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/netdev/20260721015824.45829-1-kylebot@openai.com/
Suggested-by: Eric Dumazet <edumazet@google.com>
Suggested-by: Willem de Bruijn <willemdebruijn.kernel@gmail.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Kyle Zeng <kylebot@openai.com>
Link: https://patch.msgid.link/20260816235646.76500-1-kylebot@openai.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Kyle Zeng
2026-08-16 16:56:46 -07:00
committed by Jakub Kicinski
parent 51a26bb843
commit 992cc9f94c
1 changed files with 87 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
net/packet/af_packet.c
View File

@@ -88,6 +88,7 @@
#include <linux/errqueue.h>
#include <linux/net_tstamp.h>
#include <linux/percpu.h>
#include <linux/workqueue.h>
#ifdef CONFIG_INET
#include <net/inet_common.h>
#endif
@@ -1341,6 +1342,8 @@ static void packet_sock_destruct(struct sock *sk)
WARN_ON(atomic_read(&sk->sk_rmem_alloc));
WARN_ON(refcount_read(&sk->sk_wmem_alloc));
packet_free_pending(pkt_sk(sk));
if (!sock_flag(sk, SOCK_DEAD)) {
pr_err("Attempt to release alive packet socket: %p\n", sk);
return;
@@ -2534,11 +2537,11 @@ static void tpacket_destruct_skb(struct sk_buff *skb)
__u32 ts;
ph = skb_zcopy_get_nouarg(skb);
packet_dec_pending(&po->tx_ring);
ts = __packet_set_timestamp(po, ph, skb);
__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
packet_dec_pending(&po->tx_ring);
complete(&po->skb_completion);
}
@@ -3207,7 +3210,6 @@ static int packet_release(struct socket *sock)
/* Purge queues */
skb_queue_purge(&sk->sk_receive_queue);
packet_free_pending(po);
sock_put(sk);
return 0;
@@ -4370,11 +4372,26 @@ static const struct vm_operations_struct packet_mmap_ops = {
.close = packet_mm_close,
};
struct packet_pg_vec {
struct packet_pg_vec_free *deferred;
unsigned int order;
unsigned int len;
struct pgv pg_vec[] __counted_by(len);
};
struct packet_pg_vec_free {
struct delayed_work work;
struct sock *sk;
struct packet_pg_vec *vec;
};
static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
unsigned int len)
{
struct packet_pg_vec *vec;
int i;
vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]);
for (i = 0; i < len; i++) {
if (likely(pg_vec[i].buffer)) {
if (is_vmalloc_addr(pg_vec[i].buffer))
@@ -4385,7 +4402,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
pg_vec[i].buffer = NULL;
}
}
kfree(pg_vec);
kfree(vec->deferred);
kfree(vec);
}
static void packet_free_pg_vec_work(struct work_struct *work)
{
struct packet_pg_vec_free *deferred;
struct packet_pg_vec *vec;
struct sock *sk;
deferred = container_of_const(to_delayed_work(work),
struct packet_pg_vec_free, work);
vec = deferred->vec;
sk = deferred->sk;
if (sk_wmem_alloc_get(sk)) {
queue_delayed_work(system_long_wq, &deferred->work, 1);
return;
}
free_pg_vec(vec->pg_vec, vec->order, vec->len);
sock_put(sk);
}
static void packet_free_tx_ring(struct sock *sk, struct pgv *pg_vec,
unsigned int order, unsigned int len)
{
struct packet_pg_vec_free *deferred;
struct packet_pg_vec *vec;
vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]);
deferred = vec->deferred;
if (!deferred || !sk_wmem_alloc_get(sk)) {
free_pg_vec(pg_vec, order, len);
return;
}
/* A detached ring's pending count can miss late skb destructors. */
deferred->sk = sk;
sock_hold(sk);
queue_delayed_work(system_long_wq, &deferred->work, 0);
}
static char *alloc_one_pg_vec_page(unsigned long order)
@@ -4413,20 +4469,35 @@ static char *alloc_one_pg_vec_page(unsigned long order)
return NULL;
}
static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring)
{
unsigned int block_nr = req->tp_block_nr;
struct packet_pg_vec *vec;
struct pgv *pg_vec;
int i;
pg_vec = kzalloc_objs(struct pgv, block_nr, GFP_KERNEL | __GFP_NOWARN);
if (unlikely(!pg_vec))
goto out;
vec = kzalloc_flex(*vec, pg_vec, block_nr, GFP_KERNEL | __GFP_NOWARN);
if (unlikely(!vec))
return NULL;
vec->order = order;
vec->len = block_nr;
pg_vec = vec->pg_vec;
for (i = 0; i < block_nr; i++) {
pg_vec[i].buffer = alloc_one_pg_vec_page(order);
if (unlikely(!pg_vec[i].buffer))
goto out_free_pgvec;
if (tx_ring && !vec->deferred &&
is_vmalloc_addr(pg_vec[i].buffer)) {
vec->deferred = kzalloc_obj(*vec->deferred,
GFP_KERNEL | __GFP_NOWARN);
if (!vec->deferred)
goto out_free_pgvec;
vec->deferred->vec = vec;
INIT_DELAYED_WORK(&vec->deferred->work,
packet_free_pg_vec_work);
}
}
out:
@@ -4509,7 +4580,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
err = -ENOMEM;
order = get_order(req->tp_block_size);
pg_vec = alloc_pg_vec(req, order);
pg_vec = alloc_pg_vec(req, order, tx_ring);
if (unlikely(!pg_vec))
goto out;
switch (po->tp_version) {
@@ -4561,6 +4632,9 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
err = -EBUSY;
mutex_lock(&po->pg_vec_lock);
if (closing || atomic_long_read(&po->mapped) == 0) {
if (tx_ring && !closing && packet_read_pending(rb))
goto out_unlock;
err = 0;
spin_lock_bh(&rb_queue->lock);
swap(rb->pg_vec, pg_vec);
@@ -4582,6 +4656,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
pr_err("packet_mmap: vma is busy: %ld\n",
atomic_long_read(&po->mapped));
}
out_unlock:
mutex_unlock(&po->pg_vec_lock);
spin_lock(&po->bind_lock);
@@ -4603,7 +4678,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
out_free_pg_vec:
if (pg_vec) {
bitmap_free(rx_owner_map);
free_pg_vec(pg_vec, order, req->tp_block_nr);
if (tx_ring && closing)
packet_free_tx_ring(sk, pg_vec, order, req->tp_block_nr);
else
free_pg_vec(pg_vec, order, req->tp_block_nr);
}
out:
return err;