mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-05-05 13:29:17 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'net-use-kmem_cache_free_bulk-in-kfree_skb_list'

Browse Source 
Jesper Dangaard Brouer says:

====================
net: use kmem_cache_free_bulk in kfree_skb_list

The kfree_skb_list function walks SKB (via skb->next) and frees them
individually to the SLUB/SLAB allocator (kmem_cache). It is more
efficient to bulk free them via the kmem_cache_free_bulk API.

Netstack NAPI fastpath already uses kmem_cache bulk alloc and free
APIs for SKBs.

The kfree_skb_list call got an interesting optimization in commit
520ac30f45 ("net_sched: drop packets after root qdisc lock is
released") that can create a list of SKBs "to_free" e.g. when qdisc
enqueue fails or deliberately chooses to drop . It isn't a normal data
fastpath, but the situation will likely occur when system/qdisc are
under heavy workloads, thus it makes sense to use a faster API for
freeing the SKBs.

E.g. the (often distro default) qdisc fq_codel will drop batches of
packets from fattest elephant flow, default capped at 64 packets (but
adjustable via tc argument drop_batch).

Performance measurements done in [1]:
 [1] https://github.com/xdp-project/xdp-project/blob/master/areas/mem/kfree_skb_list01.org
====================

Link: https://lore.kernel.org/r/167361788585.531803.686364041841425360.stgit@firesoul
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni
2023-01-17 10:27:31 +01:00
parent 501543b4ff eedade12f4
commit 05cb8b39ca
1 changed files with 59 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
net/core/skbuff.c
View File

@@ -930,6 +930,21 @@ void __kfree_skb(struct sk_buff *skb)
}
EXPORT_SYMBOL(__kfree_skb);
static __always_inline
bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
if (unlikely(!skb_unref(skb)))
return false;
DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX);
if (reason == SKB_CONSUMED)
trace_consume_skb(skb);
else
trace_kfree_skb(skb, __builtin_return_address(0), reason);
return true;
}
/**
* kfree_skb_reason - free an sk_buff with special reason
* @skb: buffer to free
@@ -942,28 +957,59 @@ EXPORT_SYMBOL(__kfree_skb);
void __fix_address
kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
if (unlikely(!skb_unref(skb)))
return;
DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX);
if (reason == SKB_CONSUMED)
trace_consume_skb(skb);
else
trace_kfree_skb(skb, __builtin_return_address(0), reason);
__kfree_skb(skb);
if (__kfree_skb_reason(skb, reason))
__kfree_skb(skb);
}
EXPORT_SYMBOL(kfree_skb_reason);
void kfree_skb_list_reason(struct sk_buff *segs,
enum skb_drop_reason reason)
#define KFREE_SKB_BULK_SIZE 16
struct skb_free_array {
unsigned int skb_count;
void *skb_array[KFREE_SKB_BULK_SIZE];
};
static void kfree_skb_add_bulk(struct sk_buff *skb,
struct skb_free_array *sa,
enum skb_drop_reason reason)
{
/* if SKB is a clone, don't handle this case */
if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {
__kfree_skb(skb);
return;
}
skb_release_all(skb, reason);
sa->skb_array[sa->skb_count++] = skb;
if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
kmem_cache_free_bulk(skbuff_head_cache, KFREE_SKB_BULK_SIZE,
sa->skb_array);
sa->skb_count = 0;
}
}
void __fix_address
kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
{
struct skb_free_array sa;
sa.skb_count = 0;
while (segs) {
struct sk_buff *next = segs->next;
kfree_skb_reason(segs, reason);
skb_mark_not_on_list(segs);
if (__kfree_skb_reason(segs, reason))
kfree_skb_add_bulk(segs, &sa, reason);
segs = next;
}
if (sa.skb_count)
kmem_cache_free_bulk(skbuff_head_cache, sa.skb_count,
sa.skb_array);
}
EXPORT_SYMBOL(kfree_skb_list_reason);