mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-02-28 19:56:56 -05:00
c75734b585
clang is not inlining skb_frag_unref() and __skb_frag_unref() in gro fast path. It also does not inline gro_try_pull_from_frag0(). Using __always_inline fixes this issue, makes the kernel faster _and_ smaller. Also change __skb_frag_ref(), skb_frag_ref() and skb_page_unref() to let them inlined for the last patch in this series. $ scripts/bloat-o-meter -t vmlinux.0 vmlinux.1 add/remove: 2/6 grow/shrink: 1/2 up/down: 218/-511 (-293) Function old new delta gro_pull_from_frag0 - 188 +188 __pfx_gro_pull_from_frag0 - 16 +16 skb_shift 1125 1139 +14 __pfx_skb_frag_unref 16 - -16 __pfx_gro_try_pull_from_frag0 16 - -16 __pfx___skb_frag_unref 16 - -16 __skb_frag_unref 36 - -36 skb_frag_unref 59 - -59 dev_gro_receive 1608 1514 -94 napi_gro_frags 892 771 -121 gro_try_pull_from_frag0 153 - -153 Total: Before=22566192, After=22565899, chg -0.00% Signed-off-by: Eric Dumazet <edumazet@google.com> Link: https://patch.msgid.link/20260122045720.1221017-2-edumazet@google.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
75 lines
1.8 KiB
C
75 lines
1.8 KiB
C
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
/*
|
|
* Skb ref helpers.
|
|
*
|
|
*/
|
|
|
|
#ifndef _LINUX_SKBUFF_REF_H
|
|
#define _LINUX_SKBUFF_REF_H
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
/**
|
|
* __skb_frag_ref - take an addition reference on a paged fragment.
|
|
* @frag: the paged fragment
|
|
*
|
|
* Takes an additional reference on the paged fragment @frag.
|
|
*/
|
|
static __always_inline void __skb_frag_ref(skb_frag_t *frag)
|
|
{
|
|
get_netmem(skb_frag_netmem(frag));
|
|
}
|
|
|
|
/**
|
|
* skb_frag_ref - take an addition reference on a paged fragment of an skb.
|
|
* @skb: the buffer
|
|
* @f: the fragment offset.
|
|
*
|
|
* Takes an additional reference on the @f'th paged fragment of @skb.
|
|
*/
|
|
static __always_inline void skb_frag_ref(struct sk_buff *skb, int f)
|
|
{
|
|
__skb_frag_ref(&skb_shinfo(skb)->frags[f]);
|
|
}
|
|
|
|
bool napi_pp_put_page(netmem_ref netmem);
|
|
|
|
static __always_inline void skb_page_unref(netmem_ref netmem, bool recycle)
|
|
{
|
|
#ifdef CONFIG_PAGE_POOL
|
|
if (recycle && napi_pp_put_page(netmem))
|
|
return;
|
|
#endif
|
|
put_netmem(netmem);
|
|
}
|
|
|
|
/**
|
|
* __skb_frag_unref - release a reference on a paged fragment.
|
|
* @frag: the paged fragment
|
|
* @recycle: recycle the page if allocated via page_pool
|
|
*
|
|
* Releases a reference on the paged fragment @frag
|
|
* or recycles the page via the page_pool API.
|
|
*/
|
|
static __always_inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
|
|
{
|
|
skb_page_unref(skb_frag_netmem(frag), recycle);
|
|
}
|
|
|
|
/**
|
|
* skb_frag_unref - release a reference on a paged fragment of an skb.
|
|
* @skb: the buffer
|
|
* @f: the fragment offset
|
|
*
|
|
* Releases a reference on the @f'th paged fragment of @skb.
|
|
*/
|
|
static __always_inline void skb_frag_unref(struct sk_buff *skb, int f)
|
|
{
|
|
struct skb_shared_info *shinfo = skb_shinfo(skb);
|
|
|
|
if (!skb_zcopy_managed(skb))
|
|
__skb_frag_unref(&shinfo->frags[f], skb->pp_recycle);
|
|
}
|
|
|
|
#endif /* _LINUX_SKBUFF_REF_H */
|