mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-07-16 17:57:38 -04:00
Merge tag 'net-next-6.6' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next
Pull networking updates from Paolo Abeni:
"Core:
- Increase size limits for to-be-sent skb frag allocations. This
allows tun, tap devices and packet sockets to better cope with
large writes operations
- Store netdevs in an xarray, to simplify iterating over netdevs
- Refactor nexthop selection for multipath routes
- Improve sched class lifetime handling
- Add backup nexthop ID support for bridge
- Implement drop reasons support in openvswitch
- Several data races annotations and fixes
- Constify the sk parameter of routing functions
- Prepend kernel version to netconsole message
Protocols:
- Implement support for TCP probing the peer being under memory
pressure
- Remove hard coded limitation on IPv6 specific info placement inside
the socket struct
- Get rid of sysctl_tcp_adv_win_scale and use an auto-estimated per
socket scaling factor
- Scaling-up the IPv6 expired route GC via a separated list of
expiring routes
- In-kernel support for the TLS alert protocol
- Better support for UDP reuseport with connected sockets
- Add NEXT-C-SID support for SRv6 End.X behavior, reducing the SR
header size
- Get rid of additional ancillary per MPTCP connection struct socket
- Implement support for BPF-based MPTCP packet schedulers
- Format MPTCP subtests selftests results in TAP
- Several new SMC 2.1 features including unique experimental options,
max connections per lgr negotiation, max links per lgr negotiation
BPF:
- Multi-buffer support in AF_XDP
- Add multi uprobe BPF links for attaching multiple uprobes and usdt
probes, which is significantly faster and saves extra fds
- Implement an fd-based tc BPF attach API (TCX) and BPF link support
on top of it
- Add SO_REUSEPORT support for TC bpf_sk_assign
- Support new instructions from cpu v4 to simplify the generated code
and feature completeness, for x86, arm64, riscv64
- Support defragmenting IPv(4|6) packets in BPF
- Teach verifier actual bounds of bpf_get_smp_processor_id() and fix
perf+libbpf issue related to custom section handling
- Introduce bpf map element count and enable it for all program types
- Add a BPF hook in sys_socket() to change the protocol ID from
IPPROTO_TCP to IPPROTO_MPTCP to cover migration for legacy
- Introduce bpf_me_mcache_free_rcu() and fix OOM under stress
- Add uprobe support for the bpf_get_func_ip helper
- Check skb ownership against full socket
- Support for up to 12 arguments in BPF trampoline
- Extend link_info for kprobe_multi and perf_event links
Netfilter:
- Speed-up process exit by aborting ruleset validation if a fatal
signal is pending
- Allow NLA_POLICY_MASK to be used with BE16/BE32 types
Driver API:
- Page pool optimizations, to improve data locality and cache usage
- Introduce ndo_hwtstamp_get() and ndo_hwtstamp_set() to avoid the
need for raw ioctl() handling in drivers
- Simplify genetlink dump operations (doit/dumpit) providing them the
common information already populated in struct genl_info
- Extend and use the yaml devlink specs to [re]generate the split ops
- Introduce devlink selective dumps, to allow SF filtering SF based
on handle and other attributes
- Add yaml netlink spec for netlink-raw families, allow route, link
and address related queries via the ynl tool
- Remove phylink legacy mode support
- Support offload LED blinking to phy
- Add devlink port function attributes for IPsec
New hardware / drivers:
- Ethernet:
- Broadcom ASP 2.0 (72165) ethernet controller
- MediaTek MT7988 SoC
- Texas Instruments AM654 SoC
- Texas Instruments IEP driver
- Atheros qca8081 phy
- Marvell 88Q2110 phy
- NXP TJA1120 phy
- WiFi:
- MediaTek mt7981 support
- Can:
- Kvaser SmartFusion2 PCI Express devices
- Allwinner T113 controllers
- Texas Instruments tcan4552/4553 chips
- Bluetooth:
- Intel Gale Peak
- Qualcomm WCN3988 and WCN7850
- NXP AW693 and IW624
- Mediatek MT2925
Drivers:
- Ethernet NICs:
- nVidia/Mellanox:
- mlx5:
- support UDP encapsulation in packet offload mode
- IPsec packet offload support in eswitch mode
- improve aRFS observability by adding new set of counters
- extends MACsec offload support to cover RoCE traffic
- dynamic completion EQs
- mlx4:
- convert to use auxiliary bus instead of custom interface
logic
- Intel
- ice:
- implement switchdev bridge offload, even for LAG
interfaces
- implement SRIOV support for LAG interfaces
- igc:
- add support for multiple in-flight TX timestamps
- Broadcom:
- bnxt:
- use the unified RX page pool buffers for XDP and non-XDP
- use the NAPI skb allocation cache
- OcteonTX2:
- support Round Robin scheduling HTB offload
- TC flower offload support for SPI field
- Freescale:
- add XDP_TX feature support
- AMD:
- ionic: add support for PCI FLR event
- sfc:
- basic conntrack offload
- introduce eth, ipv4 and ipv6 pedit offloads
- ST Microelectronics:
- stmmac: maximze PTP timestamping resolution
- Virtual NICs:
- Microsoft vNIC:
- batch ringing RX queue doorbell on receiving packets
- add page pool for RX buffers
- Virtio vNIC:
- add per queue interrupt coalescing support
- Google vNIC:
- add queue-page-list mode support
- Ethernet high-speed switches:
- nVidia/Mellanox (mlxsw):
- add port range matching tc-flower offload
- permit enslavement to netdevices with uppers
- Ethernet embedded switches:
- Marvell (mv88e6xxx):
- convert to phylink_pcs
- Renesas:
- r8A779fx: add speed change support
- rzn1: enables vlan support
- Ethernet PHYs:
- convert mv88e6xxx to phylink_pcs
- WiFi:
- Qualcomm Wi-Fi 7 (ath12k):
- extremely High Throughput (EHT) PHY support
- RealTek (rtl8xxxu):
- enable AP mode for: RTL8192FU, RTL8710BU (RTL8188GU),
RTL8192EU and RTL8723BU
- RealTek (rtw89):
- Introduce Time Averaged SAR (TAS) support
- Connector:
- support for event filtering"
* tag 'net-next-6.6' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (1806 commits)
net: ethernet: mtk_wed: minor change in wed_{tx,rx}info_show
net: ethernet: mtk_wed: add some more info in wed_txinfo_show handler
net: stmmac: clarify difference between "interface" and "phy_interface"
r8152: add vendor/device ID pair for D-Link DUB-E250
devlink: move devlink_notify_register/unregister() to dev.c
devlink: move small_ops definition into netlink.c
devlink: move tracepoint definitions into core.c
devlink: push linecard related code into separate file
devlink: push rate related code into separate file
devlink: push trap related code into separate file
devlink: use tracepoint_enabled() helper
devlink: push region related code into separate file
devlink: push param related code into separate file
devlink: push resource related code into separate file
devlink: push dpipe related code into separate file
devlink: move and rename devlink_dpipe_send_and_alloc_skb() helper
devlink: push shared buffer related code into separate file
devlink: push port related code into separate file
devlink: push object register/unregister notifications into separate helpers
inet: fix IP_TRANSPARENT error handling
...
This commit is contained in:
Linus Torvalds
@@ -23,6 +23,7 @@
|
||||
#include <linux/sort.h>
|
||||
#include <linux/key.h>
|
||||
#include <linux/verification.h>
|
||||
#include <linux/namei.h>
|
||||
|
||||
#include <net/bpf_sk_storage.h>
|
||||
|
||||
@@ -86,6 +87,9 @@ static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
|
||||
static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx);
|
||||
static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
|
||||
|
||||
static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx);
|
||||
static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
|
||||
|
||||
/**
|
||||
* trace_call_bpf - invoke BPF program
|
||||
* @call: tracepoint event
|
||||
@@ -223,17 +227,6 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto = {
|
||||
.arg3_type = ARG_ANYTHING,
|
||||
};
|
||||
|
||||
static __always_inline int
|
||||
bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
|
||||
if (unlikely(ret < 0))
|
||||
memset(dst, 0, size);
|
||||
return ret;
|
||||
}
|
||||
|
||||
BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size,
|
||||
const void *, unsafe_ptr)
|
||||
{
|
||||
@@ -1066,7 +1059,16 @@ static unsigned long get_entry_ip(unsigned long fentry_ip)
|
||||
|
||||
BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs)
|
||||
{
|
||||
struct kprobe *kp = kprobe_running();
|
||||
struct bpf_trace_run_ctx *run_ctx __maybe_unused;
|
||||
struct kprobe *kp;
|
||||
|
||||
#ifdef CONFIG_UPROBES
|
||||
run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
|
||||
if (run_ctx->is_uprobe)
|
||||
return ((struct uprobe_dispatch_data *)current->utask->vaddr)->bp_addr;
|
||||
#endif
|
||||
|
||||
kp = kprobe_running();
|
||||
|
||||
if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY))
|
||||
return 0;
|
||||
@@ -1105,6 +1107,30 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = {
|
||||
.arg1_type = ARG_PTR_TO_CTX,
|
||||
};
|
||||
|
||||
BPF_CALL_1(bpf_get_func_ip_uprobe_multi, struct pt_regs *, regs)
|
||||
{
|
||||
return bpf_uprobe_multi_entry_ip(current->bpf_ctx);
|
||||
}
|
||||
|
||||
static const struct bpf_func_proto bpf_get_func_ip_proto_uprobe_multi = {
|
||||
.func = bpf_get_func_ip_uprobe_multi,
|
||||
.gpl_only = false,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_PTR_TO_CTX,
|
||||
};
|
||||
|
||||
BPF_CALL_1(bpf_get_attach_cookie_uprobe_multi, struct pt_regs *, regs)
|
||||
{
|
||||
return bpf_uprobe_multi_cookie(current->bpf_ctx);
|
||||
}
|
||||
|
||||
static const struct bpf_func_proto bpf_get_attach_cookie_proto_umulti = {
|
||||
.func = bpf_get_attach_cookie_uprobe_multi,
|
||||
.gpl_only = false,
|
||||
.ret_type = RET_INTEGER,
|
||||
.arg1_type = ARG_PTR_TO_CTX,
|
||||
};
|
||||
|
||||
BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx)
|
||||
{
|
||||
struct bpf_trace_run_ctx *run_ctx;
|
||||
@@ -1547,13 +1573,17 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
||||
return &bpf_override_return_proto;
|
||||
#endif
|
||||
case BPF_FUNC_get_func_ip:
|
||||
return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ?
|
||||
&bpf_get_func_ip_proto_kprobe_multi :
|
||||
&bpf_get_func_ip_proto_kprobe;
|
||||
if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI)
|
||||
return &bpf_get_func_ip_proto_kprobe_multi;
|
||||
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
|
||||
return &bpf_get_func_ip_proto_uprobe_multi;
|
||||
return &bpf_get_func_ip_proto_kprobe;
|
||||
case BPF_FUNC_get_attach_cookie:
|
||||
return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ?
|
||||
&bpf_get_attach_cookie_proto_kmulti :
|
||||
&bpf_get_attach_cookie_proto_trace;
|
||||
if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI)
|
||||
return &bpf_get_attach_cookie_proto_kmulti;
|
||||
if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
|
||||
return &bpf_get_attach_cookie_proto_umulti;
|
||||
return &bpf_get_attach_cookie_proto_trace;
|
||||
default:
|
||||
return bpf_tracing_func_proto(func_id, prog);
|
||||
}
|
||||
@@ -2376,9 +2406,13 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
|
||||
if (is_tracepoint || is_syscall_tp) {
|
||||
*buf = is_tracepoint ? event->tp_event->tp->name
|
||||
: event->tp_event->name;
|
||||
*fd_type = BPF_FD_TYPE_TRACEPOINT;
|
||||
*probe_offset = 0x0;
|
||||
*probe_addr = 0x0;
|
||||
/* We allow NULL pointer for tracepoint */
|
||||
if (fd_type)
|
||||
*fd_type = BPF_FD_TYPE_TRACEPOINT;
|
||||
if (probe_offset)
|
||||
*probe_offset = 0x0;
|
||||
if (probe_addr)
|
||||
*probe_addr = 0x0;
|
||||
} else {
|
||||
/* kprobe/uprobe */
|
||||
err = -EOPNOTSUPP;
|
||||
@@ -2391,7 +2425,7 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
|
||||
#ifdef CONFIG_UPROBE_EVENTS
|
||||
if (flags & TRACE_EVENT_FL_UPROBE)
|
||||
err = bpf_get_uprobe_info(event, fd_type, buf,
|
||||
probe_offset,
|
||||
probe_offset, probe_addr,
|
||||
event->attr.type == PERF_TYPE_TRACEPOINT);
|
||||
#endif
|
||||
}
|
||||
@@ -2476,6 +2510,7 @@ struct bpf_kprobe_multi_link {
|
||||
u32 cnt;
|
||||
u32 mods_cnt;
|
||||
struct module **mods;
|
||||
u32 flags;
|
||||
};
|
||||
|
||||
struct bpf_kprobe_multi_run_ctx {
|
||||
@@ -2565,9 +2600,44 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
|
||||
kfree(kmulti_link);
|
||||
}
|
||||
|
||||
static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
|
||||
struct bpf_link_info *info)
|
||||
{
|
||||
u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs);
|
||||
struct bpf_kprobe_multi_link *kmulti_link;
|
||||
u32 ucount = info->kprobe_multi.count;
|
||||
int err = 0, i;
|
||||
|
||||
if (!uaddrs ^ !ucount)
|
||||
return -EINVAL;
|
||||
|
||||
kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
|
||||
info->kprobe_multi.count = kmulti_link->cnt;
|
||||
info->kprobe_multi.flags = kmulti_link->flags;
|
||||
|
||||
if (!uaddrs)
|
||||
return 0;
|
||||
if (ucount < kmulti_link->cnt)
|
||||
err = -ENOSPC;
|
||||
else
|
||||
ucount = kmulti_link->cnt;
|
||||
|
||||
if (kallsyms_show_value(current_cred())) {
|
||||
if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64)))
|
||||
return -EFAULT;
|
||||
} else {
|
||||
for (i = 0; i < ucount; i++) {
|
||||
if (put_user(0, uaddrs + i))
|
||||
return -EFAULT;
|
||||
}
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
|
||||
.release = bpf_kprobe_multi_link_release,
|
||||
.dealloc = bpf_kprobe_multi_link_dealloc,
|
||||
.fill_link_info = bpf_kprobe_multi_link_fill_link_info,
|
||||
};
|
||||
|
||||
static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)
|
||||
@@ -2881,6 +2951,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
|
||||
link->addrs = addrs;
|
||||
link->cookies = cookies;
|
||||
link->cnt = cnt;
|
||||
link->flags = flags;
|
||||
|
||||
if (cookies) {
|
||||
/*
|
||||
@@ -2931,3 +3002,301 @@ static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_UPROBES
|
||||
struct bpf_uprobe_multi_link;
|
||||
|
||||
struct bpf_uprobe {
|
||||
struct bpf_uprobe_multi_link *link;
|
||||
loff_t offset;
|
||||
u64 cookie;
|
||||
struct uprobe_consumer consumer;
|
||||
};
|
||||
|
||||
struct bpf_uprobe_multi_link {
|
||||
struct path path;
|
||||
struct bpf_link link;
|
||||
u32 cnt;
|
||||
struct bpf_uprobe *uprobes;
|
||||
struct task_struct *task;
|
||||
};
|
||||
|
||||
struct bpf_uprobe_multi_run_ctx {
|
||||
struct bpf_run_ctx run_ctx;
|
||||
unsigned long entry_ip;
|
||||
struct bpf_uprobe *uprobe;
|
||||
};
|
||||
|
||||
static void bpf_uprobe_unregister(struct path *path, struct bpf_uprobe *uprobes,
|
||||
u32 cnt)
|
||||
{
|
||||
u32 i;
|
||||
|
||||
for (i = 0; i < cnt; i++) {
|
||||
uprobe_unregister(d_real_inode(path->dentry), uprobes[i].offset,
|
||||
&uprobes[i].consumer);
|
||||
}
|
||||
}
|
||||
|
||||
static void bpf_uprobe_multi_link_release(struct bpf_link *link)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *umulti_link;
|
||||
|
||||
umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
|
||||
bpf_uprobe_unregister(&umulti_link->path, umulti_link->uprobes, umulti_link->cnt);
|
||||
}
|
||||
|
||||
static void bpf_uprobe_multi_link_dealloc(struct bpf_link *link)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *umulti_link;
|
||||
|
||||
umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
|
||||
if (umulti_link->task)
|
||||
put_task_struct(umulti_link->task);
|
||||
path_put(&umulti_link->path);
|
||||
kvfree(umulti_link->uprobes);
|
||||
kfree(umulti_link);
|
||||
}
|
||||
|
||||
static const struct bpf_link_ops bpf_uprobe_multi_link_lops = {
|
||||
.release = bpf_uprobe_multi_link_release,
|
||||
.dealloc = bpf_uprobe_multi_link_dealloc,
|
||||
};
|
||||
|
||||
static int uprobe_prog_run(struct bpf_uprobe *uprobe,
|
||||
unsigned long entry_ip,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *link = uprobe->link;
|
||||
struct bpf_uprobe_multi_run_ctx run_ctx = {
|
||||
.entry_ip = entry_ip,
|
||||
.uprobe = uprobe,
|
||||
};
|
||||
struct bpf_prog *prog = link->link.prog;
|
||||
bool sleepable = prog->aux->sleepable;
|
||||
struct bpf_run_ctx *old_run_ctx;
|
||||
int err = 0;
|
||||
|
||||
if (link->task && current != link->task)
|
||||
return 0;
|
||||
|
||||
if (sleepable)
|
||||
rcu_read_lock_trace();
|
||||
else
|
||||
rcu_read_lock();
|
||||
|
||||
migrate_disable();
|
||||
|
||||
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
|
||||
err = bpf_prog_run(link->link.prog, regs);
|
||||
bpf_reset_run_ctx(old_run_ctx);
|
||||
|
||||
migrate_enable();
|
||||
|
||||
if (sleepable)
|
||||
rcu_read_unlock_trace();
|
||||
else
|
||||
rcu_read_unlock();
|
||||
return err;
|
||||
}
|
||||
|
||||
static bool
|
||||
uprobe_multi_link_filter(struct uprobe_consumer *con, enum uprobe_filter_ctx ctx,
|
||||
struct mm_struct *mm)
|
||||
{
|
||||
struct bpf_uprobe *uprobe;
|
||||
|
||||
uprobe = container_of(con, struct bpf_uprobe, consumer);
|
||||
return uprobe->link->task->mm == mm;
|
||||
}
|
||||
|
||||
static int
|
||||
uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs)
|
||||
{
|
||||
struct bpf_uprobe *uprobe;
|
||||
|
||||
uprobe = container_of(con, struct bpf_uprobe, consumer);
|
||||
return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);
|
||||
}
|
||||
|
||||
static int
|
||||
uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs)
|
||||
{
|
||||
struct bpf_uprobe *uprobe;
|
||||
|
||||
uprobe = container_of(con, struct bpf_uprobe, consumer);
|
||||
return uprobe_prog_run(uprobe, func, regs);
|
||||
}
|
||||
|
||||
static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
|
||||
{
|
||||
struct bpf_uprobe_multi_run_ctx *run_ctx;
|
||||
|
||||
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
|
||||
return run_ctx->entry_ip;
|
||||
}
|
||||
|
||||
static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
|
||||
{
|
||||
struct bpf_uprobe_multi_run_ctx *run_ctx;
|
||||
|
||||
run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
|
||||
return run_ctx->uprobe->cookie;
|
||||
}
|
||||
|
||||
int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *link = NULL;
|
||||
unsigned long __user *uref_ctr_offsets;
|
||||
unsigned long *ref_ctr_offsets = NULL;
|
||||
struct bpf_link_primer link_primer;
|
||||
struct bpf_uprobe *uprobes = NULL;
|
||||
struct task_struct *task = NULL;
|
||||
unsigned long __user *uoffsets;
|
||||
u64 __user *ucookies;
|
||||
void __user *upath;
|
||||
u32 flags, cnt, i;
|
||||
struct path path;
|
||||
char *name;
|
||||
pid_t pid;
|
||||
int err;
|
||||
|
||||
/* no support for 32bit archs yet */
|
||||
if (sizeof(u64) != sizeof(void *))
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)
|
||||
return -EINVAL;
|
||||
|
||||
flags = attr->link_create.uprobe_multi.flags;
|
||||
if (flags & ~BPF_F_UPROBE_MULTI_RETURN)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* path, offsets and cnt are mandatory,
|
||||
* ref_ctr_offsets and cookies are optional
|
||||
*/
|
||||
upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path);
|
||||
uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets);
|
||||
cnt = attr->link_create.uprobe_multi.cnt;
|
||||
|
||||
if (!upath || !uoffsets || !cnt)
|
||||
return -EINVAL;
|
||||
|
||||
uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets);
|
||||
ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies);
|
||||
|
||||
name = strndup_user(upath, PATH_MAX);
|
||||
if (IS_ERR(name)) {
|
||||
err = PTR_ERR(name);
|
||||
return err;
|
||||
}
|
||||
|
||||
err = kern_path(name, LOOKUP_FOLLOW, &path);
|
||||
kfree(name);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if (!d_is_reg(path.dentry)) {
|
||||
err = -EBADF;
|
||||
goto error_path_put;
|
||||
}
|
||||
|
||||
pid = attr->link_create.uprobe_multi.pid;
|
||||
if (pid) {
|
||||
rcu_read_lock();
|
||||
task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
|
||||
rcu_read_unlock();
|
||||
if (!task)
|
||||
goto error_path_put;
|
||||
}
|
||||
|
||||
err = -ENOMEM;
|
||||
|
||||
link = kzalloc(sizeof(*link), GFP_KERNEL);
|
||||
uprobes = kvcalloc(cnt, sizeof(*uprobes), GFP_KERNEL);
|
||||
|
||||
if (!uprobes || !link)
|
||||
goto error_free;
|
||||
|
||||
if (uref_ctr_offsets) {
|
||||
ref_ctr_offsets = kvcalloc(cnt, sizeof(*ref_ctr_offsets), GFP_KERNEL);
|
||||
if (!ref_ctr_offsets)
|
||||
goto error_free;
|
||||
}
|
||||
|
||||
for (i = 0; i < cnt; i++) {
|
||||
if (ucookies && __get_user(uprobes[i].cookie, ucookies + i)) {
|
||||
err = -EFAULT;
|
||||
goto error_free;
|
||||
}
|
||||
if (uref_ctr_offsets && __get_user(ref_ctr_offsets[i], uref_ctr_offsets + i)) {
|
||||
err = -EFAULT;
|
||||
goto error_free;
|
||||
}
|
||||
if (__get_user(uprobes[i].offset, uoffsets + i)) {
|
||||
err = -EFAULT;
|
||||
goto error_free;
|
||||
}
|
||||
|
||||
uprobes[i].link = link;
|
||||
|
||||
if (flags & BPF_F_UPROBE_MULTI_RETURN)
|
||||
uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
|
||||
else
|
||||
uprobes[i].consumer.handler = uprobe_multi_link_handler;
|
||||
|
||||
if (pid)
|
||||
uprobes[i].consumer.filter = uprobe_multi_link_filter;
|
||||
}
|
||||
|
||||
link->cnt = cnt;
|
||||
link->uprobes = uprobes;
|
||||
link->path = path;
|
||||
link->task = task;
|
||||
|
||||
bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI,
|
||||
&bpf_uprobe_multi_link_lops, prog);
|
||||
|
||||
for (i = 0; i < cnt; i++) {
|
||||
err = uprobe_register_refctr(d_real_inode(link->path.dentry),
|
||||
uprobes[i].offset,
|
||||
ref_ctr_offsets ? ref_ctr_offsets[i] : 0,
|
||||
&uprobes[i].consumer);
|
||||
if (err) {
|
||||
bpf_uprobe_unregister(&path, uprobes, i);
|
||||
goto error_free;
|
||||
}
|
||||
}
|
||||
|
||||
err = bpf_link_prime(&link->link, &link_primer);
|
||||
if (err)
|
||||
goto error_free;
|
||||
|
||||
kvfree(ref_ctr_offsets);
|
||||
return bpf_link_settle(&link_primer);
|
||||
|
||||
error_free:
|
||||
kvfree(ref_ctr_offsets);
|
||||
kvfree(uprobes);
|
||||
kfree(link);
|
||||
if (task)
|
||||
put_task_struct(task);
|
||||
error_path_put:
|
||||
path_put(&path);
|
||||
return err;
|
||||
}
|
||||
#else /* !CONFIG_UPROBES */
|
||||
int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_UPROBES */
|
||||
|
||||
@@ -1561,15 +1561,10 @@ int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
|
||||
|
||||
*fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
|
||||
: BPF_FD_TYPE_KPROBE;
|
||||
if (tk->symbol) {
|
||||
*symbol = tk->symbol;
|
||||
*probe_offset = tk->rp.kp.offset;
|
||||
*probe_addr = 0;
|
||||
} else {
|
||||
*symbol = NULL;
|
||||
*probe_offset = 0;
|
||||
*probe_addr = (unsigned long)tk->rp.kp.addr;
|
||||
}
|
||||
*probe_offset = tk->rp.kp.offset;
|
||||
*probe_addr = kallsyms_show_value(current_cred()) ?
|
||||
(unsigned long)tk->rp.kp.addr : 0;
|
||||
*symbol = tk->symbol;
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_PERF_EVENTS */
|
||||
|
||||
@@ -519,3 +519,8 @@ void __trace_probe_log_err(int offset, int err);
|
||||
|
||||
#define trace_probe_log_err(offs, err) \
|
||||
__trace_probe_log_err(offs, TP_ERR_##err)
|
||||
|
||||
struct uprobe_dispatch_data {
|
||||
struct trace_uprobe *tu;
|
||||
unsigned long bp_addr;
|
||||
};
|
||||
|
||||
@@ -555,12 +555,15 @@ static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *re
|
||||
struct syscall_trace_enter *rec)
|
||||
{
|
||||
struct syscall_tp_t {
|
||||
unsigned long long regs;
|
||||
struct trace_entry ent;
|
||||
unsigned long syscall_nr;
|
||||
unsigned long args[SYSCALL_DEFINE_MAXARGS];
|
||||
} param;
|
||||
} __aligned(8) param;
|
||||
int i;
|
||||
|
||||
BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
|
||||
|
||||
/* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */
|
||||
*(struct pt_regs **)¶m = regs;
|
||||
param.syscall_nr = rec->nr;
|
||||
for (i = 0; i < sys_data->nb_args; i++)
|
||||
@@ -657,11 +660,12 @@ static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *reg
|
||||
struct syscall_trace_exit *rec)
|
||||
{
|
||||
struct syscall_tp_t {
|
||||
unsigned long long regs;
|
||||
struct trace_entry ent;
|
||||
unsigned long syscall_nr;
|
||||
unsigned long ret;
|
||||
} param;
|
||||
} __aligned(8) param;
|
||||
|
||||
/* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */
|
||||
*(struct pt_regs **)¶m = regs;
|
||||
param.syscall_nr = rec->nr;
|
||||
param.ret = rec->ret;
|
||||
|
||||
@@ -88,11 +88,6 @@ static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
|
||||
static int register_uprobe_event(struct trace_uprobe *tu);
|
||||
static int unregister_uprobe_event(struct trace_uprobe *tu);
|
||||
|
||||
struct uprobe_dispatch_data {
|
||||
struct trace_uprobe *tu;
|
||||
unsigned long bp_addr;
|
||||
};
|
||||
|
||||
static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
|
||||
static int uretprobe_dispatcher(struct uprobe_consumer *con,
|
||||
unsigned long func, struct pt_regs *regs);
|
||||
@@ -1352,7 +1347,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
|
||||
if (bpf_prog_array_valid(call)) {
|
||||
u32 ret;
|
||||
|
||||
ret = bpf_prog_run_array_sleepable(call->prog_array, regs, bpf_prog_run);
|
||||
ret = bpf_prog_run_array_uprobe(call->prog_array, regs, bpf_prog_run);
|
||||
if (!ret)
|
||||
return;
|
||||
}
|
||||
@@ -1418,7 +1413,7 @@ static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
|
||||
|
||||
int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
|
||||
const char **filename, u64 *probe_offset,
|
||||
bool perf_type_tracepoint)
|
||||
u64 *probe_addr, bool perf_type_tracepoint)
|
||||
{
|
||||
const char *pevent = trace_event_name(event->tp_event);
|
||||
const char *group = event->tp_event->class->system;
|
||||
@@ -1435,6 +1430,7 @@ int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
|
||||
: BPF_FD_TYPE_UPROBE;
|
||||
*filename = tu->filename;
|
||||
*probe_offset = tu->offset;
|
||||
*probe_addr = 0;
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_PERF_EVENTS */
|
||||
|
||||
Reference in New Issue
Block a user