From: Jiri Olsa <jolsa@kernel.org>
Date: Mon, 21 Mar 2022 08:01:13 +0100
Subject: bpf: Fix kprobe_multi return probe backtrace
Patch-mainline: v5.18-rc1
Git-commit: f70986902c86f88612ed45a96aa7cf4caa65f7c1
References: jsc#PED-1377
Andrii reported that backtraces from kprobe_multi program attached
as return probes are not complete and showing just initial entry [1].
It's caused by changing registers to have original function ip address
as instruction pointer even for return probe, which will screw backtrace
from return probe.
This change keeps registers intact and store original entry ip and
link address on the stack in bpf_kprobe_multi_run_ctx struct, where
bpf_get_func_ip and bpf_get_attach_cookie helpers for kprobe_multi
programs can find it.
[1] https://lore.kernel.org/bpf/CAEf4BzZDDqK24rSKwXNp7XL3ErGD4bZa1M6c_c4EvDSt3jrZcg@mail.gmail.com/T/#m8d1301c0ea0892ddf9dc6fba57a57b8cf11b8c51
Fixes: ca74823c6e16 ("bpf: Add cookie support to programs attached with kprobe multi link")
Reported-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20220321070113.1449167-3-jolsa@kernel.org
Acked-by: Shung-Hsi Yu <shung-hsi.yu@suse.com>
---
kernel/trace/bpf_trace.c | 67 +++++++++++++++++++++++++----------------------
1 file changed, 37 insertions(+), 30 deletions(-)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -80,7 +80,8 @@ u64 bpf_get_stack(u64 r1, u64 r2, u64 r3
static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
u64 flags, const struct btf **btf,
s32 *btf_id);
-static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip);
+static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx);
+static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx);
/**
* trace_call_bpf - invoke BPF program
@@ -1043,7 +1044,7 @@ static const struct bpf_func_proto bpf_g
BPF_CALL_1(bpf_get_func_ip_kprobe_multi, struct pt_regs *, regs)
{
/* This helper call is inlined by verifier on x86. */
- return instruction_pointer(regs);
+ return bpf_kprobe_multi_entry_ip(current->bpf_ctx);
}
static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = {
@@ -1055,7 +1056,7 @@ static const struct bpf_func_proto bpf_g
BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs)
{
- return bpf_kprobe_multi_cookie(current->bpf_ctx, instruction_pointer(regs));
+ return bpf_kprobe_multi_cookie(current->bpf_ctx);
}
static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = {
@@ -2220,15 +2221,16 @@ struct bpf_kprobe_multi_link {
struct bpf_link link;
struct fprobe fp;
unsigned long *addrs;
- /*
- * The run_ctx here is used to get struct bpf_kprobe_multi_link in
- * get_attach_cookie helper, so it can't be used to store data.
- */
- struct bpf_run_ctx run_ctx;
u64 *cookies;
u32 cnt;
};
+struct bpf_kprobe_multi_run_ctx {
+ struct bpf_run_ctx run_ctx;
+ struct bpf_kprobe_multi_link *link;
+ unsigned long entry_ip;
+};
+
static void bpf_kprobe_multi_link_release(struct bpf_link *link)
{
struct bpf_kprobe_multi_link *kmulti_link;
@@ -2282,18 +2284,21 @@ static int bpf_kprobe_multi_cookie_cmp(c
return __bpf_kprobe_multi_cookie_cmp(a, b);
}
-static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip)
+static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)
{
+ struct bpf_kprobe_multi_run_ctx *run_ctx;
struct bpf_kprobe_multi_link *link;
+ u64 *cookie, entry_ip;
unsigned long *addr;
- u64 *cookie;
if (WARN_ON_ONCE(!ctx))
return 0;
- link = container_of(ctx, struct bpf_kprobe_multi_link, run_ctx);
+ run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, run_ctx);
+ link = run_ctx->link;
if (!link->cookies)
return 0;
- addr = bsearch(&ip, link->addrs, link->cnt, sizeof(ip),
+ entry_ip = run_ctx->entry_ip;
+ addr = bsearch(&entry_ip, link->addrs, link->cnt, sizeof(entry_ip),
__bpf_kprobe_multi_cookie_cmp);
if (!addr)
return 0;
@@ -2301,10 +2306,22 @@ static u64 bpf_kprobe_multi_cookie(struc
return *cookie;
}
+static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
+{
+ struct bpf_kprobe_multi_run_ctx *run_ctx;
+
+ run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, run_ctx);
+ return run_ctx->entry_ip;
+}
+
static int
kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
- struct pt_regs *regs)
+ unsigned long entry_ip, struct pt_regs *regs)
{
+ struct bpf_kprobe_multi_run_ctx run_ctx = {
+ .link = link,
+ .entry_ip = entry_ip,
+ };
struct bpf_run_ctx *old_run_ctx;
int err;
@@ -2315,7 +2332,7 @@ kprobe_multi_link_prog_run(struct bpf_kp
migrate_disable();
rcu_read_lock();
- old_run_ctx = bpf_set_run_ctx(&link->run_ctx);
+ old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
err = bpf_prog_run(link->link.prog, regs);
bpf_reset_run_ctx(old_run_ctx);
rcu_read_unlock();
@@ -2330,24 +2347,10 @@ static void
kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip,
struct pt_regs *regs)
{
- unsigned long saved_ip = instruction_pointer(regs);
struct bpf_kprobe_multi_link *link;
- /*
- * Because fprobe's regs->ip is set to the next instruction of
- * dynamic-ftrace instruction, correct entry ip must be set, so
- * that the bpf program can access entry address via regs as same
- * as kprobes.
- *
- * Both kprobe and kretprobe see the entry ip of traced function
- * as instruction pointer.
- */
- instruction_pointer_set(regs, entry_ip);
-
link = container_of(fp, struct bpf_kprobe_multi_link, fp);
- kprobe_multi_link_prog_run(link, regs);
-
- instruction_pointer_set(regs, saved_ip);
+ kprobe_multi_link_prog_run(link, entry_ip, regs);
}
static int
@@ -2514,7 +2517,11 @@ int bpf_kprobe_multi_link_attach(const u
{
return -EOPNOTSUPP;
}
-static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip)
+static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)
+{
+ return 0;
+}
+static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
{
return 0;
}