From: Eric Dumazet <edumazet@google.com>
Date: Tue, 26 Oct 2021 14:41:33 -0700
Subject: bpf: Use u64_stats_t in struct bpf_prog_stats
Patch-mainline: v5.16-rc1
Git-commit: 61a0abaee2092eee69e44fe60336aa2f5b578938
References: jsc#PED-1377
Commit 316580b69d0a ("u64_stats: provide u64_stats_t type")
fixed possible load/store tearing on 64bit arches.
For instance the following C code
stats->nsecs += sched_clock() - start;
Could be rightfully implemented like this by a compiler,
confusing concurrent readers a lot:
stats->nsecs += sched_clock();
// arbitrary delay
stats->nsecs -= start;
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20211026214133.3114279-4-eric.dumazet@gmail.com
Acked-by: Shung-Hsi Yu <shung-hsi.yu@suse.com>
---
include/linux/filter.h | 10 +++++-----
kernel/bpf/syscall.c | 18 ++++++++++++------
kernel/bpf/trampoline.c | 6 +++---
3 files changed, 20 insertions(+), 14 deletions(-)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -555,9 +555,9 @@ struct bpf_binary_header {
};
struct bpf_prog_stats {
- u64 cnt;
- u64 nsecs;
- u64 misses;
+ u64_stats_t cnt;
+ u64_stats_t nsecs;
+ u64_stats_t misses;
struct u64_stats_sync syncp;
} __aligned(2 * sizeof(u64));
@@ -619,8 +619,8 @@ static __always_inline u32 __bpf_prog_ru
ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
stats = this_cpu_ptr(prog->stats);
flags = u64_stats_update_begin_irqsave(&stats->syncp);
- stats->cnt++;
- stats->nsecs += sched_clock() - start;
+ u64_stats_inc(&stats->cnt);
+ u64_stats_add(&stats->nsecs, sched_clock() - start);
u64_stats_update_end_irqrestore(&stats->syncp, flags);
} else {
ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1807,8 +1807,14 @@ static int bpf_prog_release(struct inode
return 0;
}
+struct bpf_prog_kstats {
+ u64 nsecs;
+ u64 cnt;
+ u64 misses;
+};
+
static void bpf_prog_get_stats(const struct bpf_prog *prog,
- struct bpf_prog_stats *stats)
+ struct bpf_prog_kstats *stats)
{
u64 nsecs = 0, cnt = 0, misses = 0;
int cpu;
@@ -1821,9 +1827,9 @@ static void bpf_prog_get_stats(const str
st = per_cpu_ptr(prog->stats, cpu);
do {
start = u64_stats_fetch_begin_irq(&st->syncp);
- tnsecs = st->nsecs;
- tcnt = st->cnt;
- tmisses = st->misses;
+ tnsecs = u64_stats_read(&st->nsecs);
+ tcnt = u64_stats_read(&st->cnt);
+ tmisses = u64_stats_read(&st->misses);
} while (u64_stats_fetch_retry_irq(&st->syncp, start));
nsecs += tnsecs;
cnt += tcnt;
@@ -1839,7 +1845,7 @@ static void bpf_prog_show_fdinfo(struct
{
const struct bpf_prog *prog = filp->private_data;
char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
- struct bpf_prog_stats stats;
+ struct bpf_prog_kstats stats;
bpf_prog_get_stats(prog, &stats);
bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
@@ -3580,7 +3586,7 @@ static int bpf_prog_get_info_by_fd(struc
struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
struct bpf_prog_info info;
u32 info_len = attr->info.info_len;
- struct bpf_prog_stats stats;
+ struct bpf_prog_kstats stats;
char __user *uinsns;
u32 ulen;
int err;
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -545,7 +545,7 @@ static void notrace inc_misses_counter(s
stats = this_cpu_ptr(prog->stats);
u64_stats_update_begin(&stats->syncp);
- stats->misses++;
+ u64_stats_inc(&stats->misses);
u64_stats_update_end(&stats->syncp);
}
@@ -590,8 +590,8 @@ static void notrace update_prog_stats(st
stats = this_cpu_ptr(prog->stats);
flags = u64_stats_update_begin_irqsave(&stats->syncp);
- stats->cnt++;
- stats->nsecs += sched_clock() - start;
+ u64_stats_inc(&stats->cnt);
+ u64_stats_add(&stats->nsecs, sched_clock() - start);
u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
}