From 6f9d710dd60734cf5a0c055506f6c7df9b1a0f7b Mon Sep 17 00:00:00 2001
From: Frederic Weisbecker <frederic@kernel.org>
Date: Mon, 20 Sep 2021 12:54:43 +0200
Subject: [PATCH] arm64: Implement IRQ exit preemption static call for dynamic
preemption
Patch-mainline: Not yet, debates on arm64 static call implementation
References: jsc#SLE-18857
arm64 doesn't support generic entry yet, so the architecture's own IRQ
exit preemption path needs to be exposed through the relevant static
call.
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Quentin Perret <qperret@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: David Laight <David.Laight@ACULAB.COM>
Cc: Kees Cook <keescook@chromium.org>
Cc: Sami Tolvanen <samitolvanen@google.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Frederic Weisbecker <fweisbecker@suse.de>
---
arch/arm64/include/asm/preempt.h | 7 +++++++
arch/arm64/kernel/entry-common.c | 15 ++++++++++++---
2 files changed, 19 insertions(+), 3 deletions(-)
--- a/arch/arm64/include/asm/preempt.h
+++ b/arch/arm64/include/asm/preempt.h
@@ -3,6 +3,7 @@
#define __ASM_PREEMPT_H
#include <linux/thread_info.h>
+#include <linux/static_call_types.h>
#define PREEMPT_NEED_RESCHED BIT(32)
#define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED)
@@ -86,4 +87,10 @@ void preempt_schedule_notrace(void);
#define __preempt_schedule_notrace() preempt_schedule_notrace()
#endif /* CONFIG_PREEMPTION */
+#ifdef CONFIG_PREEMPT_DYNAMIC
+void arm64_preempt_schedule_irq(void);
+#define __irqentry_exit_cond_resched_func arm64_preempt_schedule_irq
+DECLARE_STATIC_CALL(irqentry_exit_cond_resched, __irqentry_exit_cond_resched_func);
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
#endif /* __ASM_PREEMPT_H */
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -12,6 +12,7 @@
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/thread_info.h>
+#include <linux/static_call.h>
#include <asm/cpufeature.h>
#include <asm/daifflags.h>
@@ -121,7 +122,7 @@ static void noinstr exit_el1_irq_or_nmi(
exit_to_kernel_mode(regs);
}
-static void __sched arm64_preempt_schedule_irq(void)
+void __sched arm64_preempt_schedule_irq(void)
{
lockdep_assert_irqs_disabled();
@@ -145,6 +146,9 @@ static void __sched arm64_preempt_schedu
if (system_capabilities_finalized())
preempt_schedule_irq();
}
+#ifdef CONFIG_PREEMPT_DYNAMIC
+DEFINE_STATIC_CALL(irqentry_exit_cond_resched, arm64_preempt_schedule_irq);
+#endif
static void do_interrupt_handler(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
@@ -356,8 +360,13 @@ static void noinstr el1_interrupt(struct
* preempt_count().
*/
if (IS_ENABLED(CONFIG_PREEMPTION) &&
- READ_ONCE(current_thread_info()->preempt_count) == 0)
- arm64_preempt_schedule_irq();
+ READ_ONCE(current_thread_info()->preempt_count) == 0) {
+#ifdef CONFIG_PREEMPT_DYNAMIC
+ static_call(irqentry_exit_cond_resched)();
+#else
+ arm64_preempt_schedule_irq();
+#endif
+ }
exit_el1_irq_or_nmi(regs);
}