From: Peter Zijlstra <peterz@infradead.org>
Date: Tue, 14 Apr 2020 12:36:16 +0200
Subject: x86/speculation: Change FILL_RETURN_BUFFER to work with objtool
Git-commit: 089dd8e53126ebaf506e2dc0bf89d652c36bfc12
Patch-mainline: v5.8-rc1
References: bsc#1201726 CVE-2022-26373
Change FILL_RETURN_BUFFER so that objtool groks it and can generate
correct ORC unwind information.
- Since ORC is alternative invariant; that is, all alternatives
should have the same ORC entries, the __FILL_RETURN_BUFFER body
can not be part of an alternative.
Therefore, move it out of the alternative and keep the alternative
as a sort of jump_label around it.
- Use the ANNOTATE_INTRA_FUNCTION_CALL annotation to white-list
these 'funny' call instructions to nowhere.
- Use UNWIND_HINT_EMPTY to 'fill' the speculation traps, otherwise
objtool will consider them unreachable.
- Move the RSP adjustment into the loop, such that the loop has a
deterministic stack layout.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: Josh Poimboeuf <jpoimboe@redhat.com>
Link: https://lkml.kernel.org/r/20200428191700.032079304@infradead.org
Acked-by: Borislav Petkov <bp@suse.de>
[ mb: updated to fix the orc unwinder. See bsc#1202396. ]
---
arch/x86/include/asm/nospec-branch.h | 37 ++++++++++++++++++++++++++---------
1 file changed, 28 insertions(+), 9 deletions(-)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -4,6 +4,7 @@
#define _ASM_X86_NOSPEC_BRANCH_H_
#include <linux/static_key.h>
+#include <linux/frame.h>
#include <asm/alternative.h>
#include <asm/alternative-asm.h>
@@ -40,6 +41,23 @@
#define RSB_FILL_LOOPS 16 /* To avoid underflow */
/*
+ * UNWIND_HINT_EMPTY definition which can be used in __FILL_RETURN_BUFFER below
+ * and * stringified. Normally it should be in asm/unwind_hints.h, but keep it
+ * here, so it is easily found and due it is out-of-tree-ness.
+ */
+#define UNWIND_HINT_EMPTY_RSB \
+987: \
+ .pushsection .discard.unwind_hints; \
+ /* struct unwind_hint */ \
+ .long 987b - .; \
+ .short 0; /* sp_offset */ \
+ .byte 0; /* sp_reg */ \
+ .byte 0; /* type UNWIND_HINT_TYPE_CALL */ \
+ .byte 1; /* end */ \
+ .balign 4; \
+ .popsection;
+
+/*
* Google experimented with loop-unrolling and this turned out to be
* the optimal version — two calls, each with their own speculation
* trap should their return address end up getting used, in a loop.
@@ -47,21 +65,25 @@
#define __FILL_RETURN_BUFFER(reg, nr, sp) \
mov $(nr/2), reg; \
771: \
+ ANNOTATE_INTRA_FUNCTION_CALL; \
call 772f; \
773: /* speculation trap */ \
+ UNWIND_HINT_EMPTY_RSB; \
pause; \
lfence; \
jmp 773b; \
772: \
+ ANNOTATE_INTRA_FUNCTION_CALL; \
call 774f; \
775: /* speculation trap */ \
+ UNWIND_HINT_EMPTY_RSB; \
pause; \
lfence; \
jmp 775b; \
774: \
+ add $(BITS_PER_LONG/8) * 2, sp; \
dec reg; \
- jnz 771b; \
- add $(BITS_PER_LONG/8) * nr, sp;
+ jnz 771b;
#ifdef __ASSEMBLY__
@@ -138,10 +160,8 @@
*/
.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
#ifdef CONFIG_RETPOLINE
- ANNOTATE_NOSPEC_ALTERNATIVE
- ALTERNATIVE "jmp .Lskip_rsb_\@", \
- __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
- \ftr
+ ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr
+ __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
.Lskip_rsb_\@:
#endif
.endm
@@ -250,9 +270,8 @@ static inline void vmexit_fill_RSB(void)
unsigned long loops;
asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
- ALTERNATIVE("jmp 910f",
- __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
- X86_FEATURE_RETPOLINE)
+ ALTERNATIVE("jmp 910f", "", X86_FEATURE_RETPOLINE)
+ __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1))
"910:"
: "=r" (loops), ASM_CALL_CONSTRAINT
: : "memory" );