From: Peter Zijlstra <peterz@infradead.org>
Date: Sat, 4 Dec 2021 14:43:41 +0100
Subject: x86: Prepare inline-asm for straight-line-speculation
Git-commit: b17c2baa305cccbd16bafa289fd743cc2db77966
Patch-mainline: v5.17-rc1
References: bsc#1201050 CVE-2021-26341
Replace all ret/retq instructions with ASM_RET in preparation of
making it more than a single instruction.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lore.kernel.org/r/20211204134907.964635458@infradead.org
---
arch/x86/include/asm/linkage.h | 4 ++++
arch/x86/include/asm/paravirt.h | 2 +-
arch/x86/include/asm/qspinlock_paravirt.h | 4 ++--
arch/x86/kernel/alternative.c | 2 +-
arch/x86/kernel/kprobes/core.c | 2 +-
arch/x86/kernel/paravirt.c | 2 +-
arch/x86/kvm/emulate.c | 2 +-
7 files changed, 11 insertions(+), 7 deletions(-)
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -21,6 +21,10 @@
#define __ALIGN_STR __stringify(__ALIGN)
#endif
+#else /* __ASSEMBLY__ */
+
+#define ASM_RET "ret\n\t"
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_LINKAGE_H */
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -793,7 +793,7 @@ static __always_inline bool pv_vcpu_is_p
"call " #func ";" \
PV_RESTORE_ALL_CALLER_REGS \
FRAME_END \
- "ret;" \
+ ASM_RET \
".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \
".popsection")
--- a/arch/x86/include/asm/qspinlock_paravirt.h
+++ b/arch/x86/include/asm/qspinlock_paravirt.h
@@ -47,7 +47,7 @@ asm (".pushsection .text;"
"jne .slowpath;"
"pop %rdx;"
FRAME_END
- "ret;"
+ ASM_RET
".slowpath: "
"push %rsi;"
"movzbl %al,%esi;"
@@ -55,7 +55,7 @@ asm (".pushsection .text;"
"pop %rsi;"
"pop %rdx;"
FRAME_END
- "ret;"
+ ASM_RET
".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
".popsection");
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -635,7 +635,7 @@ asm (
" .type int3_magic, @function\n"
"int3_magic:\n"
" movl $1, (%" _ASM_ARG1 ")\n"
-" ret\n"
+ ASM_RET
" .size int3_magic, .-int3_magic\n"
" .popsection\n"
);
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -736,7 +736,7 @@ asm(
RESTORE_REGS_STRING
" popf\n"
#endif
- " ret\n"
+ ASM_RET
".size kretprobe_trampoline, .-kretprobe_trampoline\n"
);
NOKPROBE_SYMBOL(kretprobe_trampoline);
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -50,7 +50,7 @@ extern void _paravirt_nop(void);
asm (".pushsection .entry.text, \"ax\"\n"
".global _paravirt_nop\n"
"_paravirt_nop:\n\t"
- "ret\n\t"
+ ASM_RET
".size _paravirt_nop, . - _paravirt_nop\n\t"
".type _paravirt_nop, @function\n\t"
".popsection");
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -429,7 +429,7 @@ static int fastop(struct x86_emulate_ctx
asm(".pushsection .fixup, \"ax\"\n"
".global kvm_fastop_exception \n"
- "kvm_fastop_exception: xor %esi, %esi; ret\n"
+ "kvm_fastop_exception: xor %esi, %esi; " ASM_RET
".popsection");
FOP_START(setcc)