From: Peter Zijlstra <peterz@infradead.org>
Date: Tue, 14 Jun 2022 23:15:53 +0200
Subject: x86/entry: Add kernel IBRS implementation
Git-commit: 2dbb887e875b1de3ca8f40ddf26bcfe55798c609
Patch-mainline: Queued in tip for 5.19
Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
References: bsc#1199657 CVE-2022-29900 CVE-2022-29901

Implement Kernel IBRS - currently the only known option to mitigate RSB
underflow speculation issues on Skylake hardware.

Note: since IBRS_ENTER requires fuller context established than
UNTRAIN_RET, it must be placed after it. However, since UNTRAIN_RET
itself implies a RET, it must come after IBRS_ENTER. This means
IBRS_ENTER needs to also move UNTRAIN_RET.

Note 2: KERNEL_IBRS is sub-optimal for XenPV.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>

  [ bp: Use the IBRS implementation which is already present in the SLE kernel ]

Signed-off-by: Borislav Petkov <bp@suse.de>
---
 arch/x86/entry/entry_64.S        |   14 ++++++++------
 arch/x86/entry/entry_64_compat.S |    8 ++++----
 2 files changed, 12 insertions(+), 10 deletions(-)

--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -3,7 +3,6 @@
  *
  * Copyright 2000-2002 Andi Kleen, SuSE Labs.
  */
-#include "calling.h"
 #include <asm/asm-offsets.h>
 #include <asm/current.h>
 #include <asm/errno.h>
@@ -17,6 +16,8 @@
 #include <linux/linkage.h>
 #include <linux/err.h>
 
+#include "calling.h"
+
 	.section .entry.text, "ax"
 
 /*
@@ -54,8 +55,6 @@ ENTRY(entry_SYSENTER_compat)
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
 
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
-	/* Restrict Indirect Branch Speculation */
-	RESTRICT_IB_SPEC
 
 	/*
 	 * User tracing code (ptrace or signal handlers) might assume that
@@ -108,6 +107,8 @@ ENTRY(entry_SYSENTER_compat)
 	xorl	%r15d, %r15d		/* nospec   r15 */
 	cld
 
+
+	RESTRICT_IB_SPEC
 	UNTRAIN_RET
 
 	/*
@@ -250,7 +251,6 @@ GLOBAL(entry_SYSCALL_compat_after_hwfram
 
 	/* Restrict Indirect Branch Speculation. All registers are saved already */
 	RESTRICT_IB_SPEC_CLOBBER
-
 	UNTRAIN_RET
 
 	/* User mode is traced as though IRQs are on, and SYSENTER
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -215,8 +215,6 @@ ENTRY(entry_SYSCALL_64)
 	 */
 	movq	%rsp, PER_CPU_VAR(rsp_scratch)
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
-	/* Restrict Indirect Branch Speculation */
-	RESTRICT_IB_SPEC
 
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS			/* pt_regs->ss */
@@ -234,6 +232,9 @@ GLOBAL(entry_SYSCALL_64_after_hwframe)
 	/* IRQs are off. */
 	movq	%rsp, %rdi
 
+	/* Restrict Indirect Branch Speculation */
+	RESTRICT_IB_SPEC
+
 	UNTRAIN_RET
 
 	call	do_syscall_64		/* returns with IRQs disabled */
@@ -1192,6 +1193,7 @@ ENTRY(paranoid_entry)
 	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
 	/* Restrict Indirect Branch speculation */
 	RESTRICT_IB_SPEC_SAVE_AND_CLOBBER save_reg=%r13d
+	UNTRAIN_RET
 	/*
 	 * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
 	 * unconditional CR3 write, even in the PTI case.  So do an lfence
@@ -1420,10 +1422,6 @@ ENTRY(nmi)
 	movq	%rsp, %rdx
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
-	/* Restrict Indirect Branch Speculation */
-	RESTRICT_IB_SPEC
-	UNTRAIN_RET
-
 	UNWIND_HINT_IRET_REGS base=%rdx offset=8
 	pushq	5*8(%rdx)	/* pt_regs->ss */
 	pushq	4*8(%rdx)	/* pt_regs->rsp */
@@ -1435,6 +1433,10 @@ ENTRY(nmi)
 	PUSH_AND_CLEAR_REGS rdx=(%rdx)
 	ENCODE_FRAME_POINTER
 
+	/* Restrict Indirect Branch Speculation */
+	RESTRICT_IB_SPEC
+	UNTRAIN_RET
+
 	/*
 	 * At this point we no longer need to worry about stack damage
 	 * due to nesting -- we're on the normal thread stack and we're