From: Peter Zijlstra <peterz@infradead.org>

Date: Tue, 14 Jun 2022 23:16:02 +0200

Subject: x86/bugs: Add retbleed=ibpb

Git-commit: 3ebc170068885b6fc7bedda6c667bb2c4d533159

Patch-mainline: v5.19-rc4

References: bsc#1199657 CVE-2022-29900 CVE-2022-29901

jmp2ret mitigates the easy-to-attack case at relatively low overhead.

It mitigates the long speculation windows after a mispredicted RET, but

it does not mitigate the short speculation window from arbitrary

instruction boundaries.

On Zen2, there is a chicken bit which needs setting, which mitigates

"arbitrary instruction boundaries" down to just "basic block boundaries".

But there is no fix for the short speculation window on basic block

boundaries, other than to flush the entire BTB to evict all attacker

predictions.

On the spectrum of "fast & blurry" -> "safe", there is (on top of STIBP

or no-SMT):

  1) Nothing		System wide open

  2) jmp2ret		May stop a script kiddy

  3) jmp2ret+chickenbit  Raises the bar rather further

  4) IBPB		Only thing which can count as "safe".

Tentative numbers put IBPB-on-entry at a 2.5x hit on Zen2, and a 10x hit

on Zen1 according to lmbench.

  [ bp: Fixup feature bit comments, document option, 32-bit build fix. ]

Suggested-by: Andrew Cooper <Andrew.Cooper3@citrix.com>

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Signed-off-by: Borislav Petkov <bp@suse.de>

Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>

Signed-off-by: Borislav Petkov <bp@suse.de>

---

 Documentation/admin-guide/kernel-parameters.txt |    3 +

 arch/x86/entry/Makefile                         |    2 -

 arch/x86/entry/entry.S                          |   32 +++++++++++++++++

 arch/x86/include/asm/cpufeatures.h              |    2 -

 arch/x86/include/asm/nospec-branch.h            |    8 +++-

 arch/x86/kernel/cpu/bugs.c                      |   43 ++++++++++++++++++------

 6 files changed, 77 insertions(+), 13 deletions(-)

--- /dev/null

+++ b/arch/x86/entry/entry.S

@@ -0,0 +1,32 @@

+/* SPDX-License-Identifier: GPL-2.0 */

+/*

+ * Common place for both 32- and 64-bit entry routines.

+ */

+

+#include <linux/linkage.h>

+#include <asm/export.h>

+#include <asm/msr-index.h>

+

+.pushsection .noinstr.text, "ax"

+

+ENTRY(entry_ibpb)

+

+	push %rax

+	push %rcx

+	push %rdx

+

+	movl	$MSR_IA32_PRED_CMD, %ecx

+	movl	$PRED_CMD_IBPB, %eax

+	xorl	%edx, %edx

+	wrmsr

+

+	pop %rdx

+	pop %rcx

+	pop %rax

+

+	RET

+END(entry_ibpb)

+/* For KVM */

+EXPORT_SYMBOL_GPL(entry_ibpb);

+

+.popsection

--- a/arch/x86/entry/Makefile

+++ b/arch/x86/entry/Makefile

@@ -6,7 +6,7 @@ OBJECT_FILES_NON_STANDARD_entry_64_compa

 CFLAGS_syscall_64.o		+= $(call cc-option,-Wno-override-init,)

 CFLAGS_syscall_32.o		+= $(call cc-option,-Wno-override-init,)

-obj-y				:= entry_$(BITS).o thunk_$(BITS).o syscall_$(BITS).o

+obj-y				:= entry.o entry_$(BITS).o thunk_$(BITS).o syscall_$(BITS).o

 obj-y				+= common.o

 obj-y				+= vdso/

--- a/arch/x86/include/asm/cpufeatures.h

+++ b/arch/x86/include/asm/cpufeatures.h

@@ -283,7 +283,7 @@

 #define X86_FEATURE_CQM_MBM_LOCAL	(11*32+ 3) /* LLC Local MBM monitoring */

 #define X86_FEATURE_FENCE_SWAPGS_USER	(11*32+ 4) /* "" LFENCE in user entry SWAPGS path */

 #define X86_FEATURE_FENCE_SWAPGS_KERNEL	(11*32+ 5) /* "" LFENCE in kernel entry SWAPGS path */

-/* FREE!				(11*32+10) */

+#define X86_FEATURE_ENTRY_IBPB		(11*32+10) /* "" Issue an IBPB on kernel entry */

 /* FREE!				(11*32+11) */

 #define X86_FEATURE_RETPOLINE		(11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */

 #define X86_FEATURE_RETPOLINE_LFENCE	(11*32+13) /* "" Use LFENCE for Spectre variant 2 */

--- a/arch/x86/include/asm/nospec-branch.h

+++ b/arch/x86/include/asm/nospec-branch.h

@@ -155,14 +155,17 @@

  * return thunk isn't mapped into the userspace tables (then again, AMD

  * typically has NO_MELTDOWN).

  *

- * Doesn't clobber any registers but does require a stable stack.

+ * While zen_untrain_ret() doesn't clobber anything but requires stack,

+ * entry_ibpb() will clobber AX, CX, DX.

  *

  * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point

  * where we have a stack but before any RET instruction.

  */

 .macro UNTRAIN_RET

 #ifdef CONFIG_RETPOLINE

-	ALTERNATIVE "", "call zen_untrain_ret", X86_FEATURE_UNRET

+	ALTERNATIVE_2 "",						\

+	              "call zen_untrain_ret", X86_FEATURE_UNRET,	\

+		      "call entry_ibpb", X86_FEATURE_ENTRY_IBPB

 #endif

 .endm

@@ -185,6 +188,7 @@

 extern void __x86_return_thunk(void);

 extern void zen_untrain_ret(void);

+extern void entry_ibpb(void);

 /*

  * Inline asm uses the %V modifier which is only in newer GCC

--- a/arch/x86/kernel/cpu/bugs.c

+++ b/arch/x86/kernel/cpu/bugs.c

@@ -878,6 +878,7 @@ static enum spectre_v2_mitigation spectr

 enum retbleed_mitigation {

 	RETBLEED_MITIGATION_NONE,

 	RETBLEED_MITIGATION_UNRET,

+	RETBLEED_MITIGATION_IBPB,

 	RETBLEED_MITIGATION_IBRS,

 	RETBLEED_MITIGATION_EIBRS,

 };

@@ -886,11 +887,13 @@ enum retbleed_mitigation_cmd {

 	RETBLEED_CMD_OFF,

 	RETBLEED_CMD_AUTO,

 	RETBLEED_CMD_UNRET,

+	RETBLEED_CMD_IBPB,

 };

 const char * const retbleed_strings[] = {

 	[RETBLEED_MITIGATION_NONE]	= "Vulnerable",

 	[RETBLEED_MITIGATION_UNRET]	= "Mitigation: untrained return thunk",

+	[RETBLEED_MITIGATION_IBPB]	= "Mitigation: IBPB",

 	[RETBLEED_MITIGATION_IBRS]	= "Mitigation: IBRS",

 	[RETBLEED_MITIGATION_EIBRS]	= "Mitigation: Enhanced IBRS",

 };

@@ -920,6 +923,8 @@ static int __init retbleed_parse_cmdline

 			retbleed_cmd = RETBLEED_CMD_AUTO;

 		} else if (!strcmp(str, "unret")) {

 			retbleed_cmd = RETBLEED_CMD_UNRET;

+		} else if (!strcmp(str, "ibpb")) {

+			retbleed_cmd = RETBLEED_CMD_IBPB;

 		} else if (!strcmp(str, "nosmt")) {

 			retbleed_nosmt = true;

 		} else {

@@ -934,11 +939,13 @@ static int __init retbleed_parse_cmdline

 early_param("retbleed", retbleed_parse_cmdline);

 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"

-#define RETBLEED_COMPILER_MSG "WARNING: kernel not compiled with RETPOLINE or -mfunction-return capable compiler!\n"

+#define RETBLEED_COMPILER_MSG "WARNING: kernel not compiled with RETPOLINE or -mfunction-return capable compiler; falling back to IBPB!\n"

 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"

 static void __init retbleed_select_mitigation(void)

 {

+	bool mitigate_smt = false;

+

 	if (!boot_cpu_has_bug(X86_BUG_RETBLEED))

 		return;

@@ -950,11 +957,21 @@ static void __init retbleed_select_mitig

 		retbleed_mitigation = RETBLEED_MITIGATION_UNRET;

 		break;

+	case RETBLEED_CMD_IBPB:

+		retbleed_mitigation = RETBLEED_MITIGATION_IBPB;

+		break;

+

 	case RETBLEED_CMD_AUTO:

 	default:

 		if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||

-		    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)

-			retbleed_mitigation = RETBLEED_MITIGATION_UNRET;

+		    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {

+

+			if (IS_ENABLED(CONFIG_RETPOLINE) &&

+			    IS_ENABLED(CONFIG_CC_HAS_RETURN_THUNK))

+				retbleed_mitigation = RETBLEED_MITIGATION_UNRET;

+			else

+				retbleed_mitigation = RETBLEED_MITIGATION_IBPB;

+		}

 		/*

 		 * The Intel mitigation (IBRS) was already selected in

@@ -970,26 +987,34 @@ static void __init retbleed_select_mitig

 		if (!IS_ENABLED(CONFIG_RETPOLINE) ||

 		    !IS_ENABLED(CONFIG_CC_HAS_RETURN_THUNK)) {

 			pr_err(RETBLEED_COMPILER_MSG);

-			retbleed_mitigation = RETBLEED_MITIGATION_NONE;

-			break;

+			retbleed_mitigation = RETBLEED_MITIGATION_IBPB;

+			goto retbleed_force_ibpb;

 		}

 		setup_force_cpu_cap(X86_FEATURE_RETHUNK);

 		setup_force_cpu_cap(X86_FEATURE_UNRET);

-		if (!boot_cpu_has(X86_FEATURE_STIBP) &&

-		    (retbleed_nosmt || cpu_mitigations_auto_nosmt()))

-			cpu_smt_disable(false);

-

 		if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&

 		    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)

 			pr_err(RETBLEED_UNTRAIN_MSG);

+

+		mitigate_smt = true;

+		break;

+

+	case RETBLEED_MITIGATION_IBPB:

+retbleed_force_ibpb:

+		setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);

+		mitigate_smt = true;

 		break;

 	default:

 		break;

 	}

+	if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&

+	    (retbleed_nosmt || cpu_mitigations_auto_nosmt()))

+		cpu_smt_disable(false);

+

 	/*

 	 * Let IBRS trump all on Intel without affecting the effects of the

 	 * retbleed= cmdline option.

--- a/Documentation/admin-guide/kernel-parameters.txt

+++ b/Documentation/admin-guide/kernel-parameters.txt

@@ -3986,6 +3986,9 @@

 				       disabling SMT if necessary for

 				       the full mitigation (only on Zen1

 				       and older without STIBP).

+			ibpb	     - mitigate short speculation windows on

+				       basic block boundaries too. Safe, highest

+				       perf impact.

 			unret        - force enable untrained return thunks,

 				       only effective on AMD f15h-f17h

 				       based systems.