From: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>

Date: Thu, 19 May 2022 20:35:15 -0700

Subject: KVM: x86/speculation: Disable Fill buffer clear within guests

Git-commit: 027bbb884be006b05d9c577d6401686053aa789e

Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git

Patch-mainline: Queued in tip for v5.19

References: bsc#1199650 CVE-2022-21166 CVE-2022-21127 CVE-2022-21123 CVE-2022-21125 CVE-2022-21180

The enumeration of MD_CLEAR in CPUID(EAX=7,ECX=0).EDX{bit 10} is not an

accurate indicator on all CPUs of whether the VERW instruction will

overwrite fill buffers. FB_CLEAR enumeration in

IA32_ARCH_CAPABILITIES{bit 17} covers the case of CPUs that are not

vulnerable to MDS/TAA, indicating that microcode does overwrite fill

buffers.

Guests running in VMM environments may not be aware of all the

capabilities/vulnerabilities of the host CPU. Specifically, a guest may

apply MDS/TAA mitigations when a virtual CPU is enumerated as vulnerable

to MDS/TAA even when the physical CPU is not. On CPUs that enumerate

FB_CLEAR_CTRL the VMM may set FB_CLEAR_DIS to skip overwriting of fill

buffers by the VERW instruction. This is done by setting FB_CLEAR_DIS

during VMENTER and resetting on VMEXIT. For guests that enumerate

FB_CLEAR (explicitly asking for fill buffer clear capability) the VMM

will not use FB_CLEAR_DIS.

Irrespective of guest state, host overwrites CPU buffers before VMENTER

to protect itself from an MMIO capable guest, as part of mitigation for

MMIO Stale Data vulnerabilities.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>

Signed-off-by: Borislav Petkov <bp@suse.de>

---

 arch/x86/include/asm/msr-index.h |    6 +++

 arch/x86/kvm/vmx.c               |   73 +++++++++++++++++++++++++++++++++++++++

 arch/x86/kvm/x86.c               |    3 +

 3 files changed, 82 insertions(+)

--- a/arch/x86/include/asm/msr-index.h

+++ b/arch/x86/include/asm/msr-index.h

@@ -123,6 +123,11 @@

 						 * VERW clears CPU fill buffer

 						 * even on MDS_NO CPUs.

 						 */

+#define ARCH_CAP_FB_CLEAR_CTRL		BIT(18)	/*

+						 * MSR_IA32_MCU_OPT_CTRL[FB_CLEAR_DIS]

+						 * bit available to control VERW

+						 * behavior.

+						 */

 #define MSR_IA32_FLUSH_CMD		0x0000010b

 #define L1D_FLUSH			BIT(0)	/*

@@ -142,6 +147,7 @@

 /* SRBDS support */

 #define MSR_IA32_MCU_OPT_CTRL		0x00000123

 #define RNGDS_MITG_DIS			BIT(0)

+#define FB_CLEAR_DIS			BIT(3)	/* CPU Fill buffer clear disable */

 #define MSR_IA32_SYSENTER_CS		0x00000174

 #define MSR_IA32_SYSENTER_ESP		0x00000175

--- a/arch/x86/kvm/vmx.c

+++ b/arch/x86/kvm/vmx.c

@@ -209,6 +209,9 @@ static const struct {

 #define L1D_CACHE_ORDER 4

 static void *vmx_l1d_flush_pages;

+/* Control for disabling CPU Fill buffer clear */

+static bool __read_mostly vmx_fb_clear_ctrl_available;

+

 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)

 {

 	struct page *page;

@@ -824,6 +827,9 @@ struct vcpu_vmx {

 	 */

 	u64 msr_ia32_feature_control;

 	u64 msr_ia32_feature_control_valid_bits;

+

+	u64 msr_ia32_mcu_opt_ctrl;

+	bool disable_fb_clear;

 };

 enum segment_cache_field {

@@ -1693,6 +1699,61 @@ static void vmcs_load(struct vmcs *vmcs)

 		       vmcs, phys_addr);

 }

+static void vmx_setup_fb_clear_ctrl(void)

+{

+	u64 msr;

+

+	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES) &&

+	    !boot_cpu_has_bug(X86_BUG_MDS) &&

+	    !boot_cpu_has_bug(X86_BUG_TAA)) {

+		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);

+		if (msr & ARCH_CAP_FB_CLEAR_CTRL)

+			vmx_fb_clear_ctrl_available = true;

+	}

+}

+

+static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx)

+{

+	u64 msr;

+

+	if (!vmx->disable_fb_clear)

+		return;

+

+	rdmsrl(MSR_IA32_MCU_OPT_CTRL, msr);

+	msr |= FB_CLEAR_DIS;

+	wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr);

+	/* Cache the MSR value to avoid reading it later */

+	vmx->msr_ia32_mcu_opt_ctrl = msr;

+}

+

+static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx)

+{

+	if (!vmx->disable_fb_clear)

+		return;

+

+	vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS;

+	wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl);

+}

+

+static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx)

+{

+	vmx->disable_fb_clear = vmx_fb_clear_ctrl_available;

+

+	/*

+	 * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS

+	 * at VMEntry. Skip the MSR read/write when a guest has no use case to

+	 * execute VERW.

+	 */

+	if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) ||

+	   ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) &&

+	    (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) &&

+	    (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) &&

+	    (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) &&

+	    (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO)))

+		vmx->disable_fb_clear = false;

+}

+

+

 #ifdef CONFIG_KEXEC_CORE

 /*

  * This bitmap is used to indicate whether the vmclear

@@ -3775,6 +3836,10 @@ static int vmx_set_msr(struct kvm_vcpu *

 			ret = kvm_set_msr_common(vcpu, msr_info);

 	}

+	/* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */

+	if (msr_index == MSR_IA32_ARCH_CAPABILITIES)

+		vmx_update_fb_clear_dis(vcpu, vmx);

+

 	return ret;

 }

@@ -6109,6 +6174,8 @@ static void vmx_vcpu_reset(struct kvm_vc

 	update_exception_bitmap(vcpu);

 	vpid_sync_context(vmx->vpid);

+

+	vmx_update_fb_clear_dis(vcpu, vmx);

 }

 /*

@@ -9862,6 +9929,8 @@ static void __noclone vmx_vcpu_run(struc

 		 kvm_arch_has_assigned_device(vcpu->kvm))

 		mds_clear_cpu_buffers();

+	vmx_disable_fb_clear(vmx);

+

 	asm(

 		/* Store host registers */

 		"push %%" _ASM_DX "; push %%" _ASM_BP ";"

@@ -9991,6 +10060,8 @@ static void __noclone vmx_vcpu_run(struc

 #endif

 	      );

+	vmx_enable_fb_clear(vmx);

+

 	/*

 	 * We do not use IBRS in the kernel. If this vCPU has used the

 	 * SPEC_CTRL MSR it may have left it on; save the value and

@@ -13140,6 +13211,8 @@ static int __init vmx_init(void)

 		}

 	}

+	vmx_setup_fb_clear_ctrl();

+

 #ifdef CONFIG_KEXEC_CORE

 	rcu_assign_pointer(crash_vmclear_loaded_vmcss,

 			   crash_vmclear_local_loaded_vmcss);

--- a/arch/x86/kvm/x86.c

+++ b/arch/x86/kvm/x86.c

@@ -1125,6 +1125,9 @@ u64 kvm_get_arch_capabilities(void)

 	 */

 	data |= ARCH_CAP_PSCHANGE_MC_NO;

+	/* Guests don't need to know "Fill buffer clear control" exists */

+	data &= ~ARCH_CAP_FB_CLEAR_CTRL;

+

 	return data;

 }