|
Jiri Slaby |
57cc8d |
From: Jeremi Piotrowski <jpiotrowski@linux.microsoft.com>
|
|
Jiri Slaby |
57cc8d |
Date: Fri, 24 Mar 2023 15:52:33 +0100
|
|
Jiri Slaby |
57cc8d |
Subject: [PATCH] KVM: SVM: Flush Hyper-V TLB when required
|
|
Jiri Slaby |
57cc8d |
References: bsc#1012628
|
|
Jiri Slaby |
57cc8d |
Patch-mainline: 6.2.11
|
|
Jiri Slaby |
57cc8d |
Git-commit: e5c972c1fadacc858b6a564d056f177275238040
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
commit e5c972c1fadacc858b6a564d056f177275238040 upstream.
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
The Hyper-V "EnlightenedNptTlb" enlightenment is always enabled when KVM
|
|
Jiri Slaby |
57cc8d |
is running on top of Hyper-V and Hyper-V exposes support for it (which
|
|
Jiri Slaby |
57cc8d |
is always). On AMD CPUs this enlightenment results in ASID invalidations
|
|
Jiri Slaby |
57cc8d |
not flushing TLB entries derived from the NPT. To force the underlying
|
|
Jiri Slaby |
57cc8d |
(L0) hypervisor to rebuild its shadow page tables, an explicit hypercall
|
|
Jiri Slaby |
57cc8d |
is needed.
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
The original KVM implementation of Hyper-V's "EnlightenedNptTlb" on SVM
|
|
Jiri Slaby |
57cc8d |
only added remote TLB flush hooks. This worked out fine for a while, as
|
|
Jiri Slaby |
57cc8d |
sufficient remote TLB flushes where being issued in KVM to mask the
|
|
Jiri Slaby |
57cc8d |
problem. Since v5.17, changes in the TDP code reduced the number of
|
|
Jiri Slaby |
57cc8d |
flushes and the out-of-sync TLB prevents guests from booting
|
|
Jiri Slaby |
57cc8d |
successfully.
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
Split svm_flush_tlb_current() into separate callbacks for the 3 cases
|
|
Jiri Slaby |
57cc8d |
(guest/all/current), and issue the required Hyper-V hypercall when a
|
|
Jiri Slaby |
57cc8d |
Hyper-V TLB flush is needed. The most important case where the TLB flush
|
|
Jiri Slaby |
57cc8d |
was missing is when loading a new PGD, which is followed by what is now
|
|
Jiri Slaby |
57cc8d |
svm_flush_tlb_current().
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
Cc: stable@vger.kernel.org # v5.17+
|
|
Jiri Slaby |
57cc8d |
Fixes: 1e0c7d40758b ("KVM: SVM: hyper-v: Remote TLB flush for SVM")
|
|
Jiri Slaby |
57cc8d |
Link: https://lore.kernel.org/lkml/43980946-7bbf-dcef-7e40-af904c456250@linux.microsoft.com/
|
|
Jiri Slaby |
57cc8d |
Suggested-by: Sean Christopherson <seanjc@google.com>
|
|
Jiri Slaby |
57cc8d |
Signed-off-by: Jeremi Piotrowski <jpiotrowski@linux.microsoft.com>
|
|
Jiri Slaby |
57cc8d |
Reviewed-by: Vitaly Kuznetsov <vkuznets@redhat.com>
|
|
Jiri Slaby |
57cc8d |
Message-Id: <20230324145233.4585-1-jpiotrowski@linux.microsoft.com>
|
|
Jiri Slaby |
57cc8d |
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
|
|
Jiri Slaby |
57cc8d |
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
|
|
Jiri Slaby |
57cc8d |
Signed-off-by: Jiri Slaby <jslaby@suse.cz>
|
|
Jiri Slaby |
57cc8d |
---
|
|
Jiri Slaby |
57cc8d |
arch/x86/kvm/kvm_onhyperv.h | 5 +++++
|
|
Jiri Slaby |
57cc8d |
arch/x86/kvm/svm/svm.c | 37 ++++++++++++++++++++++++++++++---
|
|
Jiri Slaby |
57cc8d |
arch/x86/kvm/svm/svm_onhyperv.h | 15 +++++++++++++
|
|
Jiri Slaby |
57cc8d |
3 files changed, 54 insertions(+), 3 deletions(-)
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
diff --git a/arch/x86/kvm/kvm_onhyperv.h b/arch/x86/kvm/kvm_onhyperv.h
|
|
Jiri Slaby |
57cc8d |
index 287e98ef..6272dabe 100644
|
|
Jiri Slaby |
57cc8d |
--- a/arch/x86/kvm/kvm_onhyperv.h
|
|
Jiri Slaby |
57cc8d |
+++ b/arch/x86/kvm/kvm_onhyperv.h
|
|
Jiri Slaby |
57cc8d |
@@ -12,6 +12,11 @@ int hv_remote_flush_tlb_with_range(struct kvm *kvm,
|
|
Jiri Slaby |
57cc8d |
int hv_remote_flush_tlb(struct kvm *kvm);
|
|
Jiri Slaby |
57cc8d |
void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp);
|
|
Jiri Slaby |
57cc8d |
#else /* !CONFIG_HYPERV */
|
|
Jiri Slaby |
57cc8d |
+static inline int hv_remote_flush_tlb(struct kvm *kvm)
|
|
Jiri Slaby |
57cc8d |
+{
|
|
Jiri Slaby |
57cc8d |
+ return -EOPNOTSUPP;
|
|
Jiri Slaby |
57cc8d |
+}
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
static inline void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp)
|
|
Jiri Slaby |
57cc8d |
{
|
|
Jiri Slaby |
57cc8d |
}
|
|
Jiri Slaby |
57cc8d |
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
|
|
Jiri Slaby |
57cc8d |
index f03bdaf7..a682ca4c 100644
|
|
Jiri Slaby |
57cc8d |
--- a/arch/x86/kvm/svm/svm.c
|
|
Jiri Slaby |
57cc8d |
+++ b/arch/x86/kvm/svm/svm.c
|
|
Jiri Slaby |
57cc8d |
@@ -3718,7 +3718,7 @@ static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
|
|
Jiri Slaby |
57cc8d |
}
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
-static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
+static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
{
|
|
Jiri Slaby |
57cc8d |
struct vcpu_svm *svm = to_svm(vcpu);
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
@@ -3742,6 +3742,37 @@ static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
svm->current_vmcb->asid_generation--;
|
|
Jiri Slaby |
57cc8d |
}
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
+static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
+{
|
|
Jiri Slaby |
57cc8d |
+ hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
+ /*
|
|
Jiri Slaby |
57cc8d |
+ * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
|
|
Jiri Slaby |
57cc8d |
+ * flush the NPT mappings via hypercall as flushing the ASID only
|
|
Jiri Slaby |
57cc8d |
+ * affects virtual to physical mappings, it does not invalidate guest
|
|
Jiri Slaby |
57cc8d |
+ * physical to host physical mappings.
|
|
Jiri Slaby |
57cc8d |
+ */
|
|
Jiri Slaby |
57cc8d |
+ if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
|
|
Jiri Slaby |
57cc8d |
+ hyperv_flush_guest_mapping(root_tdp);
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
+ svm_flush_tlb_asid(vcpu);
|
|
Jiri Slaby |
57cc8d |
+}
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
+static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
+{
|
|
Jiri Slaby |
57cc8d |
+ /*
|
|
Jiri Slaby |
57cc8d |
+ * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
|
|
Jiri Slaby |
57cc8d |
+ * flushes should be routed to hv_remote_flush_tlb() without requesting
|
|
Jiri Slaby |
57cc8d |
+ * a "regular" remote flush. Reaching this point means either there's
|
|
Jiri Slaby |
57cc8d |
+ * a KVM bug or a prior hv_remote_flush_tlb() call failed, both of
|
|
Jiri Slaby |
57cc8d |
+ * which might be fatal to the guest. Yell, but try to recover.
|
|
Jiri Slaby |
57cc8d |
+ */
|
|
Jiri Slaby |
57cc8d |
+ if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
|
|
Jiri Slaby |
57cc8d |
+ hv_remote_flush_tlb(vcpu->kvm);
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
+ svm_flush_tlb_asid(vcpu);
|
|
Jiri Slaby |
57cc8d |
+}
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
|
|
Jiri Slaby |
57cc8d |
{
|
|
Jiri Slaby |
57cc8d |
struct vcpu_svm *svm = to_svm(vcpu);
|
|
Jiri Slaby |
57cc8d |
@@ -4747,10 +4778,10 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
|
|
Jiri Slaby |
57cc8d |
.set_rflags = svm_set_rflags,
|
|
Jiri Slaby |
57cc8d |
.get_if_flag = svm_get_if_flag,
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
- .flush_tlb_all = svm_flush_tlb_current,
|
|
Jiri Slaby |
57cc8d |
+ .flush_tlb_all = svm_flush_tlb_all,
|
|
Jiri Slaby |
57cc8d |
.flush_tlb_current = svm_flush_tlb_current,
|
|
Jiri Slaby |
57cc8d |
.flush_tlb_gva = svm_flush_tlb_gva,
|
|
Jiri Slaby |
57cc8d |
- .flush_tlb_guest = svm_flush_tlb_current,
|
|
Jiri Slaby |
57cc8d |
+ .flush_tlb_guest = svm_flush_tlb_asid,
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
.vcpu_pre_run = svm_vcpu_pre_run,
|
|
Jiri Slaby |
57cc8d |
.vcpu_run = svm_vcpu_run,
|
|
Jiri Slaby |
57cc8d |
diff --git a/arch/x86/kvm/svm/svm_onhyperv.h b/arch/x86/kvm/svm/svm_onhyperv.h
|
|
Jiri Slaby |
57cc8d |
index 65c355b4..345b87ac 100644
|
|
Jiri Slaby |
57cc8d |
--- a/arch/x86/kvm/svm/svm_onhyperv.h
|
|
Jiri Slaby |
57cc8d |
+++ b/arch/x86/kvm/svm/svm_onhyperv.h
|
|
Jiri Slaby |
57cc8d |
@@ -6,6 +6,8 @@
|
|
Jiri Slaby |
57cc8d |
#ifndef __ARCH_X86_KVM_SVM_ONHYPERV_H__
|
|
Jiri Slaby |
57cc8d |
#define __ARCH_X86_KVM_SVM_ONHYPERV_H__
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
+#include <asm/mshyperv.h>
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
#if IS_ENABLED(CONFIG_HYPERV)
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
#include "kvm_onhyperv.h"
|
|
Jiri Slaby |
57cc8d |
@@ -15,6 +17,14 @@ static struct kvm_x86_ops svm_x86_ops;
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
int svm_hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu);
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
+static inline bool svm_hv_is_enlightened_tlb_enabled(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
+{
|
|
Jiri Slaby |
57cc8d |
+ struct hv_vmcb_enlightenments *hve = &to_svm(vcpu)->vmcb->control.hv_enlightenments;
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
+ return ms_hyperv.nested_features & HV_X64_NESTED_ENLIGHTENED_TLB &&
|
|
Jiri Slaby |
57cc8d |
+ !!hve->hv_enlightenments_control.enlightened_npt_tlb;
|
|
Jiri Slaby |
57cc8d |
+}
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
static inline void svm_hv_init_vmcb(struct vmcb *vmcb)
|
|
Jiri Slaby |
57cc8d |
{
|
|
Jiri Slaby |
57cc8d |
struct hv_vmcb_enlightenments *hve = &vmcb->control.hv_enlightenments;
|
|
Jiri Slaby |
57cc8d |
@@ -80,6 +90,11 @@ static inline void svm_hv_update_vp_id(struct vmcb *vmcb, struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
}
|
|
Jiri Slaby |
57cc8d |
#else
|
|
Jiri Slaby |
57cc8d |
|
|
Jiri Slaby |
57cc8d |
+static inline bool svm_hv_is_enlightened_tlb_enabled(struct kvm_vcpu *vcpu)
|
|
Jiri Slaby |
57cc8d |
+{
|
|
Jiri Slaby |
57cc8d |
+ return false;
|
|
Jiri Slaby |
57cc8d |
+}
|
|
Jiri Slaby |
57cc8d |
+
|
|
Jiri Slaby |
57cc8d |
static inline void svm_hv_init_vmcb(struct vmcb *vmcb)
|
|
Jiri Slaby |
57cc8d |
{
|
|
Jiri Slaby |
57cc8d |
}
|
|
Jiri Slaby |
57cc8d |
--
|
|
Jiri Slaby |
57cc8d |
2.35.3
|
|
Jiri Slaby |
57cc8d |
|