From 955dd30fd47b0ff094532f11d6b83a7ddba0b582 Mon Sep 17 00:00:00 2001
From: James Morse <james.morse@arm.com>
Date: Mon, 28 Feb 2022 18:42:00 +0100
Subject: [PATCH 22/22] arm64: Use the clearbhb instruction in mitigations
Git-commit: 228a26b912287934789023b4132ba76065d9491c
Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
Patch-mainline: Queued in a subsystem tree
References: bsc#1191580 CVE-2022-0001 CVE-2022-000 CVE-2022-23960
Future CPUs may implement a clearbhb instruction that is sufficient
to mitigate SpectreBHB. CPUs that implement this instruction, but
not CSV2.3 must be affected by Spectre-BHB.
Add support to use this instruction as the BHB mitigation on CPUs
that support it. The instruction is in the hint space, so it will
be treated by a NOP as older CPUs.
Reviewed-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
[ modified for stable: Use a KVM vector template instead of alternatives ]
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Matthias Brugger <mbrugger@suse.com>
---
arch/arm64/include/asm/assembler.h | 7 +++++++
arch/arm64/include/asm/cpufeature.h | 13 +++++++++++++
arch/arm64/include/asm/sysreg.h | 3 +++
arch/arm64/include/asm/vectors.h | 7 +++++++
arch/arm64/kernel/bpi.S | 5 +++++
arch/arm64/kernel/cpu_errata.c | 14 ++++++++++++++
arch/arm64/kernel/cpufeature.c | 1 +
arch/arm64/kernel/entry.S | 8 ++++++++
8 files changed, 58 insertions(+)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 73fc138067855..7f4e835ab05a3 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -121,6 +121,13 @@
hint #20
.endm
+/*
+ * Clear Branch History instruction
+ */
+ .macro clearbhb
+ hint #22
+ .endm
+
/*
* Sanitise a 64-bit bounded index wrt speculation, returning zero if out
* of bounds.
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 43ee1b3183710..2f04f1243fa71 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -498,6 +498,19 @@ static inline bool supports_csv2p3(int scope)
return csv2_val == 3;
}
+static inline bool supports_clearbhb(int scope)
+{
+ u64 isar2;
+
+ if (scope == SCOPE_LOCAL_CPU)
+ isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
+ else
+ isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
+
+ return cpuid_feature_extract_unsigned_field(isar2,
+ ID_AA64ISAR2_CLEARBHB_SHIFT);
+}
+
static inline bool system_supports_32bit_el0(void)
{
return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 2ea85c600ae10..e037171677730 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -522,6 +522,9 @@
#define ID_AA64ISAR1_FCMA_SHIFT 16
#define ID_AA64ISAR1_JSCVT_SHIFT 12
+/* id_aa64isar2 */
+#define ID_AA64ISAR2_CLEARBHB_SHIFT 28
+
/* id_aa64pfr0 */
#define ID_AA64PFR0_CSV3_SHIFT 60
#define ID_AA64PFR0_CSV2_SHIFT 56
diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h
index f222d8e033b32..695583b9a145b 100644
--- a/arch/arm64/include/asm/vectors.h
+++ b/arch/arm64/include/asm/vectors.h
@@ -33,6 +33,12 @@ enum arm64_bp_harden_el1_vectors {
* canonical vectors.
*/
EL1_VECTOR_BHB_FW,
+
+ /*
+ * Use the ClearBHB instruction, before branching to the canonical
+ * vectors.
+ */
+ EL1_VECTOR_BHB_CLEAR_INSN,
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
/*
@@ -44,6 +50,7 @@ enum arm64_bp_harden_el1_vectors {
#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
#define EL1_VECTOR_BHB_LOOP -1
#define EL1_VECTOR_BHB_FW -1
+#define EL1_VECTOR_BHB_CLEAR_INSN -1
#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
/* The vectors to use on return from EL0. e.g. to remap the kernel */
diff --git a/arch/arm64/kernel/bpi.S b/arch/arm64/kernel/bpi.S
index cdcf2197b72be..7060ee8e411ad 100644
--- a/arch/arm64/kernel/bpi.S
+++ b/arch/arm64/kernel/bpi.S
@@ -115,3 +115,8 @@ ENTRY(__spectre_bhb_loop_k32_start)
ldp x0, x1, [sp, #(8 * 0)]
add sp, sp, #(8 * 2)
ENTRY(__spectre_bhb_loop_k32_end)
+
+ENTRY(__spectre_bhb_clearbhb_start)
+ hint #22 /* aka clearbhb */
+ isb
+ENTRY(__spectre_bhb_clearbhb_end)
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 058b5ea86b0af..6305c10a877ee 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -119,6 +119,8 @@ extern char __spectre_bhb_loop_k24_start[];
extern char __spectre_bhb_loop_k24_end[];
extern char __spectre_bhb_loop_k32_start[];
extern char __spectre_bhb_loop_k32_end[];
+extern char __spectre_bhb_clearbhb_start[];
+extern char __spectre_bhb_clearbhb_end[];
static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
const char *hyp_vecs_end)
@@ -939,6 +941,7 @@ static void update_mitigation_state(enum mitigation_state *oldp,
* - Mitigated by a branchy loop a CPU specific number of times, and listed
* in our "loop mitigated list".
* - Mitigated in software by the firmware Spectre v2 call.
+ * - Has the ClearBHB instruction to perform the mitigation.
* - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
* software mitigation in the vectors is needed.
* - Has CSV2.3, so is unaffected.
@@ -1078,6 +1081,9 @@ bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
if (supports_csv2p3(scope))
return false;
+ if (supports_clearbhb(scope))
+ return true;
+
if (spectre_bhb_loop_affected(scope))
return true;
@@ -1118,6 +1124,8 @@ static const char *kvm_bhb_get_vecs_end(const char *start)
return __spectre_bhb_loop_k24_end;
else if (start == __spectre_bhb_loop_k32_start)
return __spectre_bhb_loop_k32_end;
+ else if (start == __spectre_bhb_clearbhb_start)
+ return __spectre_bhb_clearbhb_end;
return NULL;
}
@@ -1159,6 +1167,7 @@ static void kvm_setup_bhb_slot(const char *hyp_vecs_start)
#define __spectre_bhb_loop_k8_start NULL
#define __spectre_bhb_loop_k24_start NULL
#define __spectre_bhb_loop_k32_start NULL
+#define __spectre_bhb_clearbhb_start NULL
static void kvm_setup_bhb_slot(const char *hyp_vecs_start) { };
#endif
@@ -1177,6 +1186,11 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
} else if (cpu_mitigations_off()) {
pr_info_once("spectre-bhb mitigation disabled by command line option\n");
} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
+ state = SPECTRE_MITIGATED;
+ } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
+ kvm_setup_bhb_slot(__spectre_bhb_clearbhb_start);
+ this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
+
state = SPECTRE_MITIGATED;
} else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
switch (spectre_bhb_loop_affected(SCOPE_SYSTEM)) {
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9bba8c0b75e87..fa442d23120d1 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -155,6 +155,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
};
static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
ARM64_FTR_END,
};
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 705134e7dddef..8de59778e79f8 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -1108,6 +1108,7 @@ alternative_else_nop_endif
#define BHB_MITIGATION_NONE 0
#define BHB_MITIGATION_LOOP 1
#define BHB_MITIGATION_FW 2
+#define BHB_MITIGATION_INSN 3
.macro tramp_ventry, vector_start, regsize, kpti, bhb
.align 7
@@ -1124,6 +1125,11 @@ alternative_else_nop_endif
__mitigate_spectre_bhb_loop x30
.endif // \bhb == BHB_MITIGATION_LOOP
+ .if \bhb == BHB_MITIGATION_INSN
+ clearbhb
+ isb
+ .endif // \bhb == BHB_MITIGATION_INSN
+
.if \kpti == 1
/*
* Defend against branch aliasing attacks by pushing a dummy
@@ -1200,6 +1206,7 @@ ENTRY(tramp_vectors)
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
END(tramp_vectors)
@@ -1262,6 +1269,7 @@ ENTRY(__bp_harden_el1_vectors)
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
generate_el1_vector bhb=BHB_MITIGATION_LOOP
generate_el1_vector bhb=BHB_MITIGATION_FW
+ generate_el1_vector bhb=BHB_MITIGATION_INSN
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
END(__bp_harden_el1_vectors)
.popsection
--
2.34.1