From: Nikolay Borisov <nik.borisov@suse.com
Date: Fri, 20 Oct 2017 09:30:43 -0500
Subject: x86: Fix kabi when adding new cpuid features
Patch-mainline: Never, kABI workaround
References: bsc#1213287, CVE-2023-20569
Since cpuinfo_x86 cannot be grown because of kABI consideration implement a scheme
which allows to add new caps bit. The idea is to simply add a new x86_ext_capability
array at the end of cpuinfo_x86 thus retaining kABI. All new cpuid bits will Subsequently
be recorded in the new member.
Additionally cpu_caps_set is also grown by the number of additional cpuid words
and the getter/setter macros are modified to reflect this. All forced bits which
belong to extended cpuid words will be written in cpu_caps_set after NCAPINTS+NBUGINTS.
The code also supports adding additional bugsints. It works similarly to how the
additional cpu feature ints work. The only problem is that the first ordinary bugint
overlaps with the first extended cpufeature due to the way IS_EXT_CPUID_BIT is
coded. This problem is fixed by ensuring that the various macros for checking
features/bug in cpufeature.h are called with the respective set of either flags or
bug. This ensure that if IS_EXT_CPUID_BIT returns true it's because we've called it
with an extended feature int.
---
arch/x86/include/asm/cpufeature.h | 78 ++++++++++++++++++++++++-------
arch/x86/include/asm/cpufeatures.h | 5 +
arch/x86/include/asm/disabled-features.h | 4 -
arch/x86/include/asm/processor.h | 5 +
arch/x86/include/asm/required-features.h | 4 -
arch/x86/kernel/alternative.c | 2
arch/x86/include/asm/cpufeature.h | 92 ++++++++++++++++++++++++-------
arch/x86/include/asm/cpufeatures.h | 5 +
arch/x86/include/asm/disabled-features.h | 4 -
arch/x86/include/asm/processor.h | 5 +
arch/x86/include/asm/required-features.h | 4 -
arch/x86/kernel/alternative.c | 2
arch/x86/include/asm/cpufeature.h | 99 ++++++++++++++++++++++++-------
arch/x86/include/asm/cpufeatures.h | 5 +
arch/x86/include/asm/disabled-features.h | 4 -
arch/x86/include/asm/processor.h | 5 +
arch/x86/include/asm/required-features.h | 4 -
arch/x86/kernel/alternative.c | 5 -
arch/x86/kernel/cpu/common.c | 12 +++
arch/x86/kernel/cpu/mkcapflags.sh | 4 -
arch/x86/kernel/cpu/proc.c | 4 -
arch/x86/kvm/cpuid.c | 4 -
arch/x86/kvm/svm.c | 2
arch/x86/include/asm/cpufeature.h | 99 ++++++++++++++++++++++++-------
arch/x86/include/asm/cpufeatures.h | 5 +
arch/x86/include/asm/disabled-features.h | 4 -
arch/x86/include/asm/processor.h | 5 +
arch/x86/include/asm/required-features.h | 4 -
arch/x86/kernel/alternative.c | 5 -
arch/x86/kernel/cpu/common.c | 12 +++
arch/x86/kernel/cpu/mkcapflags.sh | 4 -
arch/x86/kernel/cpu/proc.c | 4 -
arch/x86/kvm/cpuid.c | 4 -
arch/x86/kvm/svm.c | 2
11 files changed, 107 insertions(+), 41 deletions(-)
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -29,11 +29,28 @@ enum cpuid_leafs
CPUID_7_ECX,
CPUID_8000_0007_EBX,
CPUID_7_EDX,
+ CPUID_MAX = CPUID_7_EDX,
+ /*
+ * Everything below should go into the extended caps array to preserve
+ * kABI
+ */
CPUID_8000_0021_EAX,
CPUID_LNX_5,
CPUID_LNX_6,
};
+#define CPUID_IDX(x) \
+ __builtin_choose_expr((x) > CPUID_MAX, (x) - CPUID_MAX - 1, (x))
+#define IS_EXT_CPUID_BIT(bit) ((bit>>5) >= NCAPINTS)
+
+/*
+ * This macro must be called with bit belonging to one of the extended bug bits,
+ * this required because the first bug word (19) aliases with the first extended
+ * cap word (19) as such the check would be ambiguous if this macro is called
+ * with a bit which represents an extended cpuid.
+ */
+#define IS_EXT_BUG_BIT(bit) ((bit>>5) >= (NCAPINTS + NBUGINTS))
+
#ifdef CONFIG_X86_FEATURE_NAMES
extern const char * const x86_cap_flags[NCAPINTS*32];
extern const char * const x86_power_flags[32];
@@ -48,11 +65,12 @@ extern const char * const x86_power_flag
* In order to save room, we index into this array by doing
* X86_BUG_<name> - NCAPINTS*32.
*/
-extern const char * const x86_bug_flags[NBUGINTS*32];
-
-#define test_cpu_cap(c, bit) \
- test_bit(bit, (unsigned long *)((c)->x86_capability))
+extern const char * const x86_bug_flags[(NBUGINTS+NEXTBUGINTS)*32];
+#define test_cpu_cap(c, bit) \
+ (IS_EXT_CPUID_BIT((bit)) ? test_bit((bit) - (NCAPINTS*32), \
+ (unsigned long *)((c)->x86_ext_capability)) : \
+ test_bit(bit, (unsigned long *)((c)->x86_capability)))
/*
* There are 32 bits/features in each mask word. The high bits
* (selected with (bit>>5) give us the word number and the low 5
@@ -83,10 +101,8 @@ extern const char * const x86_bug_flags[
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) || \
- CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 19, feature_bit) || \
- CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 20, feature_bit) || \
REQUIRED_MASK_CHECK || \
- BUILD_BUG_ON_ZERO(NCAPINTS != 21))
+ BUILD_BUG_ON_ZERO(NCAPINTS != 19))
#define DISABLED_MASK_BIT_SET(feature_bit) \
( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 0, feature_bit) || \
@@ -108,10 +124,8 @@ extern const char * const x86_bug_flags[
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) || \
- CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 19, feature_bit) || \
- CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 20, feature_bit) || \
DISABLED_MASK_CHECK || \
- BUILD_BUG_ON_ZERO(NCAPINTS != 21))
+ BUILD_BUG_ON_ZERO(NCAPINTS != 19))
#define cpu_has(c, bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
@@ -134,17 +148,46 @@ extern const char * const x86_bug_flags[
#define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit)
-#define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability))
+#define set_cpu_cap(c, bit) do { \
+ if (IS_EXT_CPUID_BIT(bit)) { \
+ set_bit(bit - (NCAPINTS*32), \
+ (unsigned long *)(c)->x86_ext_capability); \
+ } else { \
+ set_bit(bit, (unsigned long *)((c)->x86_capability)); \
+ } \
+} while (0)
extern void setup_clear_cpu_cap(unsigned int bit);
extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
#define setup_force_cpu_cap(bit) do { \
- set_cpu_cap(&boot_cpu_data, bit); \
- set_bit(bit, (unsigned long *)cpu_caps_set); \
+ if (IS_EXT_CPUID_BIT(bit)) { \
+ set_bit(bit - (NCAPINTS*32), \
+ (unsigned long *)&cpu_caps_set[NCAPINTS + NBUGINTS]); \
+ set_bit(bit - (NCAPINTS*32), \
+ (unsigned long *)(&boot_cpu_data)->x86_ext_capability); \
+ } else { \
+ set_cpu_cap(&boot_cpu_data, bit); \
+ set_bit(bit, (unsigned long *)cpu_caps_set); \
+ } \
} while (0)
-#define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)
+
+/*
+ * This has to be re-implemented because of the aliasing issues between
+ * extended capability bits and bug bits, see comment above IS_EXT_BUG_BIT.
+ */
+#define setup_force_cpu_bug(bit) do { \
+ if (IS_EXT_BUG_BIT(bit)) { \
+ set_bit(bit - ((NCAPINTS+NBUGINTS)*32), \
+ (unsigned long *)&cpu_caps_set[NCAPINTS + NBUGINTS + NEXTCAPINTS]); \
+ set_bit(bit - ((NCAPINTS+NBUGINTS)*32), \
+ (unsigned long *)&(&boot_cpu_data)->x86_ext_capability[NEXTCAPINTS]); \
+ } else { \
+ set_bit(bit, (unsigned long *)((&boot_cpu_data)->x86_capability)); \
+ set_bit(bit, (unsigned long *)cpu_caps_set); \
+ } \
+} while (0)
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_X86_FAST_FEATURE_TESTS)
/*
@@ -196,11 +239,12 @@ static __always_inline __pure bool _stat
}
#define static_cpu_has(bit) \
-( \
- __builtin_constant_p(boot_cpu_has(bit)) ? \
+({ \
+ BUILD_BUG_ON_MSG((bit >> 5) >= NCAPINTS+NBUGINTS, "extended bits/bugs not supported"); \
+ (__builtin_constant_p(boot_cpu_has(bit)) ? \
boot_cpu_has(bit) : \
- _static_cpu_has(bit) \
-)
+ _static_cpu_has(bit)); \
+})
#else
/*
* Fall back to dynamic for gcc versions which don't support asm goto. Should be
@@ -209,15 +253,26 @@ static __always_inline __pure bool _stat
#define static_cpu_has(bit) boot_cpu_has(bit)
#endif
-#define cpu_has_bug(c, bit) cpu_has(c, (bit))
-#define set_cpu_bug(c, bit) set_cpu_cap(c, (bit))
+#define cpu_has_bug(c, bit) (IS_EXT_BUG_BIT((bit)) ? test_bit((bit) - ((NCAPINTS+NBUGINTS)*32), \
+ (unsigned long *)(&((c)->x86_ext_capability[NEXTCAPINTS]))) : \
+ test_bit(bit, (unsigned long *)((c)->x86_capability)))
+
+#define set_cpu_bug(c, bit) do { \
+ if (IS_EXT_BUG_BIT(bit)) { \
+ set_bit(bit - ((NCAPINTS+NBUGINTS)*32), \
+ (unsigned long *)(&((c)->x86_ext_capability[NEXTCAPINTS]))); \
+ } else { \
+ set_bit(bit, (unsigned long *)((c)->x86_capability)); \
+ } \
+} while (0)
+
#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit))
#define static_cpu_has_bug(bit) static_cpu_has((bit))
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit))
-#define boot_cpu_set_bug(bit) set_cpu_cap(&boot_cpu_data, (bit))
+#define boot_cpu_set_bug(bit) set_cpu_bug(&boot_cpu_data, (bit))
-#define MAX_CPU_FEATURES (NCAPINTS * 32)
+#define MAX_CPU_FEATURES ((NCAPINTS + NEXTCAPINTS) * 32)
#define cpu_have_feature boot_cpu_has
#define CPU_FEATURE_TYPEFMT "x86,ven%04Xfam%04Xmod%04X"
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -12,8 +12,11 @@
/*
* Defines x86 CPU feature bits
*/
-#define NCAPINTS 23 /* N 32-bit words worth of info */
+#define NCAPINTS 19 /* N 32-bit words worth of info */
#define NBUGINTS 1 /* N 32-bit bug flags */
+#define NEXTCAPINTS 4 /* N 32-bit words which come after NCAPINTS */
+#define NEXTBUGINTS 1 /* N 32-bit words which come after NEXTCAPINTS */
+
/*
* Note: If the comment begins with a quoted string, that string is used
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -76,8 +76,6 @@
#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP)
#define DISABLED_MASK17 0
#define DISABLED_MASK18 0
-#define DISABLED_MASK19 0
-#define DISABLED_MASK20 0
-#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 21)
+#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
#endif /* _ASM_X86_DISABLED_FEATURES_H */
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -139,6 +139,9 @@ struct cpuinfo_x86 {
*/
u8 x86_cache_bits;
unsigned initialized : 1;
+#ifndef __GENKSYMS__
+ __u32 x86_ext_capability[NEXTCAPINTS+NEXTBUGINTS];
+#endif
};
struct cpuid_regs {
@@ -172,7 +175,7 @@ extern struct cpuinfo_x86 new_cpu_data;
extern struct x86_hw_tss doublefault_tss;
extern __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
-extern __u32 cpu_caps_set[NCAPINTS + NBUGINTS];
+extern __u32 cpu_caps_set[NCAPINTS + NBUGINTS + NEXTCAPINTS + NEXTBUGINTS];
#ifdef CONFIG_SMP
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
--- a/arch/x86/include/asm/required-features.h
+++ b/arch/x86/include/asm/required-features.h
@@ -107,8 +107,6 @@
#define REQUIRED_MASK16 (NEED_LA57)
#define REQUIRED_MASK17 0
#define REQUIRED_MASK18 0
-#define REQUIRED_MASK19 0
-#define REQUIRED_MASK20 0
-#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 21)
+#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
#endif /* _ASM_X86_REQUIRED_FEATURES_H */
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -385,8 +385,9 @@ void __init_or_module noinline apply_alt
instr = (u8 *)&a->instr_offset + a->instr_offset;
replacement = (u8 *)&a->repl_offset + a->repl_offset;
BUG_ON(a->instrlen > sizeof(insnbuf));
- BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
- if (!boot_cpu_has(a->cpuid)) {
+ BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS + NEXTCAPINTS + NEXTBUGINTS) * 32);
+ if ((a->cpuid < NCAPINTS && !boot_cpu_has(a->cpuid)) ||
+ (a->cpuid > NCAPINTS && !boot_cpu_has_bug(a->cpuid))) {
if (a->padlen > 1)
optimize_nops(a, instr);
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -486,7 +486,7 @@ static const char *table_lookup_model(st
}
__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
-__u32 cpu_caps_set[NCAPINTS + NBUGINTS];
+__u32 cpu_caps_set[NCAPINTS + NBUGINTS + NEXTCAPINTS + NEXTBUGINTS];
void load_percpu_segment(int cpu)
{
@@ -749,6 +749,10 @@ static void apply_forced_caps(struct cpu
c->x86_capability[i] &= ~cpu_caps_cleared[i];
c->x86_capability[i] |= cpu_caps_set[i];
}
+
+ for (i = 0; i < NEXTCAPINTS + NEXTBUGINTS; i++) {
+ c->x86_ext_capability[i] |= cpu_caps_set[NCAPINTS + NBUGINTS + i];
+ }
}
static void init_speculation_control(struct cpuinfo_x86 *c)
@@ -888,8 +892,12 @@ void get_cpu_cap(struct cpuinfo_x86 *c)
if (c->extended_cpuid_level >= 0x8000000a)
c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
+ /*
+ * Every new leaf should go into the extended caps array, consult
+ * cpuid_leafs enum for more info
+ */
if (c->extended_cpuid_level >= 0x80000021)
- c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
+ c->x86_ext_capability[CPUID_IDX(CPUID_8000_0021_EAX)] = cpuid_eax(0x80000021);
init_scattered_cpuid_features(c);
init_speculation_control(c);
--- a/arch/x86/kernel/cpu/mkcapflags.sh
+++ b/arch/x86/kernel/cpu/mkcapflags.sh
@@ -56,10 +56,10 @@ trap 'rm "$OUT"' EXIT
echo "#endif"
echo ""
- dump_array "x86_cap_flags" "NCAPINTS*32" "X86_FEATURE_" ""
+ dump_array "x86_cap_flags" "(NCAPINTS+NEXTCAPINTS)*32" "X86_FEATURE_" ""
echo ""
- dump_array "x86_bug_flags" "NBUGINTS*32" "X86_BUG_" "NCAPINTS*32"
+ dump_array "x86_bug_flags" "(NBUGINTS+NEXTBUGINTS)*32" "X86_BUG_" "NCAPINTS*32"
) > $OUT
--- a/arch/x86/kernel/cpu/proc.c
+++ b/arch/x86/kernel/cpu/proc.c
@@ -93,12 +93,12 @@ static int show_cpuinfo(struct seq_file
show_cpuinfo_misc(m, c);
seq_puts(m, "flags\t\t:");
- for (i = 0; i < 32*NCAPINTS; i++)
+ for (i = 0; i < 32*(NCAPINTS + NEXTCAPINTS); i++)
if (cpu_has(c, i) && x86_cap_flags[i] != NULL)
seq_printf(m, " %s", x86_cap_flags[i]);
seq_puts(m, "\nbugs\t\t:");
- for (i = 0; i < 32*NBUGINTS; i++) {
+ for (i = 0; i < 32*(NBUGINTS+NEXTBUGINTS); i++) {
unsigned int bug_bit = 32*NCAPINTS + i;
if (cpu_has_bug(c, bug_bit) && x86_bug_flags[i])
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -676,7 +676,7 @@ static inline int __do_cpuid_ent(struct
* loop if said highest leaf has no subleaves indexed by ECX.
*/
if (entry->eax >= 0x8000001d &&
- (static_cpu_has(X86_FEATURE_LFENCE_RDTSC)
+ (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)
|| !static_cpu_has_bug(X86_BUG_NULL_SEG)))
entry->eax = max(entry->eax, 0x80000021);
break;
@@ -769,7 +769,7 @@ static inline int __do_cpuid_ent(struct
* the mask with the raw host CPUID, and reporting support in AMD's
* leaf can make it easier for userspace to detect the feature.
*/
- if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
+ if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
entry->eax |= F(LFENCE_RDTSC);
if (!static_cpu_has_bug(X86_BUG_NULL_SEG))
entry->eax |= BIT(6);
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -4107,7 +4107,7 @@ static int svm_get_msr_feature(struct kv
switch (msr->index) {
case MSR_AMD64_DE_CFG:
- if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
+ if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
msr->data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
break;
default: