From: Peter Zijlstra <peterz@infradead.org>

Date: Thu, 27 Oct 2022 14:54:41 -0700

Subject: x86/mm: Randomize per-cpu entry area

Git-commit: 97e3d26b5e5f371b3ee223d94dd123e6c442ba80

Patch-mainline: v6.2-rc1

References: bsc#1207845 CVE-2023-0597

Seth found that the CPU-entry-area; the piece of per-cpu data that is

mapped into the userspace page-tables for kPTI is not subject to any

randomization -- irrespective of kASLR settings.

On x86_64 a whole P4D (512 GB) of virtual address space is reserved for

this structure, which is plenty large enough to randomize things a

little.

As such, use a straight forward randomization scheme that avoids

duplicates to spread the existing CPUs over the available space.

  [ bp: Fix le build. ]

Reported-by: Seth Jenkins <sethjenkins@google.com>

Reviewed-by: Kees Cook <keescook@chromium.org>

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>

Signed-off-by: Borislav Petkov <bp@suse.de>

[mkoutny: v5.14 backport: init_cea_offsets() is called way before

	  prandom_init_early() initcall, prandom_u32_max() is not properly

	  seeded yet. Use KASLR seed and local state to generate CPU entry

	  areas offsets, this is based on the approach in

	  kernel_randomize_memory() and should provide same randomness

	  guarantees -- beware we don't get cryptographically secure random

	  offsets. This reduces effective entropy in exfiltrating *all* CPU

	  entry areas by log2(nr_cpus) bits, entropy for *any* CPU is

	  unaffected.

	  This was chosen instead of backporting f62384995e4c ("random: split

	  initialization into early step and later step") and crng related

	  reworks.]

[mkoutny: v5.3 backport: dropped hw_breakpoint hunk without 24ae0c91cbc5

	  ("x86/hw_breakpoint: Prevent data breakpoints on cpu_entry_area"),

	  adjusted context for missing doublefault_stack on 32b]

Acked-by: Michal Koutný <mkoutny@suse.com>

---

 arch/x86/include/asm/cpu_entry_area.h |   13 ++++-----

 arch/x86/mm/cpu_entry_area.c          |   48 ++++++++++++++++++++++++++++++++--

 2 files changed, 53 insertions(+), 8 deletions(-)

--- a/arch/x86/include/asm/cpu_entry_area.h

+++ b/arch/x86/include/asm/cpu_entry_area.h

@@ -111,10 +111,6 @@ struct cpu_entry_area {

 };

 #define CPU_ENTRY_AREA_SIZE		(sizeof(struct cpu_entry_area))

-#define CPU_ENTRY_AREA_ARRAY_SIZE	(CPU_ENTRY_AREA_SIZE * NR_CPUS)

-

-/* Total size includes the readonly IDT mapping page as well: */

-#define CPU_ENTRY_AREA_TOTAL_SIZE	(CPU_ENTRY_AREA_ARRAY_SIZE + PAGE_SIZE)

 DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);

 DECLARE_PER_CPU(struct cea_exception_stacks *, cea_exception_stacks);

@@ -128,8 +124,13 @@ extern void cea_set_pte(void *cea_vaddr,

 #define CPU_ENTRY_AREA_RO_IDT_VADDR	((void *)CPU_ENTRY_AREA_RO_IDT)

-#define CPU_ENTRY_AREA_MAP_SIZE			\

-	(CPU_ENTRY_AREA_PER_CPU + CPU_ENTRY_AREA_ARRAY_SIZE - CPU_ENTRY_AREA_BASE)

+#ifdef CONFIG_X86_32

+#define CPU_ENTRY_AREA_MAP_SIZE		(CPU_ENTRY_AREA_PER_CPU +		\

+					 (CPU_ENTRY_AREA_SIZE * NR_CPUS) -	\

+					 CPU_ENTRY_AREA_BASE)

+#else

+#define CPU_ENTRY_AREA_MAP_SIZE		P4D_SIZE

+#endif

 extern struct cpu_entry_area *get_cpu_entry_area(int cpu);

--- a/arch/x86/mm/cpu_entry_area.c

+++ b/arch/x86/mm/cpu_entry_area.c

@@ -4,6 +4,7 @@

 #include <linux/percpu.h>

 #include <linux/kallsyms.h>

 #include <linux/kcore.h>

+#include <linux/prandom.h>

 #include <asm/cpu_entry_area.h>

 #include <asm/pgtable.h>

@@ -15,11 +16,53 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struc

 #ifdef CONFIG_X86_64

 static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks);

 DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks);

+

+static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, _cea_offset);

+

+static __always_inline unsigned int cea_offset(unsigned int cpu)

+{

+	return per_cpu(_cea_offset, cpu);

+}

+

+static __init void init_cea_offsets(void)

+{

+	struct rnd_state rand_state;

+	unsigned int max_cea, rand;

+	unsigned int i, j;

+

+	max_cea = (CPU_ENTRY_AREA_MAP_SIZE - PAGE_SIZE) / CPU_ENTRY_AREA_SIZE;

+	prandom_seed_state(&rand_state, kaslr_get_random_long("CPU entry"));

+

+	/* O(sodding terrible) */

+	for_each_possible_cpu(i) {

+		unsigned int cea;

+

+again:

+		prandom_bytes_state(&rand_state, &rand, sizeof(rand));

+		cea = rand % max_cea;

+

+		for_each_possible_cpu(j) {

+			if (cea_offset(j) == cea)

+				goto again;

+

+			if (i == j)

+				break;

+		}

+

+		per_cpu(_cea_offset, i) = cea;

+	}

+}

+#else /* !X86_64 */

+static __always_inline unsigned int cea_offset(unsigned int cpu)

+{

+	return cpu;

+}

+static inline void init_cea_offsets(void) { }

 #endif

 struct cpu_entry_area *get_cpu_entry_area(int cpu)

 {

-	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;

+	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cea_offset(cpu) * CPU_ENTRY_AREA_SIZE;

 	BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);

 	return (struct cpu_entry_area *) va;

@@ -180,7 +223,6 @@ static __init void setup_cpu_entry_area_

 	/* The +1 is for the readonly IDT: */

 	BUILD_BUG_ON((CPU_ENTRY_AREA_PAGES+1)*PAGE_SIZE != CPU_ENTRY_AREA_MAP_SIZE);

-	BUILD_BUG_ON(CPU_ENTRY_AREA_TOTAL_SIZE != CPU_ENTRY_AREA_MAP_SIZE);

 	BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);

 	start = CPU_ENTRY_AREA_BASE;

@@ -196,6 +238,8 @@ void __init setup_cpu_entry_areas(void)

 {

 	unsigned int cpu;

+	init_cea_offsets();

+

 	setup_cpu_entry_area_ptes();

 	for_each_possible_cpu(cpu)