From 7bc0584cd2a3422cd04779ad97def0e05419868f Mon Sep 17 00:00:00 2001
From: Dave Martin <Dave.Martin@arm.com>
Date: Tue, 31 Oct 2017 15:51:12 +0000
Subject: [PATCH] arm64/sve: Preserve SVE registers around EFI runtime service
 calls
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Git-commit: fdfa976cae5cbd46aa4b4f9e554a93c9f8b35c62
Patch-mainline: v4.15-rc1
References: fate#325866

The EFI runtime services ABI allows EFI to make free use of the
FPSIMD registers during EFI runtime service calls, subject to the
callee-save requirements of the AArch64 procedure call standard.

However, the SVE architecture allows upper bits of the SVE vector
registers to be zeroed as a side-effect of FPSIMD V-register
writes.  This means that the SVE vector registers must be saved in
their entirety in order to avoid data loss: non-SVE-aware EFI
implementations cannot restore them correctly.

The non-IRQ case is already handled gracefully by
kernel_neon_begin().  For the IRQ case, this patch allocates a
suitable per-CPU stash buffer for the full SVE register state and
uses it to preserve the affected registers around EFI calls.  It is
currently unclear how the EFI runtime services ABI will be
clarified with respect to SVE, so it safest to assume that the
predicate registers and FFR must be saved and restored too.

No attempt is made to restore the restore the vector length after
a call, for now.  It is deemed rather insane for EFI to change it,
and contemporary EFI implementations certainly won't.

Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Mian Yousaf Kaukab <yousaf.kaukab@suse.com>
---
 arch/arm64/kernel/fpsimd.c | 67 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 61 insertions(+), 6 deletions(-)

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index cab6235c6963..a618b4e2034f 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -123,11 +123,13 @@ static int sve_default_vl = -1;
 int __ro_after_init sve_max_vl = -1;
 /* Set of available vector lengths, as vq_to_bit(vq): */
 static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
+static void __percpu *efi_sve_state;
 
 #else /* ! CONFIG_ARM64_SVE */
 
 /* Dummy declaration for code that will be optimised out: */
 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
+extern void __percpu *efi_sve_state;
 
 #endif /* ! CONFIG_ARM64_SVE */
 
@@ -552,6 +554,30 @@ int sve_verify_vq_map(void)
 	return ret;
 }
 
+static void __init sve_efi_setup(void)
+{
+	if (!IS_ENABLED(CONFIG_EFI))
+		return;
+
+	/*
+	 * alloc_percpu() warns and prints a backtrace if this goes wrong.
+	 * This is evidence of a crippled system and we are returning void,
+	 * so no attempt is made to handle this situation here.
+	 */
+	if (!sve_vl_valid(sve_max_vl))
+		goto fail;
+
+	efi_sve_state = __alloc_percpu(
+		SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
+	if (!efi_sve_state)
+		goto fail;
+
+	return;
+
+fail:
+	panic("Cannot allocate percpu memory for EFI SVE save/restore");
+}
+
 /*
  * Enable SVE for EL1.
  * Intended for use by the cpufeatures code during CPU boot.
@@ -599,6 +625,8 @@ void __init sve_setup(void)
 		sve_max_vl);
 	pr_info("SVE: default vector length %u bytes per vector\n",
 		sve_default_vl);
+
+	sve_efi_setup();
 }
 
 /*
@@ -926,6 +954,7 @@ EXPORT_SYMBOL(kernel_neon_end);
 
 static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);
 static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
+static DEFINE_PER_CPU(bool, efi_sve_state_used);
 
 /*
  * EFI runtime services support functions
@@ -951,10 +980,24 @@ void __efi_fpsimd_begin(void)
 
 	WARN_ON(preemptible());
 
-	if (may_use_simd())
+	if (may_use_simd()) {
 		kernel_neon_begin();
-	else {
-		fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
+	} else {
+		/*
+		 * If !efi_sve_state, SVE can't be in use yet and doesn't need
+		 * preserving:
+		 */
+		if (system_supports_sve() && likely(efi_sve_state)) {
+			char *sve_state = this_cpu_ptr(efi_sve_state);
+
+			__this_cpu_write(efi_sve_state_used, true);
+
+			sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
+				       &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
+		} else {
+			fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
+		}
+
 		__this_cpu_write(efi_fpsimd_state_used, true);
 	}
 }
@@ -967,10 +1010,22 @@ void __efi_fpsimd_end(void)
 	if (!system_supports_fpsimd())
 		return;
 
-	if (__this_cpu_xchg(efi_fpsimd_state_used, false))
-		fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
-	else
+	if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
 		kernel_neon_end();
+	} else {
+		if (system_supports_sve() &&
+		    likely(__this_cpu_read(efi_sve_state_used))) {
+			char const *sve_state = this_cpu_ptr(efi_sve_state);
+
+			sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
+				       &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
+				       sve_vq_from_vl(sve_get_vl()) - 1);
+
+			__this_cpu_write(efi_sve_state_used, false);
+		} else {
+			fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
+		}
+	}
 }
 
 #endif /* CONFIG_EFI */
-- 
2.11.0