From: Martin Schwidefsky <schwidefsky@de.ibm.com>
Subject: s390/mm: add guest ASCE TLB flush optimization
Patch-mainline: v4.14-rc1
Git-commit: 28c807e5132ecc9f1607461eabfa1fc67b21e163
References: FATE#321512, bsc#1061009
Summary: kernel: guest TLB purge enhancements
Description: Add support for the KVM guest TLB purge enhancements
introduced with IBM z14.
Upstream-Description:
s390/mm: add guest ASCE TLB flush optimization
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Acked-by: Hannes Reinecke <hare@suse.com>
---
arch/s390/include/asm/pgtable.h | 58 ++++++++++++++++++++++++++++-----------
arch/s390/include/asm/tlbflush.h | 2 +
arch/s390/mm/pageattr.c | 2 -
arch/s390/mm/pgtable.c | 36 ++++++++++++++++--------
4 files changed, 70 insertions(+), 28 deletions(-)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -900,9 +900,11 @@ static inline pte_t pte_mkhuge(pte_t pte
#define IPTE_LOCAL 1
#define IPTE_NODAT 0x400
+#define IPTE_GUEST_ASCE 0x800
static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
- unsigned long opt, int local)
+ unsigned long opt, unsigned long asce,
+ int local)
{
unsigned long pto = (unsigned long) ptep;
@@ -916,6 +918,7 @@ static inline void __ptep_ipte(unsigned
}
/* Invalidate ptes with options + TLB flush of the ptes */
+ opt = opt | (asce & _ASCE_ORIGIN);
asm volatile(
" .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
: [r2] "+a" (address), [r3] "+a" (opt)
@@ -1288,34 +1291,59 @@ static inline void __pmdp_csp(pmd_t *pmd
#define IDTE_PTOA 0x0800
#define IDTE_NODAT 0x1000
+#define IDTE_GUEST_ASCE 0x2000
static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
- unsigned long opt, int local)
+ unsigned long opt, unsigned long asce,
+ int local)
{
unsigned long sto;
sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);
- asm volatile(
- " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
- : "+m" (*pmdp)
- : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
- [m4] "i" (local)
- : "cc" );
+ if (__builtin_constant_p(opt) && opt == 0) {
+ /* flush without guest asce */
+ asm volatile(
+ " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
+ : "+m" (*pmdp)
+ : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
+ [m4] "i" (local)
+ : "cc" );
+ } else {
+ /* flush with guest asce */
+ asm volatile(
+ " .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
+ : "+m" (*pmdp)
+ : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
+ [r3] "a" (asce), [m4] "i" (local)
+ : "cc" );
+ }
}
static inline void __pudp_idte(unsigned long addr, pud_t *pudp,
- unsigned long opt, int local)
+ unsigned long opt, unsigned long asce,
+ int local)
{
unsigned long r3o;
r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);
r3o |= _ASCE_TYPE_REGION3;
- asm volatile(
- " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
- : "+m" (*pudp)
- : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
- [m4] "i" (local)
- : "cc" );
+ if (__builtin_constant_p(opt) && opt == 0) {
+ /* flush without guest asce */
+ asm volatile(
+ " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
+ : "+m" (*pudp)
+ : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
+ [m4] "i" (local)
+ : "cc");
+ } else {
+ /* flush with guest asce */
+ asm volatile(
+ " .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
+ : "+m" (*pudp)
+ : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
+ [r3] "a" (asce), [m4] "i" (local)
+ : "cc" );
+ }
}
pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
--- a/arch/s390/include/asm/tlbflush.h
+++ b/arch/s390/include/asm/tlbflush.h
@@ -23,6 +23,8 @@ static inline void __tlb_flush_idte(unsi
unsigned long opt;
opt = IDTE_PTOA;
+ if (MACHINE_HAS_TLB_GUEST)
+ opt |= IDTE_GUEST_ASCE;
/* Global TLB flush for the mm */
asm volatile(
" .insn rrf,0xb98e0000,0,%0,%1,0"
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -308,7 +308,7 @@ static void ipte_range(pte_t *pte, unsig
return;
}
for (i = 0; i < nr; i++) {
- __ptep_ipte(address, pte, 0, IPTE_GLOBAL);
+ __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL);
address += PAGE_SIZE;
pte++;
}
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -35,9 +35,13 @@ static inline void ptep_ipte_local(struc
asce = READ_ONCE(mm->context.gmap_asce);
if (asce == 0UL)
opt |= IPTE_NODAT;
- __ptep_ipte(addr, ptep, opt, IPTE_LOCAL);
+ if (asce != -1UL) {
+ asce = asce ? : mm->context.asce;
+ opt |= IPTE_GUEST_ASCE;
+ }
+ __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
} else {
- __ptep_ipte(addr, ptep, 0, IPTE_LOCAL);
+ __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
}
}
@@ -51,9 +55,13 @@ static inline void ptep_ipte_global(stru
asce = READ_ONCE(mm->context.gmap_asce);
if (asce == 0UL)
opt |= IPTE_NODAT;
- __ptep_ipte(addr, ptep, opt, IPTE_GLOBAL);
+ if (asce != -1UL) {
+ asce = asce ? : mm->context.asce;
+ opt |= IPTE_GUEST_ASCE;
+ }
+ __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
} else {
- __ptep_ipte(addr, ptep, 0, IPTE_GLOBAL);
+ __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
}
}
@@ -326,18 +334,20 @@ static inline void pmdp_idte_local(struc
unsigned long addr, pmd_t *pmdp)
{
if (MACHINE_HAS_TLB_GUEST)
- __pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_LOCAL);
+ __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
+ mm->context.asce, IDTE_LOCAL);
else
- __pmdp_idte(addr, pmdp, 0, IDTE_LOCAL);
+ __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
}
static inline void pmdp_idte_global(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
if (MACHINE_HAS_TLB_GUEST)
- __pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_GLOBAL);
+ __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
+ mm->context.asce, IDTE_GLOBAL);
else if (MACHINE_HAS_IDTE)
- __pmdp_idte(addr, pmdp, 0, IDTE_GLOBAL);
+ __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
else
__pmdp_csp(pmdp);
}
@@ -410,18 +420,20 @@ static inline void pudp_idte_local(struc
unsigned long addr, pud_t *pudp)
{
if (MACHINE_HAS_TLB_GUEST)
- __pudp_idte(addr, pudp, IDTE_NODAT, IDTE_LOCAL);
+ __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
+ mm->context.asce, IDTE_LOCAL);
else
- __pudp_idte(addr, pudp, 0, IDTE_LOCAL);
+ __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
}
static inline void pudp_idte_global(struct mm_struct *mm,
unsigned long addr, pud_t *pudp)
{
if (MACHINE_HAS_TLB_GUEST)
- __pudp_idte(addr, pudp, IDTE_NODAT, IDTE_GLOBAL);
+ __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
+ mm->context.asce, IDTE_GLOBAL);
else if (MACHINE_HAS_IDTE)
- __pudp_idte(addr, pudp, 0, IDTE_GLOBAL);
+ __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
else
/*
* Invalid bit position is the same for pmd and pud, so we can