From: Martin Schwidefsky <schwidefsky@de.ibm.com>
Subject: s390/mm,kvm: use nodat PGSTE tag to optimize TLB flushing
Patch-mainline: v4.14-rc1
Git-commit: cd774b9076845312caf76ad52523bd9219e89ae1
References: FATE#321512, bsc#1061009
Summary: kernel: guest TLB purge enhancements
Description: Add support for the KVM guest TLB purge enhancements
introduced with IBM z14.
Upstream-Description:
s390/mm,kvm: use nodat PGSTE tag to optimize TLB flushing
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Acked-by: Hannes Reinecke <hare@suse.com>
---
arch/s390/include/asm/pgtable.h | 1
arch/s390/mm/pgtable.c | 47 +++++++++++++++++++++++++---------------
2 files changed, 31 insertions(+), 17 deletions(-)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -373,6 +373,7 @@ static inline int is_module_addr(void *a
/* Guest Page State used for virtualization */
#define _PGSTE_GPS_ZERO 0x0000000080000000UL
+#define _PGSTE_GPS_NODAT 0x0000000040000000UL
#define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL
#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -26,14 +26,14 @@
#include <asm/page-states.h>
static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep)
+ pte_t *ptep, int nodat)
{
unsigned long opt, asce;
if (MACHINE_HAS_TLB_GUEST) {
opt = 0;
asce = READ_ONCE(mm->context.gmap_asce);
- if (asce == 0UL)
+ if (asce == 0UL || nodat)
opt |= IPTE_NODAT;
if (asce != -1UL) {
asce = asce ? : mm->context.asce;
@@ -46,14 +46,14 @@ static inline void ptep_ipte_local(struc
}
static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep)
+ pte_t *ptep, int nodat)
{
unsigned long opt, asce;
if (MACHINE_HAS_TLB_GUEST) {
opt = 0;
asce = READ_ONCE(mm->context.gmap_asce);
- if (asce == 0UL)
+ if (asce == 0UL || nodat)
opt |= IPTE_NODAT;
if (asce != -1UL) {
asce = asce ? : mm->context.asce;
@@ -66,7 +66,8 @@ static inline void ptep_ipte_global(stru
}
static inline pte_t ptep_flush_direct(struct mm_struct *mm,
- unsigned long addr, pte_t *ptep)
+ unsigned long addr, pte_t *ptep,
+ int nodat)
{
pte_t old;
@@ -76,15 +77,16 @@ static inline pte_t ptep_flush_direct(st
atomic_inc(&mm->context.flush_count);
if (MACHINE_HAS_TLB_LC &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
- ptep_ipte_local(mm, addr, ptep);
+ ptep_ipte_local(mm, addr, ptep, nodat);
else
- ptep_ipte_global(mm, addr, ptep);
+ ptep_ipte_global(mm, addr, ptep, nodat);
atomic_dec(&mm->context.flush_count);
return old;
}
static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
- unsigned long addr, pte_t *ptep)
+ unsigned long addr, pte_t *ptep,
+ int nodat)
{
pte_t old;
@@ -97,7 +99,7 @@ static inline pte_t ptep_flush_lazy(stru
pte_val(*ptep) |= _PAGE_INVALID;
mm->context.flush_mm = 1;
} else
- ptep_ipte_global(mm, addr, ptep);
+ ptep_ipte_global(mm, addr, ptep, nodat);
atomic_dec(&mm->context.flush_count);
return old;
}
@@ -269,10 +271,12 @@ pte_t ptep_xchg_direct(struct mm_struct
{
pgste_t pgste;
pte_t old;
+ int nodat;
preempt_disable();
pgste = ptep_xchg_start(mm, addr, ptep);
- old = ptep_flush_direct(mm, addr, ptep);
+ nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+ old = ptep_flush_direct(mm, addr, ptep, nodat);
old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
preempt_enable();
return old;
@@ -284,10 +288,12 @@ pte_t ptep_xchg_lazy(struct mm_struct *m
{
pgste_t pgste;
pte_t old;
+ int nodat;
preempt_disable();
pgste = ptep_xchg_start(mm, addr, ptep);
- old = ptep_flush_lazy(mm, addr, ptep);
+ nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+ old = ptep_flush_lazy(mm, addr, ptep, nodat);
old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
preempt_enable();
return old;
@@ -299,10 +305,12 @@ pte_t ptep_modify_prot_start(struct mm_s
{
pgste_t pgste;
pte_t old;
+ int nodat;
preempt_disable();
pgste = ptep_xchg_start(mm, addr, ptep);
- old = ptep_flush_lazy(mm, addr, ptep);
+ nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+ old = ptep_flush_lazy(mm, addr, ptep, nodat);
if (mm_has_pgste(mm)) {
pgste = pgste_update_all(old, pgste, mm);
pgste_set(ptep, pgste);
@@ -557,7 +565,7 @@ int ptep_force_prot(struct mm_struct *mm
{
pte_t entry;
pgste_t pgste;
- int pte_i, pte_p;
+ int pte_i, pte_p, nodat;
pgste = pgste_get_lock(ptep);
entry = *ptep;
@@ -570,13 +578,14 @@ int ptep_force_prot(struct mm_struct *mm
return -EAGAIN;
}
/* Change access rights and set pgste bit */
+ nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
if (prot == PROT_NONE && !pte_i) {
- ptep_flush_direct(mm, addr, ptep);
+ ptep_flush_direct(mm, addr, ptep, nodat);
pgste = pgste_update_all(entry, pgste, mm);
pte_val(entry) |= _PAGE_INVALID;
}
if (prot == PROT_READ && !pte_p) {
- ptep_flush_direct(mm, addr, ptep);
+ ptep_flush_direct(mm, addr, ptep, nodat);
pte_val(entry) &= ~_PAGE_INVALID;
pte_val(entry) |= _PAGE_PROTECT;
}
@@ -616,10 +625,12 @@ int ptep_shadow_pte(struct mm_struct *mm
void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
{
pgste_t pgste;
+ int nodat;
pgste = pgste_get_lock(ptep);
/* notifier is called by the caller */
- ptep_flush_direct(mm, saddr, ptep);
+ nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+ ptep_flush_direct(mm, saddr, ptep, nodat);
/* don't touch the storage key - it belongs to parent pgste */
pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
pgste_set_unlock(ptep, pgste);
@@ -691,6 +702,7 @@ bool test_and_clear_guest_dirty(struct m
pte_t *ptep;
pte_t pte;
bool dirty;
+ int nodat;
pgd = pgd_offset(mm, addr);
pud = pud_alloc(mm, pgd, addr);
@@ -716,7 +728,8 @@ bool test_and_clear_guest_dirty(struct m
pte = *ptep;
if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
pgste = pgste_pte_notify(mm, addr, ptep, pgste);
- ptep_ipte_global(mm, addr, ptep);
+ nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
+ ptep_ipte_global(mm, addr, ptep, nodat);
if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
pte_val(pte) |= _PAGE_PROTECT;
else