Subject: mm, rt: kmap_atomic scheduling
From: Peter Zijlstra <peterz@infradead.org>
Date: Thu, 28 Jul 2011 10:43:51 +0200
Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git
Git-commit: da77ceac3d20f27310a07a7c346a4ee6b40d6c28
Patch-mainline: Queued in subsystem maintainer repository
References: SLE Realtime Extension
In fact, with migrate_disable() existing one could play games with
kmap_atomic. You could save/restore the kmap_atomic slots on context
switch (if there are any in use of course), this should be esp easy now
that we have a kmap_atomic stack.
Something like the below.. it wants replacing all the preempt_disable()
stuff with pagefault_disable() && migrate_disable() of course, but then
you can flip kmaps around like below.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
[dvhart@linux.intel.com: build fix]
Link: http://lkml.kernel.org/r/1311842631.5890.208.camel@twins
[tglx@linutronix.de: Get rid of the per cpu variable and store the idx
and the pte content right away in the task struct.
Shortens the context switch code. ]
Signed-off-by: Daniel Wagner <dwagner@suse.de>
---
arch/x86/kernel/process_32.c | 32 ++++++++++++++++++++++++++++++++
arch/x86/mm/highmem_32.c | 13 ++++++++++---
arch/x86/mm/iomap_32.c | 9 ++++++++-
include/linux/highmem.h | 31 +++++++++++++++++++++++++------
include/linux/sched.h | 7 +++++++
include/linux/uaccess.h | 2 ++
mm/highmem.c | 6 +++---
7 files changed, 87 insertions(+), 13 deletions(-)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -38,6 +38,7 @@
#include <linux/io.h>
#include <linux/kdebug.h>
#include <linux/syscalls.h>
+#include <linux/highmem.h>
#include <asm/pgtable.h>
#include <asm/ldt.h>
@@ -196,6 +197,35 @@ start_thread(struct pt_regs *regs, unsig
}
EXPORT_SYMBOL_GPL(start_thread);
+#ifdef CONFIG_PREEMPT_RT
+static void switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p)
+{
+ int i;
+
+ /*
+ * Clear @prev's kmap_atomic mappings
+ */
+ for (i = 0; i < prev_p->kmap_idx; i++) {
+ int idx = i + KM_TYPE_NR * smp_processor_id();
+ pte_t *ptep = kmap_pte - idx;
+
+ kpte_clear_flush(ptep, __fix_to_virt(FIX_KMAP_BEGIN + idx));
+ }
+ /*
+ * Restore @next_p's kmap_atomic mappings
+ */
+ for (i = 0; i < next_p->kmap_idx; i++) {
+ int idx = i + KM_TYPE_NR * smp_processor_id();
+
+ if (!pte_none(next_p->kmap_pte[i]))
+ set_pte(kmap_pte - idx, next_p->kmap_pte[i]);
+ }
+}
+#else
+static inline void
+switch_kmaps(struct task_struct *prev_p, struct task_struct *next_p) { }
+#endif
+
/*
* switch_to(x,y) should switch tasks from x to y.
@@ -266,6 +296,8 @@ EXPORT_SYMBOL_GPL(start_thread);
switch_to_extra(prev_p, next_p);
+ switch_kmaps(prev_p, next_p);
+
/*
* Leave lazy mode, flushing any hypercalls made here.
* This must be done before restoring TLS segments so
--- a/arch/x86/mm/highmem_32.c
+++ b/arch/x86/mm/highmem_32.c
@@ -33,10 +33,11 @@ EXPORT_SYMBOL(kunmap);
*/
void *kmap_atomic_prot(struct page *page, pgprot_t prot)
{
+ pte_t pte = mk_pte(page, prot);
unsigned long vaddr;
int idx, type;
- preempt_disable();
+ preempt_disable_nort();
pagefault_disable();
if (!PageHighMem(page))
@@ -46,7 +47,10 @@ void *kmap_atomic_prot(struct page *page
idx = type + KM_TYPE_NR*smp_processor_id();
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
BUG_ON(!pte_none(*(kmap_pte-idx)));
- set_pte(kmap_pte-idx, mk_pte(page, prot));
+#ifdef CONFIG_PREEMPT_RT
+ current->kmap_pte[type] = pte;
+#endif
+ set_pte(kmap_pte-idx, pte);
arch_flush_lazy_mmu_mode();
return (void *)vaddr;
@@ -89,6 +93,9 @@ void __kunmap_atomic(void *kvaddr)
* is a bad idea also, in case the page changes cacheability
* attributes or becomes a protected page in a hypervisor.
*/
+#ifdef CONFIG_PREEMPT_RT
+ current->kmap_pte[type] = __pte(0);
+#endif
kpte_clear_flush(kmap_pte-idx, vaddr);
kmap_atomic_idx_pop();
arch_flush_lazy_mmu_mode();
@@ -101,7 +108,7 @@ void __kunmap_atomic(void *kvaddr)
#endif
pagefault_enable();
- preempt_enable();
+ preempt_enable_nort();
}
EXPORT_SYMBOL(__kunmap_atomic);
--- a/arch/x86/mm/iomap_32.c
+++ b/arch/x86/mm/iomap_32.c
@@ -46,6 +46,7 @@ EXPORT_SYMBOL_GPL(iomap_free);
void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot)
{
+ pte_t pte = pfn_pte(pfn, prot);
unsigned long vaddr;
int idx, type;
@@ -55,7 +56,10 @@ void *kmap_atomic_prot_pfn(unsigned long
type = kmap_atomic_idx_push();
idx = type + KM_TYPE_NR * smp_processor_id();
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
- set_pte(kmap_pte - idx, pfn_pte(pfn, prot));
+#ifdef CONFIG_PREEMPT_RT
+ current->kmap_pte[type] = pte;
+#endif
+ set_pte(kmap_pte - idx, pte);
arch_flush_lazy_mmu_mode();
return (void *)vaddr;
@@ -106,6 +110,9 @@ iounmap_atomic(void __iomem *kvaddr)
* is a bad idea also, in case the page changes cacheability
* attributes or becomes a protected page in a hypervisor.
*/
+#ifdef CONFIG_PREEMPT_RT
+ current->kmap_pte[type] = __pte(0);
+#endif
kpte_clear_flush(kmap_pte-idx, vaddr);
kmap_atomic_idx_pop();
}
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -90,7 +90,7 @@ static inline void kunmap(struct page *p
static inline void *kmap_atomic(struct page *page)
{
- preempt_disable();
+ preempt_disable_nort();
pagefault_disable();
return page_address(page);
}
@@ -99,7 +99,7 @@ static inline void *kmap_atomic(struct p
static inline void __kunmap_atomic(void *addr)
{
pagefault_enable();
- preempt_enable();
+ preempt_enable_nort();
}
#define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn))
@@ -111,32 +111,51 @@ static inline void __kunmap_atomic(void
#if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
+#ifndef CONFIG_PREEMPT_RT
DECLARE_PER_CPU(int, __kmap_atomic_idx);
+#endif
static inline int kmap_atomic_idx_push(void)
{
+#ifndef CONFIG_PREEMPT_RT
int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1;
-#ifdef CONFIG_DEBUG_HIGHMEM
+# ifdef CONFIG_DEBUG_HIGHMEM
WARN_ON_ONCE(in_irq() && !irqs_disabled());
BUG_ON(idx >= KM_TYPE_NR);
-#endif
+# endif
return idx;
+#else
+ current->kmap_idx++;
+ BUG_ON(current->kmap_idx > KM_TYPE_NR);
+ return current->kmap_idx - 1;
+#endif
}
static inline int kmap_atomic_idx(void)
{
+#ifndef CONFIG_PREEMPT_RT
return __this_cpu_read(__kmap_atomic_idx) - 1;
+#else
+ return current->kmap_idx - 1;
+#endif
}
static inline void kmap_atomic_idx_pop(void)
{
-#ifdef CONFIG_DEBUG_HIGHMEM
+#ifndef CONFIG_PREEMPT_RT
+# ifdef CONFIG_DEBUG_HIGHMEM
int idx = __this_cpu_dec_return(__kmap_atomic_idx);
BUG_ON(idx < 0);
-#else
+# else
__this_cpu_dec(__kmap_atomic_idx);
+# endif
+#else
+ current->kmap_idx--;
+# ifdef CONFIG_DEBUG_HIGHMEM
+ BUG_ON(current->kmap_idx < 0);
+# endif
#endif
}
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -32,6 +32,7 @@
#include <linux/posix-timers.h>
#include <linux/rseq.h>
#include <linux/seqlock.h>
+#include <asm/kmap_types.h>
/* task_struct member predeclarations (sorted alphabetically): */
struct audit_context;
@@ -1266,6 +1267,12 @@ struct task_struct {
unsigned int sequential_io;
unsigned int sequential_io_avg;
#endif
+#ifdef CONFIG_PREEMPT_RT
+# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32
+ int kmap_idx;
+ pte_t kmap_pte[KM_TYPE_NR];
+# endif
+#endif
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
unsigned long task_state_change;
#endif
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -182,6 +182,7 @@ static __always_inline void pagefault_di
*/
static inline void pagefault_disable(void)
{
+ migrate_disable();
pagefault_disabled_inc();
/*
* make sure to have issued the store before a pagefault
@@ -198,6 +199,7 @@ static inline void pagefault_enable(void
*/
barrier();
pagefault_disabled_dec();
+ migrate_enable();
}
/*
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -30,9 +30,11 @@
#include <linux/kgdb.h>
#include <asm/tlbflush.h>
-
+#ifndef CONFIG_PREEMPT_RT
#if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
DEFINE_PER_CPU(int, __kmap_atomic_idx);
+EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
+#endif
#endif
/*
@@ -108,8 +110,6 @@ static inline wait_queue_head_t *get_pkm
atomic_long_t _totalhigh_pages __read_mostly;
EXPORT_SYMBOL(_totalhigh_pages);
-EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
-
unsigned int nr_free_highpages (void)
{
struct zone *zone;