From: jbeulich@novell.com
Subject: Go into polling mode early if lock owner is not running
Patch-mainline: n/a
This could be merged into the original ticket spinlock code once
validated, if there wasn't the dependency on smp-processor-id.h, which
only gets introduced in the 2.6.32 merge.
--- head.orig/arch/x86/Kconfig 2014-01-07 17:00:36.000000000 +0100
+++ head/arch/x86/Kconfig 2014-01-07 17:00:40.000000000 +0100
@@ -16,7 +16,7 @@ config X86_64
def_bool y
depends on 64BIT
select X86_DEV_DMA_OPS
- select ARCH_USE_CMPXCHG_LOCKREF
+ select ARCH_USE_CMPXCHG_LOCKREF if !XEN
### Arch settings
config X86
@@ -380,6 +380,8 @@ config X86_64_XEN
bool "Enable Xen compatible kernel"
depends on X86_64
select XEN
+ select ARCH_USE_CMPXCHG_LOCKREF if XEN_SPINLOCK_ACQUIRE_NESTING = ""
+# select ARCH_USE_CMPXCHG_LOCKREF if (XEN_SPINLOCK_ACQUIRE_NESTING + 1) * NR_CPUS < 256
help
This option will compile a kernel compatible with Xen hypervisor
--- head.orig/arch/x86/include/mach-xen/asm/spinlock.h 2013-09-26 15:42:51.000000000 +0200
+++ head/arch/x86/include/mach-xen/asm/spinlock.h 2013-10-01 16:48:41.000000000 +0200
@@ -44,6 +44,7 @@
#define SPIN_THRESHOLD (1 << 15)
#include <asm/irqflags.h>
+#include <asm/smp-processor-id.h>
int xen_spinlock_init(unsigned int cpu);
void xen_spinlock_cleanup(unsigned int cpu);
@@ -54,8 +55,8 @@ struct __raw_tickets xen_spin_adjust(con
#define xen_spin_adjust(lock, raw_tickets) (raw_tickets)
#define xen_spin_wait(l, t, f) xen_spin_wait(l, t)
#endif
-bool xen_spin_wait(arch_spinlock_t *, struct __raw_tickets *,
- unsigned int flags);
+unsigned int xen_spin_wait(arch_spinlock_t *, struct __raw_tickets *,
+ unsigned int flags);
void xen_spin_kick(const arch_spinlock_t *, unsigned int ticket);
static __always_inline int __ticket_spin_value_unlocked(arch_spinlock_t lock)
@@ -76,6 +77,8 @@ static __always_inline int __ticket_spin
* in the high part, because a wide xadd increment of the low part would carry
* up and contaminate the high part.
*/
+#define __spin_count_dec(c, l) (vcpu_running((l)->owner) ? --(c) : ((c) >>= 1))
+
#if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
{
@@ -83,21 +86,23 @@ static __always_inline void __ticket_spi
unsigned int count, flags = arch_local_irq_save();
inc = xadd(&lock->tickets, inc);
- if (likely(inc.head == inc.tail)) {
+ if (likely(inc.head == inc.tail))
+ arch_local_irq_restore(flags);
+ else {
+ inc = xen_spin_adjust(lock, inc);
arch_local_irq_restore(flags);
- return;
- }
- inc = xen_spin_adjust(lock, inc);
- arch_local_irq_restore(flags);
-
- do {
count = SPIN_THRESHOLD;
- while (inc.head != inc.tail && --count) {
- cpu_relax();
- inc.head = ACCESS_ONCE(lock->tickets.head);
- }
- } while (unlikely(!count) && !xen_spin_wait(lock, &inc, flags));
+ do {
+ while (inc.head != inc.tail
+ && __spin_count_dec(count, lock)) {
+ cpu_relax();
+ inc.head = ACCESS_ONCE(lock->tickets.head);
+ }
+ } while (unlikely(!count)
+ && (count = xen_spin_wait(lock, &inc, flags)));
+ }
barrier(); /* make sure nothing creeps before the lock is taken */
+ lock->owner = raw_smp_processor_id();
}
#else
#define __ticket_spin_lock(lock) __ticket_spin_lock_flags(lock, -1)
@@ -107,35 +112,41 @@ static __always_inline void __ticket_spi
unsigned long flags)
{
struct __raw_tickets inc = { .tail = TICKET_LOCK_INC };
- unsigned int count;
inc = xadd(&lock->tickets, inc);
- if (likely(inc.head == inc.tail))
- return;
- inc = xen_spin_adjust(lock, inc);
+ if (unlikely(inc.head != inc.tail)) {
+ unsigned int count = SPIN_THRESHOLD;
- do {
- count = SPIN_THRESHOLD;
- while (inc.head != inc.tail && --count) {
- cpu_relax();
- inc.head = ACCESS_ONCE(lock->tickets.head);
- }
- } while (unlikely(!count) && !xen_spin_wait(lock, &inc, flags));
+ inc = xen_spin_adjust(lock, inc);
+ do {
+ while (inc.head != inc.tail
+ && __spin_count_dec(count, lock)) {
+ cpu_relax();
+ inc.head = ACCESS_ONCE(lock->tickets.head);
+ }
+ } while (unlikely(!count)
+ && (count = xen_spin_wait(lock, &inc, flags)));
+ }
barrier(); /* make sure nothing creeps before the lock is taken */
+ lock->owner = raw_smp_processor_id();
}
+#undef __spin_count_dec
+
static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
{
- arch_spinlock_t old, new;
+ arch_spinlock_t old;
old.tickets = ACCESS_ONCE(lock->tickets);
if (old.tickets.head != old.tickets.tail)
return 0;
- new.head_tail = old.head_tail + (TICKET_LOCK_INC << TICKET_SHIFT);
-
/* cmpxchg is a full barrier, so nothing can move before it */
- return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
+ if (cmpxchg(&lock->head_tail, old.head_tail,
+ old.head_tail + (TICKET_LOCK_INC << TICKET_SHIFT)) != old.head_tail)
+ return 0;
+ lock->owner = raw_smp_processor_id();
+ return 1;
}
static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
--- head.orig/arch/x86/include/mach-xen/asm/spinlock_types.h 2013-09-26 15:35:09.000000000 +0200
+++ head/arch/x86/include/mach-xen/asm/spinlock_types.h 2012-02-01 09:32:19.000000000 +0100
@@ -27,9 +27,16 @@ typedef u32 __ticketpair_t;
typedef union {
__ticketpair_t head_tail;
- struct __raw_tickets {
- __ticket_t head, tail;
- } tickets;
+ struct {
+ struct __raw_tickets {
+ __ticket_t head, tail;
+ } tickets;
+#if CONFIG_NR_CPUS <= 256
+ u8 owner;
+#else
+ u16 owner;
+#endif
+ };
#else /* ndef CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING */
typedef struct {
/*
--- head.orig/drivers/xen/core/spinlock.c 2013-08-08 11:52:23.000000000 +0200
+++ head/drivers/xen/core/spinlock.c 2014-01-07 17:30:10.000000000 +0100
@@ -46,6 +46,8 @@ int xen_spinlock_init(unsigned int cpu)
struct evtchn_bind_ipi bind_ipi;
int rc;
+ setup_runstate_area(cpu);
+
if (nopoll)
return 0;
@@ -154,6 +156,7 @@ static unsigned int ticket_drop(struct s
if (cmpxchg(&spinning->ticket, ticket, -1) != ticket)
return -1;
+ lock->owner = cpu;
__add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
ticket = (__ticket_t)(ticket + 1);
return ticket != lock->tickets.tail ? ticket : -1;
@@ -233,21 +236,24 @@ void xen_spin_irq_exit(void)
if (spinning->ticket + 1)
continue;
spinning->ticket = ticket_get(lock, spinning->prev);
+ if (ACCESS_ONCE(lock->tickets.head) == spinning->ticket)
+ lock->owner = raw_smp_processor_id();
}
}
#endif
-bool xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
- unsigned int flags)
+unsigned int xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
+ unsigned int flags)
{
+ unsigned int cpu = raw_smp_processor_id();
typeof(vcpu_info(0)->evtchn_upcall_mask) upcall_mask
= arch_local_save_flags();
struct spinning spinning;
/* If kicker interrupt not initialized yet, just spin. */
- if (unlikely(!cpu_online(raw_smp_processor_id()))
+ if (unlikely(!cpu_online(cpu))
|| unlikely(!__this_cpu_read(poll_evtchn)))
- return false;
+ return UINT_MAX;
/* announce we're spinning */
spinning.ticket = ptok->tail;
@@ -324,13 +330,15 @@ bool xen_spin_wait(arch_spinlock_t *lock
sequence();
arch_local_irq_restore(upcall_mask);
smp_rmb();
- if (lock->tickets.head == spinning.ticket)
- return true;
+ if (lock->tickets.head == spinning.ticket) {
+ lock->owner = cpu;
+ return 0;
+ }
BUG_ON(CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING && !(spinning.ticket + 1));
ptok->head = lock->tickets.head;
ptok->tail = spinning.ticket;
- return false;
+ return SPIN_THRESHOLD >> 2;
}
static inline unsigned int cpumask_cycle(int n, const struct cpumask *srcp)