From: jbeulich@novell.com
Subject: Go into polling mode early if lock owner is not running
Patch-mainline: n/a
This could be merged into the original ticket spinlock code once
validated, if there wasn't the dependency on smp-processor-id.h, which
only gets introduced in the 2.6.32 merge.
--- head.orig/arch/x86/include/mach-xen/asm/spinlock.h 2012-02-09 12:49:39.000000000 +0100
+++ head/arch/x86/include/mach-xen/asm/spinlock.h 2012-02-02 13:51:42.000000000 +0100
@@ -44,6 +44,7 @@
#ifdef TICKET_SHIFT
#include <asm/irqflags.h>
+#include <asm/smp-processor-id.h>
int xen_spinlock_init(unsigned int cpu);
void xen_spinlock_cleanup(unsigned int cpu);
@@ -54,8 +55,8 @@ struct __raw_tickets xen_spin_adjust(con
#define xen_spin_adjust(lock, raw_tickets) (raw_tickets)
#define xen_spin_wait(l, t, f) xen_spin_wait(l, t)
#endif
-bool xen_spin_wait(arch_spinlock_t *, struct __raw_tickets *,
- unsigned int flags);
+unsigned int xen_spin_wait(arch_spinlock_t *, struct __raw_tickets *,
+ unsigned int flags);
void xen_spin_kick(const arch_spinlock_t *, unsigned int ticket);
/*
@@ -71,6 +72,8 @@ void xen_spin_kick(const arch_spinlock_t
* in the high part, because a wide xadd increment of the low part would carry
* up and contaminate the high part.
*/
+#define __spin_count_dec(c, l) (vcpu_running((l)->owner) ? --(c) : ((c) >>= 1))
+
#if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
{
@@ -78,21 +81,23 @@ static __always_inline void __ticket_spi
unsigned int count, flags = arch_local_irq_save();
inc = xadd(&lock->tickets, inc);
- if (likely(inc.head == inc.tail)) {
+ if (likely(inc.head == inc.tail))
+ arch_local_irq_restore(flags);
+ else {
+ inc = xen_spin_adjust(lock, inc);
arch_local_irq_restore(flags);
- return;
- }
- inc = xen_spin_adjust(lock, inc);
- arch_local_irq_restore(flags);
-
- do {
count = 1 << 12;
- while (inc.head != inc.tail && --count) {
- cpu_relax();
- inc.head = ACCESS_ONCE(lock->tickets.head);
- }
- } while (unlikely(!count) && !xen_spin_wait(lock, &inc, flags));
+ do {
+ while (inc.head != inc.tail
+ && __spin_count_dec(count, lock)) {
+ cpu_relax();
+ inc.head = ACCESS_ONCE(lock->tickets.head);
+ }
+ } while (unlikely(!count)
+ && (count = xen_spin_wait(lock, &inc, flags)));
+ }
barrier(); /* make sure nothing creeps before the lock is taken */
+ lock->owner = raw_smp_processor_id();
}
#else
#define __ticket_spin_lock(lock) __ticket_spin_lock_flags(lock, -1)
@@ -102,35 +107,41 @@ static __always_inline void __ticket_spi
unsigned long flags)
{
struct __raw_tickets inc = { .tail = 1 };
- unsigned int count;
inc = xadd(&lock->tickets, inc);
- if (likely(inc.head == inc.tail))
- return;
- inc = xen_spin_adjust(lock, inc);
+ if (unlikely(inc.head != inc.tail)) {
+ unsigned int count = 1 << 12;
- do {
- count = 1 << 12;
- while (inc.head != inc.tail && --count) {
- cpu_relax();
- inc.head = ACCESS_ONCE(lock->tickets.head);
- }
- } while (unlikely(!count) && !xen_spin_wait(lock, &inc, flags));
+ inc = xen_spin_adjust(lock, inc);
+ do {
+ while (inc.head != inc.tail
+ && __spin_count_dec(count, lock)) {
+ cpu_relax();
+ inc.head = ACCESS_ONCE(lock->tickets.head);
+ }
+ } while (unlikely(!count)
+ && (count = xen_spin_wait(lock, &inc, flags)));
+ }
barrier(); /* make sure nothing creeps before the lock is taken */
+ lock->owner = raw_smp_processor_id();
}
+#undef __spin_count_dec
+
static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
{
- arch_spinlock_t old, new;
+ arch_spinlock_t old;
old.tickets = ACCESS_ONCE(lock->tickets);
if (old.tickets.head != old.tickets.tail)
return 0;
- new.head_tail = old.head_tail + (1 << TICKET_SHIFT);
-
/* cmpxchg is a full barrier, so nothing can move before it */
- return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
+ if (cmpxchg(&lock->head_tail, old.head_tail,
+ old.head_tail + (1 << TICKET_SHIFT)) != old.head_tail)
+ return 0;
+ lock->owner = raw_smp_processor_id();
+ return 1;
}
static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
--- head.orig/arch/x86/include/mach-xen/asm/spinlock_types.h 2012-01-26 13:51:28.000000000 +0100
+++ head/arch/x86/include/mach-xen/asm/spinlock_types.h 2012-02-01 09:32:19.000000000 +0100
@@ -27,9 +27,16 @@ typedef u32 __ticketpair_t;
typedef union {
__ticketpair_t head_tail;
- struct __raw_tickets {
- __ticket_t head, tail;
- } tickets;
+ struct {
+ struct __raw_tickets {
+ __ticket_t head, tail;
+ } tickets;
+#if CONFIG_NR_CPUS <= 256
+ u8 owner;
+#else
+ u16 owner;
+#endif
+ };
#else /* ndef CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING */
typedef struct {
/*
--- head.orig/drivers/xen/core/spinlock.c 2012-02-01 09:18:36.000000000 +0100
+++ head/drivers/xen/core/spinlock.c 2012-02-10 11:52:12.000000000 +0100
@@ -43,6 +43,8 @@ int __cpuinit xen_spinlock_init(unsigned
struct evtchn_bind_ipi bind_ipi;
int rc;
+ setup_runstate_area(cpu);
+
WARN_ON(per_cpu(poll_evtchn, cpu));
bind_ipi.vcpu = cpu;
rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
@@ -147,6 +149,7 @@ static unsigned int ticket_drop(struct s
if (cmpxchg(&spinning->ticket, ticket, -1) != ticket)
return -1;
+ lock->owner = cpu;
__add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
ticket = (__ticket_t)(ticket + 1);
return ticket != lock->tickets.tail ? ticket : -1;
@@ -227,21 +230,24 @@ void xen_spin_irq_exit(void)
if (spinning->ticket + 1)
continue;
spinning->ticket = ticket_get(lock, spinning->prev);
+ if (ACCESS_ONCE(lock->tickets.head) == spinning->ticket)
+ lock->owner = cpu;
}
}
#endif
-bool xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
- unsigned int flags)
+unsigned int xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
+ unsigned int flags)
{
+ unsigned int cpu = raw_smp_processor_id();
typeof(vcpu_info(0)->evtchn_upcall_mask) upcall_mask
= arch_local_save_flags();
struct spinning spinning;
/* If kicker interrupt not initialized yet, just spin. */
- if (unlikely(!cpu_online(raw_smp_processor_id()))
- || unlikely(!percpu_read(poll_evtchn)))
- return false;
+ if (unlikely(!cpu_online(cpu))
+ || unlikely(!this_cpu_read(poll_evtchn)))
+ return UINT_MAX;
/* announce we're spinning */
spinning.ticket = ptok->tail;
@@ -318,13 +323,15 @@ bool xen_spin_wait(arch_spinlock_t *lock
sequence(SEQ_REMOVE_BIAS);
arch_local_irq_restore(upcall_mask);
smp_rmb();
- if (lock->tickets.head == spinning.ticket)
- return true;
+ if (lock->tickets.head == spinning.ticket) {
+ lock->owner = cpu;
+ return 0;
+ }
BUG_ON(CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING && !(spinning.ticket + 1));
ptok->head = lock->tickets.head;
ptok->tail = spinning.ticket;
- return false;
+ return 1 << 10;
}
void xen_spin_kick(const arch_spinlock_t *lock, unsigned int ticket)