From: jbeulich@novell.com
Subject: Go into polling mode early if lock owner is not running
Patch-mainline: n/a

This could be merged into the original ticket spinlock code once
validated, if there wasn't the dependency on smp-processor-id.h, which
only gets introduced in the 2.6.32 merge.

--- head.orig/arch/x86/include/mach-xen/asm/spinlock.h	2012-02-09 12:49:39.000000000 +0100
+++ head/arch/x86/include/mach-xen/asm/spinlock.h	2012-02-02 13:51:42.000000000 +0100
@@ -44,6 +44,7 @@
 #ifdef TICKET_SHIFT
 
 #include <asm/irqflags.h>
+#include <asm/smp-processor-id.h>
 
 int xen_spinlock_init(unsigned int cpu);
 void xen_spinlock_cleanup(unsigned int cpu);
@@ -54,8 +55,8 @@ struct __raw_tickets xen_spin_adjust(con
 #define xen_spin_adjust(lock, raw_tickets) (raw_tickets)
 #define xen_spin_wait(l, t, f) xen_spin_wait(l, t)
 #endif
-bool xen_spin_wait(arch_spinlock_t *, struct __raw_tickets *,
-		   unsigned int flags);
+unsigned int xen_spin_wait(arch_spinlock_t *, struct __raw_tickets *,
+			   unsigned int flags);
 void xen_spin_kick(const arch_spinlock_t *, unsigned int ticket);
 
 /*
@@ -71,6 +72,8 @@ void xen_spin_kick(const arch_spinlock_t
  * in the high part, because a wide xadd increment of the low part would carry
  * up and contaminate the high part.
  */
+#define __spin_count_dec(c, l) (vcpu_running((l)->owner) ? --(c) : ((c) >>= 1))
+
 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
 {
@@ -78,21 +81,23 @@ static __always_inline void __ticket_spi
 	unsigned int count, flags = arch_local_irq_save();
 
 	inc = xadd(&lock->tickets, inc);
-	if (likely(inc.head == inc.tail)) {
+	if (likely(inc.head == inc.tail))
+		arch_local_irq_restore(flags);
+	else {
+		inc = xen_spin_adjust(lock, inc);
 		arch_local_irq_restore(flags);
-		return;
-	}
-	inc = xen_spin_adjust(lock, inc);
-	arch_local_irq_restore(flags);
-
-	do {
 		count = 1 << 12;
-		while (inc.head != inc.tail && --count) {
-			cpu_relax();
-			inc.head = ACCESS_ONCE(lock->tickets.head);
-		}
-	} while (unlikely(!count) && !xen_spin_wait(lock, &inc, flags));
+		do {
+			while (inc.head != inc.tail
+			       && __spin_count_dec(count, lock)) {
+				cpu_relax();
+				inc.head = ACCESS_ONCE(lock->tickets.head);
+			}
+		} while (unlikely(!count)
+			 && (count = xen_spin_wait(lock, &inc, flags)));
+	}
 	barrier();		/* make sure nothing creeps before the lock is taken */
+	lock->owner = raw_smp_processor_id();
 }
 #else
 #define __ticket_spin_lock(lock) __ticket_spin_lock_flags(lock, -1)
@@ -102,35 +107,41 @@ static __always_inline void __ticket_spi
 						     unsigned long flags)
 {
 	struct __raw_tickets inc = { .tail = 1 };
-	unsigned int count;
 
 	inc = xadd(&lock->tickets, inc);
-	if (likely(inc.head == inc.tail))
-		return;
-	inc = xen_spin_adjust(lock, inc);
+	if (unlikely(inc.head != inc.tail)) {
+		unsigned int count = 1 << 12;
 
-	do {
-		count = 1 << 12;
-		while (inc.head != inc.tail && --count) {
-			cpu_relax();
-			inc.head = ACCESS_ONCE(lock->tickets.head);
-		}
-	} while (unlikely(!count) && !xen_spin_wait(lock, &inc, flags));
+		inc = xen_spin_adjust(lock, inc);
+		do {
+			while (inc.head != inc.tail
+			       && __spin_count_dec(count, lock)) {
+				cpu_relax();
+				inc.head = ACCESS_ONCE(lock->tickets.head);
+			}
+		} while (unlikely(!count)
+			 && (count = xen_spin_wait(lock, &inc, flags)));
+	}
 	barrier();		/* make sure nothing creeps before the lock is taken */
+	lock->owner = raw_smp_processor_id();
 }
 
+#undef __spin_count_dec
+
 static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
 {
-	arch_spinlock_t old, new;
+	arch_spinlock_t old;
 
 	old.tickets = ACCESS_ONCE(lock->tickets);
 	if (old.tickets.head != old.tickets.tail)
 		return 0;
 
-	new.head_tail = old.head_tail + (1 << TICKET_SHIFT);
-
 	/* cmpxchg is a full barrier, so nothing can move before it */
-	return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
+	if (cmpxchg(&lock->head_tail, old.head_tail,
+		    old.head_tail + (1 << TICKET_SHIFT)) != old.head_tail)
+		return 0;
+	lock->owner = raw_smp_processor_id();
+	return 1;
 }
 
 static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
--- head.orig/arch/x86/include/mach-xen/asm/spinlock_types.h	2012-01-26 13:51:28.000000000 +0100
+++ head/arch/x86/include/mach-xen/asm/spinlock_types.h	2012-02-01 09:32:19.000000000 +0100
@@ -27,9 +27,16 @@ typedef u32 __ticketpair_t;
 
 typedef union {
 	__ticketpair_t head_tail;
-	struct __raw_tickets {
-		__ticket_t head, tail;
-	} tickets;
+	struct {
+		struct __raw_tickets {
+			__ticket_t head, tail;
+		} tickets;
+#if CONFIG_NR_CPUS <= 256
+		u8 owner;
+#else
+		u16 owner;
+#endif
+	};
 #else /* ndef CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING */
 typedef struct {
 /*
--- head.orig/drivers/xen/core/spinlock.c	2012-02-01 09:18:36.000000000 +0100
+++ head/drivers/xen/core/spinlock.c	2012-02-10 11:52:12.000000000 +0100
@@ -43,6 +43,8 @@ int __cpuinit xen_spinlock_init(unsigned
 	struct evtchn_bind_ipi bind_ipi;
 	int rc;
 
+	setup_runstate_area(cpu);
+
  	WARN_ON(per_cpu(poll_evtchn, cpu));
 	bind_ipi.vcpu = cpu;
 	rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
@@ -147,6 +149,7 @@ static unsigned int ticket_drop(struct s
 
 	if (cmpxchg(&spinning->ticket, ticket, -1) != ticket)
 		return -1;
+	lock->owner = cpu;
 	__add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
 	ticket = (__ticket_t)(ticket + 1);
 	return ticket != lock->tickets.tail ? ticket : -1;
@@ -227,21 +230,24 @@ void xen_spin_irq_exit(void)
 		if (spinning->ticket + 1)
 			continue;
 		spinning->ticket = ticket_get(lock, spinning->prev);
+		if (ACCESS_ONCE(lock->tickets.head) == spinning->ticket)
+			lock->owner = cpu;
 	}
 }
 #endif
 
-bool xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
-		   unsigned int flags)
+unsigned int xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
+			   unsigned int flags)
 {
+	unsigned int cpu = raw_smp_processor_id();
 	typeof(vcpu_info(0)->evtchn_upcall_mask) upcall_mask
 		= arch_local_save_flags();
 	struct spinning spinning;
 
 	/* If kicker interrupt not initialized yet, just spin. */
-	if (unlikely(!cpu_online(raw_smp_processor_id()))
-	    || unlikely(!percpu_read(poll_evtchn)))
-		return false;
+	if (unlikely(!cpu_online(cpu))
+	    || unlikely(!this_cpu_read(poll_evtchn)))
+		return UINT_MAX;
 
 	/* announce we're spinning */
 	spinning.ticket = ptok->tail;
@@ -318,13 +323,15 @@ bool xen_spin_wait(arch_spinlock_t *lock
 	sequence(SEQ_REMOVE_BIAS);
 	arch_local_irq_restore(upcall_mask);
 	smp_rmb();
-	if (lock->tickets.head == spinning.ticket)
-		return true;
+	if (lock->tickets.head == spinning.ticket) {
+		lock->owner = cpu;
+		return 0;
+	}
 	BUG_ON(CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING && !(spinning.ticket + 1));
 	ptok->head = lock->tickets.head;
 	ptok->tail = spinning.ticket;
 
-	return false;
+	return 1 << 10;
 }
 
 void xen_spin_kick(const arch_spinlock_t *lock, unsigned int ticket)