From: John Stultz <johnstul@us.ibm.com>

Date: Fri, 3 Jul 2009 08:29:58 -0500

Subject: posix-timers: Thread posix-cpu-timers on -rt

Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git

Git-commit: da77ceac3d20f27310a07a7c346a4ee6b40d6c28

Patch-mainline: Queued in subsystem maintainer repository

References: SLE Realtime Extension

posix-cpu-timer code takes non -rt safe locks in hard irq

context. Move it to a thread.

[ 3.0 fixes from Peter Zijlstra <peterz@infradead.org> ]

Signed-off-by: John Stultz <johnstul@us.ibm.com>

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

Signed-off-by: Daniel Wagner <dwagner@suse.de>

---

 include/linux/posix-timers.h   |   10 ++

 kernel/time/posix-cpu-timers.c |  175 ++++++++++++++++++++++++++++++++++++++---

 2 files changed, 174 insertions(+), 11 deletions(-)

--- a/include/linux/posix-timers.h

+++ b/include/linux/posix-timers.h

@@ -123,6 +123,9 @@ struct posix_cputimers {

 	struct posix_cputimer_base	bases[CPUCLOCK_MAX];

 	unsigned int			timers_active;

 	unsigned int			expiry_active;

+#ifdef CONFIG_PREEMPT_RT

+	struct task_struct		*posix_timer_list;

+#endif

 };

 static inline void posix_cputimers_init(struct posix_cputimers *pct)

@@ -152,9 +155,16 @@ static inline void posix_cputimers_rt_wa

 	INIT_CPU_TIMERBASE(b[2]),					\

 }

+#ifdef CONFIG_PREEMPT_RT

+# define INIT_TIMER_LIST	.posix_timer_list = NULL,

+#else

+# define INIT_TIMER_LIST

+#endif

+

 #define INIT_CPU_TIMERS(s)						\

 	.posix_cputimers = {						\

 		.bases = INIT_CPU_TIMERBASES(s.posix_cputimers.bases),	\

+		INIT_TIMER_LIST						\

 	},

 #else

 struct posix_cputimers { };

--- a/kernel/time/posix-cpu-timers.c

+++ b/kernel/time/posix-cpu-timers.c

@@ -3,8 +3,10 @@

  * Implement CPU time clocks for the POSIX clock interface.

  */

+#include <uapi/linux/sched/types.h>

 #include <linux/sched/signal.h>

 #include <linux/sched/cputime.h>

+#include <linux/sched/rt.h>

 #include <linux/posix-timers.h>

 #include <linux/errno.h>

 #include <linux/math64.h>

@@ -15,6 +17,7 @@

 #include <linux/workqueue.h>

 #include <linux/compat.h>

 #include <linux/sched/deadline.h>

+#include <linux/smpboot.h>

 #include "posix-timers.h"

@@ -27,6 +30,9 @@ void posix_cputimers_group_init(struct p

 		pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC;

 		pct->timers_active = true;

 	}

+#ifdef CONFIG_PREEMPT_RT

+	pct->posix_timer_list = NULL;

+#endif

 }

 /*

@@ -804,7 +810,8 @@ static inline void check_dl_overrun(stru

 	}

 }

-static bool check_rlimit(u64 time, u64 limit, int signo, bool rt, bool hard)

+static bool check_rlimit(struct task_struct *tsk, u64 time, u64 limit,

+			 int signo, bool rt, bool hard)

 {

 	if (time < limit)

 		return false;

@@ -812,9 +819,9 @@ static bool check_rlimit(u64 time, u64 l

 	if (print_fatal_signals) {

 		pr_info("%s Watchdog Timeout (%s): %s[%d]\n",

 			rt ? "RT" : "CPU", hard ? "hard" : "soft",

-			current->comm, task_pid_nr(current));

+			tsk->comm, task_pid_nr(tsk));

 	}

-	__group_send_sig_info(signo, SEND_SIG_PRIV, current);

+	__group_send_sig_info(signo, SEND_SIG_PRIV, tsk);

 	return true;

 }

@@ -850,11 +857,11 @@ static void check_thread_timers(struct t

 		/* At the hard limit, send SIGKILL. No further action. */

 		if (hard != RLIM_INFINITY &&

-		    check_rlimit(rttime, hard, SIGKILL, true, true))

+		    check_rlimit(tsk, rttime, hard, SIGKILL, true, true))

 			return;

 		/* At the soft limit, send a SIGXCPU every second */

-		if (check_rlimit(rttime, soft, SIGXCPU, true, false)) {

+		if (check_rlimit(tsk, rttime, soft, SIGXCPU, true, false)) {

 			soft += USEC_PER_SEC;

 			tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur = soft;

 		}

@@ -949,11 +956,11 @@ static void check_process_timers(struct

 		/* At the hard limit, send SIGKILL. No further action. */

 		if (hard != RLIM_INFINITY &&

-		    check_rlimit(ptime, hardns, SIGKILL, false, true))

+		    check_rlimit(tsk, ptime, hardns, SIGKILL, false, true))

 			return;

 		/* At the soft limit, send a SIGXCPU every second */

-		if (check_rlimit(ptime, softns, SIGXCPU, false, false)) {

+		if (check_rlimit(tsk, ptime, softns, SIGXCPU, false, false)) {

 			sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1;

 			softns += NSEC_PER_SEC;

 		}

@@ -1110,15 +1117,12 @@ static inline bool fastpath_timer_check(

  * already updated our counts.  We need to check if any timers fire now.

  * Interrupts are disabled.

  */

-void run_posix_cpu_timers(void)

+static void __run_posix_cpu_timers(struct task_struct *tsk)

 {

-	struct task_struct *tsk = current;

 	struct k_itimer *timer, *next;

 	unsigned long flags;

 	LIST_HEAD(firing);

-	lockdep_assert_irqs_disabled();

-

 	/*

 	 * The fast path checks that there are no expired thread or thread

 	 * group timers.  If that's so, just return.

@@ -1171,6 +1175,155 @@ void run_posix_cpu_timers(void)

 	}

 }

+#ifdef CONFIG_PREEMPT_RT

+#include <linux/kthread.h>

+#include <linux/cpu.h>

+DEFINE_PER_CPU(struct task_struct *, posix_timer_task);

+DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist);

+DEFINE_PER_CPU(bool, posix_timer_th_active);

+

+static void posix_cpu_kthread_fn(unsigned int cpu)

+{

+	struct task_struct *tsk = NULL;

+	struct task_struct *next = NULL;

+

+	BUG_ON(per_cpu(posix_timer_task, cpu) != current);

+

+	/* grab task list */

+	raw_local_irq_disable();

+	tsk = per_cpu(posix_timer_tasklist, cpu);

+	per_cpu(posix_timer_tasklist, cpu) = NULL;

+	raw_local_irq_enable();

+

+	/* its possible the list is empty, just return */

+	if (!tsk)

+		return;

+

+	/* Process task list */

+	while (1) {

+		/* save next */

+		next = tsk->posix_cputimers.posix_timer_list;

+

+		/* run the task timers, clear its ptr and

+		 * unreference it

+		 */

+		__run_posix_cpu_timers(tsk);

+		tsk->posix_cputimers.posix_timer_list = NULL;

+		put_task_struct(tsk);

+

+		/* check if this is the last on the list */

+		if (next == tsk)

+			break;

+		tsk = next;

+	}

+}

+

+static inline int __fastpath_timer_check(struct task_struct *tsk)

+{

+	/* tsk == current, ensure it is safe to use ->signal/sighand */

+	if (unlikely(tsk->exit_state))

+		return 0;

+

+	if (!expiry_cache_is_inactive(&tsk->posix_cputimers))

+		return 1;

+

+	if (!expiry_cache_is_inactive(&tsk->signal->posix_cputimers))

+		return 1;

+

+	return 0;

+}

+

+void run_posix_cpu_timers(void)

+{

+	unsigned int cpu = smp_processor_id();

+	struct task_struct *tsk = current;

+	struct task_struct *tasklist;

+

+	BUG_ON(!irqs_disabled());

+

+	if (per_cpu(posix_timer_th_active, cpu) != true)

+		return;

+

+	/* get per-cpu references */

+	tasklist = per_cpu(posix_timer_tasklist, cpu);

+

+	/* check to see if we're already queued */

+	if (!tsk->posix_cputimers.posix_timer_list && __fastpath_timer_check(tsk)) {

+		get_task_struct(tsk);

+		if (tasklist) {

+			tsk->posix_cputimers.posix_timer_list = tasklist;

+		} else {

+			/*

+			 * The list is terminated by a self-pointing

+			 * task_struct

+			 */

+			tsk->posix_cputimers.posix_timer_list = tsk;

+		}

+		per_cpu(posix_timer_tasklist, cpu) = tsk;

+

+		wake_up_process(per_cpu(posix_timer_task, cpu));

+	}

+}

+

+static int posix_cpu_kthread_should_run(unsigned int cpu)

+{

+	return __this_cpu_read(posix_timer_tasklist) != NULL;

+}

+

+static void posix_cpu_kthread_park(unsigned int cpu)

+{

+	this_cpu_write(posix_timer_th_active, false);

+}

+

+static void posix_cpu_kthread_unpark(unsigned int cpu)

+{

+	this_cpu_write(posix_timer_th_active, true);

+}

+

+static void posix_cpu_kthread_setup(unsigned int cpu)

+{

+	struct sched_param sp;

+

+	sp.sched_priority = MAX_RT_PRIO - 1;

+	sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);

+	posix_cpu_kthread_unpark(cpu);

+}

+

+static struct smp_hotplug_thread posix_cpu_thread = {

+	.store			= &posix_timer_task,

+	.thread_should_run	= posix_cpu_kthread_should_run,

+	.thread_fn		= posix_cpu_kthread_fn,

+	.thread_comm		= "posixcputmr/%u",

+	.setup			= posix_cpu_kthread_setup,

+	.park			= posix_cpu_kthread_park,

+	.unpark			= posix_cpu_kthread_unpark,

+};

+

+static int __init posix_cpu_thread_init(void)

+{

+	/* Start one for boot CPU. */

+	unsigned long cpu;

+	int ret;

+

+	/* init the per-cpu posix_timer_tasklets */

+	for_each_possible_cpu(cpu)

+		per_cpu(posix_timer_tasklist, cpu) = NULL;

+

+	ret = smpboot_register_percpu_thread(&posix_cpu_thread);

+	WARN_ON(ret);

+

+	return 0;

+}

+early_initcall(posix_cpu_thread_init);

+

+#else /* CONFIG_PREEMPT_RT */

+void run_posix_cpu_timers(void)

+{

+	lockdep_assert_irqs_disabled();

+	__run_posix_cpu_timers(current);

+}

+#endif /* CONFIG_PREEMPT_RT */

+

 /*

  * Set one of the process-wide special case CPU timers or RLIMIT_CPU.

  * The tsk->sighand->siglock must be held by the caller.