From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Thu, 29 Jan 2015 15:10:08 +0100
Subject: block/mq: don't complete requests via IPI
Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git
Git-commit: f0f1f94aa26aa65d87de00dc8ae85752a0f99813
Patch-mainline: Queued in subsystem maintainer repository
References: SLE Realtime Extension

The IPI runs in hardirq context and there are sleeping locks. This patch
moves the completion into a workqueue.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Mike Galbraith <mgalbraith@suse.de>
---
 block/blk-core.c       |    3 +++
 block/blk-mq.c         |   24 ++++++++++++++++++++++++
 include/linux/blk-mq.h |    2 +-
 include/linux/blkdev.h |    3 +++
 4 files changed, 31 insertions(+), 1 deletion(-)

--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -188,6 +188,9 @@ void blk_rq_init(struct request_queue *q
 
 	INIT_LIST_HEAD(&rq->queuelist);
 	INIT_LIST_HEAD(&rq->timeout_list);
+#ifdef CONFIG_PREEMPT_RT_FULL
+	INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
+#endif
 	rq->cpu = -1;
 	rq->q = q;
 	rq->__sector = (sector_t) -1;
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -326,6 +326,9 @@ static struct request *blk_mq_rq_ctx_ini
 	rq->extra_len = 0;
 	rq->__deadline = 0;
 
+#ifdef CONFIG_PREEMPT_RT_FULL
+	INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
+#endif
 	INIT_LIST_HEAD(&rq->timeout_list);
 	rq->timeout = 0;
 
@@ -556,6 +559,17 @@ void blk_mq_end_request(struct request *
 }
 EXPORT_SYMBOL(blk_mq_end_request);
 
+#ifdef CONFIG_PREEMPT_RT_FULL
+
+void __blk_mq_complete_request_remote_work(struct work_struct *work)
+{
+	struct request *rq = container_of(work, struct request, work);
+
+	rq->q->softirq_done_fn(rq);
+}
+
+#else
+
 static void __blk_mq_complete_request_remote(void *data)
 {
 	struct request *rq = data;
@@ -563,6 +577,8 @@ static void __blk_mq_complete_request_re
 	rq->q->softirq_done_fn(rq);
 }
 
+#endif
+
 void blk_mq_complete_request_sync(struct request *rq)
 {
 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
@@ -603,10 +619,18 @@ static void __blk_mq_complete_request(st
 		shared = cpus_share_cache(cpu, ctx->cpu);
 
 	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
+#ifdef CONFIG_PREEMPT_RT_FULL
+		/*
+		 * We could force QUEUE_FLAG_SAME_FORCE then we would not get in
+		 * here. But we could try to invoke it one the CPU like this.
+		 */
+		schedule_work_on(ctx->cpu, &rq->work);
+#else
 		rq->csd.func = __blk_mq_complete_request_remote;
 		rq->csd.info = rq;
 		rq->csd.flags = 0;
 		smp_call_function_single_async(ctx->cpu, &rq->csd);
+#endif
 	} else {
 		rq->q->softirq_done_fn(rq);
 	}
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -248,7 +248,7 @@ static inline u16 blk_mq_unique_tag_to_t
 	return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;
 }
 
-
+void __blk_mq_complete_request_remote_work(struct work_struct *work);
 int blk_mq_request_started(struct request *rq);
 int blk_mq_request_completed(struct request *rq);
 void blk_mq_start_request(struct request *rq);
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -167,6 +167,9 @@ struct request {
 	struct bio *biotail;
 
 	struct list_head queuelist;
+#ifdef CONFIG_PREEMPT_RT_FULL
+	struct work_struct work;
+#endif
 
 	/*
 	 * The hash is used inside the scheduler, and killed once the