From 9cc25a7b48aabd633080d6242378ba708cf47ee3 Mon Sep 17 00:00:00 2001

From: Peter Zijlstra <peterz@infradead.org>

Date: Tue, 23 Jan 2018 20:34:30 +0100

Subject: [PATCH] sched/core: Optimize update_stats_*()

References: bnc#1101669 optimise numa balancing for fast migrate

Patch-mainline: v4.16-rc1

Git-commit: 2ed41a55023dc5be6742ca0eb8df5cb20e8dcaae

These functions are already gated by schedstats_enabled(), there is no

point in then issuing another static_branch for every individual

update in them.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Cc: Linus Torvalds <torvalds@linux-foundation.org>

Cc: Mike Galbraith <efault@gmx.de>

Cc: Peter Zijlstra <peterz@infradead.org>

Cc: Thomas Gleixner <tglx@linutronix.de>

Cc: linux-kernel@vger.kernel.org

Signed-off-by: Ingo Molnar <mingo@kernel.org>

Signed-off-by: Mel Gorman <mgorman@suse.de>

---

 kernel/sched/fair.c  | 32 ++++++++++++++++----------------

 kernel/sched/stats.h |  4 ++++

 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c

index 4b656b1ee149..0ac3f9c7e787 100644

--- a/kernel/sched/fair.c

+++ b/kernel/sched/fair.c

@@ -882,7 +882,7 @@ update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)

 	    likely(wait_start > prev_wait_start))

 		wait_start -= prev_wait_start;

-	schedstat_set(se->statistics.wait_start, wait_start);

+	__schedstat_set(se->statistics.wait_start, wait_start);

 }

 static inline void

@@ -904,17 +904,17 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)

 			 * time stamp can be adjusted to accumulate wait time

 			 * prior to migration.

 			 */

-			schedstat_set(se->statistics.wait_start, delta);

+			__schedstat_set(se->statistics.wait_start, delta);

 			return;

 		}

 		trace_sched_stat_wait(p, delta);

 	}

-	schedstat_set(se->statistics.wait_max,

+	__schedstat_set(se->statistics.wait_max,

 		      max(schedstat_val(se->statistics.wait_max), delta));

-	schedstat_inc(se->statistics.wait_count);

-	schedstat_add(se->statistics.wait_sum, delta);

-	schedstat_set(se->statistics.wait_start, 0);

+	__schedstat_inc(se->statistics.wait_count);

+	__schedstat_add(se->statistics.wait_sum, delta);

+	__schedstat_set(se->statistics.wait_start, 0);

 }

 static inline void

@@ -939,10 +939,10 @@ update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)

 			delta = 0;

 		if (unlikely(delta > schedstat_val(se->statistics.sleep_max)))

-			schedstat_set(se->statistics.sleep_max, delta);

+			__schedstat_set(se->statistics.sleep_max, delta);

-		schedstat_set(se->statistics.sleep_start, 0);

-		schedstat_add(se->statistics.sum_sleep_runtime, delta);

+		__schedstat_set(se->statistics.sleep_start, 0);

+		__schedstat_add(se->statistics.sum_sleep_runtime, delta);

 		if (tsk) {

 			account_scheduler_latency(tsk, delta >> 10, 1);

@@ -956,15 +956,15 @@ update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)

 			delta = 0;

 		if (unlikely(delta > schedstat_val(se->statistics.block_max)))

-			schedstat_set(se->statistics.block_max, delta);

+			__schedstat_set(se->statistics.block_max, delta);

-		schedstat_set(se->statistics.block_start, 0);

-		schedstat_add(se->statistics.sum_sleep_runtime, delta);

+		__schedstat_set(se->statistics.block_start, 0);

+		__schedstat_add(se->statistics.sum_sleep_runtime, delta);

 		if (tsk) {

 			if (tsk->in_iowait) {

-				schedstat_add(se->statistics.iowait_sum, delta);

-				schedstat_inc(se->statistics.iowait_count);

+				__schedstat_add(se->statistics.iowait_sum, delta);

+				__schedstat_inc(se->statistics.iowait_count);

 				trace_sched_stat_iowait(tsk, delta);

 			}

@@ -1023,10 +1023,10 @@ update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)

 		struct task_struct *tsk = task_of(se);

 		if (tsk->state & TASK_INTERRUPTIBLE)

-			schedstat_set(se->statistics.sleep_start,

+			__schedstat_set(se->statistics.sleep_start,

 				      rq_clock(rq_of(cfs_rq)));

 		if (tsk->state & TASK_UNINTERRUPTIBLE)

-			schedstat_set(se->statistics.block_start,

+			__schedstat_set(se->statistics.block_start,

 				      rq_clock(rq_of(cfs_rq)));

 	}

 }

diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h

index ebee041977b3..cf7308d69d9a 100644

--- a/kernel/sched/stats.h

+++ b/kernel/sched/stats.h

@@ -32,7 +32,9 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)

 #define schedstat_enabled()		static_branch_unlikely(&sched_schedstats)

 #define __schedstat_inc(var)		do { var++; } while (0)

 #define schedstat_inc(var)		do { if (schedstat_enabled()) { var++; } } while (0)

+#define __schedstat_add(var, amt)	do { var += (amt); } while (0)

 #define schedstat_add(var, amt)		do { if (schedstat_enabled()) { var += (amt); } } while (0)

+#define __schedstat_set(var, val)		do { var = (val); } while (0)

 #define schedstat_set(var, val)		do { if (schedstat_enabled()) { var = (val); } } while (0)

 #define schedstat_val(var)		(var)

 #define schedstat_val_or_zero(var)	((schedstat_enabled()) ? (var) : 0)

@@ -50,7 +52,9 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)

 #define schedstat_enabled()		0

 #define __schedstat_inc(var)		do { } while (0)

 #define schedstat_inc(var)		do { } while (0)

+#define __schedstat_add(var, amt)	do { } while (0)

 #define schedstat_add(var, amt)		do { } while (0)

+#define __schedstat_set(var, val)	do { } while (0)

 #define schedstat_set(var, val)		do { } while (0)

 #define schedstat_val(var)		0

 #define schedstat_val_or_zero(var)	0