From 580c785ca3c60509d9e489a3d71deacb068973f0 Mon Sep 17 00:00:00 2001

From: Valentin Schneider <valentin.schneider@arm.com>

Date: Wed, 18 Aug 2021 13:13:33 +0530

Subject: [PATCH] sched/topology: Skip updating masks for non-online nodes

Patch-mainline: v5.15-rc1

References: bsc#1197446 ltc#183000

Git-commit: 0083242c93759dde353a963a90cb351c5c283379

The scheduler currently expects NUMA node distances to be stable from

init onwards, and as a consequence builds the related data structures

once-and-for-all at init (see sched_init_numa()).

Unfortunately, on some architectures node distance is unreliable for

offline nodes and may very well change upon onlining.

Skip over offline nodes during sched_init_numa(). Track nodes that have

been onlined at least once, and trigger a build of a node's NUMA masks

when it is first onlined post-init.

Reported-by: Geetika Moolchandani <Geetika.Moolchandani1@ibm.com>

Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>

Signed-off-by: Valentin Schneider <valentin.schneider@arm.com>

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

Link: https://lkml.kernel.org/r/20210818074333.48645-1-srikar@linux.vnet.ibm.com

Signed-off-by: Mel Gorman <mgorman@suse.de>

---

 kernel/sched/topology.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++

 1 file changed, 65 insertions(+)

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c

--- a/kernel/sched/topology.c

+++ b/kernel/sched/topology.c

@@ -1290,6 +1290,8 @@ int				sched_max_numa_distance;

 static int			*sched_domains_numa_distance;

 static struct cpumask		***sched_domains_numa_masks;

 int __read_mostly		node_reclaim_distance = RECLAIM_DISTANCE;

+

+static unsigned long __read_mostly *sched_numa_onlined_nodes;

 #endif

 /*

@@ -1655,6 +1657,16 @@ void sched_init_numa(void)

 			sched_domains_numa_masks[i][j] = mask;

 			for_each_node(k) {

+				/*

+				 * Distance information can be unreliable for

+				 * offline nodes, defer building the node

+				 * masks to its bringup.

+				 * This relies on all unique distance values

+				 * still being visible at init time.

+				 */

+				if (!node_online(j))

+					continue;

+

 				if (node_distance(j, k) > sched_domains_numa_distance[i])

 					continue;

@@ -1705,6 +1717,53 @@ void sched_init_numa(void)

 	sched_max_numa_distance = sched_domains_numa_distance[level - 1];

 	init_numa_topology_type();

+

+	sched_numa_onlined_nodes = bitmap_alloc(nr_node_ids, GFP_KERNEL);

+	if (!sched_numa_onlined_nodes)

+		return;

+

+	bitmap_zero(sched_numa_onlined_nodes, nr_node_ids);

+	for_each_online_node(i)

+		bitmap_set(sched_numa_onlined_nodes, i, 1);

+}

+

+static void __sched_domains_numa_masks_set(unsigned int node)

+{

+	int i, j;

+

+	/*

+	 * NUMA masks are not built for offline nodes in sched_init_numa().

+	 * Thus, when a CPU of a never-onlined-before node gets plugged in,

+	 * adding that new CPU to the right NUMA masks is not sufficient: the

+	 * masks of that CPU's node must also be updated.

+	 */

+	if (test_bit(node, sched_numa_onlined_nodes))

+		return;

+

+	bitmap_set(sched_numa_onlined_nodes, node, 1);

+

+	for (i = 0; i < sched_domains_numa_levels; i++) {

+		for (j = 0; j < nr_node_ids; j++) {

+			if (!node_online(j) || node == j)

+				continue;

+

+			if (node_distance(j, node) > sched_domains_numa_distance[i])

+				continue;

+

+			/* Add remote nodes in our masks */

+			cpumask_or(sched_domains_numa_masks[i][node],

+				   sched_domains_numa_masks[i][node],

+				   sched_domains_numa_masks[0][j]);

+		}

+	}

+

+	/*

+	 * A new node has been brought up, potentially changing the topology

+	 * classification.

+	 *

+	 * Note that this is racy vs any use of sched_numa_topology_type :/

+	 */

+	init_numa_topology_type();

 }

 void sched_domains_numa_masks_set(unsigned int cpu)

@@ -1712,8 +1771,14 @@ void sched_domains_numa_masks_set(unsign

 	int node = cpu_to_node(cpu);

 	int i, j;

+	__sched_domains_numa_masks_set(node);

+

 	for (i = 0; i < sched_domains_numa_levels; i++) {

 		for (j = 0; j < nr_node_ids; j++) {

+			if (!node_online(j))

+				continue;

+

+			/* Set ourselves in the remote node's masks */

 			if (node_distance(j, node) <= sched_domains_numa_distance[i])

 				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);

 		}