From 92125c3a602454824d70edb1b2abb382811cab4f Mon Sep 17 00:00:00 2001
From: Nick Child <nnac123@linux.ibm.com>
Date: Thu, 10 Nov 2022 15:32:17 -0600
Subject: [PATCH] ibmvnic: Add hotpluggable CPU callbacks to reassign affinity
hints
References: jsc#PED-2322
Patch-mainline: v6.2-rc1
Git-commit: 92125c3a602454824d70edb1b2abb382811cab4f
When CPU's are added and removed, ibmvnic devices will reassign
hint values. Introduce a new cpu hotplug state CPUHP_IBMVNIC_DEAD
to signal to ibmvnic devices that the CPU has been removed and it
is time to reset affinity hint assignments. On the other hand,
when CPU's are being added, add a state instance to
CPUHP_AP_ONLINE_DYN which will trigger a reassignment of affinity
hints once the new CPU's are online. This implementation is based
on the virtio_net driver.
Signed-off-by: Thomas Falcon <tlfalcon@linux.ibm.com>
Signed-off-by: Dany Madden <drt@linux.ibm.com>
Signed-off-by: Nick Child <nnac123@linux.ibm.com>
Reviewed-by: Rick Lindsley <ricklind@linux.ibm.com>
Reviewed-by: Haren Myneni <haren@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Michal Suchanek <msuchanek@suse.de>
---
drivers/net/ethernet/ibm/ibmvnic.c | 89 +++++++++++++++++++++++++++++-
drivers/net/ethernet/ibm/ibmvnic.h | 4 ++
include/linux/cpuhotplug.h | 1 +
3 files changed, 93 insertions(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 0c969bdaf94d..2fc0d50dbb86 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -298,6 +298,57 @@ static void ibmvnic_set_affinity(struct ibmvnic_adapter *adapter)
}
}
+static int ibmvnic_cpu_online(unsigned int cpu, struct hlist_node *node)
+{
+ struct ibmvnic_adapter *adapter;
+
+ adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);
+ ibmvnic_set_affinity(adapter);
+ return 0;
+}
+
+static int ibmvnic_cpu_dead(unsigned int cpu, struct hlist_node *node)
+{
+ struct ibmvnic_adapter *adapter;
+
+ adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node_dead);
+ ibmvnic_set_affinity(adapter);
+ return 0;
+}
+
+static int ibmvnic_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
+{
+ struct ibmvnic_adapter *adapter;
+
+ adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);
+ ibmvnic_clean_affinity(adapter);
+ return 0;
+}
+
+static enum cpuhp_state ibmvnic_online;
+
+static int ibmvnic_cpu_notif_add(struct ibmvnic_adapter *adapter)
+{
+ int ret;
+
+ ret = cpuhp_state_add_instance_nocalls(ibmvnic_online, &adapter->node);
+ if (ret)
+ return ret;
+ ret = cpuhp_state_add_instance_nocalls(CPUHP_IBMVNIC_DEAD,
+ &adapter->node_dead);
+ if (!ret)
+ return ret;
+ cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);
+ return ret;
+}
+
+static void ibmvnic_cpu_notif_remove(struct ibmvnic_adapter *adapter)
+{
+ cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);
+ cpuhp_state_remove_instance_nocalls(CPUHP_IBMVNIC_DEAD,
+ &adapter->node_dead);
+}
+
static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
unsigned long length, unsigned long *number,
unsigned long *irq)
@@ -6292,10 +6343,19 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
}
dev_info(&dev->dev, "ibmvnic registered\n");
+ rc = ibmvnic_cpu_notif_add(adapter);
+ if (rc) {
+ netdev_err(netdev, "Registering cpu notifier failed\n");
+ goto cpu_notif_add_failed;
+ }
+
complete(&adapter->probe_done);
return 0;
+cpu_notif_add_failed:
+ unregister_netdev(netdev);
+
ibmvnic_register_fail:
device_remove_file(&dev->dev, &dev_attr_failover);
@@ -6346,6 +6406,8 @@ static void ibmvnic_remove(struct vio_dev *dev)
spin_unlock_irqrestore(&adapter->state_lock, flags);
+ ibmvnic_cpu_notif_remove(adapter);
+
flush_work(&adapter->ibmvnic_reset);
flush_delayed_work(&adapter->ibmvnic_delayed_reset);
@@ -6476,15 +6538,40 @@ static struct vio_driver ibmvnic_driver = {
/* module functions */
static int __init ibmvnic_module_init(void)
{
+ int ret;
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/ibmvnic:online",
+ ibmvnic_cpu_online,
+ ibmvnic_cpu_down_prep);
+ if (ret < 0)
+ goto out;
+ ibmvnic_online = ret;
+ ret = cpuhp_setup_state_multi(CPUHP_IBMVNIC_DEAD, "net/ibmvnic:dead",
+ NULL, ibmvnic_cpu_dead);
+ if (ret)
+ goto err_dead;
+
+ ret = vio_register_driver(&ibmvnic_driver);
+ if (ret)
+ goto err_vio_register;
+
pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,
IBMVNIC_DRIVER_VERSION);
- return vio_register_driver(&ibmvnic_driver);
+ return 0;
+err_vio_register:
+ cpuhp_remove_multi_state(CPUHP_IBMVNIC_DEAD);
+err_dead:
+ cpuhp_remove_multi_state(ibmvnic_online);
+out:
+ return ret;
}
static void __exit ibmvnic_module_exit(void)
{
vio_unregister_driver(&ibmvnic_driver);
+ cpuhp_remove_multi_state(CPUHP_IBMVNIC_DEAD);
+ cpuhp_remove_multi_state(ibmvnic_online);
}
module_init(ibmvnic_module_init);
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index 6720fec1ae67..b35c9b6f913b 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -984,6 +984,10 @@ struct ibmvnic_adapter {
int reset_done_rc;
bool wait_for_reset;
+ /* CPU hotplug instances for online & dead */
+ struct hlist_node node;
+ struct hlist_node node_dead;
+
/* partner capabilities */
u64 min_tx_queues;
u64 min_rx_queues;
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index f61447913db9..c8bc85a87b1e 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -69,6 +69,7 @@ enum cpuhp_state {
CPUHP_X86_APB_DEAD,
CPUHP_X86_MCE_DEAD,
CPUHP_VIRT_NET_DEAD,
+ CPUHP_IBMVNIC_DEAD,
CPUHP_SLUB_DEAD,
CPUHP_DEBUG_OBJ_DEAD,
CPUHP_MM_WRITEBACK_DEAD,
--
2.35.3