From a98d1a0ca6d3fd6197f18749972d4cc21195b724 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 24 Dec 2016 12:34:02 +0100 Subject: scsi: qedi: Convert to hotplug state machine The CPU hotplug code is a trainwreck. It leaks a notifier in case of driver registration error and the per cpu loop is racy against cpu hotplug. Aside of that the driver should have been written and merged with the new state machine interfaces in the first place. Mop up the mess and Convert it to the hotplug state machine. Signed-off-by: Thomas Grumpy Gleixner Cc: Nilesh Javali Cc: Adheer Chandravanshi Cc: Chad Dupuis Cc: Saurav Kashyap Cc: Arun Easi Cc: Manish Rangankar Cc: Johannes Thumshirn Cc: Hannes Reinecke Cc: Martin K. Petersen Cc: James Bottomley --- drivers/scsi/qedi/qedi_main.c | 96 +++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 64 deletions(-) (limited to 'drivers/scsi') diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c index 19ead8d17e55..5eda21d903e9 100644 --- a/drivers/scsi/qedi/qedi_main.c +++ b/drivers/scsi/qedi/qedi_main.c @@ -1612,30 +1612,29 @@ static int qedi_percpu_io_thread(void *arg) return 0; } -static void qedi_percpu_thread_create(unsigned int cpu) +static int qedi_cpu_online(unsigned int cpu) { - struct qedi_percpu_s *p; + struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu); struct task_struct *thread; - p = &per_cpu(qedi_percpu, cpu); - thread = kthread_create_on_node(qedi_percpu_io_thread, (void *)p, cpu_to_node(cpu), "qedi_thread/%d", cpu); - if (likely(!IS_ERR(thread))) { - kthread_bind(thread, cpu); - p->iothread = thread; - wake_up_process(thread); - } + if (IS_ERR(thread)) + return PTR_ERR(thread); + + kthread_bind(thread, cpu); + p->iothread = thread; + wake_up_process(thread); + return 0; } -static void qedi_percpu_thread_destroy(unsigned int cpu) +static int qedi_cpu_offline(unsigned int cpu) { - struct qedi_percpu_s *p; - struct task_struct *thread; + struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu); struct qedi_work *work, *tmp; + struct task_struct *thread; - p = &per_cpu(qedi_percpu, cpu); spin_lock_bh(&p->p_work_lock); thread = p->iothread; p->iothread = NULL; @@ -1650,35 +1649,9 @@ static void qedi_percpu_thread_destroy(unsigned int cpu) spin_unlock_bh(&p->p_work_lock); if (thread) kthread_stop(thread); + return 0; } -static int qedi_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - QEDI_ERR(NULL, "CPU %d online.\n", cpu); - qedi_percpu_thread_create(cpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - QEDI_ERR(NULL, "CPU %d offline.\n", cpu); - qedi_percpu_thread_destroy(cpu); - break; - default: - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block qedi_cpu_notifier = { - .notifier_call = qedi_cpu_callback, -}; - void qedi_reset_host_mtu(struct qedi_ctx *qedi, u16 mtu) { struct qed_ll2_params params; @@ -2038,6 +2011,8 @@ static struct pci_device_id qedi_pci_tbl[] = { }; MODULE_DEVICE_TABLE(pci, qedi_pci_tbl); +static enum cpuhp_state qedi_cpuhp_state; + static struct pci_driver qedi_pci_driver = { .name = QEDI_MODULE_NAME, .id_table = qedi_pci_tbl, @@ -2047,16 +2022,13 @@ static struct pci_driver qedi_pci_driver = { static int __init qedi_init(void) { - int rc = 0; - int ret; struct qedi_percpu_s *p; - unsigned int cpu = 0; + int cpu, rc = 0; qedi_ops = qed_get_iscsi_ops(); if (!qedi_ops) { QEDI_ERR(NULL, "Failed to get qed iSCSI operations\n"); - rc = -EINVAL; - goto exit_qedi_init_0; + return -EINVAL; } #ifdef CONFIG_DEBUG_FS @@ -2070,15 +2042,6 @@ static int __init qedi_init(void) goto exit_qedi_init_1; } - register_hotcpu_notifier(&qedi_cpu_notifier); - - ret = pci_register_driver(&qedi_pci_driver); - if (ret) { - QEDI_ERR(NULL, "Failed to register driver\n"); - rc = -EINVAL; - goto exit_qedi_init_2; - } - for_each_possible_cpu(cpu) { p = &per_cpu(qedi_percpu, cpu); INIT_LIST_HEAD(&p->work_list); @@ -2086,11 +2049,22 @@ static int __init qedi_init(void) p->iothread = NULL; } - for_each_online_cpu(cpu) - qedi_percpu_thread_create(cpu); + rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "scsi/qedi:online", + qedi_cpu_online, qedi_cpu_offline); + if (rc < 0) + goto exit_qedi_init_2; + qedi_cpuhp_state = rc; - return rc; + rc = pci_register_driver(&qedi_pci_driver); + if (rc) { + QEDI_ERR(NULL, "Failed to register driver\n"); + goto exit_qedi_hp; + } + + return 0; +exit_qedi_hp: + cpuhp_remove_state(qedi_cpuhp_state); exit_qedi_init_2: iscsi_unregister_transport(&qedi_iscsi_transport); exit_qedi_init_1: @@ -2098,19 +2072,13 @@ exit_qedi_init_1: qedi_dbg_exit(); #endif qed_put_iscsi_ops(); -exit_qedi_init_0: return rc; } static void __exit qedi_cleanup(void) { - unsigned int cpu = 0; - - for_each_online_cpu(cpu) - qedi_percpu_thread_destroy(cpu); - pci_unregister_driver(&qedi_pci_driver); - unregister_hotcpu_notifier(&qedi_cpu_notifier); + cpuhp_remove_state(qedi_cpuhp_state); iscsi_unregister_transport(&qedi_iscsi_transport); #ifdef CONFIG_DEBUG_FS -- cgit v1.2.1 From c53b005dd64bdcf5acac00bd55ecf94dda22dc4f Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 21 Dec 2016 20:19:50 +0100 Subject: scsi/bnx2fc: Convert to hotplug state machine Install the callbacks via the state machine. No functional change. This is the minimal fixup so we can remove the hotplug notifier mess completely. The real rework of this driver to use work queues is still stuck in review/testing on the SCSI mailing list. Signed-off-by: Sebastian Andrzej Siewior Cc: "James E.J. Bottomley" Cc: linux-scsi@vger.kernel.org Cc: "Martin K. Petersen" Cc: Peter Zijlstra Cc: Chad Dupuis Cc: QLogic-Storage-Upstream@qlogic.com Cc: Johannes Thumshirn Cc: Christoph Hellwig Link: http://lkml.kernel.org/r/20161221192111.757309869@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 79 ++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 46 deletions(-) (limited to 'drivers/scsi') diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c index 0990130821fa..c639d5a02656 100644 --- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c +++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c @@ -127,13 +127,6 @@ module_param_named(log_fka, bnx2fc_log_fka, uint, S_IRUGO|S_IWUSR); MODULE_PARM_DESC(log_fka, " Print message to kernel log when fcoe is " "initiating a FIP keep alive when debug logging is enabled."); -static int bnx2fc_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu); -/* notification function for CPU hotplug events */ -static struct notifier_block bnx2fc_cpu_notifier = { - .notifier_call = bnx2fc_cpu_callback, -}; - static inline struct net_device *bnx2fc_netdev(const struct fc_lport *lport) { return ((struct bnx2fc_interface *) @@ -2622,37 +2615,19 @@ static void bnx2fc_percpu_thread_destroy(unsigned int cpu) kthread_stop(thread); } -/** - * bnx2fc_cpu_callback - Handler for CPU hotplug events - * - * @nfb: The callback data block - * @action: The event triggering the callback - * @hcpu: The index of the CPU that the event is for - * - * This creates or destroys per-CPU data for fcoe - * - * Returns NOTIFY_OK always. - */ -static int bnx2fc_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) + +static int bnx2fc_cpu_online(unsigned int cpu) { - unsigned cpu = (unsigned long)hcpu; + printk(PFX "CPU %x online: Create Rx thread\n", cpu); + bnx2fc_percpu_thread_create(cpu); + return 0; +} - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - printk(PFX "CPU %x online: Create Rx thread\n", cpu); - bnx2fc_percpu_thread_create(cpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - printk(PFX "CPU %x offline: Remove Rx thread\n", cpu); - bnx2fc_percpu_thread_destroy(cpu); - break; - default: - break; - } - return NOTIFY_OK; +static int bnx2fc_cpu_dead(unsigned int cpu) +{ + printk(PFX "CPU %x offline: Remove Rx thread\n", cpu); + bnx2fc_percpu_thread_destroy(cpu); + return 0; } static int bnx2fc_slave_configure(struct scsi_device *sdev) @@ -2664,6 +2639,8 @@ static int bnx2fc_slave_configure(struct scsi_device *sdev) return 0; } +static enum cpuhp_state bnx2fc_online_state; + /** * bnx2fc_mod_init - module init entry point * @@ -2724,21 +2701,31 @@ static int __init bnx2fc_mod_init(void) spin_lock_init(&p->fp_work_lock); } - cpu_notifier_register_begin(); + get_online_cpus(); - for_each_online_cpu(cpu) { + for_each_online_cpu(cpu) bnx2fc_percpu_thread_create(cpu); - } - /* Initialize per CPU interrupt thread */ - __register_hotcpu_notifier(&bnx2fc_cpu_notifier); + rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "scsi/bnx2fc:online", + bnx2fc_cpu_online, NULL); + if (rc < 0) + goto stop_threads; + bnx2fc_online_state = rc; - cpu_notifier_register_done(); + cpuhp_setup_state_nocalls(CPUHP_SCSI_BNX2FC_DEAD, "scsi/bnx2fc:dead", + NULL, bnx2fc_cpu_dead); + put_online_cpus(); cnic_register_driver(CNIC_ULP_FCOE, &bnx2fc_cnic_cb); return 0; +stop_threads: + for_each_online_cpu(cpu) + bnx2fc_percpu_thread_destroy(cpu); + put_online_cpus(); + kthread_stop(l2_thread); free_wq: destroy_workqueue(bnx2fc_wq); release_bt: @@ -2797,16 +2784,16 @@ static void __exit bnx2fc_mod_exit(void) if (l2_thread) kthread_stop(l2_thread); - cpu_notifier_register_begin(); - + get_online_cpus(); /* Destroy per cpu threads */ for_each_online_cpu(cpu) { bnx2fc_percpu_thread_destroy(cpu); } - __unregister_hotcpu_notifier(&bnx2fc_cpu_notifier); + cpuhp_remove_state_nocalls(bnx2fc_online_state); + cpuhp_remove_state_nocalls(CPUHP_SCSI_BNX2FC_DEAD); - cpu_notifier_register_done(); + put_online_cpus(); destroy_workqueue(bnx2fc_wq); /* -- cgit v1.2.1 From e210faa2359f92eb2e417cd8462eb980a4dbb172 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 21 Dec 2016 20:19:51 +0100 Subject: scsi/bnx2i: Convert to hotplug state machine Install the callbacks via the state machine. No functional change. This is the minimal fixup so we can remove the hotplug notifier mess completely. The real rework of this driver to use work queues is still stuck in review/testing on the SCSI mailing list. Signed-off-by: Sebastian Andrzej Siewior Cc: "James E.J. Bottomley" Cc: linux-scsi@vger.kernel.org Cc: "Martin K. Petersen" Cc: Peter Zijlstra Cc: Chad Dupuis Cc: QLogic-Storage-Upstream@qlogic.com Cc: Johannes Thumshirn Cc: Christoph Hellwig Link: http://lkml.kernel.org/r/20161221192111.836895753@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/scsi/bnx2i/bnx2i_init.c | 78 ++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 48 deletions(-) (limited to 'drivers/scsi') diff --git a/drivers/scsi/bnx2i/bnx2i_init.c b/drivers/scsi/bnx2i/bnx2i_init.c index c8b410c24cf0..86afc002814c 100644 --- a/drivers/scsi/bnx2i/bnx2i_init.c +++ b/drivers/scsi/bnx2i/bnx2i_init.c @@ -70,14 +70,6 @@ u64 iscsi_error_mask = 0x00; DEFINE_PER_CPU(struct bnx2i_percpu_s, bnx2i_percpu); -static int bnx2i_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu); -/* notification function for CPU hotplug events */ -static struct notifier_block bnx2i_cpu_notifier = { - .notifier_call = bnx2i_cpu_callback, -}; - - /** * bnx2i_identify_device - identifies NetXtreme II device type * @hba: Adapter structure pointer @@ -461,41 +453,21 @@ static void bnx2i_percpu_thread_destroy(unsigned int cpu) kthread_stop(thread); } - -/** - * bnx2i_cpu_callback - Handler for CPU hotplug events - * - * @nfb: The callback data block - * @action: The event triggering the callback - * @hcpu: The index of the CPU that the event is for - * - * This creates or destroys per-CPU data for iSCSI - * - * Returns NOTIFY_OK always. - */ -static int bnx2i_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int bnx2i_cpu_online(unsigned int cpu) { - unsigned cpu = (unsigned long)hcpu; + pr_info("bnx2i: CPU %x online: Create Rx thread\n", cpu); + bnx2i_percpu_thread_create(cpu); + return 0; +} - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - printk(KERN_INFO "bnx2i: CPU %x online: Create Rx thread\n", - cpu); - bnx2i_percpu_thread_create(cpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - printk(KERN_INFO "CPU %x offline: Remove Rx thread\n", cpu); - bnx2i_percpu_thread_destroy(cpu); - break; - default: - break; - } - return NOTIFY_OK; +static int bnx2i_cpu_dead(unsigned int cpu) +{ + pr_info("CPU %x offline: Remove Rx thread\n", cpu); + bnx2i_percpu_thread_destroy(cpu); + return 0; } +static enum cpuhp_state bnx2i_online_state; /** * bnx2i_mod_init - module init entry point @@ -539,18 +511,28 @@ static int __init bnx2i_mod_init(void) p->iothread = NULL; } - cpu_notifier_register_begin(); + get_online_cpus(); for_each_online_cpu(cpu) bnx2i_percpu_thread_create(cpu); - /* Initialize per CPU interrupt thread */ - __register_hotcpu_notifier(&bnx2i_cpu_notifier); - - cpu_notifier_register_done(); + err = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "scsi/bnx2i:online", + bnx2i_cpu_online, NULL); + if (err < 0) + goto remove_threads; + bnx2i_online_state = err; + cpuhp_setup_state_nocalls(CPUHP_SCSI_BNX2I_DEAD, "scsi/bnx2i:dead", + NULL, bnx2i_cpu_dead); + put_online_cpus(); return 0; +remove_threads: + for_each_online_cpu(cpu) + bnx2i_percpu_thread_destroy(cpu); + put_online_cpus(); + cnic_unregister_driver(CNIC_ULP_ISCSI); unreg_xport: iscsi_unregister_transport(&bnx2i_iscsi_transport); out: @@ -587,14 +569,14 @@ static void __exit bnx2i_mod_exit(void) } mutex_unlock(&bnx2i_dev_lock); - cpu_notifier_register_begin(); + get_online_cpus(); for_each_online_cpu(cpu) bnx2i_percpu_thread_destroy(cpu); - __unregister_hotcpu_notifier(&bnx2i_cpu_notifier); - - cpu_notifier_register_done(); + cpuhp_remove_state_nocalls(bnx2i_online_state); + cpuhp_remove_state_nocalls(CPUHP_SCSI_BNX2I_DEAD); + put_online_cpus(); iscsi_unregister_transport(&bnx2i_iscsi_transport); cnic_unregister_driver(CNIC_ULP_ISCSI); -- cgit v1.2.1