From a19423b98704aa85e84097be6d1d44a8615c2340 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Tue, 11 Mar 2014 02:04:03 +0530 Subject: CPU hotplug: Add lockdep annotations to get/put_online_cpus() Add lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin()/cpu_hotplug_end(). Cc: Ingo Molnar Reviewed-by: Oleg Nesterov Signed-off-by: Gautham R. Shenoy Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- kernel/cpu.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kernel/cpu.c b/kernel/cpu.c index deff2e693766..33caf5e97701 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "smpboot.h" @@ -57,17 +58,30 @@ static struct { * an ongoing cpu hotplug operation. */ int refcount; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif } cpu_hotplug = { .active_writer = NULL, .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), .refcount = 0, +#ifdef CONFIG_DEBUG_LOCK_ALLOC + .dep_map = {.name = "cpu_hotplug.lock" }, +#endif }; +/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ +#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) +#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) +#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) + void get_online_cpus(void) { might_sleep(); if (cpu_hotplug.active_writer == current) return; + cpuhp_lock_acquire_read(); mutex_lock(&cpu_hotplug.lock); cpu_hotplug.refcount++; mutex_unlock(&cpu_hotplug.lock); @@ -87,6 +101,7 @@ void put_online_cpus(void) if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer)) wake_up_process(cpu_hotplug.active_writer); mutex_unlock(&cpu_hotplug.lock); + cpuhp_lock_release(); } EXPORT_SYMBOL_GPL(put_online_cpus); @@ -117,6 +132,7 @@ void cpu_hotplug_begin(void) { cpu_hotplug.active_writer = current; + cpuhp_lock_acquire(); for (;;) { mutex_lock(&cpu_hotplug.lock); if (likely(!cpu_hotplug.refcount)) @@ -131,6 +147,7 @@ void cpu_hotplug_done(void) { cpu_hotplug.active_writer = NULL; mutex_unlock(&cpu_hotplug.lock); + cpuhp_lock_release(); } /* -- cgit From 93ae4f978ca7f26d17df915ac7afc919c1dd0353 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:04:14 +0530 Subject: CPU hotplug: Provide lockless versions of callback registration functions The following method of CPU hotplug callback registration is not safe due to the possibility of an ABBA deadlock involving the cpu_add_remove_lock and the cpu_hotplug.lock. get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); The deadlock is shown below: CPU 0 CPU 1 ----- ----- Acquire cpu_hotplug.lock [via get_online_cpus()] CPU online/offline operation takes cpu_add_remove_lock [via cpu_maps_update_begin()] Try to acquire cpu_add_remove_lock [via register_cpu_notifier()] CPU online/offline operation tries to acquire cpu_hotplug.lock [via cpu_hotplug_begin()] *** DEADLOCK! *** The problem here is that callback registration takes the locks in one order whereas the CPU hotplug operations take the same locks in the opposite order. To avoid this issue and to provide a race-free method to register CPU hotplug callbacks (along with initialization of already online CPUs), introduce new variants of the callback registration APIs that simply register the callbacks without holding the cpu_add_remove_lock during the registration. That way, we can avoid the ABBA scenario. However, we will need to hold the cpu_add_remove_lock throughout the entire critical section, to protect updates to the callback/notifier chain. This can be achieved by writing the callback registration code as follows: cpu_maps_update_begin(); [ or cpu_notifier_register_begin(); see below ] for_each_online_cpu(cpu) init_cpu(cpu); /* This doesn't take the cpu_add_remove_lock */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_maps_update_done(); [ or cpu_notifier_register_done(); see below ] Note that we can't use get_online_cpus() here instead of cpu_maps_update_begin() because the cpu_hotplug.lock is dropped during the invocation of CPU_POST_DEAD notifiers, and hence get_online_cpus() cannot provide the necessary synchronization to protect the callback/notifier chains against concurrent reads and writes. On the other hand, since the cpu_add_remove_lock protects the entire hotplug operation (including CPU_POST_DEAD), we can use cpu_maps_update_begin/done() to guarantee proper synchronization. Also, since cpu_maps_update_begin/done() is like a super-set of get/put_online_cpus(), the former naturally protects the critical sections from concurrent hotplug operations. Since the names cpu_maps_update_begin/done() don't make much sense in CPU hotplug callback registration scenarios, we'll introduce new APIs named cpu_notifier_register_begin/done() and map them to cpu_maps_update_begin/done(). In summary, introduce the lockless variants of un/register_cpu_notifier() and also export the cpu_notifier_register_begin/done() APIs for use by modules. This way, we provide a race-free way to register hotplug callbacks as well as perform initialization for the CPUs that are already online. Cc: Thomas Gleixner Cc: Andrew Morton Cc: Peter Zijlstra Cc: Ingo Molnar Acked-by: Oleg Nesterov Acked-by: Toshi Kani Reviewed-by: Gautham R. Shenoy Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- include/linux/cpu.h | 47 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/cpu.c | 21 +++++++++++++++++++-- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 03e235ad1bba..488d6ebcf6a1 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -122,26 +122,46 @@ enum { { .notifier_call = fn, .priority = pri }; \ register_cpu_notifier(&fn##_nb); \ } + +#define __cpu_notifier(fn, pri) { \ + static struct notifier_block fn##_nb = \ + { .notifier_call = fn, .priority = pri }; \ + __register_cpu_notifier(&fn##_nb); \ +} #else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ #define cpu_notifier(fn, pri) do { (void)(fn); } while (0) +#define __cpu_notifier(fn, pri) do { (void)(fn); } while (0) #endif /* #else #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ + #ifdef CONFIG_HOTPLUG_CPU extern int register_cpu_notifier(struct notifier_block *nb); +extern int __register_cpu_notifier(struct notifier_block *nb); extern void unregister_cpu_notifier(struct notifier_block *nb); +extern void __unregister_cpu_notifier(struct notifier_block *nb); #else #ifndef MODULE extern int register_cpu_notifier(struct notifier_block *nb); +extern int __register_cpu_notifier(struct notifier_block *nb); #else static inline int register_cpu_notifier(struct notifier_block *nb) { return 0; } + +static inline int __register_cpu_notifier(struct notifier_block *nb) +{ + return 0; +} #endif static inline void unregister_cpu_notifier(struct notifier_block *nb) { } + +static inline void __unregister_cpu_notifier(struct notifier_block *nb) +{ +} #endif int cpu_up(unsigned int cpu); @@ -149,19 +169,32 @@ void notify_cpu_starting(unsigned int cpu); extern void cpu_maps_update_begin(void); extern void cpu_maps_update_done(void); +#define cpu_notifier_register_begin cpu_maps_update_begin +#define cpu_notifier_register_done cpu_maps_update_done + #else /* CONFIG_SMP */ #define cpu_notifier(fn, pri) do { (void)(fn); } while (0) +#define __cpu_notifier(fn, pri) do { (void)(fn); } while (0) static inline int register_cpu_notifier(struct notifier_block *nb) { return 0; } +static inline int __register_cpu_notifier(struct notifier_block *nb) +{ + return 0; +} + static inline void unregister_cpu_notifier(struct notifier_block *nb) { } +static inline void __unregister_cpu_notifier(struct notifier_block *nb) +{ +} + static inline void cpu_maps_update_begin(void) { } @@ -170,6 +203,14 @@ static inline void cpu_maps_update_done(void) { } +static inline void cpu_notifier_register_begin(void) +{ +} + +static inline void cpu_notifier_register_done(void) +{ +} + #endif /* CONFIG_SMP */ extern struct bus_type cpu_subsys; @@ -183,8 +224,11 @@ extern void put_online_cpus(void); extern void cpu_hotplug_disable(void); extern void cpu_hotplug_enable(void); #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) +#define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri) #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) +#define __register_hotcpu_notifier(nb) __register_cpu_notifier(nb) #define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb) +#define __unregister_hotcpu_notifier(nb) __unregister_cpu_notifier(nb) void clear_tasks_mm_cpumask(int cpu); int cpu_down(unsigned int cpu); @@ -197,9 +241,12 @@ static inline void cpu_hotplug_done(void) {} #define cpu_hotplug_disable() do { } while (0) #define cpu_hotplug_enable() do { } while (0) #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) +#define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) /* These aren't inline functions due to a GCC bug. */ #define register_hotcpu_notifier(nb) ({ (void)(nb); 0; }) +#define __register_hotcpu_notifier(nb) ({ (void)(nb); 0; }) #define unregister_hotcpu_notifier(nb) ({ (void)(nb); }) +#define __unregister_hotcpu_notifier(nb) ({ (void)(nb); }) #endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_PM_SLEEP_SMP diff --git a/kernel/cpu.c b/kernel/cpu.c index 33caf5e97701..a9e710eef0e2 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -28,18 +28,23 @@ static DEFINE_MUTEX(cpu_add_remove_lock); /* - * The following two API's must be used when attempting - * to serialize the updates to cpu_online_mask, cpu_present_mask. + * The following two APIs (cpu_maps_update_begin/done) must be used when + * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. + * The APIs cpu_notifier_register_begin/done() must be used to protect CPU + * hotplug callback (un)registration performed using __register_cpu_notifier() + * or __unregister_cpu_notifier(). */ void cpu_maps_update_begin(void) { mutex_lock(&cpu_add_remove_lock); } +EXPORT_SYMBOL(cpu_notifier_register_begin); void cpu_maps_update_done(void) { mutex_unlock(&cpu_add_remove_lock); } +EXPORT_SYMBOL(cpu_notifier_register_done); static RAW_NOTIFIER_HEAD(cpu_chain); @@ -183,6 +188,11 @@ int __ref register_cpu_notifier(struct notifier_block *nb) return ret; } +int __ref __register_cpu_notifier(struct notifier_block *nb) +{ + return raw_notifier_chain_register(&cpu_chain, nb); +} + static int __cpu_notify(unsigned long val, void *v, int nr_to_call, int *nr_calls) { @@ -206,6 +216,7 @@ static void cpu_notify_nofail(unsigned long val, void *v) BUG_ON(cpu_notify(val, v)); } EXPORT_SYMBOL(register_cpu_notifier); +EXPORT_SYMBOL(__register_cpu_notifier); void __ref unregister_cpu_notifier(struct notifier_block *nb) { @@ -215,6 +226,12 @@ void __ref unregister_cpu_notifier(struct notifier_block *nb) } EXPORT_SYMBOL(unregister_cpu_notifier); +void __ref __unregister_cpu_notifier(struct notifier_block *nb) +{ + raw_notifier_chain_unregister(&cpu_chain, nb); +} +EXPORT_SYMBOL(__unregister_cpu_notifier); + /** * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU * @cpu: a CPU id -- cgit From 8489d90b5f32effa402be2a4ee7636ba0c652145 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:04:32 +0530 Subject: Doc/cpu-hotplug: Specify race-free way to register CPU hotplug callbacks Recommend the usage of the new CPU hotplug callback registration APIs (__register_cpu_notifier() etc), when subsystems need to also perform initialization for already online CPUs. Provide examples of correct and race-free ways of achieving this, and point out the kinds of code that are error-prone. Cc: Rob Landley Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- Documentation/cpu-hotplug.txt | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/Documentation/cpu-hotplug.txt b/Documentation/cpu-hotplug.txt index be675d2d15a7..a0b005d2bd95 100644 --- a/Documentation/cpu-hotplug.txt +++ b/Documentation/cpu-hotplug.txt @@ -312,12 +312,57 @@ things will happen if a notifier in path sent a BAD notify code. Q: I don't see my action being called for all CPUs already up and running? A: Yes, CPU notifiers are called only when new CPUs are on-lined or offlined. If you need to perform some action for each cpu already in the system, then + do this: for_each_online_cpu(i) { foobar_cpu_callback(&foobar_cpu_notifier, CPU_UP_PREPARE, i); foobar_cpu_callback(&foobar_cpu_notifier, CPU_ONLINE, i); } + However, if you want to register a hotplug callback, as well as perform + some initialization for CPUs that are already online, then do this: + + Version 1: (Correct) + --------- + + cpu_notifier_register_begin(); + + for_each_online_cpu(i) { + foobar_cpu_callback(&foobar_cpu_notifier, + CPU_UP_PREPARE, i); + foobar_cpu_callback(&foobar_cpu_notifier, + CPU_ONLINE, i); + } + + /* Note the use of the double underscored version of the API */ + __register_cpu_notifier(&foobar_cpu_notifier); + + cpu_notifier_register_done(); + + Note that the following code is *NOT* the right way to achieve this, + because it is prone to an ABBA deadlock between the cpu_add_remove_lock + and the cpu_hotplug.lock. + + Version 2: (Wrong!) + --------- + + get_online_cpus(); + + for_each_online_cpu(i) { + foobar_cpu_callback(&foobar_cpu_notifier, + CPU_UP_PREPARE, i); + foobar_cpu_callback(&foobar_cpu_notifier, + CPU_ONLINE, i); + } + + register_cpu_notifier(&foobar_cpu_notifier); + + put_online_cpus(); + + So always use the first version shown above when you want to register + callbacks as well as initialize the already online CPUs. + + Q: If i would like to develop cpu hotplug support for a new architecture, what do i need at a minimum? A: The following are what is required for CPU hotplug infrastructure to work -- cgit From f0bdb5e0c72b7347c867da539367138ad95c6b24 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:04:39 +0530 Subject: CPU hotplug, perf: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the perf subsystem's hotplug notifier by using this latter form of callback registration. Also provide a bare-bones version of perf_cpu_notifier() that doesn't invoke the notifiers for the already online CPUs. This would be useful for subsystems that need to perform a different set of initialization for the already online CPUs, or don't need the initialization altogether. Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- include/linux/perf_event.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index e56b07f5c9b6..3356abcfff18 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -835,6 +835,8 @@ do { \ { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ unsigned long cpu = smp_processor_id(); \ unsigned long flags; \ + \ + cpu_notifier_register_begin(); \ fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ (void *)(unsigned long)cpu); \ local_irq_save(flags); \ @@ -843,9 +845,21 @@ do { \ local_irq_restore(flags); \ fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ (void *)(unsigned long)cpu); \ - register_cpu_notifier(&fn##_nb); \ + __register_cpu_notifier(&fn##_nb); \ + cpu_notifier_register_done(); \ } while (0) +/* + * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the + * callback for already online CPUs. + */ +#define __perf_cpu_notifier(fn) \ +do { \ + static struct notifier_block fn##_nb = \ + { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ + \ + __register_cpu_notifier(&fn##_nb); \ +} while (0) struct perf_pmu_events_attr { struct device_attribute attr; -- cgit From eff722b06bdd6de326894ea2353ec0eda3e5ed34 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:04:45 +0530 Subject: ia64, salinfo: Fix hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the salinfo code in ia64 by using this latter form of callback registration. Cc: Tony Luck Cc: Fenghua Yu Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/ia64/kernel/salinfo.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c index 960a396f5929..ee9719eebb1e 100644 --- a/arch/ia64/kernel/salinfo.c +++ b/arch/ia64/kernel/salinfo.c @@ -635,6 +635,8 @@ salinfo_init(void) (void *)salinfo_entries[i].feature); } + cpu_notifier_register_begin(); + for (i = 0; i < ARRAY_SIZE(salinfo_log_name); i++) { data = salinfo_data + i; data->type = i; @@ -669,7 +671,9 @@ salinfo_init(void) salinfo_timer.function = &salinfo_timeout; add_timer(&salinfo_timer); - register_hotcpu_notifier(&salinfo_cpu_notifier); + __register_hotcpu_notifier(&salinfo_cpu_notifier); + + cpu_notifier_register_done(); return 0; } -- cgit From 9f37bca9a9e97fcd7589492aba7b8a5310026c47 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:04:52 +0530 Subject: ia64, palinfo: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the palinfo code in ia64 by using this latter form of callback registration. Cc: Tony Luck Cc: Fenghua Yu Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/ia64/kernel/palinfo.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c index ab333284f4b2..c39c3cd3ac34 100644 --- a/arch/ia64/kernel/palinfo.c +++ b/arch/ia64/kernel/palinfo.c @@ -996,13 +996,17 @@ palinfo_init(void) if (!palinfo_dir) return -ENOMEM; + cpu_notifier_register_begin(); + /* Create palinfo dirs in /proc for all online cpus */ for_each_online_cpu(i) { create_palinfo_proc_entries(i); } /* Register for future delivery via notify registration */ - register_hotcpu_notifier(&palinfo_cpu_notifier); + __register_hotcpu_notifier(&palinfo_cpu_notifier); + + cpu_notifier_register_done(); return 0; } -- cgit From f5a7d445ffb6b251dc41942ffe41f515e8d639a4 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:04:58 +0530 Subject: ia64, topology: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the topology code in ia64 by using this latter form of callback registration. Cc: Tony Luck Cc: Fenghua Yu Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/ia64/kernel/topology.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index ca69a5a96dcc..f295f9abba4b 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -454,12 +454,16 @@ static int __init cache_sysfs_init(void) { int i; + cpu_notifier_register_begin(); + for_each_online_cpu(i) { struct device *sys_dev = get_cpu_device((unsigned int)i); cache_add_dev(sys_dev); } - register_hotcpu_notifier(&cache_cpu_notifier); + __register_hotcpu_notifier(&cache_cpu_notifier); + + cpu_notifier_register_done(); return 0; } -- cgit From f2e48a89053dea259d7f96cf0fd1cfc7c4b34d80 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:05:10 +0530 Subject: ia64, err-inject: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the error injection code in ia64 by using this latter form of callback registration. Cc: Tony Luck Cc: Fenghua Yu Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/ia64/kernel/err_inject.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c index f59c0b844e88..0c161ed6d18e 100644 --- a/arch/ia64/kernel/err_inject.c +++ b/arch/ia64/kernel/err_inject.c @@ -269,12 +269,17 @@ err_inject_init(void) #ifdef ERR_INJ_DEBUG printk(KERN_INFO "Enter error injection driver.\n"); #endif + + cpu_notifier_register_begin(); + for_each_online_cpu(i) { err_inject_cpu_callback(&err_inject_cpu_notifier, CPU_ONLINE, (void *)(long)i); } - register_hotcpu_notifier(&err_inject_cpu_notifier); + __register_hotcpu_notifier(&err_inject_cpu_notifier); + + cpu_notifier_register_done(); return 0; } @@ -288,11 +293,17 @@ err_inject_exit(void) #ifdef ERR_INJ_DEBUG printk(KERN_INFO "Exit error injection driver.\n"); #endif + + cpu_notifier_register_begin(); + for_each_online_cpu(i) { sys_dev = get_cpu_device(i); sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group); } - unregister_hotcpu_notifier(&err_inject_cpu_notifier); + + __unregister_hotcpu_notifier(&err_inject_cpu_notifier); + + cpu_notifier_register_done(); } module_init(err_inject_init); -- cgit From c5929bd3a9920432dfb485253c64163fdfc90faf Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:05:21 +0530 Subject: arm, hw-breakpoint: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the hw-breakpoint code in arm by using this latter form of callback registration. Cc: Russell King Cc: Ingo Molnar Acked-by: Will Deacon Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/arm/kernel/hw_breakpoint.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 3d446605cbf8..3702de8ab9b9 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -1072,6 +1072,8 @@ static int __init arch_hw_breakpoint_init(void) core_num_brps = get_num_brps(); core_num_wrps = get_num_wrps(); + cpu_notifier_register_begin(); + /* * We need to tread carefully here because DBGSWENABLE may be * driven low on this core and there isn't an architected way to @@ -1088,6 +1090,7 @@ static int __init arch_hw_breakpoint_init(void) if (!cpumask_empty(&debug_err_mask)) { core_num_brps = 0; core_num_wrps = 0; + cpu_notifier_register_done(); return 0; } @@ -1107,7 +1110,10 @@ static int __init arch_hw_breakpoint_init(void) TRAP_HWBKPT, "breakpoint debug exception"); /* Register hotplug and PM notifiers. */ - register_cpu_notifier(&dbg_reset_nb); + __register_cpu_notifier(&dbg_reset_nb); + + cpu_notifier_register_done(); + pm_init(); return 0; } -- cgit From 8146875de7d47c632921ea8284097a1ff0601ee7 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 18 Mar 2014 15:53:05 +0530 Subject: arm, kvm: Fix CPU hotplug callback registration On 03/15/2014 12:40 AM, Christoffer Dall wrote: > On Fri, Mar 14, 2014 at 11:13:29AM +0530, Srivatsa S. Bhat wrote: >> On 03/13/2014 04:51 AM, Christoffer Dall wrote: >>> On Tue, Mar 11, 2014 at 02:05:38AM +0530, Srivatsa S. Bhat wrote: >>>> Subsystems that want to register CPU hotplug callbacks, as well as perform >>>> initialization for the CPUs that are already online, often do it as shown >>>> below: >>>> [...] >>> Just so we're clear, the existing code was simply racy as not prone to >>> deadlocks, right? >>> >>> This makes it clear that the test above for compatible CPUs can be quite >>> easily evaded by using CPU hotplug, but we don't really have a good >>> solution for handling that yet... Hmmm, grumble grumble, I guess if you >>> hotplug unsupported CPUs on a KVM/ARM system for now, stuff will break. >>> >> >> In this particular case, there was no deadlock possibility, rather the >> existing code had insufficient synchronization against CPU hotplug. >> >> init_hyp_mode() would invoke cpu_init_hyp_mode() on currently online CPUs >> using on_each_cpu(). If a CPU came online after this point and before calling >> register_cpu_notifier(), that CPU would remain uninitialized because this >> subsystem would miss the hot-online event. This patch fixes this bug and >> also uses the new synchronization method (instead of get/put_online_cpus()) >> to ensure that we don't deadlock with CPU hotplug. >> > > Yes, that was my conclusion as well. Thanks for clarifying. (It could > be noted in the commit message as well if you should feel so inclined). > Please find the patch with updated changelog (and your Ack) below. (No changes in code). From: Srivatsa S. Bhat Subject: [PATCH] arm, kvm: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); In the existing arm kvm code, there is no synchronization with CPU hotplug to avoid missing the hotplug events that might occur after invoking init_hyp_mode() and before calling register_cpu_notifier(). Fix this bug and also use the new synchronization method (instead of get/put_online_cpus()) to ensure that we don't deadlock with CPU hotplug. Cc: Gleb Natapov Cc: Russell King Cc: Ingo Molnar Acked-by: Paolo Bonzini Acked-by: Christoffer Dall Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/arm/kvm/arm.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index bd18bb8b2770..f0e50a0f3a65 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -1051,21 +1051,26 @@ int kvm_arch_init(void *opaque) } } + cpu_notifier_register_begin(); + err = init_hyp_mode(); if (err) goto out_err; - err = register_cpu_notifier(&hyp_init_cpu_nb); + err = __register_cpu_notifier(&hyp_init_cpu_nb); if (err) { kvm_err("Cannot register HYP init CPU notifier (%d)\n", err); goto out_err; } + cpu_notifier_register_done(); + hyp_cpu_pm_init(); kvm_coproc_table_init(); return 0; out_err: + cpu_notifier_register_done(); return err; } -- cgit From 6575080e671f3675a4c47b61c38556d54e028ca8 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:05:46 +0530 Subject: s390, cacheinfo: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the cacheinfo code in s390 by using this latter form of callback registration. Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/s390/kernel/cache.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 3a414c0f93ed..c0b03c28d157 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -378,9 +378,12 @@ static int __init cache_init(void) if (!test_facility(34)) return 0; cache_build_info(); + + cpu_notifier_register_begin(); for_each_online_cpu(cpu) cache_add_cpu(cpu); - hotcpu_notifier(cache_hotplug, 0); + __hotcpu_notifier(cache_hotplug, 0); + cpu_notifier_register_done(); return 0; } device_initcall(cache_init); -- cgit From f4edbcd5d1f350b90583c8f3e0e7112b28f65e62 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:05:58 +0530 Subject: s390, smp: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the smp code in s390 by using this latter form of callback registration. Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: Thomas Gleixner Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/s390/kernel/smp.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index a7125b62a9a6..e10be3541816 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1057,19 +1057,24 @@ static DEVICE_ATTR(rescan, 0200, NULL, rescan_store); static int __init s390_smp_init(void) { - int cpu, rc; + int cpu, rc = 0; - hotcpu_notifier(smp_cpu_notify, 0); #ifdef CONFIG_HOTPLUG_CPU rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan); if (rc) return rc; #endif + cpu_notifier_register_begin(); for_each_present_cpu(cpu) { rc = smp_add_present_cpu(cpu); if (rc) - return rc; + goto out; } - return 0; + + __hotcpu_notifier(smp_cpu_notify, 0); + +out: + cpu_notifier_register_done(); + return rc; } subsys_initcall(s390_smp_init); -- cgit From 1e0b0c4c6b9241b66c6032e14962a149f852f23a Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:06:09 +0530 Subject: sparc, sysfs: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the sysfs code in sparc by using this latter form of callback registration. Cc: Ingo Molnar Acked-by: David S. Miller Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/sparc/kernel/sysfs.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index c21c673e5f7c..a364000ca1aa 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -300,7 +300,7 @@ static int __init topology_init(void) check_mmu_stats(); - register_cpu_notifier(&sysfs_cpu_nb); + cpu_notifier_register_begin(); for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); @@ -310,6 +310,10 @@ static int __init topology_init(void) register_cpu_online(cpu); } + __register_cpu_notifier(&sysfs_cpu_nb); + + cpu_notifier_register_done(); + return 0; } -- cgit From d1a5511390b0eb242c70ab977abff28644f66a5a Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:06:31 +0530 Subject: powerpc, sysfs: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the sysfs code in powerpc by using this latter form of callback registration. Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Olof Johansson Cc: Wang Dongsheng Cc: Ingo Molnar Acked-by: Madhavan Srinivasan Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/powerpc/kernel/sysfs.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index 97e1dc917683..d90d4b7810d6 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -975,7 +975,8 @@ static int __init topology_init(void) int cpu; register_nodes(); - register_cpu_notifier(&sysfs_cpu_nb); + + cpu_notifier_register_begin(); for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); @@ -999,6 +1000,11 @@ static int __init topology_init(void) if (cpu_online(cpu)) register_cpu_online(cpu); } + + __register_cpu_notifier(&sysfs_cpu_nb); + + cpu_notifier_register_done(); + #ifdef CONFIG_PPC64 sysfs_create_dscr_default(); #endif /* CONFIG_PPC64 */ -- cgit From de82a01befdd33f6a1c050c7f888e2fa9949f48e Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:06:37 +0530 Subject: x86, msr: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the msr code in x86 by using this latter form of callback registration. Cc: "H. Peter Anvin" Cc: Thomas Gleixner Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/msr.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 05266b5aae22..c9603ac80de5 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -259,14 +259,15 @@ static int __init msr_init(void) goto out_chrdev; } msr_class->devnode = msr_devnode; - get_online_cpus(); + + cpu_notifier_register_begin(); for_each_online_cpu(i) { err = msr_device_create(i); if (err != 0) goto out_class; } - register_hotcpu_notifier(&msr_class_cpu_notifier); - put_online_cpus(); + __register_hotcpu_notifier(&msr_class_cpu_notifier); + cpu_notifier_register_done(); err = 0; goto out; @@ -275,7 +276,7 @@ out_class: i = 0; for_each_online_cpu(i) msr_device_destroy(i); - put_online_cpus(); + cpu_notifier_register_done(); class_destroy(msr_class); out_chrdev: __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr"); @@ -286,13 +287,14 @@ out: static void __exit msr_exit(void) { int cpu = 0; - get_online_cpus(); + + cpu_notifier_register_begin(); for_each_online_cpu(cpu) msr_device_destroy(cpu); class_destroy(msr_class); __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr"); - unregister_hotcpu_notifier(&msr_class_cpu_notifier); - put_online_cpus(); + __unregister_hotcpu_notifier(&msr_class_cpu_notifier); + cpu_notifier_register_done(); } module_init(msr_init); -- cgit From 4b660b384da7b47996436cc3151ca5995c121c76 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:06:44 +0530 Subject: x86, cpuid: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the cpuid code in x86 by using this latter form of callback registration. Cc: "H. Peter Anvin" Cc: Thomas Gleixner Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpuid.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index 7d9481c743f8..3225ae6c5180 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -198,14 +198,15 @@ static int __init cpuid_init(void) goto out_chrdev; } cpuid_class->devnode = cpuid_devnode; - get_online_cpus(); + + cpu_notifier_register_begin(); for_each_online_cpu(i) { err = cpuid_device_create(i); if (err != 0) goto out_class; } - register_hotcpu_notifier(&cpuid_class_cpu_notifier); - put_online_cpus(); + __register_hotcpu_notifier(&cpuid_class_cpu_notifier); + cpu_notifier_register_done(); err = 0; goto out; @@ -215,7 +216,7 @@ out_class: for_each_online_cpu(i) { cpuid_device_destroy(i); } - put_online_cpus(); + cpu_notifier_register_done(); class_destroy(cpuid_class); out_chrdev: __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); @@ -227,13 +228,13 @@ static void __exit cpuid_exit(void) { int cpu = 0; - get_online_cpus(); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) cpuid_device_destroy(cpu); class_destroy(cpuid_class); __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); - unregister_hotcpu_notifier(&cpuid_class_cpu_notifier); - put_online_cpus(); + __unregister_hotcpu_notifier(&cpuid_class_cpu_notifier); + cpu_notifier_register_done(); } module_init(cpuid_init); -- cgit From 42112a0f5d34601e5d0a1bc7c769ba39adda0769 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:06:50 +0530 Subject: x86, vsyscall: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the vsyscall code in x86 by using this latter form of callback registration. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/vsyscall_64.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 1f96f9347ed9..556eaf2a3555 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -393,9 +393,13 @@ static int __init vsyscall_init(void) { BUG_ON(VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)); + cpu_notifier_register_begin(); + on_each_cpu(cpu_vsyscall_init, NULL, 1); /* notifier priority > KVM */ - hotcpu_notifier(cpu_vsyscall_notifier, 30); + __hotcpu_notifier(cpu_vsyscall_notifier, 30); + + cpu_notifier_register_done(); return 0; } -- cgit From 2c666adacc9e4b00b918553ca136b8eef0ec3c2c Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:06:57 +0530 Subject: x86, intel, uncore: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the uncore code in intel-x86 by using this latter form of callback registration. Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Thomas Gleixner Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/perf_event_intel_uncore.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c index 047f540cf3f7..28922f62eb29 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c @@ -3817,7 +3817,7 @@ static int __init uncore_cpu_init(void) if (ret) return ret; - get_online_cpus(); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) { int i, phys_id = topology_physical_package_id(cpu); @@ -3836,9 +3836,9 @@ static int __init uncore_cpu_init(void) } on_each_cpu(uncore_cpu_setup, NULL, 1); - register_cpu_notifier(&uncore_cpu_nb); + __register_cpu_notifier(&uncore_cpu_nb); - put_online_cpus(); + cpu_notifier_register_done(); return 0; } -- cgit From 82a8f131aadf55ac7fbc8c6f65f34d83101160de Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:07:04 +0530 Subject: x86, mce: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the mce code in x86 by using this latter form of callback registration. Cc: Tony Luck Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/mcheck/mce.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 4d5419b249da..9b7734b1f975 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -2434,14 +2434,18 @@ static __init int mcheck_init_device(void) if (err) return err; + cpu_notifier_register_begin(); for_each_online_cpu(i) { err = mce_device_create(i); - if (err) + if (err) { + cpu_notifier_register_done(); return err; + } } register_syscore_ops(&mce_syscore_ops); - register_hotcpu_notifier(&mce_cpu_notifier); + __register_hotcpu_notifier(&mce_cpu_notifier); + cpu_notifier_register_done(); /* register character device /dev/mcelog */ misc_register(&mce_chrdev_device); -- cgit From 4e6192bbec2ec21d2a6685987674a1c73e810de1 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:07:21 +0530 Subject: x86, therm_throt.c: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the thermal throttle code in x86 by using this latter form of callback registration. Cc: Tony Luck Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/mcheck/therm_throt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index 3eec7de76efb..e05dfa398dc2 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -319,7 +319,7 @@ static __init int thermal_throttle_init_device(void) if (!atomic_read(&therm_throt_en)) return 0; - register_hotcpu_notifier(&thermal_throttle_cpu_notifier); + cpu_notifier_register_begin(); #ifdef CONFIG_HOTPLUG_CPU mutex_lock(&therm_cpu_lock); @@ -333,6 +333,9 @@ static __init int thermal_throttle_init_device(void) mutex_unlock(&therm_cpu_lock); #endif + __register_hotcpu_notifier(&thermal_throttle_cpu_notifier); + cpu_notifier_register_done(); + return 0; } device_initcall(thermal_throttle_init_device); -- cgit From 7b7139d4abe8a94446db6a91882f1400920891d5 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:07:28 +0530 Subject: x86, therm_throt.c: Remove unused therm_cpu_lock After fixing the CPU hotplug callback registration code, the callbacks invoked for each online CPU, during the initialization phase in thermal_throttle_init_device(), can no longer race with the actual CPU hotplug notifier callbacks (in thermal_throttle_cpu_callback). Hence the therm_cpu_lock is unnecessary now. Remove it. Cc: Tony Luck Cc: Borislav Petkov Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Suggested-by: Oleg Nesterov Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/mcheck/therm_throt.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index e05dfa398dc2..d921b7ee6595 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -271,9 +271,6 @@ static void thermal_throttle_remove_dev(struct device *dev) sysfs_remove_group(&dev->kobj, &thermal_attr_group); } -/* Mutex protecting device creation against CPU hotplug: */ -static DEFINE_MUTEX(therm_cpu_lock); - /* Get notified when a cpu comes on/off. Be hotplug friendly. */ static int thermal_throttle_cpu_callback(struct notifier_block *nfb, @@ -289,18 +286,14 @@ thermal_throttle_cpu_callback(struct notifier_block *nfb, switch (action) { case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: - mutex_lock(&therm_cpu_lock); err = thermal_throttle_add_dev(dev, cpu); - mutex_unlock(&therm_cpu_lock); WARN_ON(err); break; case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: case CPU_DEAD: case CPU_DEAD_FROZEN: - mutex_lock(&therm_cpu_lock); thermal_throttle_remove_dev(dev); - mutex_unlock(&therm_cpu_lock); break; } return notifier_from_errno(err); @@ -321,17 +314,11 @@ static __init int thermal_throttle_init_device(void) cpu_notifier_register_begin(); -#ifdef CONFIG_HOTPLUG_CPU - mutex_lock(&therm_cpu_lock); -#endif /* connect live CPUs to sysfs */ for_each_online_cpu(cpu) { err = thermal_throttle_add_dev(get_cpu_device(cpu), cpu); WARN_ON(err); } -#ifdef CONFIG_HOTPLUG_CPU - mutex_unlock(&therm_cpu_lock); -#endif __register_hotcpu_notifier(&thermal_throttle_cpu_notifier); cpu_notifier_register_done(); -- cgit From 047868ce2952bb6a86ccea3ecc6fd59faa9062a7 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:07:45 +0530 Subject: x86, amd, ibs: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the amd-ibs code in x86 by using this latter form of callback registration. Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Thomas Gleixner Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/perf_event_amd_ibs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c index 4b8e4d3cd6ea..4c36bbe3173a 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c +++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c @@ -926,13 +926,13 @@ static __init int amd_ibs_init(void) goto out; perf_ibs_pm_init(); - get_online_cpus(); + cpu_notifier_register_begin(); ibs_caps = caps; /* make ibs_caps visible to other cpus: */ smp_mb(); - perf_cpu_notifier(perf_ibs_cpu_notifier); smp_call_function(setup_APIC_ibs, NULL, 1); - put_online_cpus(); + __perf_cpu_notifier(perf_ibs_cpu_notifier); + cpu_notifier_register_done(); ret = perf_event_ibs_init(); out: -- cgit From 8c60ea146499b9d2a81ceb5e3e0bd215ef0b6287 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:07:57 +0530 Subject: x86, intel, cacheinfo: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the intel cacheinfo code in x86 by using this latter form of callback registration. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/intel_cacheinfo.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 0641113e2965..a952e9c85b6f 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -1225,21 +1225,24 @@ static struct notifier_block cacheinfo_cpu_notifier = { static int __init cache_sysfs_init(void) { - int i; + int i, err = 0; if (num_cache_leaves == 0) return 0; + cpu_notifier_register_begin(); for_each_online_cpu(i) { - int err; struct device *dev = get_cpu_device(i); err = cache_add_dev(dev); if (err) - return err; + goto out; } - register_hotcpu_notifier(&cacheinfo_cpu_notifier); - return 0; + __register_hotcpu_notifier(&cacheinfo_cpu_notifier); + +out: + cpu_notifier_register_done(); + return err; } device_initcall(cache_sysfs_init); -- cgit From fd537e56f65c61bb4cece99ac3a1a1bef6676df9 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:08:09 +0530 Subject: x86, intel, rapl: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the intel rapl code in x86 by using this latter form of callback registration. Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Thomas Gleixner Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/perf_event_intel_rapl.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/perf_event_intel_rapl.c b/arch/x86/kernel/cpu/perf_event_intel_rapl.c index 5ad35ad94d0f..059218ed5208 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_rapl.c +++ b/arch/x86/kernel/cpu/perf_event_intel_rapl.c @@ -646,19 +646,20 @@ static int __init rapl_pmu_init(void) /* unsupported */ return 0; } - get_online_cpus(); + + cpu_notifier_register_begin(); for_each_online_cpu(cpu) { rapl_cpu_prepare(cpu); rapl_cpu_init(cpu); } - perf_cpu_notifier(rapl_cpu_notifier); + __perf_cpu_notifier(rapl_cpu_notifier); ret = perf_pmu_register(&rapl_pmu_class, "power", -1); if (WARN_ON(ret)) { pr_info("RAPL PMU detected, registration failed (%d), RAPL PMU disabled\n", ret); - put_online_cpus(); + cpu_notifier_register_done(); return -1; } @@ -672,7 +673,7 @@ static int __init rapl_pmu_init(void) hweight32(rapl_cntr_mask), ktime_to_ms(pmu->timer_interval)); - put_online_cpus(); + cpu_notifier_register_done(); return 0; } -- cgit From a8c17c2951f3e198982dd2416b068183fc9fcc7e Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:08:29 +0530 Subject: x86, amd, uncore: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the amd-uncore code in x86 by using this latter form of callback registration. Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Thomas Gleixner Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/cpu/perf_event_amd_uncore.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/perf_event_amd_uncore.c b/arch/x86/kernel/cpu/perf_event_amd_uncore.c index 754291adec33..3bbdf4cd38b9 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_amd_uncore.c @@ -531,15 +531,16 @@ static int __init amd_uncore_init(void) if (ret) return -ENODEV; - get_online_cpus(); + cpu_notifier_register_begin(); + /* init cpus already online before registering for hotplug notifier */ for_each_online_cpu(cpu) { amd_uncore_cpu_up_prepare(cpu); smp_call_function_single(cpu, init_cpu_already_online, NULL, 1); } - register_cpu_notifier(&amd_uncore_cpu_notifier_block); - put_online_cpus(); + __register_cpu_notifier(&amd_uncore_cpu_notifier_block); + cpu_notifier_register_done(); return 0; } -- cgit From 9014ad2a500cb3d260987cb5d63fd940f6bbbbe1 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:08:36 +0530 Subject: x86, hpet: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the hpet code in x86 by using this latter form of callback registration. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kernel/hpet.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index da85a8e830a1..d89382be899d 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -943,12 +943,14 @@ static __init int hpet_late_init(void) if (boot_cpu_has(X86_FEATURE_ARAT)) return 0; + cpu_notifier_register_begin(); for_each_online_cpu(cpu) { hpet_cpuhp_notify(NULL, CPU_ONLINE, (void *)(long)cpu); } /* This notifier should be called after workqueue is ready */ - hotcpu_notifier(hpet_cpuhp_notify, -20); + __hotcpu_notifier(hpet_cpuhp_notify, -20); + cpu_notifier_register_done(); return 0; } -- cgit From 9f668f6661698d320df552fccc7a3dab0de89d13 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:08:43 +0530 Subject: x86, pci, amd-bus: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the amd-bus code in x86 by using this latter form of callback registration. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Acked-by: Bjorn Helgaas Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/pci/amd_bus.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index a48be98e9ded..f05cbf0764d3 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -380,10 +380,13 @@ static int __init pci_io_ecs_init(void) if (early_pci_allowed()) pci_enable_pci_io_ecs(); - register_cpu_notifier(&amd_cpu_notifier); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) amd_cpu_notify(&amd_cpu_notifier, (unsigned long)CPU_ONLINE, (void *)(long)cpu); + __register_cpu_notifier(&amd_cpu_notifier); + cpu_notifier_register_done(); + pci_probe |= PCI_HAS_IO_ECS; return 0; -- cgit From 76902e3d9ce88ebd70e362e4d3ff173afa5ce575 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:08:49 +0530 Subject: x86, oprofile, nmi: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the oprofile code in x86 by using this latter form of callback registration. But retain the calls to get/put_online_cpus(), since they are used in other places as well, to protect the variables 'nmi_enabled' and 'ctr_running'. Strictly speaking, this is not necessary since cpu_notifier_register_begin/done() provide a stronger synchronization with CPU hotplug than get/put_online_cpus(). However, let's retain the calls to get/put_online_cpus() to be consistent with the other call-sites. By nesting get/put_online_cpus() *inside* cpu_notifier_register_begin/done(), we avoid the ABBA deadlock possibility mentioned above. Cc: Robert Richter Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/oprofile/nmi_int.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index 6890d8498e0b..379e8bd0deea 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c @@ -494,14 +494,19 @@ static int nmi_setup(void) if (err) goto fail; + cpu_notifier_register_begin(); + + /* Use get/put_online_cpus() to protect 'nmi_enabled' */ get_online_cpus(); - register_cpu_notifier(&oprofile_cpu_nb); nmi_enabled = 1; /* make nmi_enabled visible to the nmi handler: */ smp_mb(); on_each_cpu(nmi_cpu_setup, NULL, 1); + __register_cpu_notifier(&oprofile_cpu_nb); put_online_cpus(); + cpu_notifier_register_done(); + return 0; fail: free_msrs(); @@ -512,12 +517,18 @@ static void nmi_shutdown(void) { struct op_msrs *msrs; + cpu_notifier_register_begin(); + + /* Use get/put_online_cpus() to protect 'nmi_enabled' & 'ctr_running' */ get_online_cpus(); - unregister_cpu_notifier(&oprofile_cpu_nb); on_each_cpu(nmi_cpu_shutdown, NULL, 1); nmi_enabled = 0; ctr_running = 0; + __unregister_cpu_notifier(&oprofile_cpu_nb); put_online_cpus(); + + cpu_notifier_register_done(); + /* make variables visible to the nmi handler: */ smp_mb(); unregister_nmi_handler(NMI_LOCAL, "oprofile"); -- cgit From 460dd42e1187047427e265edc451cb115341b74b Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:01 +0530 Subject: x86, kvm: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the kvm code in x86 by using this latter form of callback registration. Cc: Gleb Natapov Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Acked-by: Paolo Bonzini Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/x86/kvm/x86.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2b8578432d5b..4db6f56e7b30 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5365,7 +5365,8 @@ static void kvm_timer_init(void) int cpu; max_tsc_khz = tsc_khz; - register_hotcpu_notifier(&kvmclock_cpu_notifier_block); + + cpu_notifier_register_begin(); if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { #ifdef CONFIG_CPU_FREQ struct cpufreq_policy policy; @@ -5382,6 +5383,10 @@ static void kvm_timer_init(void) pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz); for_each_online_cpu(cpu) smp_call_function_single(cpu, tsc_khz_changed, NULL, 1); + + __register_hotcpu_notifier(&kvmclock_cpu_notifier_block); + cpu_notifier_register_done(); + } static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu); -- cgit From 3d0dc643f81ed64e2a7f839e8491728addbc51b9 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:08 +0530 Subject: arm64, hw_breakpoint.c: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the hw-breakpoint code in arm64 by using this latter form of callback registration. Cc: Catalin Marinas Cc: Lorenzo Pieralisi Cc: Ingo Molnar Acked-by: Will Deacon Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/arm64/kernel/hw_breakpoint.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index f17f581116fc..bee789757806 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -913,6 +913,8 @@ static int __init arch_hw_breakpoint_init(void) pr_info("found %d breakpoint and %d watchpoint registers.\n", core_num_brps, core_num_wrps); + cpu_notifier_register_begin(); + /* * Reset the breakpoint resources. We assume that a halting * debugger will leave the world in a nice state for us. @@ -927,7 +929,10 @@ static int __init arch_hw_breakpoint_init(void) TRAP_HWBKPT, "hw-watchpoint handler"); /* Register hotplug notifier. */ - register_cpu_notifier(&hw_breakpoint_reset_nb); + __register_cpu_notifier(&hw_breakpoint_reset_nb); + + cpu_notifier_register_done(); + /* Register cpu_suspend hw breakpoint restore hook */ cpu_suspend_set_dbg_restorer(hw_breakpoint_reset); -- cgit From 4b0b68af37b930403cf9074c0cf504fc2387c2fa Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:20 +0530 Subject: arm64, debug-monitors: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the debug-monitors code in arm64 by using this latter form of callback registration. Cc: Catalin Marinas Cc: Russell King Cc: Ingo Molnar Acked-by: Will Deacon Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- arch/arm64/kernel/debug-monitors.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 636ba8b6240b..c9855313eb27 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -155,12 +155,16 @@ static struct notifier_block os_lock_nb = { static int debug_monitors_init(void) { + cpu_notifier_register_begin(); + /* Clear the OS lock. */ smp_call_function(clear_os_lock, NULL, 1); clear_os_lock(NULL); /* Register hotplug handler. */ - register_cpu_notifier(&os_lock_nb); + __register_cpu_notifier(&os_lock_nb); + + cpu_notifier_register_done(); return 0; } postcore_initcall(debug_monitors_init); -- cgit From 009f225ef050d231ebb7b87264a21a7daac0f175 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:26 +0530 Subject: powercap, intel-rapl: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the intel-rapl code in the powercap driver by using this latter form of callback registration. But retain the calls to get/put_online_cpus(), since they also protect the function rapl_cleanup_data(). By nesting get/put_online_cpus() *inside* cpu_notifier_register_begin/done(), we avoid the ABBA deadlock possibility mentioned above. Cc: Srinivas Pandruvada Cc: Ingo Molnar Tested-by: Jacob Pan Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c index 3c6768378a94..d6c74c1b5ea7 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl.c @@ -1369,6 +1369,9 @@ static int __init rapl_init(void) return -ENODEV; } + + cpu_notifier_register_begin(); + /* prevent CPU hotplug during detection */ get_online_cpus(); ret = rapl_detect_topology(); @@ -1380,20 +1383,23 @@ static int __init rapl_init(void) ret = -ENODEV; goto done; } - register_hotcpu_notifier(&rapl_cpu_notifier); + __register_hotcpu_notifier(&rapl_cpu_notifier); done: put_online_cpus(); + cpu_notifier_register_done(); return ret; } static void __exit rapl_exit(void) { + cpu_notifier_register_begin(); get_online_cpus(); - unregister_hotcpu_notifier(&rapl_cpu_notifier); + __unregister_hotcpu_notifier(&rapl_cpu_notifier); rapl_unregister_powercap(); rapl_cleanup_data(); put_online_cpus(); + cpu_notifier_register_done(); } module_init(rapl_init); -- cgit From bc0003c93f0e68b5383fb36579f7efa267b7dc16 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:39 +0530 Subject: scsi, bnx2i: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the bnx2i code in scsi by using this latter form of callback registration. Cc: Eddie Wai Cc: "James E.J. Bottomley" Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/scsi/bnx2i/bnx2i_init.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/bnx2i/bnx2i_init.c b/drivers/scsi/bnx2i/bnx2i_init.c index 34c294b42c84..80c03b452d61 100644 --- a/drivers/scsi/bnx2i/bnx2i_init.c +++ b/drivers/scsi/bnx2i/bnx2i_init.c @@ -537,11 +537,15 @@ static int __init bnx2i_mod_init(void) p->iothread = NULL; } + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) bnx2i_percpu_thread_create(cpu); /* Initialize per CPU interrupt thread */ - register_hotcpu_notifier(&bnx2i_cpu_notifier); + __register_hotcpu_notifier(&bnx2i_cpu_notifier); + + cpu_notifier_register_done(); return 0; @@ -581,11 +585,15 @@ static void __exit bnx2i_mod_exit(void) } mutex_unlock(&bnx2i_dev_lock); - unregister_hotcpu_notifier(&bnx2i_cpu_notifier); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) bnx2i_percpu_thread_destroy(cpu); + __unregister_hotcpu_notifier(&bnx2i_cpu_notifier); + + cpu_notifier_register_done(); + iscsi_unregister_transport(&bnx2i_iscsi_transport); cnic_unregister_driver(CNIC_ULP_ISCSI); } -- cgit From 7229b6d0b2a9e62d5eb4886dcd0137664c735718 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:45 +0530 Subject: scsi, bnx2fc: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the bnx2fc code in scsi by using this latter form of callback registration. Cc: Eddie Wai Cc: "James E.J. Bottomley" Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c index 9b948505d118..c4ec235baead 100644 --- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c +++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c @@ -2586,12 +2586,16 @@ static int __init bnx2fc_mod_init(void) spin_lock_init(&p->fp_work_lock); } + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) { bnx2fc_percpu_thread_create(cpu); } /* Initialize per CPU interrupt thread */ - register_hotcpu_notifier(&bnx2fc_cpu_notifier); + __register_hotcpu_notifier(&bnx2fc_cpu_notifier); + + cpu_notifier_register_done(); cnic_register_driver(CNIC_ULP_FCOE, &bnx2fc_cnic_cb); @@ -2656,13 +2660,17 @@ static void __exit bnx2fc_mod_exit(void) if (l2_thread) kthread_stop(l2_thread); - unregister_hotcpu_notifier(&bnx2fc_cpu_notifier); + cpu_notifier_register_begin(); /* Destroy per cpu threads */ for_each_online_cpu(cpu) { bnx2fc_percpu_thread_destroy(cpu); } + __unregister_hotcpu_notifier(&bnx2fc_cpu_notifier); + + cpu_notifier_register_done(); + destroy_workqueue(bnx2fc_wq); /* * detach from scsi transport -- cgit From cd45ae38035d1b7f98dd92429b59bbc8ba9443e9 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:52 +0530 Subject: scsi, fcoe: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the fcoe code in scsi by using this latter form of callback registration. Cc: Robert Love Cc: "James E.J. Bottomley" Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/scsi/fcoe/fcoe.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c index f3170008ae71..d5e105b173f0 100644 --- a/drivers/scsi/fcoe/fcoe.c +++ b/drivers/scsi/fcoe/fcoe.c @@ -2633,14 +2633,18 @@ static int __init fcoe_init(void) skb_queue_head_init(&p->fcoe_rx_list); } + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) fcoe_percpu_thread_create(cpu); /* Initialize per CPU interrupt thread */ - rc = register_hotcpu_notifier(&fcoe_cpu_notifier); + rc = __register_hotcpu_notifier(&fcoe_cpu_notifier); if (rc) goto out_free; + cpu_notifier_register_done(); + /* Setup link change notification */ fcoe_dev_setup(); @@ -2655,6 +2659,9 @@ out_free: for_each_online_cpu(cpu) { fcoe_percpu_thread_destroy(cpu); } + + cpu_notifier_register_done(); + mutex_unlock(&fcoe_config_mutex); destroy_workqueue(fcoe_wq); return rc; @@ -2687,11 +2694,15 @@ static void __exit fcoe_exit(void) } rtnl_unlock(); - unregister_hotcpu_notifier(&fcoe_cpu_notifier); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) fcoe_percpu_thread_destroy(cpu); + __unregister_hotcpu_notifier(&fcoe_cpu_notifier); + + cpu_notifier_register_done(); + mutex_unlock(&fcoe_config_mutex); /* -- cgit From f0e71fcd0fa6f3f5495cd9ad3f1e4acd94446a55 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:09:59 +0530 Subject: zsmalloc: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the zsmalloc code by using this latter form of callback registration. Cc: Nitin Gupta Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Acked-by: Minchan Kim Signed-off-by: Rafael J. Wysocki --- mm/zsmalloc.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index c03ca5e9fe15..36b4591a7a2d 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -814,21 +814,32 @@ static void zs_exit(void) { int cpu; + cpu_notifier_register_begin(); + for_each_online_cpu(cpu) zs_cpu_notifier(NULL, CPU_DEAD, (void *)(long)cpu); - unregister_cpu_notifier(&zs_cpu_nb); + __unregister_cpu_notifier(&zs_cpu_nb); + + cpu_notifier_register_done(); } static int zs_init(void) { int cpu, ret; - register_cpu_notifier(&zs_cpu_nb); + cpu_notifier_register_begin(); + + __register_cpu_notifier(&zs_cpu_nb); for_each_online_cpu(cpu) { ret = zs_cpu_notifier(NULL, CPU_UP_PREPARE, (void *)(long)cpu); - if (notifier_to_errno(ret)) + if (notifier_to_errno(ret)) { + cpu_notifier_register_done(); goto fail; + } } + + cpu_notifier_register_done(); + return 0; fail: zs_exit(); -- cgit From 0197fbd212461f25666272b9e4654f2ccd94cff8 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:06 +0530 Subject: acpi-cpufreq: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the acpi-cpufreq code by using this latter form of callback registration. Cc: Ingo Molnar Acked-by: Viresh Kumar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/acpi-cpufreq.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 18448a7e9f86..245ae078e68e 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -907,15 +907,16 @@ static void __init acpi_cpufreq_boost_init(void) acpi_cpufreq_driver.boost_supported = true; acpi_cpufreq_driver.boost_enabled = boost_state(0); - get_online_cpus(); + + cpu_notifier_register_begin(); /* Force all MSRs to the same value */ boost_set_msrs(acpi_cpufreq_driver.boost_enabled, cpu_online_mask); - register_cpu_notifier(&boost_nb); + __register_cpu_notifier(&boost_nb); - put_online_cpus(); + cpu_notifier_register_done(); } } -- cgit From e12b711196a158a475350ee67876a1e9e2601661 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:12 +0530 Subject: drivers/base/topology.c: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the topology code by using this latter form of callback registration. Cc: Ingo Molnar Acked-by: Greg Kroah-Hartman Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/base/topology.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 94ffee378f10..a738d1069d6b 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -161,16 +161,20 @@ static int topology_cpu_callback(struct notifier_block *nfb, static int topology_sysfs_init(void) { int cpu; - int rc; + int rc = 0; + + cpu_notifier_register_begin(); for_each_online_cpu(cpu) { rc = topology_add_dev(cpu); if (rc) - return rc; + goto out; } - hotcpu_notifier(topology_cpu_callback, 0); + __hotcpu_notifier(topology_cpu_callback, 0); - return 0; +out: + cpu_notifier_register_done(); + return rc; } device_initcall(topology_sysfs_init); -- cgit From 8daa127f4e857427c66f365f650dda90f459aa54 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:23 +0530 Subject: clocksource, dummy-timer: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the clocksource dummy-timer code by using this latter form of callback registration. Cc: Daniel Lezcano Cc: Thomas Gleixner Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/clocksource/dummy_timer.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/clocksource/dummy_timer.c b/drivers/clocksource/dummy_timer.c index b3eb582d6a6f..ad3572541728 100644 --- a/drivers/clocksource/dummy_timer.c +++ b/drivers/clocksource/dummy_timer.c @@ -56,14 +56,19 @@ static struct notifier_block dummy_timer_cpu_nb = { static int __init dummy_timer_register(void) { - int err = register_cpu_notifier(&dummy_timer_cpu_nb); + int err = 0; + + cpu_notifier_register_begin(); + err = __register_cpu_notifier(&dummy_timer_cpu_nb); if (err) - return err; + goto out; /* We won't get a call on the boot CPU, so register immediately */ if (num_possible_cpus() > 1) dummy_timer_setup(); - return 0; +out: + cpu_notifier_register_done(); + return err; } early_initcall(dummy_timer_register); -- cgit From 07494d547e92bde6857522d2a92ff70896aecadb Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:30 +0530 Subject: intel-idle: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the intel-idle code by using this latter form of callback registration. Cc: Len Brown Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/idle/intel_idle.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 8e1939f564f4..51493ed4643b 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -681,14 +681,19 @@ static int __init intel_idle_init(void) if (intel_idle_cpuidle_devices == NULL) return -ENOMEM; + cpu_notifier_register_begin(); + for_each_online_cpu(i) { retval = intel_idle_cpu_init(i); if (retval) { + cpu_notifier_register_done(); cpuidle_unregister_driver(&intel_idle_driver); return retval; } } - register_cpu_notifier(&cpu_hotplug_notifier); + __register_cpu_notifier(&cpu_hotplug_notifier); + + cpu_notifier_register_done(); return 0; } @@ -698,10 +703,13 @@ static void __exit intel_idle_exit(void) intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); + cpu_notifier_register_begin(); if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) on_each_cpu(__setup_broadcast_timer, (void *)false, 1); - unregister_cpu_notifier(&cpu_hotplug_notifier); + __unregister_cpu_notifier(&cpu_hotplug_notifier); + + cpu_notifier_register_done(); return; } -- cgit From 180d86463257812dc17e5df912f3bddcc96abb00 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:36 +0530 Subject: oprofile, nmi-timer: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the nmi-timer code in oprofile by using this latter form of callback registration. Cc: Robert Richter Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/oprofile/nmi_timer_int.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/oprofile/nmi_timer_int.c b/drivers/oprofile/nmi_timer_int.c index 76f1c9357f39..9559829fb234 100644 --- a/drivers/oprofile/nmi_timer_int.c +++ b/drivers/oprofile/nmi_timer_int.c @@ -108,8 +108,8 @@ static void nmi_timer_shutdown(void) struct perf_event *event; int cpu; - get_online_cpus(); - unregister_cpu_notifier(&nmi_timer_cpu_nb); + cpu_notifier_register_begin(); + __unregister_cpu_notifier(&nmi_timer_cpu_nb); for_each_possible_cpu(cpu) { event = per_cpu(nmi_timer_events, cpu); if (!event) @@ -119,7 +119,7 @@ static void nmi_timer_shutdown(void) perf_event_release_kernel(event); } - put_online_cpus(); + cpu_notifier_register_done(); } static int nmi_timer_setup(void) @@ -132,20 +132,23 @@ static int nmi_timer_setup(void) do_div(period, HZ); nmi_timer_attr.sample_period = period; - get_online_cpus(); - err = register_cpu_notifier(&nmi_timer_cpu_nb); + cpu_notifier_register_begin(); + err = __register_cpu_notifier(&nmi_timer_cpu_nb); if (err) goto out; + /* can't attach events to offline cpus: */ for_each_online_cpu(cpu) { err = nmi_timer_start_cpu(cpu); - if (err) - break; + if (err) { + cpu_notifier_register_done(); + nmi_timer_shutdown(); + return err; + } } - if (err) - nmi_timer_shutdown(); + out: - put_online_cpus(); + cpu_notifier_register_done(); return err; } -- cgit From 99c3bf361a4afd2ad331e767b3e5e6fa3488a096 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:43 +0530 Subject: octeon, watchdog: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the watchdog code in octeon by using this latter form of callback registration. Cc: Wim Van Sebroeck Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/watchdog/octeon-wdt-main.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c index 461208831428..4baf2d788920 100644 --- a/drivers/watchdog/octeon-wdt-main.c +++ b/drivers/watchdog/octeon-wdt-main.c @@ -708,10 +708,13 @@ static int __init octeon_wdt_init(void) cpumask_clear(&irq_enabled_cpus); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) octeon_wdt_setup_interrupt(cpu); - register_hotcpu_notifier(&octeon_wdt_cpu_notifier); + __register_hotcpu_notifier(&octeon_wdt_cpu_notifier); + cpu_notifier_register_done(); + out: return ret; } @@ -725,7 +728,8 @@ static void __exit octeon_wdt_cleanup(void) misc_deregister(&octeon_wdt_miscdev); - unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier); + cpu_notifier_register_begin(); + __unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier); for_each_online_cpu(cpu) { int core = cpu2core(cpu); @@ -734,6 +738,9 @@ static void __exit octeon_wdt_cleanup(void) /* Free the interrupt handler */ free_irq(OCTEON_IRQ_WDOG0 + core, octeon_wdt_poke_irq); } + + cpu_notifier_register_done(); + /* * Disable the boot-bus memory, the code it points to is soon * to go missing. -- cgit From cf0485a2ac70acb1bc83f6310a7ebef3070f0333 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:10:54 +0530 Subject: thermal, x86-pkg-temp: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the thermal x86-pkg-temp code by using this latter form of callback registration. Cc: Zhang Rui Cc: Eduardo Valentin Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/thermal/x86_pkg_temp_thermal.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/thermal/x86_pkg_temp_thermal.c b/drivers/thermal/x86_pkg_temp_thermal.c index 081fd7e6a9f0..9ea3d9d49ffc 100644 --- a/drivers/thermal/x86_pkg_temp_thermal.c +++ b/drivers/thermal/x86_pkg_temp_thermal.c @@ -590,12 +590,12 @@ static int __init pkg_temp_thermal_init(void) platform_thermal_package_rate_control = pkg_temp_thermal_platform_thermal_rate_control; - get_online_cpus(); + cpu_notifier_register_begin(); for_each_online_cpu(i) if (get_core_online(i)) goto err_ret; - register_hotcpu_notifier(&pkg_temp_thermal_notifier); - put_online_cpus(); + __register_hotcpu_notifier(&pkg_temp_thermal_notifier); + cpu_notifier_register_done(); pkg_temp_debugfs_init(); /* Don't care if fails */ @@ -604,7 +604,7 @@ static int __init pkg_temp_thermal_init(void) err_ret: for_each_online_cpu(i) put_core_offline(i); - put_online_cpus(); + cpu_notifier_register_done(); kfree(pkg_work_scheduled); platform_thermal_package_notify = NULL; platform_thermal_package_rate_control = NULL; @@ -617,8 +617,8 @@ static void __exit pkg_temp_thermal_exit(void) struct phy_dev_entry *phdev, *n; int i; - get_online_cpus(); - unregister_hotcpu_notifier(&pkg_temp_thermal_notifier); + cpu_notifier_register_begin(); + __unregister_hotcpu_notifier(&pkg_temp_thermal_notifier); mutex_lock(&phy_dev_list_mutex); list_for_each_entry_safe(phdev, n, &phy_dev_list, list) { /* Retore old MSR value for package thermal interrupt */ @@ -636,7 +636,7 @@ static void __exit pkg_temp_thermal_exit(void) for_each_online_cpu(i) cancel_delayed_work_sync( &per_cpu(pkg_temp_thermal_threshold_work, i)); - put_online_cpus(); + cpu_notifier_register_done(); kfree(pkg_work_scheduled); -- cgit From 3289705fe2b429569f37730ecf660719b8924420 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:11:11 +0530 Subject: hwmon, coretemp: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the hwmon coretemp code by using this latter form of callback registration. Cc: Fenghua Yu Cc: Jean Delvare Cc: Ingo Molnar Acked-by: Guenter Roeck Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/hwmon/coretemp.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index bbb0b0d463f7..746a6ad8525f 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -849,20 +849,20 @@ static int __init coretemp_init(void) if (err) goto exit; - get_online_cpus(); + cpu_notifier_register_begin(); for_each_online_cpu(i) get_core_online(i); #ifndef CONFIG_HOTPLUG_CPU if (list_empty(&pdev_list)) { - put_online_cpus(); + cpu_notifier_register_done(); err = -ENODEV; goto exit_driver_unreg; } #endif - register_hotcpu_notifier(&coretemp_cpu_notifier); - put_online_cpus(); + __register_hotcpu_notifier(&coretemp_cpu_notifier); + cpu_notifier_register_done(); return 0; #ifndef CONFIG_HOTPLUG_CPU @@ -877,8 +877,8 @@ static void __exit coretemp_exit(void) { struct pdev_entry *p, *n; - get_online_cpus(); - unregister_hotcpu_notifier(&coretemp_cpu_notifier); + cpu_notifier_register_begin(); + __unregister_hotcpu_notifier(&coretemp_cpu_notifier); mutex_lock(&pdev_list_mutex); list_for_each_entry_safe(p, n, &pdev_list, list) { platform_device_unregister(p->pdev); @@ -886,7 +886,7 @@ static void __exit coretemp_exit(void) kfree(p); } mutex_unlock(&pdev_list_mutex); - put_online_cpus(); + cpu_notifier_register_done(); platform_driver_unregister(&coretemp_driver); } -- cgit From 2480b6a3e5e80d778455f8138ae6d6efb568cd59 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:11:33 +0530 Subject: hwmon, via-cputemp: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the hwmon via-cputemp code by using this latter form of callback registration. Cc: Jean Delvare Cc: Ingo Molnar Acked-by: Guenter Roeck Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/hwmon/via-cputemp.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c index 38944e94f65f..8df43c51de2c 100644 --- a/drivers/hwmon/via-cputemp.c +++ b/drivers/hwmon/via-cputemp.c @@ -319,7 +319,7 @@ static int __init via_cputemp_init(void) if (err) goto exit; - get_online_cpus(); + cpu_notifier_register_begin(); for_each_online_cpu(i) { struct cpuinfo_x86 *c = &cpu_data(i); @@ -339,14 +339,14 @@ static int __init via_cputemp_init(void) #ifndef CONFIG_HOTPLUG_CPU if (list_empty(&pdev_list)) { - put_online_cpus(); + cpu_notifier_register_done(); err = -ENODEV; goto exit_driver_unreg; } #endif - register_hotcpu_notifier(&via_cputemp_cpu_notifier); - put_online_cpus(); + __register_hotcpu_notifier(&via_cputemp_cpu_notifier); + cpu_notifier_register_done(); return 0; #ifndef CONFIG_HOTPLUG_CPU @@ -361,8 +361,8 @@ static void __exit via_cputemp_exit(void) { struct pdev_entry *p, *n; - get_online_cpus(); - unregister_hotcpu_notifier(&via_cputemp_cpu_notifier); + cpu_notifier_register_begin(); + __unregister_hotcpu_notifier(&via_cputemp_cpu_notifier); mutex_lock(&pdev_list_mutex); list_for_each_entry_safe(p, n, &pdev_list, list) { platform_device_unregister(p->pdev); @@ -370,7 +370,7 @@ static void __exit via_cputemp_exit(void) kfree(p); } mutex_unlock(&pdev_list_mutex); - put_online_cpus(); + cpu_notifier_register_done(); platform_driver_unregister(&via_cputemp_driver); } -- cgit From 43ea9536499e80d858e26cecb7abcc893f86d222 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:11:45 +0530 Subject: xen, balloon: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). The xen balloon driver doesn't take get/put_online_cpus() around this code, but that is also buggy, since it can miss CPU hotplug events in between the initialization and callback registration: for_each_online_cpu(cpu) init_cpu(cpu); ^ | Race window; Can miss CPU hotplug events here. v register_cpu_notifier(&foobar_cpu_notifier); Interestingly, the balloon code in xen can simply be reorganized as shown below, to have a race-free method to register hotplug callbacks, without even taking get/put_online_cpus(). This is because the initialization performed for already online CPUs is exactly the same as that performed for CPUs that come online later. Moreover, the code has checks in place to avoid double initialization. register_cpu_notifier(&foobar_cpu_notifier); get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); put_online_cpus(); A hotplug operation that occurs between registering the notifier and calling get_online_cpus(), won't disrupt anything, because the code takes care to perform the memory allocations only once. So reorganize the balloon code in xen this way to fix the issues with CPU hotplug callback registration. Cc: Konrad Rzeszutek Wilk Cc: David Vrabel Cc: Ingo Molnar Reviewed-by: Boris Ostrovsky Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- drivers/xen/balloon.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 37d06ea624aa..dd7954922942 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -592,19 +592,29 @@ static void __init balloon_add_region(unsigned long start_pfn, } } +static int alloc_balloon_scratch_page(int cpu) +{ + if (per_cpu(balloon_scratch_page, cpu) != NULL) + return 0; + + per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL); + if (per_cpu(balloon_scratch_page, cpu) == NULL) { + pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu); + return -ENOMEM; + } + + return 0; +} + + static int balloon_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; switch (action) { case CPU_UP_PREPARE: - if (per_cpu(balloon_scratch_page, cpu) != NULL) - break; - per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL); - if (per_cpu(balloon_scratch_page, cpu) == NULL) { - pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu); + if (alloc_balloon_scratch_page(cpu)) return NOTIFY_BAD; - } break; default: break; @@ -624,15 +634,17 @@ static int __init balloon_init(void) return -ENODEV; if (!xen_feature(XENFEAT_auto_translated_physmap)) { - for_each_online_cpu(cpu) - { - per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL); - if (per_cpu(balloon_scratch_page, cpu) == NULL) { - pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu); + register_cpu_notifier(&balloon_cpu_notifier); + + get_online_cpus(); + for_each_online_cpu(cpu) { + if (alloc_balloon_scratch_page(cpu)) { + put_online_cpus(); + unregister_cpu_notifier(&balloon_cpu_notifier); return -ENOMEM; } } - register_cpu_notifier(&balloon_cpu_notifier); + put_online_cpus(); } pr_info("Initialising balloon driver\n"); -- cgit From d39ad278a3001c860da4d7c13e51259b1904bec5 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:11:56 +0530 Subject: trace, ring-buffer: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the tracing ring-buffer code by using this latter form of callback registration. Cc: Frederic Weisbecker Cc: Ingo Molnar Acked-by: Steven Rostedt Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- kernel/trace/ring_buffer.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index fc4da2d97f9b..c634868c2921 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1301,7 +1301,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, * In that off case, we need to allocate for all possible cpus. */ #ifdef CONFIG_HOTPLUG_CPU - get_online_cpus(); + cpu_notifier_register_begin(); cpumask_copy(buffer->cpumask, cpu_online_mask); #else cpumask_copy(buffer->cpumask, cpu_possible_mask); @@ -1324,10 +1324,10 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, #ifdef CONFIG_HOTPLUG_CPU buffer->cpu_notify.notifier_call = rb_cpu_notify; buffer->cpu_notify.priority = 0; - register_cpu_notifier(&buffer->cpu_notify); + __register_cpu_notifier(&buffer->cpu_notify); + cpu_notifier_register_done(); #endif - put_online_cpus(); mutex_init(&buffer->mutex); return buffer; @@ -1341,7 +1341,9 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, fail_free_cpumask: free_cpumask_var(buffer->cpumask); - put_online_cpus(); +#ifdef CONFIG_HOTPLUG_CPU + cpu_notifier_register_done(); +#endif fail_free_buffer: kfree(buffer); @@ -1358,16 +1360,17 @@ ring_buffer_free(struct ring_buffer *buffer) { int cpu; - get_online_cpus(); - #ifdef CONFIG_HOTPLUG_CPU - unregister_cpu_notifier(&buffer->cpu_notify); + cpu_notifier_register_begin(); + __unregister_cpu_notifier(&buffer->cpu_notify); #endif for_each_buffer_cpu(buffer, cpu) rb_free_cpu_buffer(buffer->buffers[cpu]); - put_online_cpus(); +#ifdef CONFIG_HOTPLUG_CPU + cpu_notifier_register_done(); +#endif kfree(buffer->buffers); free_cpumask_var(buffer->cpumask); -- cgit From c270a817196a9374a2dc730624d1501dced40b4d Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:12:08 +0530 Subject: profile: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the profile code by using this latter form of callback registration. Cc: Al Viro Cc: Mauro Carvalho Chehab Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- kernel/profile.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/kernel/profile.c b/kernel/profile.c index ebdd9c1a86b4..93b2a3fe0a64 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -591,18 +591,28 @@ out_cleanup: int __ref create_proc_profile(void) /* false positive from hotcpu_notifier */ { struct proc_dir_entry *entry; + int err = 0; if (!prof_on) return 0; - if (create_hash_tables()) - return -ENOMEM; + + cpu_notifier_register_begin(); + + if (create_hash_tables()) { + err = -ENOMEM; + goto out; + } + entry = proc_create("profile", S_IWUSR | S_IRUGO, NULL, &proc_profile_operations); if (!entry) - return 0; + goto out; proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t)); - hotcpu_notifier(profile_cpu_callback, 0); - return 0; + __hotcpu_notifier(profile_cpu_callback, 0); + +out: + cpu_notifier_register_done(); + return err; } module_init(create_proc_profile); #endif /* CONFIG_PROC_FS */ -- cgit From 0be94bad0b601df94b8558c0cbd28f7e6633c9e8 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:12:27 +0530 Subject: mm, vmstat: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the vmstat code in the MM subsystem by using this latter form of callback registration. Cc: Andrew Morton Cc: Johannes Weiner Cc: Cody P Schafer Cc: Toshi Kani Cc: Dave Hansen Cc: Ingo Molnar Acked-by: Christoph Lameter Acked-by: Rik van Riel Reviewed-by: Yasuaki Ishimatsu Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- mm/vmstat.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/vmstat.c b/mm/vmstat.c index def5dd2fbe61..58c6f3d09735 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1292,14 +1292,14 @@ static int __init setup_vmstat(void) #ifdef CONFIG_SMP int cpu; - register_cpu_notifier(&vmstat_notifier); + cpu_notifier_register_begin(); + __register_cpu_notifier(&vmstat_notifier); - get_online_cpus(); for_each_online_cpu(cpu) { start_cpu_timer(cpu); node_set_state(cpu_to_node(cpu), N_CPU); } - put_online_cpus(); + cpu_notifier_register_done(); #endif #ifdef CONFIG_PROC_FS proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations); -- cgit From 576378249c8e0a020aafeaa702c834dff81dd596 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:12:40 +0530 Subject: mm, zswap: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the zswap code by using this latter form of callback registration. Cc: Ingo Molnar Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- mm/zswap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index e55bab9dc41f..d7337fbf6605 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -387,18 +387,18 @@ static int zswap_cpu_init(void) { unsigned long cpu; - get_online_cpus(); + cpu_notifier_register_begin(); for_each_online_cpu(cpu) if (__zswap_cpu_notifier(CPU_UP_PREPARE, cpu) != NOTIFY_OK) goto cleanup; - register_cpu_notifier(&zswap_cpu_notifier_block); - put_online_cpus(); + __register_cpu_notifier(&zswap_cpu_notifier_block); + cpu_notifier_register_done(); return 0; cleanup: for_each_online_cpu(cpu) __zswap_cpu_notifier(CPU_UP_CANCELED, cpu); - put_online_cpus(); + cpu_notifier_register_done(); return -ENOMEM; } -- cgit From e30a293e8ad7e6048d6d88bcc114094f964bd67b Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:12:51 +0530 Subject: net/core/flow.c: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the code in net/core/flow.c by using this latter form of callback registration. Cc: Li RongQing Cc: Sasha Levin Cc: Andrew Morton Cc: Chris Metcalf Cc: Ingo Molnar Acked-by: David S. Miller Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- net/core/flow.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/net/core/flow.c b/net/core/flow.c index dfa602ceb8cd..9a2151f5d593 100644 --- a/net/core/flow.c +++ b/net/core/flow.c @@ -456,6 +456,8 @@ static int __init flow_cache_init(struct flow_cache *fc) if (!fc->percpu) return -ENOMEM; + cpu_notifier_register_begin(); + for_each_online_cpu(i) { if (flow_cache_cpu_prepare(fc, i)) goto err; @@ -463,7 +465,9 @@ static int __init flow_cache_init(struct flow_cache *fc) fc->hotcpu_notifier = (struct notifier_block){ .notifier_call = flow_cache_cpu, }; - register_hotcpu_notifier(&fc->hotcpu_notifier); + __register_hotcpu_notifier(&fc->hotcpu_notifier); + + cpu_notifier_register_done(); setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd, (unsigned long) fc); @@ -479,6 +483,8 @@ err: fcp->hash_table = NULL; } + cpu_notifier_register_done(); + free_percpu(fc->percpu); fc->percpu = NULL; -- cgit From a0e247a8059223593f9c5c3d5c1fd50eedf415c0 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Tue, 11 Mar 2014 02:12:57 +0530 Subject: net/iucv/iucv.c: Fix CPU hotplug callback registration Subsystems that want to register CPU hotplug callbacks, as well as perform initialization for the CPUs that are already online, often do it as shown below: get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); This is wrong, since it is prone to ABBA deadlocks involving the cpu_add_remove_lock and the cpu_hotplug.lock (when running concurrently with CPU hotplug operations). Instead, the correct and race-free way of performing the callback registration is: cpu_notifier_register_begin(); for_each_online_cpu(cpu) init_cpu(cpu); /* Note the use of the double underscored version of the API */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_notifier_register_done(); Fix the code in net/iucv/iucv.c by using this latter form of callback registration. Also, provide helper functions to perform the common memory allocations and frees, to condense repetitive code. Cc: Ursula Braun Cc: Ingo Molnar Acked-by: David S. Miller Signed-off-by: Srivatsa S. Bhat Signed-off-by: Rafael J. Wysocki --- net/iucv/iucv.c | 121 ++++++++++++++++++++++++++------------------------------ 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index cd5b8ec9be04..79a0ce95799f 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -621,6 +621,42 @@ static void iucv_disable(void) put_online_cpus(); } +static void free_iucv_data(int cpu) +{ + kfree(iucv_param_irq[cpu]); + iucv_param_irq[cpu] = NULL; + kfree(iucv_param[cpu]); + iucv_param[cpu] = NULL; + kfree(iucv_irq_data[cpu]); + iucv_irq_data[cpu] = NULL; +} + +static int alloc_iucv_data(int cpu) +{ + /* Note: GFP_DMA used to get memory below 2G */ + iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data), + GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); + if (!iucv_irq_data[cpu]) + goto out_free; + + /* Allocate parameter blocks. */ + iucv_param[cpu] = kmalloc_node(sizeof(union iucv_param), + GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); + if (!iucv_param[cpu]) + goto out_free; + + iucv_param_irq[cpu] = kmalloc_node(sizeof(union iucv_param), + GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); + if (!iucv_param_irq[cpu]) + goto out_free; + + return 0; + +out_free: + free_iucv_data(cpu); + return -ENOMEM; +} + static int iucv_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -630,38 +666,14 @@ static int iucv_cpu_notify(struct notifier_block *self, switch (action) { case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: - iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data), - GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); - if (!iucv_irq_data[cpu]) - return notifier_from_errno(-ENOMEM); - - iucv_param[cpu] = kmalloc_node(sizeof(union iucv_param), - GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); - if (!iucv_param[cpu]) { - kfree(iucv_irq_data[cpu]); - iucv_irq_data[cpu] = NULL; + if (alloc_iucv_data(cpu)) return notifier_from_errno(-ENOMEM); - } - iucv_param_irq[cpu] = kmalloc_node(sizeof(union iucv_param), - GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); - if (!iucv_param_irq[cpu]) { - kfree(iucv_param[cpu]); - iucv_param[cpu] = NULL; - kfree(iucv_irq_data[cpu]); - iucv_irq_data[cpu] = NULL; - return notifier_from_errno(-ENOMEM); - } break; case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: case CPU_DEAD: case CPU_DEAD_FROZEN: - kfree(iucv_param_irq[cpu]); - iucv_param_irq[cpu] = NULL; - kfree(iucv_param[cpu]); - iucv_param[cpu] = NULL; - kfree(iucv_irq_data[cpu]); - iucv_irq_data[cpu] = NULL; + free_iucv_data(cpu); break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: @@ -2025,33 +2037,20 @@ static int __init iucv_init(void) goto out_int; } - for_each_online_cpu(cpu) { - /* Note: GFP_DMA used to get memory below 2G */ - iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data), - GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); - if (!iucv_irq_data[cpu]) { - rc = -ENOMEM; - goto out_free; - } + cpu_notifier_register_begin(); - /* Allocate parameter blocks. */ - iucv_param[cpu] = kmalloc_node(sizeof(union iucv_param), - GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); - if (!iucv_param[cpu]) { - rc = -ENOMEM; - goto out_free; - } - iucv_param_irq[cpu] = kmalloc_node(sizeof(union iucv_param), - GFP_KERNEL|GFP_DMA, cpu_to_node(cpu)); - if (!iucv_param_irq[cpu]) { + for_each_online_cpu(cpu) { + if (alloc_iucv_data(cpu)) { rc = -ENOMEM; goto out_free; } - } - rc = register_hotcpu_notifier(&iucv_cpu_notifier); + rc = __register_hotcpu_notifier(&iucv_cpu_notifier); if (rc) goto out_free; + + cpu_notifier_register_done(); + rc = register_reboot_notifier(&iucv_reboot_notifier); if (rc) goto out_cpu; @@ -2069,16 +2068,14 @@ static int __init iucv_init(void) out_reboot: unregister_reboot_notifier(&iucv_reboot_notifier); out_cpu: - unregister_hotcpu_notifier(&iucv_cpu_notifier); + cpu_notifier_register_begin(); + __unregister_hotcpu_notifier(&iucv_cpu_notifier); out_free: - for_each_possible_cpu(cpu) { - kfree(iucv_param_irq[cpu]); - iucv_param_irq[cpu] = NULL; - kfree(iucv_param[cpu]); - iucv_param[cpu] = NULL; - kfree(iucv_irq_data[cpu]); - iucv_irq_data[cpu] = NULL; - } + for_each_possible_cpu(cpu) + free_iucv_data(cpu); + + cpu_notifier_register_done(); + root_device_unregister(iucv_root); out_int: unregister_external_interrupt(0x4000, iucv_external_interrupt); @@ -2105,15 +2102,11 @@ static void __exit iucv_exit(void) kfree(p); spin_unlock_irq(&iucv_queue_lock); unregister_reboot_notifier(&iucv_reboot_notifier); - unregister_hotcpu_notifier(&iucv_cpu_notifier); - for_each_possible_cpu(cpu) { - kfree(iucv_param_irq[cpu]); - iucv_param_irq[cpu] = NULL; - kfree(iucv_param[cpu]); - iucv_param[cpu] = NULL; - kfree(iucv_irq_data[cpu]); - iucv_irq_data[cpu] = NULL; - } + cpu_notifier_register_begin(); + __unregister_hotcpu_notifier(&iucv_cpu_notifier); + for_each_possible_cpu(cpu) + free_iucv_data(cpu); + cpu_notifier_register_done(); root_device_unregister(iucv_root); bus_unregister(&iucv_bus); unregister_external_interrupt(0x4000, iucv_external_interrupt); -- cgit