summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--mm/page_alloc.c40
1 files changed, 22 insertions, 18 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 46c30fa26acd..41985aa4672d 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -92,6 +92,10 @@ EXPORT_PER_CPU_SYMBOL(_numa_mem_);
int _node_numa_mem_[MAX_NUMNODES];
#endif
+/* work_structs for global per-cpu drains */
+DEFINE_MUTEX(pcpu_drain_mutex);
+DEFINE_PER_CPU(struct work_struct, pcpu_drain);
+
#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
volatile unsigned long latent_entropy __latent_entropy;
EXPORT_SYMBOL(latent_entropy);
@@ -2360,7 +2364,6 @@ static void drain_local_pages_wq(struct work_struct *work)
*/
void drain_all_pages(struct zone *zone)
{
- struct work_struct __percpu *works;
int cpu;
/*
@@ -2373,7 +2376,16 @@ void drain_all_pages(struct zone *zone)
if (current->flags & PF_WQ_WORKER)
return;
- works = alloc_percpu_gfp(struct work_struct, GFP_ATOMIC);
+ /*
+ * Do not drain if one is already in progress unless it's specific to
+ * a zone. Such callers are primarily CMA and memory hotplug and need
+ * the drain to be complete when the call returns.
+ */
+ if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
+ if (!zone)
+ return;
+ mutex_lock(&pcpu_drain_mutex);
+ }
/*
* We don't care about racing with CPU hotplug event
@@ -2406,23 +2418,15 @@ void drain_all_pages(struct zone *zone)
cpumask_clear_cpu(cpu, &cpus_with_pcps);
}
- if (works) {
- for_each_cpu(cpu, &cpus_with_pcps) {
- struct work_struct *work = per_cpu_ptr(works, cpu);
- INIT_WORK(work, drain_local_pages_wq);
- schedule_work_on(cpu, work);
- }
- for_each_cpu(cpu, &cpus_with_pcps)
- flush_work(per_cpu_ptr(works, cpu));
- } else {
- for_each_cpu(cpu, &cpus_with_pcps) {
- struct work_struct work;
-
- INIT_WORK(&work, drain_local_pages_wq);
- schedule_work_on(cpu, &work);
- flush_work(&work);
- }
+ for_each_cpu(cpu, &cpus_with_pcps) {
+ struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu);
+ INIT_WORK(work, drain_local_pages_wq);
+ schedule_work_on(cpu, work);
}
+ for_each_cpu(cpu, &cpus_with_pcps)
+ flush_work(per_cpu_ptr(&pcpu_drain, cpu));
+
+ mutex_unlock(&pcpu_drain_mutex);
}
#ifdef CONFIG_HIBERNATION