summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRoman Gushchin <guro@fb.com>2019-05-07 10:01:50 -0700
committerDennis Zhou <dennis@kernel.org>2019-05-09 10:51:06 -0700
commit7d9ab9b6adffd9c474c1274acb5f6208f9a09cf3 (patch)
tree461ee811d60454e5e21fb6f567ec2e2435575a12
parentddde2af747ad79010f14691f381522987fbcb860 (diff)
percpu_ref: release percpu memory early without PERCPU_REF_ALLOW_REINIT
Release percpu memory after finishing the switch to the atomic mode if only PERCPU_REF_ALLOW_REINIT isn't set. Signed-off-by: Roman Gushchin <guro@fb.com> Acked-by: Tejun Heo <tj@kernel.org> Signed-off-by: Dennis Zhou <dennis@kernel.org>
-rw-r--r--include/linux/percpu-refcount.h1
-rw-r--r--lib/percpu-refcount.c13
2 files changed, 12 insertions, 2 deletions
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index 0f0240af8520..7aef0abc194a 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -102,6 +102,7 @@ struct percpu_ref {
percpu_ref_func_t *release;
percpu_ref_func_t *confirm_switch;
bool force_atomic:1;
+ bool allow_reinit:1;
struct rcu_head rcu;
};
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 9877682e49c7..501b517bd3db 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -69,11 +69,14 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
return -ENOMEM;
ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;
+ ref->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT;
- if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD))
+ if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) {
ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
- else
+ ref->allow_reinit = true;
+ } else {
start_count += PERCPU_COUNT_BIAS;
+ }
if (flags & PERCPU_REF_INIT_DEAD)
ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
@@ -119,6 +122,9 @@ static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu)
ref->confirm_switch = NULL;
wake_up_all(&percpu_ref_switch_waitq);
+ if (!ref->allow_reinit)
+ percpu_ref_exit(ref);
+
/* drop ref from percpu_ref_switch_to_atomic() */
percpu_ref_put(ref);
}
@@ -194,6 +200,9 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref)
if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))
return;
+ if (WARN_ON_ONCE(!ref->allow_reinit))
+ return;
+
atomic_long_add(PERCPU_COUNT_BIAS, &ref->count);
/*