summaryrefslogtreecommitdiff
path: root/include/linux/list.h
diff options
context:
space:
mode:
authorEric Dumazet <edumazet@google.com>2019-11-07 11:37:37 -0800
committerPaul E. McKenney <paulmck@kernel.org>2019-12-09 12:36:58 -0800
commitc54a2744497db4b6887b9c905ef7aa0b3620c956 (patch)
tree467a0ddf6de4ee7d036e9748a4c6c075473932a2 /include/linux/list.h
parentf452ee096d95482892b101bde4fd037fa025d3cc (diff)
list: Add hlist_unhashed_lockless()
We would like to use hlist_unhashed() from timer_pending(), which runs without protection of a lock. Note that other callers might also want to use this variant. Instead of forcing a READ_ONCE() for all hlist_unhashed() callers, add a new helper with an explicit _lockless suffix in the name to better document what is going on. Also add various WRITE_ONCE() in __hlist_del(), hlist_add_head() and hlist_add_before()/hlist_add_behind() to pair with the READ_ONCE(). Signed-off-by: Eric Dumazet <edumazet@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> [ paulmck: Also add WRITE_ONCE() to rculist.h. ] Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Diffstat (limited to 'include/linux/list.h')
-rw-r--r--include/linux/list.h32
1 files changed, 21 insertions, 11 deletions
diff --git a/include/linux/list.h b/include/linux/list.h
index 85c92555e31f..61f5aaf96192 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -749,6 +749,16 @@ static inline int hlist_unhashed(const struct hlist_node *h)
return !h->pprev;
}
+/* This variant of hlist_unhashed() must be used in lockless contexts
+ * to avoid potential load-tearing.
+ * The READ_ONCE() is paired with the various WRITE_ONCE() in hlist
+ * helpers that are defined below.
+ */
+static inline int hlist_unhashed_lockless(const struct hlist_node *h)
+{
+ return !READ_ONCE(h->pprev);
+}
+
static inline int hlist_empty(const struct hlist_head *h)
{
return !READ_ONCE(h->first);
@@ -761,7 +771,7 @@ static inline void __hlist_del(struct hlist_node *n)
WRITE_ONCE(*pprev, next);
if (next)
- next->pprev = pprev;
+ WRITE_ONCE(next->pprev, pprev);
}
static inline void hlist_del(struct hlist_node *n)
@@ -782,32 +792,32 @@ static inline void hlist_del_init(struct hlist_node *n)
static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
{
struct hlist_node *first = h->first;
- n->next = first;
+ WRITE_ONCE(n->next, first);
if (first)
- first->pprev = &n->next;
+ WRITE_ONCE(first->pprev, &n->next);
WRITE_ONCE(h->first, n);
- n->pprev = &h->first;
+ WRITE_ONCE(n->pprev, &h->first);
}
/* next must be != NULL */
static inline void hlist_add_before(struct hlist_node *n,
struct hlist_node *next)
{
- n->pprev = next->pprev;
- n->next = next;
- next->pprev = &n->next;
+ WRITE_ONCE(n->pprev, next->pprev);
+ WRITE_ONCE(n->next, next);
+ WRITE_ONCE(next->pprev, &n->next);
WRITE_ONCE(*(n->pprev), n);
}
static inline void hlist_add_behind(struct hlist_node *n,
struct hlist_node *prev)
{
- n->next = prev->next;
- prev->next = n;
- n->pprev = &prev->next;
+ WRITE_ONCE(n->next, prev->next);
+ WRITE_ONCE(prev->next, n);
+ WRITE_ONCE(n->pprev, &prev->next);
if (n->next)
- n->next->pprev = &n->next;
+ WRITE_ONCE(n->next->pprev, &n->next);
}
/* after that we'll appear to be on some hlist and hlist_del will work */