diff options
Diffstat (limited to 'mm/list_lru.c')
| -rw-r--r-- | mm/list_lru.c | 51 |
1 files changed, 29 insertions, 22 deletions
diff --git a/mm/list_lru.c b/mm/list_lru.c index f93ada6a207b..ec48b5dadf51 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -60,31 +60,34 @@ list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx) return &lru->node[nid].lru; } +static inline bool lock_list_lru(struct list_lru_one *l, bool irq) +{ + if (irq) + spin_lock_irq(&l->lock); + else + spin_lock(&l->lock); + if (unlikely(READ_ONCE(l->nr_items) == LONG_MIN)) { + if (irq) + spin_unlock_irq(&l->lock); + else + spin_unlock(&l->lock); + return false; + } + return true; +} + static inline struct list_lru_one * lock_list_lru_of_memcg(struct list_lru *lru, int nid, struct mem_cgroup *memcg, bool irq, bool skip_empty) { struct list_lru_one *l; - long nr_items; rcu_read_lock(); again: l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); - if (likely(l)) { - if (irq) - spin_lock_irq(&l->lock); - else - spin_lock(&l->lock); - nr_items = READ_ONCE(l->nr_items); - if (likely(nr_items != LONG_MIN)) { - WARN_ON(nr_items < 0); - rcu_read_unlock(); - return l; - } - if (irq) - spin_unlock_irq(&l->lock); - else - spin_unlock(&l->lock); + if (likely(l) && lock_list_lru(l, irq)) { + rcu_read_unlock(); + return l; } /* * Caller may simply bail out if raced with reparenting or @@ -450,6 +453,7 @@ static void memcg_reparent_list_lru_one(struct list_lru *lru, int nid, list_splice_init(&src->list, &dst->list); if (src->nr_items) { + WARN_ON(src->nr_items < 0); dst->nr_items += src->nr_items; set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru)); } @@ -510,7 +514,7 @@ int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru, gfp_t gfp) { unsigned long flags; - struct list_lru_memcg *mlru; + struct list_lru_memcg *mlru = NULL; struct mem_cgroup *pos, *parent; XA_STATE(xas, &lru->xa, 0); @@ -535,9 +539,11 @@ int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru, parent = parent_mem_cgroup(pos); } - mlru = memcg_init_list_lru_one(lru, gfp); - if (!mlru) - return -ENOMEM; + if (!mlru) { + mlru = memcg_init_list_lru_one(lru, gfp); + if (!mlru) + return -ENOMEM; + } xas_set(&xas, pos->kmemcg_id); do { xas_lock_irqsave(&xas, flags); @@ -548,10 +554,11 @@ int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru, } xas_unlock_irqrestore(&xas, flags); } while (xas_nomem(&xas, gfp)); - if (mlru) - kfree(mlru); } while (pos != memcg && !css_is_dying(&pos->css)); + if (unlikely(mlru)) + kfree(mlru); + return xas_error(&xas); } #else |