From ac9380f6b8a6a908a9df023bf8a2bcdaf9d2d6cb Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 7 Apr 2021 20:57:31 -0700 Subject: percpu: fix a comment about the chunks ordering Since the commit 3e54097beb22 ("percpu: manage chunks based on contig_bits instead of free_bytes") chunks are sorted based on the size of the biggest continuous free area instead of the total number of free bytes. Update the corresponding comment to reflect this. Signed-off-by: Roman Gushchin Signed-off-by: Dennis Zhou --- mm/percpu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 23308113a5ff..2c459a0ee5e6 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -99,7 +99,10 @@ #include "percpu-internal.h" -/* the slots are sorted by free bytes left, 1-31 bytes share the same slot */ +/* + * The slots are sorted by the size of the biggest continuous free area. + * 1-31 bytes share the same slot. + */ #define PCPU_SLOT_BASE_SHIFT 5 /* chunks in slots below this are subject to being sidelined on failed alloc */ #define PCPU_SLOT_FAIL_THRESHOLD 3 -- cgit From 67c2669d69fb5ada0f3b5123fb6ebf6fef9faee5 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 7 Apr 2021 20:57:32 -0700 Subject: percpu: split __pcpu_balance_workfn() __pcpu_balance_workfn() became fairly big and hard to follow, but in fact it consists of two fully independent parts, responsible for the destruction of excessive free chunks and population of necessarily amount of free pages. In order to simplify the code and prepare for adding of a new functionality, split it in two functions: 1) pcpu_balance_free, 2) pcpu_balance_populated. Move the taking/releasing of the pcpu_alloc_mutex to an upper level to keep the current synchronization in place. Signed-off-by: Roman Gushchin Reviewed-by: Dennis Zhou Signed-off-by: Dennis Zhou --- mm/percpu.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 2c459a0ee5e6..61339b3d9337 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1933,31 +1933,22 @@ void __percpu *__alloc_reserved_percpu(size_t size, size_t align) } /** - * __pcpu_balance_workfn - manage the amount of free chunks and populated pages + * pcpu_balance_free - manage the amount of free chunks * @type: chunk type * - * Reclaim all fully free chunks except for the first one. This is also - * responsible for maintaining the pool of empty populated pages. However, - * it is possible that this is called when physical memory is scarce causing - * OOM killer to be triggered. We should avoid doing so until an actual - * allocation causes the failure as it is possible that requests can be - * serviced from already backed regions. + * Reclaim all fully free chunks except for the first one. */ -static void __pcpu_balance_workfn(enum pcpu_chunk_type type) +static void pcpu_balance_free(enum pcpu_chunk_type type) { - /* gfp flags passed to underlying allocators */ - const gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; LIST_HEAD(to_free); struct list_head *pcpu_slot = pcpu_chunk_list(type); struct list_head *free_head = &pcpu_slot[pcpu_nr_slots - 1]; struct pcpu_chunk *chunk, *next; - int slot, nr_to_pop, ret; /* * There's no reason to keep around multiple unused chunks and VM * areas can be scarce. Destroy all free chunks except for one. */ - mutex_lock(&pcpu_alloc_mutex); spin_lock_irq(&pcpu_lock); list_for_each_entry_safe(chunk, next, free_head, list) { @@ -1985,6 +1976,25 @@ static void __pcpu_balance_workfn(enum pcpu_chunk_type type) pcpu_destroy_chunk(chunk); cond_resched(); } +} + +/** + * pcpu_balance_populated - manage the amount of populated pages + * @type: chunk type + * + * Maintain a certain amount of populated pages to satisfy atomic allocations. + * It is possible that this is called when physical memory is scarce causing + * OOM killer to be triggered. We should avoid doing so until an actual + * allocation causes the failure as it is possible that requests can be + * serviced from already backed regions. + */ +static void pcpu_balance_populated(enum pcpu_chunk_type type) +{ + /* gfp flags passed to underlying allocators */ + const gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; + struct list_head *pcpu_slot = pcpu_chunk_list(type); + struct pcpu_chunk *chunk; + int slot, nr_to_pop, ret; /* * Ensure there are certain number of free populated pages for @@ -2054,22 +2064,24 @@ retry_pop: goto retry_pop; } } - - mutex_unlock(&pcpu_alloc_mutex); } /** * pcpu_balance_workfn - manage the amount of free chunks and populated pages * @work: unused * - * Call __pcpu_balance_workfn() for each chunk type. + * Call pcpu_balance_free() and pcpu_balance_populated() for each chunk type. */ static void pcpu_balance_workfn(struct work_struct *work) { enum pcpu_chunk_type type; - for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) - __pcpu_balance_workfn(type); + for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) { + mutex_lock(&pcpu_alloc_mutex); + pcpu_balance_free(type); + pcpu_balance_populated(type); + mutex_unlock(&pcpu_alloc_mutex); + } } /** -- cgit From 8ea2e1e35d1eb4c76290ff5d565a1bfd6c24f117 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 7 Apr 2021 20:57:35 -0700 Subject: percpu: factor out pcpu_check_block_hint() Factor out the pcpu_check_block_hint() helper, which will be useful in the future. The new function checks if the allocation can likely fit within the contig hint. Signed-off-by: Roman Gushchin Signed-off-by: Dennis Zhou Signed-off-by: Dennis Zhou --- mm/percpu.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 61339b3d9337..5edc7bd88133 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -306,6 +306,25 @@ static unsigned long pcpu_block_off_to_off(int index, int off) return index * PCPU_BITMAP_BLOCK_BITS + off; } +/** + * pcpu_check_block_hint - check against the contig hint + * @block: block of interest + * @bits: size of allocation + * @align: alignment of area (max PAGE_SIZE) + * + * Check to see if the allocation can fit in the block's contig hint. + * Note, a chunk uses the same hints as a block so this can also check against + * the chunk's contig hint. + */ +static bool pcpu_check_block_hint(struct pcpu_block_md *block, int bits, + size_t align) +{ + int bit_off = ALIGN(block->contig_hint_start, align) - + block->contig_hint_start; + + return bit_off + bits <= block->contig_hint; +} + /* * pcpu_next_hint - determine which hint to use * @block: block of interest @@ -1066,14 +1085,11 @@ static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits, int bit_off, bits, next_off; /* - * Check to see if the allocation can fit in the chunk's contig hint. - * This is an optimization to prevent scanning by assuming if it - * cannot fit in the global hint, there is memory pressure and creating - * a new chunk would happen soon. + * This is an optimization to prevent scanning by assuming if the + * allocation cannot fit in the global hint, there is memory pressure + * and creating a new chunk would happen soon. */ - bit_off = ALIGN(chunk_md->contig_hint_start, align) - - chunk_md->contig_hint_start; - if (bit_off + alloc_bits > chunk_md->contig_hint) + if (!pcpu_check_block_hint(chunk_md, alloc_bits, align)) return -1; bit_off = pcpu_next_hint(chunk_md, alloc_bits); -- cgit From 1c29a3ceaf5f02919e0a89119a70382581453dbb Mon Sep 17 00:00:00 2001 From: Dennis Zhou Date: Sun, 18 Apr 2021 22:44:16 +0000 Subject: percpu: use pcpu_free_slot instead of pcpu_nr_slots - 1 This prepares for adding a to_depopulate list and sidelined list after the free slot in the set of lists in pcpu_slot. Signed-off-by: Dennis Zhou Acked-by: Roman Gushchin Signed-off-by: Dennis Zhou --- mm/percpu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 5edc7bd88133..d462222f4adc 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -135,6 +135,7 @@ static int pcpu_unit_size __ro_after_init; static int pcpu_nr_units __ro_after_init; static int pcpu_atom_size __ro_after_init; int pcpu_nr_slots __ro_after_init; +int pcpu_free_slot __ro_after_init; static size_t pcpu_chunk_struct_size __ro_after_init; /* cpus with the lowest and highest unit addresses */ @@ -237,7 +238,7 @@ static int __pcpu_size_to_slot(int size) static int pcpu_size_to_slot(int size) { if (size == pcpu_unit_size) - return pcpu_nr_slots - 1; + return pcpu_free_slot; return __pcpu_size_to_slot(size); } @@ -1806,7 +1807,7 @@ restart: goto fail; } - if (list_empty(&pcpu_slot[pcpu_nr_slots - 1])) { + if (list_empty(&pcpu_slot[pcpu_free_slot])) { chunk = pcpu_create_chunk(type, pcpu_gfp); if (!chunk) { err = "failed to allocate new chunk"; @@ -1958,7 +1959,7 @@ static void pcpu_balance_free(enum pcpu_chunk_type type) { LIST_HEAD(to_free); struct list_head *pcpu_slot = pcpu_chunk_list(type); - struct list_head *free_head = &pcpu_slot[pcpu_nr_slots - 1]; + struct list_head *free_head = &pcpu_slot[pcpu_free_slot]; struct pcpu_chunk *chunk, *next; /* @@ -2033,7 +2034,7 @@ retry_pop: 0, PCPU_EMPTY_POP_PAGES_HIGH); } - for (slot = pcpu_size_to_slot(PAGE_SIZE); slot < pcpu_nr_slots; slot++) { + for (slot = pcpu_size_to_slot(PAGE_SIZE); slot <= pcpu_free_slot; slot++) { unsigned int nr_unpop = 0, rs, re; if (!nr_to_pop) @@ -2140,7 +2141,7 @@ void free_percpu(void __percpu *ptr) if (chunk->free_bytes == pcpu_unit_size) { struct pcpu_chunk *pos; - list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list) + list_for_each_entry(pos, &pcpu_slot[pcpu_free_slot], list) if (pos != chunk) { need_balance = true; break; @@ -2562,7 +2563,8 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, * Allocate chunk slots. The additional last slot is for * empty chunks. */ - pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2; + pcpu_free_slot = __pcpu_size_to_slot(pcpu_unit_size) + 1; + pcpu_nr_slots = pcpu_free_slot + 1; pcpu_chunk_lists = memblock_alloc(pcpu_nr_slots * sizeof(pcpu_chunk_lists[0]) * PCPU_NR_CHUNK_TYPES, -- cgit From f183324133ea535db4127f9fad3e19725ca88bf3 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 7 Apr 2021 20:57:36 -0700 Subject: percpu: implement partial chunk depopulation From Roman ("percpu: partial chunk depopulation"): In our [Facebook] production experience the percpu memory allocator is sometimes struggling with returning the memory to the system. A typical example is a creation of several thousands memory cgroups (each has several chunks of the percpu data used for vmstats, vmevents, ref counters etc). Deletion and complete releasing of these cgroups doesn't always lead to a shrinkage of the percpu memory, so that sometimes there are several GB's of memory wasted. The underlying problem is the fragmentation: to release an underlying chunk all percpu allocations should be released first. The percpu allocator tends to top up chunks to improve the utilization. It means new small-ish allocations (e.g. percpu ref counters) are placed onto almost filled old-ish chunks, effectively pinning them in memory. This patchset solves this problem by implementing a partial depopulation of percpu chunks: chunks with many empty pages are being asynchronously depopulated and the pages are returned to the system. To illustrate the problem the following script can be used: -- cd /sys/fs/cgroup mkdir percpu_test echo "+memory" > percpu_test/cgroup.subtree_control cat /proc/meminfo | grep Percpu for i in `seq 1 1000`; do mkdir percpu_test/cg_"${i}" for j in `seq 1 10`; do mkdir percpu_test/cg_"${i}"_"${j}" done done cat /proc/meminfo | grep Percpu for i in `seq 1 1000`; do for j in `seq 1 10`; do rmdir percpu_test/cg_"${i}"_"${j}" done done sleep 10 cat /proc/meminfo | grep Percpu for i in `seq 1 1000`; do rmdir percpu_test/cg_"${i}" done rmdir percpu_test -- It creates 11000 memory cgroups and removes every 10 out of 11. It prints the initial size of the percpu memory, the size after creating all cgroups and the size after deleting most of them. Results: vanilla: ./percpu_test.sh Percpu: 7488 kB Percpu: 481152 kB Percpu: 481152 kB with this patchset applied: ./percpu_test.sh Percpu: 7488 kB Percpu: 481408 kB Percpu: 135552 kB The total size of the percpu memory was reduced by more than 3.5 times. This patch: This patch implements partial depopulation of percpu chunks. As of now, a chunk can be depopulated only as a part of the final destruction, if there are no more outstanding allocations. However to minimize a memory waste it might be useful to depopulate a partially filed chunk, if a small number of outstanding allocations prevents the chunk from being fully reclaimed. This patch implements the following depopulation process: it scans over the chunk pages, looks for a range of empty and populated pages and performs the depopulation. To avoid races with new allocations, the chunk is previously isolated. After the depopulation the chunk is sidelined to a special list or freed. New allocations prefer using active chunks to sidelined chunks. If a sidelined chunk is used, it is reintegrated to the active lists. The depopulation is scheduled on the free path if the chunk is all of the following: 1) has more than 1/4 of total pages free and populated 2) the system has enough free percpu pages aside of this chunk 3) isn't the reserved chunk 4) isn't the first chunk If it's already depopulated but got free populated pages, it's a good target too. The chunk is moved to a special slot, pcpu_to_depopulate_slot, chunk->isolated is set, and the balance work item is scheduled. On isolation, these pages are removed from the pcpu_nr_empty_pop_pages. It is constantly replaced to the to_depopulate_slot when it meets these qualifications. pcpu_reclaim_populated() iterates over the to_depopulate_slot until it becomes empty. The depopulation is performed in the reverse direction to keep populated pages close to the beginning. Depopulated chunks are sidelined to preferentially avoid them for new allocations. When no active chunk can suffice a new allocation, sidelined chunks are first checked before creating a new chunk. Signed-off-by: Roman Gushchin Co-developed-by: Dennis Zhou Signed-off-by: Dennis Zhou Tested-by: Pratik Sampat Signed-off-by: Dennis Zhou --- mm/percpu.c | 180 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 15 deletions(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index d462222f4adc..79eebc80860d 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -136,6 +136,8 @@ static int pcpu_nr_units __ro_after_init; static int pcpu_atom_size __ro_after_init; int pcpu_nr_slots __ro_after_init; int pcpu_free_slot __ro_after_init; +int pcpu_sidelined_slot __ro_after_init; +int pcpu_to_depopulate_slot __ro_after_init; static size_t pcpu_chunk_struct_size __ro_after_init; /* cpus with the lowest and highest unit addresses */ @@ -562,10 +564,41 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) { int nslot = pcpu_chunk_slot(chunk); + /* leave isolated chunks in-place */ + if (chunk->isolated) + return; + if (oslot != nslot) __pcpu_chunk_move(chunk, nslot, oslot < nslot); } +static void pcpu_isolate_chunk(struct pcpu_chunk *chunk) +{ + enum pcpu_chunk_type type = pcpu_chunk_type(chunk); + struct list_head *pcpu_slot = pcpu_chunk_list(type); + + lockdep_assert_held(&pcpu_lock); + + if (!chunk->isolated) { + chunk->isolated = true; + pcpu_nr_empty_pop_pages[type] -= chunk->nr_empty_pop_pages; + } + list_move(&chunk->list, &pcpu_slot[pcpu_to_depopulate_slot]); +} + +static void pcpu_reintegrate_chunk(struct pcpu_chunk *chunk) +{ + enum pcpu_chunk_type type = pcpu_chunk_type(chunk); + + lockdep_assert_held(&pcpu_lock); + + if (chunk->isolated) { + chunk->isolated = false; + pcpu_nr_empty_pop_pages[type] += chunk->nr_empty_pop_pages; + pcpu_chunk_relocate(chunk, -1); + } +} + /* * pcpu_update_empty_pages - update empty page counters * @chunk: chunk of interest @@ -578,7 +611,7 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr) { chunk->nr_empty_pop_pages += nr; - if (chunk != pcpu_reserved_chunk) + if (chunk != pcpu_reserved_chunk && !chunk->isolated) pcpu_nr_empty_pop_pages[pcpu_chunk_type(chunk)] += nr; } @@ -1778,7 +1811,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, restart: /* search through normal chunks */ - for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { + for (slot = pcpu_size_to_slot(size); slot <= pcpu_free_slot; slot++) { list_for_each_entry_safe(chunk, next, &pcpu_slot[slot], list) { off = pcpu_find_block_fit(chunk, bits, bit_align, is_atomic); @@ -1789,9 +1822,10 @@ restart: } off = pcpu_alloc_area(chunk, bits, bit_align, off); - if (off >= 0) + if (off >= 0) { + pcpu_reintegrate_chunk(chunk); goto area_found; - + } } } @@ -1952,10 +1986,13 @@ void __percpu *__alloc_reserved_percpu(size_t size, size_t align) /** * pcpu_balance_free - manage the amount of free chunks * @type: chunk type + * @empty_only: free chunks only if there are no populated pages * - * Reclaim all fully free chunks except for the first one. + * If empty_only is %false, reclaim all fully free chunks regardless of the + * number of populated pages. Otherwise, only reclaim chunks that have no + * populated pages. */ -static void pcpu_balance_free(enum pcpu_chunk_type type) +static void pcpu_balance_free(enum pcpu_chunk_type type, bool empty_only) { LIST_HEAD(to_free); struct list_head *pcpu_slot = pcpu_chunk_list(type); @@ -1975,7 +2012,8 @@ static void pcpu_balance_free(enum pcpu_chunk_type type) if (chunk == list_first_entry(free_head, struct pcpu_chunk, list)) continue; - list_move(&chunk->list, &to_free); + if (!empty_only || chunk->nr_empty_pop_pages == 0) + list_move(&chunk->list, &to_free); } spin_unlock_irq(&pcpu_lock); @@ -2083,20 +2121,121 @@ retry_pop: } } +/** + * pcpu_reclaim_populated - scan over to_depopulate chunks and free empty pages + * @type: chunk type + * + * Scan over chunks in the depopulate list and try to release unused populated + * pages back to the system. Depopulated chunks are sidelined to prevent + * repopulating these pages unless required. Fully free chunks are reintegrated + * and freed accordingly (1 is kept around). If we drop below the empty + * populated pages threshold, reintegrate the chunk if it has empty free pages. + * Each chunk is scanned in the reverse order to keep populated pages close to + * the beginning of the chunk. + */ +static void pcpu_reclaim_populated(enum pcpu_chunk_type type) +{ + struct list_head *pcpu_slot = pcpu_chunk_list(type); + struct pcpu_chunk *chunk; + struct pcpu_block_md *block; + int i, end; + + spin_lock_irq(&pcpu_lock); + +restart: + /* + * Once a chunk is isolated to the to_depopulate list, the chunk is no + * longer discoverable to allocations whom may populate pages. The only + * other accessor is the free path which only returns area back to the + * allocator not touching the populated bitmap. + */ + while (!list_empty(&pcpu_slot[pcpu_to_depopulate_slot])) { + chunk = list_first_entry(&pcpu_slot[pcpu_to_depopulate_slot], + struct pcpu_chunk, list); + WARN_ON(chunk->immutable); + + /* + * Scan chunk's pages in the reverse order to keep populated + * pages close to the beginning of the chunk. + */ + for (i = chunk->nr_pages - 1, end = -1; i >= 0; i--) { + /* no more work to do */ + if (chunk->nr_empty_pop_pages == 0) + break; + + /* reintegrate chunk to prevent atomic alloc failures */ + if (pcpu_nr_empty_pop_pages[type] < + PCPU_EMPTY_POP_PAGES_HIGH) { + pcpu_reintegrate_chunk(chunk); + goto restart; + } + + /* + * If the page is empty and populated, start or + * extend the (i, end) range. If i == 0, decrease + * i and perform the depopulation to cover the last + * (first) page in the chunk. + */ + block = chunk->md_blocks + i; + if (block->contig_hint == PCPU_BITMAP_BLOCK_BITS && + test_bit(i, chunk->populated)) { + if (end == -1) + end = i; + if (i > 0) + continue; + i--; + } + + /* depopulate if there is an active range */ + if (end == -1) + continue; + + spin_unlock_irq(&pcpu_lock); + pcpu_depopulate_chunk(chunk, i + 1, end + 1); + cond_resched(); + spin_lock_irq(&pcpu_lock); + + pcpu_chunk_depopulated(chunk, i + 1, end + 1); + + /* reset the range and continue */ + end = -1; + } + + if (chunk->free_bytes == pcpu_unit_size) + pcpu_reintegrate_chunk(chunk); + else + list_move(&chunk->list, + &pcpu_slot[pcpu_sidelined_slot]); + } + + spin_unlock_irq(&pcpu_lock); +} + /** * pcpu_balance_workfn - manage the amount of free chunks and populated pages * @work: unused * - * Call pcpu_balance_free() and pcpu_balance_populated() for each chunk type. + * For each chunk type, manage the number of fully free chunks and the number of + * populated pages. An important thing to consider is when pages are freed and + * how they contribute to the global counts. */ static void pcpu_balance_workfn(struct work_struct *work) { enum pcpu_chunk_type type; + /* + * pcpu_balance_free() is called twice because the first time we may + * trim pages in the active pcpu_nr_empty_pop_pages which may cause us + * to grow other chunks. This then gives pcpu_reclaim_populated() time + * to move fully free chunks to the active list to be freed if + * appropriate. + */ for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) { mutex_lock(&pcpu_alloc_mutex); - pcpu_balance_free(type); + pcpu_balance_free(type, false); + pcpu_reclaim_populated(type); pcpu_balance_populated(type); + pcpu_balance_free(type, true); mutex_unlock(&pcpu_alloc_mutex); } } @@ -2137,8 +2276,12 @@ void free_percpu(void __percpu *ptr) pcpu_memcg_free_hook(chunk, off, size); - /* if there are more than one fully free chunks, wake up grim reaper */ - if (chunk->free_bytes == pcpu_unit_size) { + /* + * If there are more than one fully free chunks, wake up grim reaper. + * If the chunk is isolated, it may be in the process of being + * reclaimed. Let reclaim manage cleaning up of that chunk. + */ + if (!chunk->isolated && chunk->free_bytes == pcpu_unit_size) { struct pcpu_chunk *pos; list_for_each_entry(pos, &pcpu_slot[pcpu_free_slot], list) @@ -2146,6 +2289,9 @@ void free_percpu(void __percpu *ptr) need_balance = true; break; } + } else if (pcpu_should_reclaim_chunk(chunk)) { + pcpu_isolate_chunk(chunk); + need_balance = true; } trace_percpu_free_percpu(chunk->base_addr, off, ptr); @@ -2560,11 +2706,15 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, pcpu_stats_save_ai(ai); /* - * Allocate chunk slots. The additional last slot is for - * empty chunks. + * Allocate chunk slots. The slots after the active slots are: + * sidelined_slot - isolated, depopulated chunks + * free_slot - fully free chunks + * to_depopulate_slot - isolated, chunks to depopulate */ - pcpu_free_slot = __pcpu_size_to_slot(pcpu_unit_size) + 1; - pcpu_nr_slots = pcpu_free_slot + 1; + pcpu_sidelined_slot = __pcpu_size_to_slot(pcpu_unit_size) + 1; + pcpu_free_slot = pcpu_sidelined_slot + 1; + pcpu_to_depopulate_slot = pcpu_free_slot + 1; + pcpu_nr_slots = pcpu_to_depopulate_slot + 1; pcpu_chunk_lists = memblock_alloc(pcpu_nr_slots * sizeof(pcpu_chunk_lists[0]) * PCPU_NR_CHUNK_TYPES, -- cgit From 8d55ba5df39a2eb213da63911c1a75001e0717fc Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Fri, 14 May 2021 06:39:52 +0000 Subject: percpu: make symbol 'pcpu_free_slot' static The sparse tool complains as follows: mm/percpu.c:138:5: warning: symbol 'pcpu_free_slot' was not declared. Should it be static? This symbol is not used outside of percpu.c, so marks it static. Reported-by: Hulk Robot Signed-off-by: Wei Yongjun Signed-off-by: Dennis Zhou --- mm/percpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 79eebc80860d..3135e56ce8d4 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -135,7 +135,7 @@ static int pcpu_unit_size __ro_after_init; static int pcpu_nr_units __ro_after_init; static int pcpu_atom_size __ro_after_init; int pcpu_nr_slots __ro_after_init; -int pcpu_free_slot __ro_after_init; +static int pcpu_free_slot __ro_after_init; int pcpu_sidelined_slot __ro_after_init; int pcpu_to_depopulate_slot __ro_after_init; static size_t pcpu_chunk_struct_size __ro_after_init; -- cgit From faf65dde844affa9e360ccaa4bd231c2a04b87ea Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 2 Jun 2021 18:09:31 -0700 Subject: percpu: rework memcg accounting The current implementation of the memcg accounting of the percpu memory is based on the idea of having two separate sets of chunks for accounted and non-accounted memory. This approach has an advantage of not wasting any extra memory for memcg data for non-accounted chunks, however it complicates the code and leads to a higher chunks number due to a lower chunk utilization. Instead of having two chunk types it's possible to declare all* chunks memcg-aware unless the kernel memory accounting is disabled globally by a boot option. The size of objcg_array is usually small in comparison to chunks themselves (it obviously depends on the number of CPUs), so even if some chunk will have no accounted allocations, the memory waste isn't significant and will likely be compensated by a higher chunk utilization. Also, with time more and more percpu allocations will likely become accounted. * The first chunk is initialized before the memory cgroup subsystem, so we don't know for sure whether we need to allocate obj_cgroups. Because it's small, let's make it free for use. Then we don't need to allocate obj_cgroups for it. Signed-off-by: Roman Gushchin Signed-off-by: Dennis Zhou --- mm/percpu.c | 134 ++++++++++++++++++++++++------------------------------------ 1 file changed, 53 insertions(+), 81 deletions(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 3135e56ce8d4..e7b9ca82e9aa 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -179,10 +179,10 @@ struct list_head *pcpu_chunk_lists __ro_after_init; /* chunk list slots */ static LIST_HEAD(pcpu_map_extend_chunks); /* - * The number of empty populated pages by chunk type, protected by pcpu_lock. + * The number of empty populated pages, protected by pcpu_lock. * The reserved chunk doesn't contribute to the count. */ -int pcpu_nr_empty_pop_pages[PCPU_NR_CHUNK_TYPES]; +int pcpu_nr_empty_pop_pages; /* * The number of populated pages in use by the allocator, protected by @@ -532,13 +532,10 @@ static void __pcpu_chunk_move(struct pcpu_chunk *chunk, int slot, bool move_front) { if (chunk != pcpu_reserved_chunk) { - struct list_head *pcpu_slot; - - pcpu_slot = pcpu_chunk_list(pcpu_chunk_type(chunk)); if (move_front) - list_move(&chunk->list, &pcpu_slot[slot]); + list_move(&chunk->list, &pcpu_chunk_lists[slot]); else - list_move_tail(&chunk->list, &pcpu_slot[slot]); + list_move_tail(&chunk->list, &pcpu_chunk_lists[slot]); } } @@ -574,27 +571,22 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) static void pcpu_isolate_chunk(struct pcpu_chunk *chunk) { - enum pcpu_chunk_type type = pcpu_chunk_type(chunk); - struct list_head *pcpu_slot = pcpu_chunk_list(type); - lockdep_assert_held(&pcpu_lock); if (!chunk->isolated) { chunk->isolated = true; - pcpu_nr_empty_pop_pages[type] -= chunk->nr_empty_pop_pages; + pcpu_nr_empty_pop_pages -= chunk->nr_empty_pop_pages; } - list_move(&chunk->list, &pcpu_slot[pcpu_to_depopulate_slot]); + list_move(&chunk->list, &pcpu_chunk_lists[pcpu_to_depopulate_slot]); } static void pcpu_reintegrate_chunk(struct pcpu_chunk *chunk) { - enum pcpu_chunk_type type = pcpu_chunk_type(chunk); - lockdep_assert_held(&pcpu_lock); if (chunk->isolated) { chunk->isolated = false; - pcpu_nr_empty_pop_pages[type] += chunk->nr_empty_pop_pages; + pcpu_nr_empty_pop_pages += chunk->nr_empty_pop_pages; pcpu_chunk_relocate(chunk, -1); } } @@ -612,7 +604,7 @@ static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr) { chunk->nr_empty_pop_pages += nr; if (chunk != pcpu_reserved_chunk && !chunk->isolated) - pcpu_nr_empty_pop_pages[pcpu_chunk_type(chunk)] += nr; + pcpu_nr_empty_pop_pages += nr; } /* @@ -1405,7 +1397,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, alloc_size); #ifdef CONFIG_MEMCG_KMEM - /* first chunk isn't memcg-aware */ + /* first chunk is free to use */ chunk->obj_cgroups = NULL; #endif pcpu_init_md_blocks(chunk); @@ -1447,7 +1439,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, return chunk; } -static struct pcpu_chunk *pcpu_alloc_chunk(enum pcpu_chunk_type type, gfp_t gfp) +static struct pcpu_chunk *pcpu_alloc_chunk(gfp_t gfp) { struct pcpu_chunk *chunk; int region_bits; @@ -1476,7 +1468,7 @@ static struct pcpu_chunk *pcpu_alloc_chunk(enum pcpu_chunk_type type, gfp_t gfp) goto md_blocks_fail; #ifdef CONFIG_MEMCG_KMEM - if (pcpu_is_memcg_chunk(type)) { + if (!mem_cgroup_kmem_disabled()) { chunk->obj_cgroups = pcpu_mem_zalloc(pcpu_chunk_map_bits(chunk) * sizeof(struct obj_cgroup *), gfp); @@ -1589,8 +1581,7 @@ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int page_start, int page_end, gfp_t gfp); static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int page_start, int page_end); -static struct pcpu_chunk *pcpu_create_chunk(enum pcpu_chunk_type type, - gfp_t gfp); +static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp); static void pcpu_destroy_chunk(struct pcpu_chunk *chunk); static struct page *pcpu_addr_to_page(void *addr); static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai); @@ -1633,25 +1624,25 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr) } #ifdef CONFIG_MEMCG_KMEM -static enum pcpu_chunk_type pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp, - struct obj_cgroup **objcgp) +static bool pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp, + struct obj_cgroup **objcgp) { struct obj_cgroup *objcg; if (!memcg_kmem_enabled() || !(gfp & __GFP_ACCOUNT)) - return PCPU_CHUNK_ROOT; + return true; objcg = get_obj_cgroup_from_current(); if (!objcg) - return PCPU_CHUNK_ROOT; + return true; if (obj_cgroup_charge(objcg, gfp, size * num_possible_cpus())) { obj_cgroup_put(objcg); - return PCPU_FAIL_ALLOC; + return false; } *objcgp = objcg; - return PCPU_CHUNK_MEMCG; + return true; } static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg, @@ -1661,7 +1652,7 @@ static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg, if (!objcg) return; - if (chunk) { + if (likely(chunk && chunk->obj_cgroups)) { chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = objcg; rcu_read_lock(); @@ -1678,10 +1669,12 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) { struct obj_cgroup *objcg; - if (!pcpu_is_memcg_chunk(pcpu_chunk_type(chunk))) + if (unlikely(!chunk->obj_cgroups)) return; objcg = chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT]; + if (!objcg) + return; chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = NULL; obj_cgroup_uncharge(objcg, size * num_possible_cpus()); @@ -1695,10 +1688,10 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) } #else /* CONFIG_MEMCG_KMEM */ -static enum pcpu_chunk_type +static bool pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp, struct obj_cgroup **objcgp) { - return PCPU_CHUNK_ROOT; + return true; } static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg, @@ -1733,8 +1726,6 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, gfp_t pcpu_gfp; bool is_atomic; bool do_warn; - enum pcpu_chunk_type type; - struct list_head *pcpu_slot; struct obj_cgroup *objcg = NULL; static int warn_limit = 10; struct pcpu_chunk *chunk, *next; @@ -1770,10 +1761,8 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, return NULL; } - type = pcpu_memcg_pre_alloc_hook(size, gfp, &objcg); - if (unlikely(type == PCPU_FAIL_ALLOC)) + if (unlikely(!pcpu_memcg_pre_alloc_hook(size, gfp, &objcg))) return NULL; - pcpu_slot = pcpu_chunk_list(type); if (!is_atomic) { /* @@ -1812,7 +1801,8 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, restart: /* search through normal chunks */ for (slot = pcpu_size_to_slot(size); slot <= pcpu_free_slot; slot++) { - list_for_each_entry_safe(chunk, next, &pcpu_slot[slot], list) { + list_for_each_entry_safe(chunk, next, &pcpu_chunk_lists[slot], + list) { off = pcpu_find_block_fit(chunk, bits, bit_align, is_atomic); if (off < 0) { @@ -1841,8 +1831,8 @@ restart: goto fail; } - if (list_empty(&pcpu_slot[pcpu_free_slot])) { - chunk = pcpu_create_chunk(type, pcpu_gfp); + if (list_empty(&pcpu_chunk_lists[pcpu_free_slot])) { + chunk = pcpu_create_chunk(pcpu_gfp); if (!chunk) { err = "failed to allocate new chunk"; goto fail; @@ -1886,7 +1876,7 @@ area_found: mutex_unlock(&pcpu_alloc_mutex); } - if (pcpu_nr_empty_pop_pages[type] < PCPU_EMPTY_POP_PAGES_LOW) + if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW) pcpu_schedule_balance_work(); /* clear the areas and return address relative to base address */ @@ -1985,18 +1975,16 @@ void __percpu *__alloc_reserved_percpu(size_t size, size_t align) /** * pcpu_balance_free - manage the amount of free chunks - * @type: chunk type * @empty_only: free chunks only if there are no populated pages * * If empty_only is %false, reclaim all fully free chunks regardless of the * number of populated pages. Otherwise, only reclaim chunks that have no * populated pages. */ -static void pcpu_balance_free(enum pcpu_chunk_type type, bool empty_only) +static void pcpu_balance_free(bool empty_only) { LIST_HEAD(to_free); - struct list_head *pcpu_slot = pcpu_chunk_list(type); - struct list_head *free_head = &pcpu_slot[pcpu_free_slot]; + struct list_head *free_head = &pcpu_chunk_lists[pcpu_free_slot]; struct pcpu_chunk *chunk, *next; /* @@ -2035,7 +2023,6 @@ static void pcpu_balance_free(enum pcpu_chunk_type type, bool empty_only) /** * pcpu_balance_populated - manage the amount of populated pages - * @type: chunk type * * Maintain a certain amount of populated pages to satisfy atomic allocations. * It is possible that this is called when physical memory is scarce causing @@ -2043,11 +2030,10 @@ static void pcpu_balance_free(enum pcpu_chunk_type type, bool empty_only) * allocation causes the failure as it is possible that requests can be * serviced from already backed regions. */ -static void pcpu_balance_populated(enum pcpu_chunk_type type) +static void pcpu_balance_populated(void) { /* gfp flags passed to underlying allocators */ const gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; - struct list_head *pcpu_slot = pcpu_chunk_list(type); struct pcpu_chunk *chunk; int slot, nr_to_pop, ret; @@ -2068,7 +2054,7 @@ retry_pop: pcpu_atomic_alloc_failed = false; } else { nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH - - pcpu_nr_empty_pop_pages[type], + pcpu_nr_empty_pop_pages, 0, PCPU_EMPTY_POP_PAGES_HIGH); } @@ -2079,7 +2065,7 @@ retry_pop: break; spin_lock_irq(&pcpu_lock); - list_for_each_entry(chunk, &pcpu_slot[slot], list) { + list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) { nr_unpop = chunk->nr_pages - chunk->nr_populated; if (nr_unpop) break; @@ -2111,7 +2097,7 @@ retry_pop: if (nr_to_pop) { /* ran out of chunks to populate, create a new one and retry */ - chunk = pcpu_create_chunk(type, gfp); + chunk = pcpu_create_chunk(gfp); if (chunk) { spin_lock_irq(&pcpu_lock); pcpu_chunk_relocate(chunk, -1); @@ -2123,7 +2109,6 @@ retry_pop: /** * pcpu_reclaim_populated - scan over to_depopulate chunks and free empty pages - * @type: chunk type * * Scan over chunks in the depopulate list and try to release unused populated * pages back to the system. Depopulated chunks are sidelined to prevent @@ -2133,9 +2118,8 @@ retry_pop: * Each chunk is scanned in the reverse order to keep populated pages close to * the beginning of the chunk. */ -static void pcpu_reclaim_populated(enum pcpu_chunk_type type) +static void pcpu_reclaim_populated(void) { - struct list_head *pcpu_slot = pcpu_chunk_list(type); struct pcpu_chunk *chunk; struct pcpu_block_md *block; int i, end; @@ -2149,8 +2133,8 @@ restart: * other accessor is the free path which only returns area back to the * allocator not touching the populated bitmap. */ - while (!list_empty(&pcpu_slot[pcpu_to_depopulate_slot])) { - chunk = list_first_entry(&pcpu_slot[pcpu_to_depopulate_slot], + while (!list_empty(&pcpu_chunk_lists[pcpu_to_depopulate_slot])) { + chunk = list_first_entry(&pcpu_chunk_lists[pcpu_to_depopulate_slot], struct pcpu_chunk, list); WARN_ON(chunk->immutable); @@ -2164,8 +2148,7 @@ restart: break; /* reintegrate chunk to prevent atomic alloc failures */ - if (pcpu_nr_empty_pop_pages[type] < - PCPU_EMPTY_POP_PAGES_HIGH) { + if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_HIGH) { pcpu_reintegrate_chunk(chunk); goto restart; } @@ -2205,7 +2188,7 @@ restart: pcpu_reintegrate_chunk(chunk); else list_move(&chunk->list, - &pcpu_slot[pcpu_sidelined_slot]); + &pcpu_chunk_lists[pcpu_sidelined_slot]); } spin_unlock_irq(&pcpu_lock); @@ -2221,8 +2204,6 @@ restart: */ static void pcpu_balance_workfn(struct work_struct *work) { - enum pcpu_chunk_type type; - /* * pcpu_balance_free() is called twice because the first time we may * trim pages in the active pcpu_nr_empty_pop_pages which may cause us @@ -2230,14 +2211,12 @@ static void pcpu_balance_workfn(struct work_struct *work) * to move fully free chunks to the active list to be freed if * appropriate. */ - for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) { - mutex_lock(&pcpu_alloc_mutex); - pcpu_balance_free(type, false); - pcpu_reclaim_populated(type); - pcpu_balance_populated(type); - pcpu_balance_free(type, true); - mutex_unlock(&pcpu_alloc_mutex); - } + mutex_lock(&pcpu_alloc_mutex); + pcpu_balance_free(false); + pcpu_reclaim_populated(); + pcpu_balance_populated(); + pcpu_balance_free(true); + mutex_unlock(&pcpu_alloc_mutex); } /** @@ -2256,7 +2235,6 @@ void free_percpu(void __percpu *ptr) unsigned long flags; int size, off; bool need_balance = false; - struct list_head *pcpu_slot; if (!ptr) return; @@ -2272,8 +2250,6 @@ void free_percpu(void __percpu *ptr) size = pcpu_free_area(chunk, off); - pcpu_slot = pcpu_chunk_list(pcpu_chunk_type(chunk)); - pcpu_memcg_free_hook(chunk, off, size); /* @@ -2284,7 +2260,7 @@ void free_percpu(void __percpu *ptr) if (!chunk->isolated && chunk->free_bytes == pcpu_unit_size) { struct pcpu_chunk *pos; - list_for_each_entry(pos, &pcpu_slot[pcpu_free_slot], list) + list_for_each_entry(pos, &pcpu_chunk_lists[pcpu_free_slot], list) if (pos != chunk) { need_balance = true; break; @@ -2592,7 +2568,6 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, int map_size; unsigned long tmp_addr; size_t alloc_size; - enum pcpu_chunk_type type; #define PCPU_SETUP_BUG_ON(cond) do { \ if (unlikely(cond)) { \ @@ -2716,17 +2691,14 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, pcpu_to_depopulate_slot = pcpu_free_slot + 1; pcpu_nr_slots = pcpu_to_depopulate_slot + 1; pcpu_chunk_lists = memblock_alloc(pcpu_nr_slots * - sizeof(pcpu_chunk_lists[0]) * - PCPU_NR_CHUNK_TYPES, + sizeof(pcpu_chunk_lists[0]), SMP_CACHE_BYTES); if (!pcpu_chunk_lists) panic("%s: Failed to allocate %zu bytes\n", __func__, - pcpu_nr_slots * sizeof(pcpu_chunk_lists[0]) * - PCPU_NR_CHUNK_TYPES); + pcpu_nr_slots * sizeof(pcpu_chunk_lists[0])); - for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) - for (i = 0; i < pcpu_nr_slots; i++) - INIT_LIST_HEAD(&pcpu_chunk_list(type)[i]); + for (i = 0; i < pcpu_nr_slots; i++) + INIT_LIST_HEAD(&pcpu_chunk_lists[i]); /* * The end of the static region needs to be aligned with the @@ -2763,7 +2735,7 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, /* link the first chunk in */ pcpu_first_chunk = chunk; - pcpu_nr_empty_pop_pages[PCPU_CHUNK_ROOT] = pcpu_first_chunk->nr_empty_pop_pages; + pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages; pcpu_chunk_relocate(pcpu_first_chunk, -1); /* include all regions of the first chunk */ -- cgit From 4829c791b22f98f95339248a428caf08b5f1e3e5 Mon Sep 17 00:00:00 2001 From: Dennis Zhou Date: Mon, 14 Jun 2021 14:42:05 +0000 Subject: percpu: initialize best_upa variable Tom reported this finding from clang 10's static analysis [1]. Due to the way the code is written, it will always see a successful loop iteration. Instead of setting an initial value, check that it was set instead with BUG_ON() because 0 units per allocation is bogus. [1] https://lore.kernel.org/lkml/20210515180817.1751084-1-trix@redhat.com/ Reported-by: Tom Rix Signed-off-by: Dennis Zhou --- mm/percpu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index e7b9ca82e9aa..6667dfa0fcec 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -2888,6 +2888,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( * Related to atom_size, which could be much larger than the unit_size. */ last_allocs = INT_MAX; + best_upa = 0; for (upa = max_upa; upa; upa--) { int allocs = 0, wasted = 0; @@ -2914,6 +2915,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( last_allocs = allocs; best_upa = upa; } + BUG_ON(!best_upa); upa = best_upa; /* allocate and fill alloc_info */ -- cgit From e4d777003a43feab2e000749163e531f6c48c385 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Thu, 17 Jun 2021 12:03:22 -0700 Subject: percpu: optimize locking in pcpu_balance_workfn() pcpu_balance_workfn() unconditionally calls pcpu_balance_free(), pcpu_reclaim_populated(), pcpu_balance_populated() and pcpu_balance_free() again. Each call to pcpu_balance_free() and pcpu_reclaim_populated() will cause at least one acquisition of the pcpu_lock. So even if the balancing was scheduled because of a failed atomic allocation, pcpu_lock will be acquired at least 4 times. This obviously increases the contention on the pcpu_lock. To optimize the scheme let's grab the pcpu_lock on the upper level (in pcpu_balance_workfn()) and keep it generally locked for the whole duration of the scheduled work, but release conditionally to perform any slow operations like chunk (de)population and creation of new chunks. Signed-off-by: Roman Gushchin Signed-off-by: Dennis Zhou --- mm/percpu.c | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) (limited to 'mm/percpu.c') diff --git a/mm/percpu.c b/mm/percpu.c index 6667dfa0fcec..f4c83217f217 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1980,6 +1980,9 @@ void __percpu *__alloc_reserved_percpu(size_t size, size_t align) * If empty_only is %false, reclaim all fully free chunks regardless of the * number of populated pages. Otherwise, only reclaim chunks that have no * populated pages. + * + * CONTEXT: + * pcpu_lock (can be dropped temporarily) */ static void pcpu_balance_free(bool empty_only) { @@ -1987,12 +1990,12 @@ static void pcpu_balance_free(bool empty_only) struct list_head *free_head = &pcpu_chunk_lists[pcpu_free_slot]; struct pcpu_chunk *chunk, *next; + lockdep_assert_held(&pcpu_lock); + /* * There's no reason to keep around multiple unused chunks and VM * areas can be scarce. Destroy all free chunks except for one. */ - spin_lock_irq(&pcpu_lock); - list_for_each_entry_safe(chunk, next, free_head, list) { WARN_ON(chunk->immutable); @@ -2004,8 +2007,10 @@ static void pcpu_balance_free(bool empty_only) list_move(&chunk->list, &to_free); } - spin_unlock_irq(&pcpu_lock); + if (list_empty(&to_free)) + return; + spin_unlock_irq(&pcpu_lock); list_for_each_entry_safe(chunk, next, &to_free, list) { unsigned int rs, re; @@ -2019,6 +2024,7 @@ static void pcpu_balance_free(bool empty_only) pcpu_destroy_chunk(chunk); cond_resched(); } + spin_lock_irq(&pcpu_lock); } /** @@ -2029,6 +2035,9 @@ static void pcpu_balance_free(bool empty_only) * OOM killer to be triggered. We should avoid doing so until an actual * allocation causes the failure as it is possible that requests can be * serviced from already backed regions. + * + * CONTEXT: + * pcpu_lock (can be dropped temporarily) */ static void pcpu_balance_populated(void) { @@ -2037,6 +2046,8 @@ static void pcpu_balance_populated(void) struct pcpu_chunk *chunk; int slot, nr_to_pop, ret; + lockdep_assert_held(&pcpu_lock); + /* * Ensure there are certain number of free populated pages for * atomic allocs. Fill up from the most packed so that atomic @@ -2064,13 +2075,11 @@ retry_pop: if (!nr_to_pop) break; - spin_lock_irq(&pcpu_lock); list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) { nr_unpop = chunk->nr_pages - chunk->nr_populated; if (nr_unpop) break; } - spin_unlock_irq(&pcpu_lock); if (!nr_unpop) continue; @@ -2080,12 +2089,13 @@ retry_pop: chunk->nr_pages) { int nr = min_t(int, re - rs, nr_to_pop); + spin_unlock_irq(&pcpu_lock); ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp); + cond_resched(); + spin_lock_irq(&pcpu_lock); if (!ret) { nr_to_pop -= nr; - spin_lock_irq(&pcpu_lock); pcpu_chunk_populated(chunk, rs, rs + nr); - spin_unlock_irq(&pcpu_lock); } else { nr_to_pop = 0; } @@ -2097,11 +2107,12 @@ retry_pop: if (nr_to_pop) { /* ran out of chunks to populate, create a new one and retry */ + spin_unlock_irq(&pcpu_lock); chunk = pcpu_create_chunk(gfp); + cond_resched(); + spin_lock_irq(&pcpu_lock); if (chunk) { - spin_lock_irq(&pcpu_lock); pcpu_chunk_relocate(chunk, -1); - spin_unlock_irq(&pcpu_lock); goto retry_pop; } } @@ -2117,6 +2128,10 @@ retry_pop: * populated pages threshold, reintegrate the chunk if it has empty free pages. * Each chunk is scanned in the reverse order to keep populated pages close to * the beginning of the chunk. + * + * CONTEXT: + * pcpu_lock (can be dropped temporarily) + * */ static void pcpu_reclaim_populated(void) { @@ -2124,7 +2139,7 @@ static void pcpu_reclaim_populated(void) struct pcpu_block_md *block; int i, end; - spin_lock_irq(&pcpu_lock); + lockdep_assert_held(&pcpu_lock); restart: /* @@ -2190,8 +2205,6 @@ restart: list_move(&chunk->list, &pcpu_chunk_lists[pcpu_sidelined_slot]); } - - spin_unlock_irq(&pcpu_lock); } /** @@ -2212,10 +2225,14 @@ static void pcpu_balance_workfn(struct work_struct *work) * appropriate. */ mutex_lock(&pcpu_alloc_mutex); + spin_lock_irq(&pcpu_lock); + pcpu_balance_free(false); pcpu_reclaim_populated(); pcpu_balance_populated(); pcpu_balance_free(true); + + spin_unlock_irq(&pcpu_lock); mutex_unlock(&pcpu_alloc_mutex); } -- cgit