From f2bcc99a5e901a13b754648d1dbab60f4adf9375 Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Mon, 20 Nov 2023 15:26:31 +0100 Subject: mm/mempool: replace kmap_atomic() with kmap_local_page() kmap_atomic() has been deprecated in favor of kmap_local_page(). Therefore, replace kmap_atomic() with kmap_local_page(). kmap_atomic() is implemented like a kmap_local_page() which also disables page-faults and preemption (the latter only in !PREEMPT_RT kernels). The kernel virtual addresses returned by these two API are only valid in the context of the callers (i.e., they cannot be handed to other threads). With kmap_local_page() the mappings are per thread and CPU local like in kmap_atomic(); however, they can handle page-faults and can be called from any context (including interrupts). The tasks that call kmap_local_page() can be preempted and, when they are scheduled to run again, the kernel virtual addresses are restored and are still valid. The code blocks between the mappings and un-mappings don't rely on the above-mentioned side effects of kmap_atomic(), so that mere replacements of the old API with the new one is all that they require (i.e., there is no need to explicitly call pagefault_disable() and/or preempt_disable()). Link: https://lkml.kernel.org/r/20231120142640.7077-1-fabio.maria.de.francesco@linux.intel.com Signed-off-by: Fabio M. De Francesco Cc: Ira Weiny Signed-off-by: Andrew Morton --- mm/mempool.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'mm/mempool.c') diff --git a/mm/mempool.c b/mm/mempool.c index 734bcf5afbb7..b3d2084fd989 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -64,10 +64,10 @@ static void check_element(mempool_t *pool, void *element) } else if (pool->free == mempool_free_pages) { /* Mempools backed by page allocator */ int order = (int)(long)pool->pool_data; - void *addr = kmap_atomic((struct page *)element); + void *addr = kmap_local_page((struct page *)element); __check_element(pool, addr, 1UL << (PAGE_SHIFT + order)); - kunmap_atomic(addr); + kunmap_local(addr); } } @@ -89,10 +89,10 @@ static void poison_element(mempool_t *pool, void *element) } else if (pool->alloc == mempool_alloc_pages) { /* Mempools backed by page allocator */ int order = (int)(long)pool->pool_data; - void *addr = kmap_atomic((struct page *)element); + void *addr = kmap_local_page((struct page *)element); __poison_element(addr, 1UL << (PAGE_SHIFT + order)); - kunmap_atomic(addr); + kunmap_local(addr); } } #else /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */ -- cgit From 280ec6ccb6422aa4a04f9ac4216ddcf055acc95d Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Tue, 19 Dec 2023 23:28:45 +0100 Subject: kasan: rename kasan_slab_free_mempool to kasan_mempool_poison_object Patch series "kasan: save mempool stack traces". This series updates KASAN to save alloc and free stack traces for secondary-level allocators that cache and reuse allocations internally instead of giving them back to the underlying allocator (e.g. mempool). As a part of this change, introduce and document a set of KASAN hooks: bool kasan_mempool_poison_pages(struct page *page, unsigned int order); void kasan_mempool_unpoison_pages(struct page *page, unsigned int order); bool kasan_mempool_poison_object(void *ptr); void kasan_mempool_unpoison_object(void *ptr, size_t size); and use them in the mempool code. Besides mempool, skbuff and io_uring also cache allocations and already use KASAN hooks to poison those. Their code is updated to use the new mempool hooks. The new hooks save alloc and free stack traces (for normal kmalloc and slab objects; stack traces for large kmalloc objects and page_alloc are not supported by KASAN yet), improve the readability of the users' code, and also allow the users to prevent double-free and invalid-free bugs; see the patches for the details. This patch (of 21): Rename kasan_slab_free_mempool to kasan_mempool_poison_object. kasan_slab_free_mempool is a slightly confusing name: it is unclear whether this function poisons the object when it is freed into mempool or does something when the object is freed from mempool to the underlying allocator. The new name also aligns with other mempool-related KASAN hooks added in the following patches in this series. Link: https://lkml.kernel.org/r/cover.1703024586.git.andreyknvl@google.com Link: https://lkml.kernel.org/r/c5618685abb7cdbf9fb4897f565e7759f601da84.1703024586.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Alexander Lobakin Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Breno Leitao Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Marco Elver Signed-off-by: Andrew Morton --- mm/mempool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mm/mempool.c') diff --git a/mm/mempool.c b/mm/mempool.c index b3d2084fd989..7e1c729f292b 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -107,7 +107,7 @@ static inline void poison_element(mempool_t *pool, void *element) static __always_inline void kasan_poison_element(mempool_t *pool, void *element) { if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) - kasan_slab_free_mempool(element); + kasan_mempool_poison_object(element); else if (pool->alloc == mempool_alloc_pages) kasan_poison_pages(element, (unsigned long)pool->pool_data, false); -- cgit From 7d4847ded24775a01cbe1e1a5292f132d27f158b Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Tue, 19 Dec 2023 23:28:57 +0100 Subject: mempool: skip slub_debug poisoning when KASAN is enabled With the changes in the following patch, KASAN starts saving its metadata within freed mempool elements. Thus, skip slub_debug poisoning and checking of mempool elements when KASAN is enabled. Corruptions of freed mempool elements will be detected by KASAN anyway. Link: https://lkml.kernel.org/r/98a4b1617e8ceeb266ef9a46f5e8c7f67a563ad2.1703024586.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Alexander Lobakin Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Breno Leitao Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Marco Elver Signed-off-by: Andrew Morton --- mm/mempool.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'mm/mempool.c') diff --git a/mm/mempool.c b/mm/mempool.c index 7e1c729f292b..1fd39478c85e 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -56,6 +56,10 @@ static void __check_element(mempool_t *pool, void *element, size_t size) static void check_element(mempool_t *pool, void *element) { + /* Skip checking: KASAN might save its metadata in the element. */ + if (kasan_enabled()) + return; + /* Mempools backed by slab allocator */ if (pool->free == mempool_kfree) { __check_element(pool, element, (size_t)pool->pool_data); @@ -81,6 +85,10 @@ static void __poison_element(void *element, size_t size) static void poison_element(mempool_t *pool, void *element) { + /* Skip poisoning: KASAN might save its metadata in the element. */ + if (kasan_enabled()) + return; + /* Mempools backed by slab allocator */ if (pool->alloc == mempool_kmalloc) { __poison_element(element, (size_t)pool->pool_data); -- cgit From 413643f3a3e2fe378dbd9f078ec119abc4539a38 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Tue, 19 Dec 2023 23:28:58 +0100 Subject: mempool: use new mempool KASAN hooks Update the mempool code to use the new mempool KASAN hooks. Rely on the return value of kasan_mempool_poison_object and kasan_mempool_poison_pages to prevent double-free and invalid-free bugs. Link: https://lkml.kernel.org/r/d36fc4a6865bdbd297cadb46b67641d436849f4c.1703024586.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Alexander Lobakin Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Breno Leitao Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Marco Elver Signed-off-by: Andrew Morton --- mm/mempool.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'mm/mempool.c') diff --git a/mm/mempool.c b/mm/mempool.c index 1fd39478c85e..103dc4770cfb 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -112,32 +112,34 @@ static inline void poison_element(mempool_t *pool, void *element) } #endif /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */ -static __always_inline void kasan_poison_element(mempool_t *pool, void *element) +static __always_inline bool kasan_poison_element(mempool_t *pool, void *element) { if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) - kasan_mempool_poison_object(element); + return kasan_mempool_poison_object(element); else if (pool->alloc == mempool_alloc_pages) - kasan_poison_pages(element, (unsigned long)pool->pool_data, - false); + return kasan_mempool_poison_pages(element, + (unsigned long)pool->pool_data); + return true; } static void kasan_unpoison_element(mempool_t *pool, void *element) { if (pool->alloc == mempool_kmalloc) - kasan_unpoison_range(element, (size_t)pool->pool_data); + kasan_mempool_unpoison_object(element, (size_t)pool->pool_data); else if (pool->alloc == mempool_alloc_slab) - kasan_unpoison_range(element, kmem_cache_size(pool->pool_data)); + kasan_mempool_unpoison_object(element, + kmem_cache_size(pool->pool_data)); else if (pool->alloc == mempool_alloc_pages) - kasan_unpoison_pages(element, (unsigned long)pool->pool_data, - false); + kasan_mempool_unpoison_pages(element, + (unsigned long)pool->pool_data); } static __always_inline void add_element(mempool_t *pool, void *element) { BUG_ON(pool->curr_nr >= pool->min_nr); poison_element(pool, element); - kasan_poison_element(pool, element); - pool->elements[pool->curr_nr++] = element; + if (kasan_poison_element(pool, element)) + pool->elements[pool->curr_nr++] = element; } static void *remove_element(mempool_t *pool) -- cgit From 37dcc69ad17a008d2b720bdc39f070ef2a959430 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Tue, 19 Dec 2023 23:28:59 +0100 Subject: mempool: introduce mempool_use_prealloc_only Introduce a new mempool_alloc_preallocated API that asks the mempool to only use the elements preallocated during the mempool's creation when allocating and to not attempt allocating new ones from the underlying allocator. This API is required to test the KASAN poisoning/unpoisoning functionality in KASAN tests, but it might be also useful on its own. Link: https://lkml.kernel.org/r/a14d809dbdfd04cc33bcacc632fee2abd6b83c00.1703024586.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Alexander Lobakin Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Breno Leitao Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Marco Elver Signed-off-by: Andrew Morton --- mm/mempool.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'mm/mempool.c') diff --git a/mm/mempool.c b/mm/mempool.c index 103dc4770cfb..cb7b4b56cec1 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -456,6 +456,43 @@ repeat_alloc: } EXPORT_SYMBOL(mempool_alloc); +/** + * mempool_alloc_preallocated - allocate an element from preallocated elements + * belonging to a specific memory pool + * @pool: pointer to the memory pool which was allocated via + * mempool_create(). + * + * This function is similar to mempool_alloc, but it only attempts allocating + * an element from the preallocated elements. It does not sleep and immediately + * returns if no preallocated elements are available. + * + * Return: pointer to the allocated element or %NULL if no elements are + * available. + */ +void *mempool_alloc_preallocated(mempool_t *pool) +{ + void *element; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + if (likely(pool->curr_nr)) { + element = remove_element(pool); + spin_unlock_irqrestore(&pool->lock, flags); + /* paired with rmb in mempool_free(), read comment there */ + smp_wmb(); + /* + * Update the allocation stack trace as this is more useful + * for debugging. + */ + kmemleak_update_trace(element); + return element; + } + spin_unlock_irqrestore(&pool->lock, flags); + + return NULL; +} +EXPORT_SYMBOL(mempool_alloc_preallocated); + /** * mempool_free - return an element to the pool. * @element: pool element pointer. -- cgit