1 files changed, 40 insertions, 25 deletions

diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h
index 729793ae9712..d33ce159ef33 100644
--- a/io_uring/alloc_cache.h
+++ b/io_uring/alloc_cache.h
@@ -1,53 +1,68 @@
 #ifndef IOU_ALLOC_CACHE_H
 #define IOU_ALLOC_CACHE_H
 
+#include <linux/io_uring_types.h>
+
 /*
  * Don't allow the cache to grow beyond this size.
  */
-#define IO_ALLOC_CACHE_MAX	512
+#define IO_ALLOC_CACHE_MAX	128
+
+void io_alloc_cache_free(struct io_alloc_cache *cache,
+			 void (*free)(const void *));
+bool io_alloc_cache_init(struct io_alloc_cache *cache,
+			 unsigned max_nr, unsigned int size,
+			 unsigned int init_bytes);
 
-struct io_cache_entry {
-	struct hlist_node	node;
-};
+void *io_cache_alloc_new(struct io_alloc_cache *cache, gfp_t gfp);
 
 static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,
-				      struct io_cache_entry *entry)
+				      void *entry)
 {
-	if (cache->nr_cached < IO_ALLOC_CACHE_MAX) {
-		cache->nr_cached++;
-		hlist_add_head(&entry->node, &cache->list);
+	if (cache->nr_cached < cache->max_cached) {
+		if (!kasan_mempool_poison_object(entry))
+			return false;
+		cache->entries[cache->nr_cached++] = entry;
 		return true;
 	}
 	return false;
 }
 
-static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache)
+static inline void *io_alloc_cache_get(struct io_alloc_cache *cache)
 {
-	if (!hlist_empty(&cache->list)) {
-		struct hlist_node *node = cache->list.first;
+	if (cache->nr_cached) {
+		void *entry = cache->entries[--cache->nr_cached];
 
-		hlist_del(node);
-		return container_of(node, struct io_cache_entry, node);
+		/*
+		 * If KASAN is enabled, always clear the initial bytes that
+		 * must be zeroed post alloc, in case any of them overlap
+		 * with KASAN storage.
+		 */
+#if defined(CONFIG_KASAN)
+		kasan_mempool_unpoison_object(entry, cache->elem_size);
+		if (cache->init_clear)
+			memset(entry, 0, cache->init_clear);
+#endif
+		return entry;
 	}
 
 	return NULL;
 }
 
-static inline void io_alloc_cache_init(struct io_alloc_cache *cache)
+static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp)
 {
-	INIT_HLIST_HEAD(&cache->list);
-	cache->nr_cached = 0;
+	void *obj;
+
+	obj = io_alloc_cache_get(cache);
+	if (obj)
+		return obj;
+	return io_cache_alloc_new(cache, gfp);
 }
 
-static inline void io_alloc_cache_free(struct io_alloc_cache *cache,
-					void (*free)(struct io_cache_entry *))
+static inline void io_cache_free(struct io_alloc_cache *cache, void *obj)
 {
-	while (!hlist_empty(&cache->list)) {
-		struct hlist_node *node = cache->list.first;
-
-		hlist_del(node);
-		free(container_of(node, struct io_cache_entry, node));
-	}
-	cache->nr_cached = 0;
+	if (!io_alloc_cache_put(cache, obj))
+		kfree(obj);
 }
+
 #endif