From 0fcb100d50835d6823723ef0898cd565b3796e0a Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 22 Jul 2022 09:38:21 +0000 Subject: dm bufio: Add flags argument to dm_bufio_client_create Add a flags argument to dm_bufio_client_create and update all the callers. This is in preparation to add the DM_BUFIO_NO_SLEEP flag. Signed-off-by: Nathan Huckleberry Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/md/dm-bufio.c') diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 5ffa1dcf84cf..ad5603eb12e3 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1717,7 +1717,8 @@ static unsigned long dm_bufio_shrink_count(struct shrinker *shrink, struct shrin struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned block_size, unsigned reserved_buffers, unsigned aux_size, void (*alloc_callback)(struct dm_buffer *), - void (*write_callback)(struct dm_buffer *)) + void (*write_callback)(struct dm_buffer *), + unsigned int flags) { int r; struct dm_bufio_client *c; -- cgit From b32d45824aa7e07a0c3257a16e3a2a691b11b39a Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Fri, 22 Jul 2022 09:38:22 +0000 Subject: dm bufio: Add DM_BUFIO_CLIENT_NO_SLEEP flag Add an optional flag that ensures dm_bufio_client does not sleep (primary focus is to service dm_bufio_get without sleeping). This allows the dm-bufio cache to be queried from interrupt context. To ensure that dm-bufio does not sleep, dm-bufio must use a spinlock instead of a mutex. Additionally, to avoid deadlocks, special care must be taken so that dm-bufio does not sleep while holding the spinlock. But again: the scope of this no_sleep is initially confined to dm_bufio_get, so __alloc_buffer_wait_no_callback is _not_ changed to avoid sleeping because __bufio_new avoids allocation for NF_GET. Signed-off-by: Nathan Huckleberry Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'drivers/md/dm-bufio.c') diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index ad5603eb12e3..486179d1831c 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -81,6 +81,8 @@ */ struct dm_bufio_client { struct mutex lock; + spinlock_t spinlock; + unsigned long spinlock_flags; struct list_head lru[LIST_SIZE]; unsigned long n_buffers[LIST_SIZE]; @@ -90,6 +92,7 @@ struct dm_bufio_client { s8 sectors_per_block_bits; void (*alloc_callback)(struct dm_buffer *); void (*write_callback)(struct dm_buffer *); + bool no_sleep; struct kmem_cache *slab_buffer; struct kmem_cache *slab_cache; @@ -167,17 +170,26 @@ struct dm_buffer { static void dm_bufio_lock(struct dm_bufio_client *c) { - mutex_lock_nested(&c->lock, dm_bufio_in_request()); + if (c->no_sleep) + spin_lock_irqsave_nested(&c->spinlock, c->spinlock_flags, dm_bufio_in_request()); + else + mutex_lock_nested(&c->lock, dm_bufio_in_request()); } static int dm_bufio_trylock(struct dm_bufio_client *c) { - return mutex_trylock(&c->lock); + if (c->no_sleep) + return spin_trylock_irqsave(&c->spinlock, c->spinlock_flags); + else + return mutex_trylock(&c->lock); } static void dm_bufio_unlock(struct dm_bufio_client *c) { - mutex_unlock(&c->lock); + if (c->no_sleep) + spin_unlock_irqrestore(&c->spinlock, c->spinlock_flags); + else + mutex_unlock(&c->lock); } /*----------------------------------------------------------------*/ @@ -1748,12 +1760,16 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign c->alloc_callback = alloc_callback; c->write_callback = write_callback; + if (flags & DM_BUFIO_CLIENT_NO_SLEEP) + c->no_sleep = true; + for (i = 0; i < LIST_SIZE; i++) { INIT_LIST_HEAD(&c->lru[i]); c->n_buffers[i] = 0; } mutex_init(&c->lock); + spin_lock_init(&c->spinlock); INIT_LIST_HEAD(&c->reserved_buffers); c->need_reserved_buffers = reserved_buffers; -- cgit From 3c1c875d058669b4dee68f877ec8a65ad4ae8b5b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 25 Jul 2022 18:26:48 -0400 Subject: dm bufio: conditionally enable branching for DM_BUFIO_CLIENT_NO_SLEEP Use jump_label to limit the need for branching unless the optional DM_BUFIO_CLIENT_NO_SLEEP is used. Signed-off-by: Mike Snitzer --- drivers/md/dm-bufio.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'drivers/md/dm-bufio.c') diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 486179d1831c..0a38a7aef49c 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -18,6 +18,7 @@ #include #include #include +#include #define DM_MSG_PREFIX "bufio" @@ -164,13 +165,15 @@ struct dm_buffer { #endif }; +static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled); + /*----------------------------------------------------------------*/ #define dm_bufio_in_request() (!!current->bio_list) static void dm_bufio_lock(struct dm_bufio_client *c) { - if (c->no_sleep) + if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) spin_lock_irqsave_nested(&c->spinlock, c->spinlock_flags, dm_bufio_in_request()); else mutex_lock_nested(&c->lock, dm_bufio_in_request()); @@ -178,7 +181,7 @@ static void dm_bufio_lock(struct dm_bufio_client *c) static int dm_bufio_trylock(struct dm_bufio_client *c) { - if (c->no_sleep) + if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) return spin_trylock_irqsave(&c->spinlock, c->spinlock_flags); else return mutex_trylock(&c->lock); @@ -186,7 +189,7 @@ static int dm_bufio_trylock(struct dm_bufio_client *c) static void dm_bufio_unlock(struct dm_bufio_client *c) { - if (c->no_sleep) + if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) spin_unlock_irqrestore(&c->spinlock, c->spinlock_flags); else mutex_unlock(&c->lock); @@ -1760,8 +1763,10 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign c->alloc_callback = alloc_callback; c->write_callback = write_callback; - if (flags & DM_BUFIO_CLIENT_NO_SLEEP) + if (flags & DM_BUFIO_CLIENT_NO_SLEEP) { c->no_sleep = true; + static_branch_inc(&no_sleep_enabled); + } for (i = 0; i < LIST_SIZE; i++) { INIT_LIST_HEAD(&c->lru[i]); @@ -1895,6 +1900,8 @@ void dm_bufio_client_destroy(struct dm_bufio_client *c) kmem_cache_destroy(c->slab_buffer); dm_io_client_destroy(c->dm_io); mutex_destroy(&c->lock); + if (c->no_sleep) + static_branch_dec(&no_sleep_enabled); kfree(c); } EXPORT_SYMBOL_GPL(dm_bufio_client_destroy); -- cgit