From 47757ea83a545536cdd418fec84b7a970710e48b Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 20 Nov 2023 15:29:09 +0000 Subject: netfs, fscache: Move fs/fscache/* into fs/netfs/ There's a problem with dependencies between netfslib and fscache as each wants to access some functions of the other. Deal with this by moving fs/fscache/* into fs/netfs/ and renaming those files to begin with "fscache-". For the moment, the moved files are changed as little as possible and an fscache module is still built. A subsequent patch will integrate them. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: Christian Brauner cc: linux-fsdevel@vger.kernel.org cc: linux-cachefs@redhat.com --- fs/Kconfig | 1 - fs/Makefile | 1 - fs/fscache/Kconfig | 40 -- fs/fscache/Makefile | 16 - fs/fscache/cache.c | 428 ---------------- fs/fscache/cookie.c | 1180 ------------------------------------------- fs/fscache/internal.h | 277 ---------- fs/fscache/io.c | 327 ------------ fs/fscache/main.c | 121 ----- fs/fscache/proc.c | 54 -- fs/fscache/stats.c | 102 ---- fs/fscache/volume.c | 520 ------------------- fs/netfs/Kconfig | 39 ++ fs/netfs/Makefile | 14 +- fs/netfs/fscache_cache.c | 428 ++++++++++++++++ fs/netfs/fscache_cookie.c | 1180 +++++++++++++++++++++++++++++++++++++++++++ fs/netfs/fscache_internal.h | 277 ++++++++++ fs/netfs/fscache_io.c | 327 ++++++++++++ fs/netfs/fscache_main.c | 121 +++++ fs/netfs/fscache_proc.c | 54 ++ fs/netfs/fscache_stats.c | 102 ++++ fs/netfs/fscache_volume.c | 520 +++++++++++++++++++ fs/netfs/internal.h | 5 + fs/netfs/main.c | 5 +- 24 files changed, 3069 insertions(+), 3070 deletions(-) delete mode 100644 fs/fscache/Kconfig delete mode 100644 fs/fscache/Makefile delete mode 100644 fs/fscache/cache.c delete mode 100644 fs/fscache/cookie.c delete mode 100644 fs/fscache/internal.h delete mode 100644 fs/fscache/io.c delete mode 100644 fs/fscache/main.c delete mode 100644 fs/fscache/proc.c delete mode 100644 fs/fscache/stats.c delete mode 100644 fs/fscache/volume.c create mode 100644 fs/netfs/fscache_cache.c create mode 100644 fs/netfs/fscache_cookie.c create mode 100644 fs/netfs/fscache_internal.h create mode 100644 fs/netfs/fscache_io.c create mode 100644 fs/netfs/fscache_main.c create mode 100644 fs/netfs/fscache_proc.c create mode 100644 fs/netfs/fscache_stats.c create mode 100644 fs/netfs/fscache_volume.c (limited to 'fs') diff --git a/fs/Kconfig b/fs/Kconfig index 42837617a55b..c935c341eb6e 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -140,7 +140,6 @@ source "fs/overlayfs/Kconfig" menu "Caches" source "fs/netfs/Kconfig" -source "fs/fscache/Kconfig" source "fs/cachefiles/Kconfig" endmenu diff --git a/fs/Makefile b/fs/Makefile index 75522f88e763..af7632368e98 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -60,7 +60,6 @@ obj-$(CONFIG_DLM) += dlm/ # Do not add any filesystems before this line obj-$(CONFIG_NETFS_SUPPORT) += netfs/ -obj-$(CONFIG_FSCACHE) += fscache/ obj-$(CONFIG_REISERFS_FS) += reiserfs/ obj-$(CONFIG_EXT4_FS) += ext4/ # We place ext4 before ext2 so that clean ext3 root fs's do NOT mount using the diff --git a/fs/fscache/Kconfig b/fs/fscache/Kconfig deleted file mode 100644 index b313a978ae0a..000000000000 --- a/fs/fscache/Kconfig +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -config FSCACHE - tristate "General filesystem local caching manager" - select NETFS_SUPPORT - help - This option enables a generic filesystem caching manager that can be - used by various network and other filesystems to cache data locally. - Different sorts of caches can be plugged in, depending on the - resources available. - - See Documentation/filesystems/caching/fscache.rst for more information. - -config FSCACHE_STATS - bool "Gather statistical information on local caching" - depends on FSCACHE && PROC_FS - select NETFS_STATS - help - This option causes statistical information to be gathered on local - caching and exported through file: - - /proc/fs/fscache/stats - - The gathering of statistics adds a certain amount of overhead to - execution as there are a quite a few stats gathered, and on a - multi-CPU system these may be on cachelines that keep bouncing - between CPUs. On the other hand, the stats are very useful for - debugging purposes. Saying 'Y' here is recommended. - - See Documentation/filesystems/caching/fscache.rst for more information. - -config FSCACHE_DEBUG - bool "Debug FS-Cache" - depends on FSCACHE - help - This permits debugging to be dynamically enabled in the local caching - management module. If this is set, the debugging output may be - enabled by setting bits in /sys/modules/fscache/parameter/debug. - - See Documentation/filesystems/caching/fscache.rst for more information. diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile deleted file mode 100644 index afb090ea16c4..000000000000 --- a/fs/fscache/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for general filesystem caching code -# - -fscache-y := \ - cache.o \ - cookie.o \ - io.o \ - main.o \ - volume.o - -fscache-$(CONFIG_PROC_FS) += proc.o -fscache-$(CONFIG_FSCACHE_STATS) += stats.o - -obj-$(CONFIG_FSCACHE) := fscache.o diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c deleted file mode 100644 index d645f8b302a2..000000000000 --- a/fs/fscache/cache.c +++ /dev/null @@ -1,428 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache cache handling - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include -#include "internal.h" - -static LIST_HEAD(fscache_caches); -DECLARE_RWSEM(fscache_addremove_sem); -EXPORT_SYMBOL(fscache_addremove_sem); -DECLARE_WAIT_QUEUE_HEAD(fscache_clearance_waiters); -EXPORT_SYMBOL(fscache_clearance_waiters); - -static atomic_t fscache_cache_debug_id; - -/* - * Allocate a cache cookie. - */ -static struct fscache_cache *fscache_alloc_cache(const char *name) -{ - struct fscache_cache *cache; - - cache = kzalloc(sizeof(*cache), GFP_KERNEL); - if (cache) { - if (name) { - cache->name = kstrdup(name, GFP_KERNEL); - if (!cache->name) { - kfree(cache); - return NULL; - } - } - refcount_set(&cache->ref, 1); - INIT_LIST_HEAD(&cache->cache_link); - cache->debug_id = atomic_inc_return(&fscache_cache_debug_id); - } - return cache; -} - -static bool fscache_get_cache_maybe(struct fscache_cache *cache, - enum fscache_cache_trace where) -{ - bool success; - int ref; - - success = __refcount_inc_not_zero(&cache->ref, &ref); - if (success) - trace_fscache_cache(cache->debug_id, ref + 1, where); - return success; -} - -/* - * Look up a cache cookie. - */ -struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache) -{ - struct fscache_cache *candidate, *cache, *unnamed = NULL; - - /* firstly check for the existence of the cache under read lock */ - down_read(&fscache_addremove_sem); - - list_for_each_entry(cache, &fscache_caches, cache_link) { - if (cache->name && name && strcmp(cache->name, name) == 0 && - fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) - goto got_cache_r; - if (!cache->name && !name && - fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) - goto got_cache_r; - } - - if (!name) { - list_for_each_entry(cache, &fscache_caches, cache_link) { - if (cache->name && - fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) - goto got_cache_r; - } - } - - up_read(&fscache_addremove_sem); - - /* the cache does not exist - create a candidate */ - candidate = fscache_alloc_cache(name); - if (!candidate) - return ERR_PTR(-ENOMEM); - - /* write lock, search again and add if still not present */ - down_write(&fscache_addremove_sem); - - list_for_each_entry(cache, &fscache_caches, cache_link) { - if (cache->name && name && strcmp(cache->name, name) == 0 && - fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) - goto got_cache_w; - if (!cache->name) { - unnamed = cache; - if (!name && - fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) - goto got_cache_w; - } - } - - if (unnamed && is_cache && - fscache_get_cache_maybe(unnamed, fscache_cache_get_acquire)) - goto use_unnamed_cache; - - if (!name) { - list_for_each_entry(cache, &fscache_caches, cache_link) { - if (cache->name && - fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) - goto got_cache_w; - } - } - - list_add_tail(&candidate->cache_link, &fscache_caches); - trace_fscache_cache(candidate->debug_id, - refcount_read(&candidate->ref), - fscache_cache_new_acquire); - up_write(&fscache_addremove_sem); - return candidate; - -got_cache_r: - up_read(&fscache_addremove_sem); - return cache; -use_unnamed_cache: - cache = unnamed; - cache->name = candidate->name; - candidate->name = NULL; -got_cache_w: - up_write(&fscache_addremove_sem); - kfree(candidate->name); - kfree(candidate); - return cache; -} - -/** - * fscache_acquire_cache - Acquire a cache-level cookie. - * @name: The name of the cache. - * - * Get a cookie to represent an actual cache. If a name is given and there is - * a nameless cache record available, this will acquire that and set its name, - * directing all the volumes using it to this cache. - * - * The cache will be switched over to the preparing state if not currently in - * use, otherwise -EBUSY will be returned. - */ -struct fscache_cache *fscache_acquire_cache(const char *name) -{ - struct fscache_cache *cache; - - ASSERT(name); - cache = fscache_lookup_cache(name, true); - if (IS_ERR(cache)) - return cache; - - if (!fscache_set_cache_state_maybe(cache, - FSCACHE_CACHE_IS_NOT_PRESENT, - FSCACHE_CACHE_IS_PREPARING)) { - pr_warn("Cache tag %s in use\n", name); - fscache_put_cache(cache, fscache_cache_put_cache); - return ERR_PTR(-EBUSY); - } - - return cache; -} -EXPORT_SYMBOL(fscache_acquire_cache); - -/** - * fscache_put_cache - Release a cache-level cookie. - * @cache: The cache cookie to be released - * @where: An indication of where the release happened - * - * Release the caller's reference on a cache-level cookie. The @where - * indication should give information about the circumstances in which the call - * occurs and will be logged through a tracepoint. - */ -void fscache_put_cache(struct fscache_cache *cache, - enum fscache_cache_trace where) -{ - unsigned int debug_id = cache->debug_id; - bool zero; - int ref; - - if (IS_ERR_OR_NULL(cache)) - return; - - zero = __refcount_dec_and_test(&cache->ref, &ref); - trace_fscache_cache(debug_id, ref - 1, where); - - if (zero) { - down_write(&fscache_addremove_sem); - list_del_init(&cache->cache_link); - up_write(&fscache_addremove_sem); - kfree(cache->name); - kfree(cache); - } -} - -/** - * fscache_relinquish_cache - Reset cache state and release cookie - * @cache: The cache cookie to be released - * - * Reset the state of a cache and release the caller's reference on a cache - * cookie. - */ -void fscache_relinquish_cache(struct fscache_cache *cache) -{ - enum fscache_cache_trace where = - (cache->state == FSCACHE_CACHE_IS_PREPARING) ? - fscache_cache_put_prep_failed : - fscache_cache_put_relinquish; - - cache->ops = NULL; - cache->cache_priv = NULL; - fscache_set_cache_state(cache, FSCACHE_CACHE_IS_NOT_PRESENT); - fscache_put_cache(cache, where); -} -EXPORT_SYMBOL(fscache_relinquish_cache); - -/** - * fscache_add_cache - Declare a cache as being open for business - * @cache: The cache-level cookie representing the cache - * @ops: Table of cache operations to use - * @cache_priv: Private data for the cache record - * - * Add a cache to the system, making it available for netfs's to use. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -int fscache_add_cache(struct fscache_cache *cache, - const struct fscache_cache_ops *ops, - void *cache_priv) -{ - int n_accesses; - - _enter("{%s,%s}", ops->name, cache->name); - - BUG_ON(fscache_cache_state(cache) != FSCACHE_CACHE_IS_PREPARING); - - /* Get a ref on the cache cookie and keep its n_accesses counter raised - * by 1 to prevent wakeups from transitioning it to 0 until we're - * withdrawing caching services from it. - */ - n_accesses = atomic_inc_return(&cache->n_accesses); - trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), - n_accesses, fscache_access_cache_pin); - - down_write(&fscache_addremove_sem); - - cache->ops = ops; - cache->cache_priv = cache_priv; - fscache_set_cache_state(cache, FSCACHE_CACHE_IS_ACTIVE); - - up_write(&fscache_addremove_sem); - pr_notice("Cache \"%s\" added (type %s)\n", cache->name, ops->name); - _leave(" = 0 [%s]", cache->name); - return 0; -} -EXPORT_SYMBOL(fscache_add_cache); - -/** - * fscache_begin_cache_access - Pin a cache so it can be accessed - * @cache: The cache-level cookie - * @why: An indication of the circumstances of the access for tracing - * - * Attempt to pin the cache to prevent it from going away whilst we're - * accessing it and returns true if successful. This works as follows: - * - * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), - * then we return false to indicate access was not permitted. - * - * (2) If the cache tests as live, then we increment the n_accesses count and - * then recheck the liveness, ending the access if it ceased to be live. - * - * (3) When we end the access, we decrement n_accesses and wake up the any - * waiters if it reaches 0. - * - * (4) Whilst the cache is caching, n_accesses is kept artificially - * incremented to prevent wakeups from happening. - * - * (5) When the cache is taken offline, the state is changed to prevent new - * accesses, n_accesses is decremented and we wait for n_accesses to - * become 0. - */ -bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) -{ - int n_accesses; - - if (!fscache_cache_is_live(cache)) - return false; - - n_accesses = atomic_inc_return(&cache->n_accesses); - smp_mb__after_atomic(); /* Reread live flag after n_accesses */ - trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), - n_accesses, why); - if (!fscache_cache_is_live(cache)) { - fscache_end_cache_access(cache, fscache_access_unlive); - return false; - } - return true; -} - -/** - * fscache_end_cache_access - Unpin a cache at the end of an access. - * @cache: The cache-level cookie - * @why: An indication of the circumstances of the access for tracing - * - * Unpin a cache after we've accessed it. The @why indicator is merely - * provided for tracing purposes. - */ -void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) -{ - int n_accesses; - - smp_mb__before_atomic(); - n_accesses = atomic_dec_return(&cache->n_accesses); - trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), - n_accesses, why); - if (n_accesses == 0) - wake_up_var(&cache->n_accesses); -} - -/** - * fscache_io_error - Note a cache I/O error - * @cache: The record describing the cache - * - * Note that an I/O error occurred in a cache and that it should no longer be - * used for anything. This also reports the error into the kernel log. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -void fscache_io_error(struct fscache_cache *cache) -{ - if (fscache_set_cache_state_maybe(cache, - FSCACHE_CACHE_IS_ACTIVE, - FSCACHE_CACHE_GOT_IOERROR)) - pr_err("Cache '%s' stopped due to I/O error\n", - cache->name); -} -EXPORT_SYMBOL(fscache_io_error); - -/** - * fscache_withdraw_cache - Withdraw a cache from the active service - * @cache: The cache cookie - * - * Begin the process of withdrawing a cache from service. This stops new - * cache-level and volume-level accesses from taking place and waits for - * currently ongoing cache-level accesses to end. - */ -void fscache_withdraw_cache(struct fscache_cache *cache) -{ - int n_accesses; - - pr_notice("Withdrawing cache \"%s\" (%u objs)\n", - cache->name, atomic_read(&cache->object_count)); - - fscache_set_cache_state(cache, FSCACHE_CACHE_IS_WITHDRAWN); - - /* Allow wakeups on dec-to-0 */ - n_accesses = atomic_dec_return(&cache->n_accesses); - trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), - n_accesses, fscache_access_cache_unpin); - - wait_var_event(&cache->n_accesses, - atomic_read(&cache->n_accesses) == 0); -} -EXPORT_SYMBOL(fscache_withdraw_cache); - -#ifdef CONFIG_PROC_FS -static const char fscache_cache_states[NR__FSCACHE_CACHE_STATE] = "-PAEW"; - -/* - * Generate a list of caches in /proc/fs/fscache/caches - */ -static int fscache_caches_seq_show(struct seq_file *m, void *v) -{ - struct fscache_cache *cache; - - if (v == &fscache_caches) { - seq_puts(m, - "CACHE REF VOLS OBJS ACCES S NAME\n" - "======== ===== ===== ===== ===== = ===============\n" - ); - return 0; - } - - cache = list_entry(v, struct fscache_cache, cache_link); - seq_printf(m, - "%08x %5d %5d %5d %5d %c %s\n", - cache->debug_id, - refcount_read(&cache->ref), - atomic_read(&cache->n_volumes), - atomic_read(&cache->object_count), - atomic_read(&cache->n_accesses), - fscache_cache_states[cache->state], - cache->name ?: "-"); - return 0; -} - -static void *fscache_caches_seq_start(struct seq_file *m, loff_t *_pos) - __acquires(fscache_addremove_sem) -{ - down_read(&fscache_addremove_sem); - return seq_list_start_head(&fscache_caches, *_pos); -} - -static void *fscache_caches_seq_next(struct seq_file *m, void *v, loff_t *_pos) -{ - return seq_list_next(v, &fscache_caches, _pos); -} - -static void fscache_caches_seq_stop(struct seq_file *m, void *v) - __releases(fscache_addremove_sem) -{ - up_read(&fscache_addremove_sem); -} - -const struct seq_operations fscache_caches_seq_ops = { - .start = fscache_caches_seq_start, - .next = fscache_caches_seq_next, - .stop = fscache_caches_seq_stop, - .show = fscache_caches_seq_show, -}; -#endif /* CONFIG_PROC_FS */ diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c deleted file mode 100644 index bce2492186d0..000000000000 --- a/fs/fscache/cookie.c +++ /dev/null @@ -1,1180 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* netfs cookie management - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - * - * See Documentation/filesystems/caching/netfs-api.rst for more information on - * the netfs API. - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include -#include -#include "internal.h" - -struct kmem_cache *fscache_cookie_jar; - -static void fscache_cookie_lru_timed_out(struct timer_list *timer); -static void fscache_cookie_lru_worker(struct work_struct *work); -static void fscache_cookie_worker(struct work_struct *work); -static void fscache_unhash_cookie(struct fscache_cookie *cookie); -static void fscache_perform_invalidation(struct fscache_cookie *cookie); - -#define fscache_cookie_hash_shift 15 -static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; -static LIST_HEAD(fscache_cookies); -static DEFINE_RWLOCK(fscache_cookies_lock); -static LIST_HEAD(fscache_cookie_lru); -static DEFINE_SPINLOCK(fscache_cookie_lru_lock); -DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); -static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); -static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD"; -static unsigned int fscache_lru_cookie_timeout = 10 * HZ; - -void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) -{ - const u8 *k; - - pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", - prefix, - cookie->debug_id, - cookie->flags, - atomic_read(&cookie->n_active), - atomic_read(&cookie->n_accesses), - fscache_cookie_states[cookie->state]); - pr_err("%c-cookie V=%08x [%s]\n", - prefix, - cookie->volume->debug_id, - cookie->volume->key); - - k = (cookie->key_len <= sizeof(cookie->inline_key)) ? - cookie->inline_key : cookie->key; - pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); -} - -static void fscache_free_cookie(struct fscache_cookie *cookie) -{ - if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { - spin_lock(&fscache_cookie_lru_lock); - list_del_init(&cookie->commit_link); - spin_unlock(&fscache_cookie_lru_lock); - fscache_stat_d(&fscache_n_cookies_lru); - fscache_stat(&fscache_n_cookies_lru_removed); - } - - if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { - fscache_print_cookie(cookie, 'F'); - return; - } - - write_lock(&fscache_cookies_lock); - list_del(&cookie->proc_link); - write_unlock(&fscache_cookies_lock); - if (cookie->aux_len > sizeof(cookie->inline_aux)) - kfree(cookie->aux); - if (cookie->key_len > sizeof(cookie->inline_key)) - kfree(cookie->key); - fscache_stat_d(&fscache_n_cookies); - kmem_cache_free(fscache_cookie_jar, cookie); -} - -static void __fscache_queue_cookie(struct fscache_cookie *cookie) -{ - if (!queue_work(fscache_wq, &cookie->work)) - fscache_put_cookie(cookie, fscache_cookie_put_over_queued); -} - -static void fscache_queue_cookie(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - fscache_get_cookie(cookie, where); - __fscache_queue_cookie(cookie); -} - -/* - * Initialise the access gate on a cookie by setting a flag to prevent the - * state machine from being queued when the access counter transitions to 0. - * We're only interested in this when we withdraw caching services from the - * cookie. - */ -static void fscache_init_access_gate(struct fscache_cookie *cookie) -{ - int n_accesses; - - n_accesses = atomic_read(&cookie->n_accesses); - trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), - n_accesses, fscache_access_cache_pin); - set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); -} - -/** - * fscache_end_cookie_access - Unpin a cache at the end of an access. - * @cookie: A data file cookie - * @why: An indication of the circumstances of the access for tracing - * - * Unpin a cache cookie after we've accessed it and bring a deferred - * relinquishment or withdrawal state into effect. - * - * The @why indicator is provided for tracing purposes. - */ -void fscache_end_cookie_access(struct fscache_cookie *cookie, - enum fscache_access_trace why) -{ - int n_accesses; - - smp_mb__before_atomic(); - n_accesses = atomic_dec_return(&cookie->n_accesses); - trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), - n_accesses, why); - if (n_accesses == 0 && - !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) - fscache_queue_cookie(cookie, fscache_cookie_get_end_access); -} -EXPORT_SYMBOL(fscache_end_cookie_access); - -/* - * Pin the cache behind a cookie so that we can access it. - */ -static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, - enum fscache_access_trace why) -{ - int n_accesses; - - n_accesses = atomic_inc_return(&cookie->n_accesses); - smp_mb__after_atomic(); /* (Future) read state after is-caching. - * Reread n_accesses after is-caching - */ - trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), - n_accesses, why); -} - -/** - * fscache_begin_cookie_access - Pin a cache so data can be accessed - * @cookie: A data file cookie - * @why: An indication of the circumstances of the access for tracing - * - * Attempt to pin the cache to prevent it from going away whilst we're - * accessing data and returns true if successful. This works as follows: - * - * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not - * set), we return false to indicate access was not permitted. - * - * (2) If the cookie is being cached, we increment its n_accesses count and - * then recheck the IS_CACHING flag, ending the access if it got cleared. - * - * (3) When we end the access, we decrement the cookie's n_accesses and wake - * up the any waiters if it reaches 0. - * - * (4) Whilst the cookie is actively being cached, its n_accesses is kept - * artificially incremented to prevent wakeups from happening. - * - * (5) When the cache is taken offline or if the cookie is culled, the flag is - * cleared to prevent new accesses, the cookie's n_accesses is decremented - * and we wait for it to become 0. - * - * The @why indicator are merely provided for tracing purposes. - */ -bool fscache_begin_cookie_access(struct fscache_cookie *cookie, - enum fscache_access_trace why) -{ - if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) - return false; - __fscache_begin_cookie_access(cookie, why); - if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || - !fscache_cache_is_live(cookie->volume->cache)) { - fscache_end_cookie_access(cookie, fscache_access_unlive); - return false; - } - return true; -} - -static inline void wake_up_cookie_state(struct fscache_cookie *cookie) -{ - /* Use a barrier to ensure that waiters see the state variable - * change, as spin_unlock doesn't guarantee a barrier. - * - * See comments over wake_up_bit() and waitqueue_active(). - */ - smp_mb(); - wake_up_var(&cookie->state); -} - -/* - * Change the state a cookie is at and wake up anyone waiting for that. Impose - * an ordering between the stuff stored in the cookie and the state member. - * Paired with fscache_cookie_state(). - */ -static void __fscache_set_cookie_state(struct fscache_cookie *cookie, - enum fscache_cookie_state state) -{ - smp_store_release(&cookie->state, state); -} - -static void fscache_set_cookie_state(struct fscache_cookie *cookie, - enum fscache_cookie_state state) -{ - spin_lock(&cookie->lock); - __fscache_set_cookie_state(cookie, state); - spin_unlock(&cookie->lock); - wake_up_cookie_state(cookie); -} - -/** - * fscache_cookie_lookup_negative - Note negative lookup - * @cookie: The cookie that was being looked up - * - * Note that some part of the metadata path in the cache doesn't exist and so - * we can release any waiting readers in the certain knowledge that there's - * nothing for them to actually read. - * - * This function uses no locking and must only be called from the state machine. - */ -void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) -{ - set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); - fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); -} -EXPORT_SYMBOL(fscache_cookie_lookup_negative); - -/** - * fscache_resume_after_invalidation - Allow I/O to resume after invalidation - * @cookie: The cookie that was invalidated - * - * Tell fscache that invalidation is sufficiently complete that I/O can be - * allowed again. - */ -void fscache_resume_after_invalidation(struct fscache_cookie *cookie) -{ - fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); -} -EXPORT_SYMBOL(fscache_resume_after_invalidation); - -/** - * fscache_caching_failed - Report that a failure stopped caching on a cookie - * @cookie: The cookie that was affected - * - * Tell fscache that caching on a cookie needs to be stopped due to some sort - * of failure. - * - * This function uses no locking and must only be called from the state machine. - */ -void fscache_caching_failed(struct fscache_cookie *cookie) -{ - clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); - fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); - trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), - fscache_cookie_failed); -} -EXPORT_SYMBOL(fscache_caching_failed); - -/* - * Set the index key in a cookie. The cookie struct has space for a 16-byte - * key plus length and hash, but if that's not big enough, it's instead a - * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then - * the key data. - */ -static int fscache_set_key(struct fscache_cookie *cookie, - const void *index_key, size_t index_key_len) -{ - void *buf; - size_t buf_size; - - buf_size = round_up(index_key_len, sizeof(__le32)); - - if (index_key_len > sizeof(cookie->inline_key)) { - buf = kzalloc(buf_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - cookie->key = buf; - } else { - buf = cookie->inline_key; - } - - memcpy(buf, index_key, index_key_len); - cookie->key_hash = fscache_hash(cookie->volume->key_hash, - buf, buf_size); - return 0; -} - -static bool fscache_cookie_same(const struct fscache_cookie *a, - const struct fscache_cookie *b) -{ - const void *ka, *kb; - - if (a->key_hash != b->key_hash || - a->volume != b->volume || - a->key_len != b->key_len) - return false; - - if (a->key_len <= sizeof(a->inline_key)) { - ka = &a->inline_key; - kb = &b->inline_key; - } else { - ka = a->key; - kb = b->key; - } - return memcmp(ka, kb, a->key_len) == 0; -} - -static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); - -/* - * Allocate a cookie. - */ -static struct fscache_cookie *fscache_alloc_cookie( - struct fscache_volume *volume, - u8 advice, - const void *index_key, size_t index_key_len, - const void *aux_data, size_t aux_data_len, - loff_t object_size) -{ - struct fscache_cookie *cookie; - - /* allocate and initialise a cookie */ - cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); - if (!cookie) - return NULL; - fscache_stat(&fscache_n_cookies); - - cookie->volume = volume; - cookie->advice = advice; - cookie->key_len = index_key_len; - cookie->aux_len = aux_data_len; - cookie->object_size = object_size; - if (object_size == 0) - __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); - - if (fscache_set_key(cookie, index_key, index_key_len) < 0) - goto nomem; - - if (cookie->aux_len <= sizeof(cookie->inline_aux)) { - memcpy(cookie->inline_aux, aux_data, cookie->aux_len); - } else { - cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); - if (!cookie->aux) - goto nomem; - } - - refcount_set(&cookie->ref, 1); - cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); - spin_lock_init(&cookie->lock); - INIT_LIST_HEAD(&cookie->commit_link); - INIT_WORK(&cookie->work, fscache_cookie_worker); - __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); - - write_lock(&fscache_cookies_lock); - list_add_tail(&cookie->proc_link, &fscache_cookies); - write_unlock(&fscache_cookies_lock); - fscache_see_cookie(cookie, fscache_cookie_new_acquire); - return cookie; - -nomem: - fscache_free_cookie(cookie); - return NULL; -} - -static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie) -{ - return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED; -} - -static void fscache_wait_on_collision(struct fscache_cookie *candidate, - struct fscache_cookie *wait_for) -{ - enum fscache_cookie_state *statep = &wait_for->state; - - wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for), - 20 * HZ); - if (!fscache_cookie_is_dropped(wait_for)) { - pr_notice("Potential collision c=%08x old: c=%08x", - candidate->debug_id, wait_for->debug_id); - wait_var_event(statep, fscache_cookie_is_dropped(wait_for)); - } -} - -/* - * Attempt to insert the new cookie into the hash. If there's a collision, we - * wait for the old cookie to complete if it's being relinquished and an error - * otherwise. - */ -static bool fscache_hash_cookie(struct fscache_cookie *candidate) -{ - struct fscache_cookie *cursor, *wait_for = NULL; - struct hlist_bl_head *h; - struct hlist_bl_node *p; - unsigned int bucket; - - bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); - h = &fscache_cookie_hash[bucket]; - - hlist_bl_lock(h); - hlist_bl_for_each_entry(cursor, p, h, hash_link) { - if (fscache_cookie_same(candidate, cursor)) { - if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) - goto collision; - wait_for = fscache_get_cookie(cursor, - fscache_cookie_get_hash_collision); - break; - } - } - - fscache_get_volume(candidate->volume, fscache_volume_get_cookie); - atomic_inc(&candidate->volume->n_cookies); - hlist_bl_add_head(&candidate->hash_link, h); - set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); - hlist_bl_unlock(h); - - if (wait_for) { - fscache_wait_on_collision(candidate, wait_for); - fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); - } - return true; - -collision: - trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), - fscache_cookie_collision); - pr_err("Duplicate cookie detected\n"); - fscache_print_cookie(cursor, 'O'); - fscache_print_cookie(candidate, 'N'); - hlist_bl_unlock(h); - return false; -} - -/* - * Request a cookie to represent a data storage object within a volume. - * - * We never let on to the netfs about errors. We may set a negative cookie - * pointer, but that's okay - */ -struct fscache_cookie *__fscache_acquire_cookie( - struct fscache_volume *volume, - u8 advice, - const void *index_key, size_t index_key_len, - const void *aux_data, size_t aux_data_len, - loff_t object_size) -{ - struct fscache_cookie *cookie; - - _enter("V=%x", volume->debug_id); - - if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) - return NULL; - if (!aux_data || !aux_data_len) { - aux_data = NULL; - aux_data_len = 0; - } - - fscache_stat(&fscache_n_acquires); - - cookie = fscache_alloc_cookie(volume, advice, - index_key, index_key_len, - aux_data, aux_data_len, - object_size); - if (!cookie) { - fscache_stat(&fscache_n_acquires_oom); - return NULL; - } - - if (!fscache_hash_cookie(cookie)) { - fscache_see_cookie(cookie, fscache_cookie_discard); - fscache_free_cookie(cookie); - return NULL; - } - - trace_fscache_acquire(cookie); - fscache_stat(&fscache_n_acquires_ok); - _leave(" = c=%08x", cookie->debug_id); - return cookie; -} -EXPORT_SYMBOL(__fscache_acquire_cookie); - -/* - * Prepare a cache object to be written to. - */ -static void fscache_prepare_to_write(struct fscache_cookie *cookie) -{ - cookie->volume->cache->ops->prepare_to_write(cookie); -} - -/* - * Look up a cookie in the cache. - */ -static void fscache_perform_lookup(struct fscache_cookie *cookie) -{ - enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; - bool need_withdraw = false; - - _enter(""); - - if (!cookie->volume->cache_priv) { - fscache_create_volume(cookie->volume, true); - if (!cookie->volume->cache_priv) { - fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); - goto out; - } - } - - if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { - if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) - fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); - need_withdraw = true; - _leave(" [fail]"); - goto out; - } - - fscache_see_cookie(cookie, fscache_cookie_see_active); - spin_lock(&cookie->lock); - if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) - __fscache_set_cookie_state(cookie, - FSCACHE_COOKIE_STATE_INVALIDATING); - else - __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); - spin_unlock(&cookie->lock); - wake_up_cookie_state(cookie); - trace = fscache_access_lookup_cookie_end; - -out: - fscache_end_cookie_access(cookie, trace); - if (need_withdraw) - fscache_withdraw_cookie(cookie); - fscache_end_volume_access(cookie->volume, cookie, trace); -} - -/* - * Begin the process of looking up a cookie. We offload the actual process to - * a worker thread. - */ -static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) -{ - if (will_modify) { - set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); - set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); - } - if (!fscache_begin_volume_access(cookie->volume, cookie, - fscache_access_lookup_cookie)) - return false; - - __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); - __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); - set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); - set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); - return true; -} - -/* - * Start using the cookie for I/O. This prevents the backing object from being - * reaped by VM pressure. - */ -void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) -{ - enum fscache_cookie_state state; - bool queue = false; - int n_active; - - _enter("c=%08x", cookie->debug_id); - - if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), - "Trying to use relinquished cookie\n")) - return; - - spin_lock(&cookie->lock); - - n_active = atomic_inc_return(&cookie->n_active); - trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref), - n_active, atomic_read(&cookie->n_accesses), - will_modify ? - fscache_active_use_modify : fscache_active_use); - -again: - state = fscache_cookie_state(cookie); - switch (state) { - case FSCACHE_COOKIE_STATE_QUIESCENT: - queue = fscache_begin_lookup(cookie, will_modify); - break; - - case FSCACHE_COOKIE_STATE_LOOKING_UP: - case FSCACHE_COOKIE_STATE_CREATING: - if (will_modify) - set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); - break; - case FSCACHE_COOKIE_STATE_ACTIVE: - case FSCACHE_COOKIE_STATE_INVALIDATING: - if (will_modify && - !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { - set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); - queue = true; - } - /* - * We could race with cookie_lru which may set LRU_DISCARD bit - * but has yet to run the cookie state machine. If this happens - * and another thread tries to use the cookie, clear LRU_DISCARD - * so we don't end up withdrawing the cookie while in use. - */ - if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) - fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear); - break; - - case FSCACHE_COOKIE_STATE_FAILED: - case FSCACHE_COOKIE_STATE_WITHDRAWING: - break; - - case FSCACHE_COOKIE_STATE_LRU_DISCARDING: - spin_unlock(&cookie->lock); - wait_var_event(&cookie->state, - fscache_cookie_state(cookie) != - FSCACHE_COOKIE_STATE_LRU_DISCARDING); - spin_lock(&cookie->lock); - goto again; - - case FSCACHE_COOKIE_STATE_DROPPED: - case FSCACHE_COOKIE_STATE_RELINQUISHING: - WARN(1, "Can't use cookie in state %u\n", state); - break; - } - - spin_unlock(&cookie->lock); - if (queue) - fscache_queue_cookie(cookie, fscache_cookie_get_use_work); - _leave(""); -} -EXPORT_SYMBOL(__fscache_use_cookie); - -static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) -{ - clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); - if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) - return; - - cookie->unused_at = jiffies; - spin_lock(&fscache_cookie_lru_lock); - if (list_empty(&cookie->commit_link)) { - fscache_get_cookie(cookie, fscache_cookie_get_lru); - fscache_stat(&fscache_n_cookies_lru); - } - list_move_tail(&cookie->commit_link, &fscache_cookie_lru); - - spin_unlock(&fscache_cookie_lru_lock); - timer_reduce(&fscache_cookie_lru_timer, - jiffies + fscache_lru_cookie_timeout); -} - -/* - * Stop using the cookie for I/O. - */ -void __fscache_unuse_cookie(struct fscache_cookie *cookie, - const void *aux_data, const loff_t *object_size) -{ - unsigned int debug_id = cookie->debug_id; - unsigned int r = refcount_read(&cookie->ref); - unsigned int a = atomic_read(&cookie->n_accesses); - unsigned int c; - - if (aux_data || object_size) - __fscache_update_cookie(cookie, aux_data, object_size); - - /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ - c = atomic_fetch_add_unless(&cookie->n_active, -1, 1); - if (c != 1) { - trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse); - return; - } - - spin_lock(&cookie->lock); - r = refcount_read(&cookie->ref); - a = atomic_read(&cookie->n_accesses); - c = atomic_dec_return(&cookie->n_active); - trace_fscache_active(debug_id, r, c, a, fscache_active_unuse); - if (c == 0) - fscache_unuse_cookie_locked(cookie); - spin_unlock(&cookie->lock); -} -EXPORT_SYMBOL(__fscache_unuse_cookie); - -/* - * Perform work upon the cookie, such as committing its cache state, - * relinquishing it or withdrawing the backing cache. We're protected from the - * cache going away under us as object withdrawal must come through this - * non-reentrant work item. - */ -static void fscache_cookie_state_machine(struct fscache_cookie *cookie) -{ - enum fscache_cookie_state state; - bool wake = false; - - _enter("c=%x", cookie->debug_id); - -again: - spin_lock(&cookie->lock); -again_locked: - state = cookie->state; - switch (state) { - case FSCACHE_COOKIE_STATE_QUIESCENT: - /* The QUIESCENT state is jumped to the LOOKING_UP state by - * fscache_use_cookie(). - */ - - if (atomic_read(&cookie->n_accesses) == 0 && - test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { - __fscache_set_cookie_state(cookie, - FSCACHE_COOKIE_STATE_RELINQUISHING); - wake = true; - goto again_locked; - } - break; - - case FSCACHE_COOKIE_STATE_LOOKING_UP: - spin_unlock(&cookie->lock); - fscache_init_access_gate(cookie); - fscache_perform_lookup(cookie); - goto again; - - case FSCACHE_COOKIE_STATE_INVALIDATING: - spin_unlock(&cookie->lock); - fscache_perform_invalidation(cookie); - goto again; - - case FSCACHE_COOKIE_STATE_ACTIVE: - if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { - spin_unlock(&cookie->lock); - fscache_prepare_to_write(cookie); - spin_lock(&cookie->lock); - } - if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { - __fscache_set_cookie_state(cookie, - FSCACHE_COOKIE_STATE_LRU_DISCARDING); - wake = true; - goto again_locked; - } - fallthrough; - - case FSCACHE_COOKIE_STATE_FAILED: - if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) - fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); - - if (atomic_read(&cookie->n_accesses) != 0) - break; - if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { - __fscache_set_cookie_state(cookie, - FSCACHE_COOKIE_STATE_RELINQUISHING); - wake = true; - goto again_locked; - } - if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { - __fscache_set_cookie_state(cookie, - FSCACHE_COOKIE_STATE_WITHDRAWING); - wake = true; - goto again_locked; - } - break; - - case FSCACHE_COOKIE_STATE_LRU_DISCARDING: - case FSCACHE_COOKIE_STATE_RELINQUISHING: - case FSCACHE_COOKIE_STATE_WITHDRAWING: - if (cookie->cache_priv) { - spin_unlock(&cookie->lock); - cookie->volume->cache->ops->withdraw_cookie(cookie); - spin_lock(&cookie->lock); - } - - if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) - fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); - - switch (state) { - case FSCACHE_COOKIE_STATE_RELINQUISHING: - fscache_see_cookie(cookie, fscache_cookie_see_relinquish); - fscache_unhash_cookie(cookie); - __fscache_set_cookie_state(cookie, - FSCACHE_COOKIE_STATE_DROPPED); - wake = true; - goto out; - case FSCACHE_COOKIE_STATE_LRU_DISCARDING: - fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); - break; - case FSCACHE_COOKIE_STATE_WITHDRAWING: - fscache_see_cookie(cookie, fscache_cookie_see_withdraw); - break; - default: - BUG(); - } - - clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); - clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); - clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); - clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); - set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); - __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); - wake = true; - goto again_locked; - - case FSCACHE_COOKIE_STATE_DROPPED: - break; - - default: - WARN_ONCE(1, "Cookie %x in unexpected state %u\n", - cookie->debug_id, state); - break; - } - -out: - spin_unlock(&cookie->lock); - if (wake) - wake_up_cookie_state(cookie); - _leave(""); -} - -static void fscache_cookie_worker(struct work_struct *work) -{ - struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); - - fscache_see_cookie(cookie, fscache_cookie_see_work); - fscache_cookie_state_machine(cookie); - fscache_put_cookie(cookie, fscache_cookie_put_work); -} - -/* - * Wait for the object to become inactive. The cookie's work item will be - * scheduled when someone transitions n_accesses to 0 - but if someone's - * already done that, schedule it anyway. - */ -static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) -{ - int n_accesses; - bool unpinned; - - unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); - - /* Need to read the access count after unpinning */ - n_accesses = atomic_read(&cookie->n_accesses); - if (unpinned) - trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), - n_accesses, fscache_access_cache_unpin); - if (n_accesses == 0) - fscache_queue_cookie(cookie, fscache_cookie_get_end_access); -} - -static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) -{ - fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); - - spin_lock(&cookie->lock); - if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || - time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || - atomic_read(&cookie->n_active) > 0) { - spin_unlock(&cookie->lock); - fscache_stat(&fscache_n_cookies_lru_removed); - } else { - set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); - spin_unlock(&cookie->lock); - fscache_stat(&fscache_n_cookies_lru_expired); - _debug("lru c=%x", cookie->debug_id); - __fscache_withdraw_cookie(cookie); - } - - fscache_put_cookie(cookie, fscache_cookie_put_lru); -} - -static void fscache_cookie_lru_worker(struct work_struct *work) -{ - struct fscache_cookie *cookie; - unsigned long unused_at; - - spin_lock(&fscache_cookie_lru_lock); - - while (!list_empty(&fscache_cookie_lru)) { - cookie = list_first_entry(&fscache_cookie_lru, - struct fscache_cookie, commit_link); - unused_at = cookie->unused_at + fscache_lru_cookie_timeout; - if (time_before(jiffies, unused_at)) { - timer_reduce(&fscache_cookie_lru_timer, unused_at); - break; - } - - list_del_init(&cookie->commit_link); - fscache_stat_d(&fscache_n_cookies_lru); - spin_unlock(&fscache_cookie_lru_lock); - fscache_cookie_lru_do_one(cookie); - spin_lock(&fscache_cookie_lru_lock); - } - - spin_unlock(&fscache_cookie_lru_lock); -} - -static void fscache_cookie_lru_timed_out(struct timer_list *timer) -{ - queue_work(fscache_wq, &fscache_cookie_lru_work); -} - -static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) -{ - bool need_put = false; - - if (!list_empty(&cookie->commit_link)) { - spin_lock(&fscache_cookie_lru_lock); - if (!list_empty(&cookie->commit_link)) { - list_del_init(&cookie->commit_link); - fscache_stat_d(&fscache_n_cookies_lru); - fscache_stat(&fscache_n_cookies_lru_dropped); - need_put = true; - } - spin_unlock(&fscache_cookie_lru_lock); - if (need_put) - fscache_put_cookie(cookie, fscache_cookie_put_lru); - } -} - -/* - * Remove a cookie from the hash table. - */ -static void fscache_unhash_cookie(struct fscache_cookie *cookie) -{ - struct hlist_bl_head *h; - unsigned int bucket; - - bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); - h = &fscache_cookie_hash[bucket]; - - hlist_bl_lock(h); - hlist_bl_del(&cookie->hash_link); - clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); - hlist_bl_unlock(h); - fscache_stat(&fscache_n_relinquishes_dropped); -} - -static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) -{ - fscache_cookie_drop_from_lru(cookie); - __fscache_withdraw_cookie(cookie); -} - -/** - * fscache_withdraw_cookie - Mark a cookie for withdrawal - * @cookie: The cookie to be withdrawn. - * - * Allow the cache backend to withdraw the backing for a cookie for its own - * reasons, even if that cookie is in active use. - */ -void fscache_withdraw_cookie(struct fscache_cookie *cookie) -{ - set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); - fscache_drop_withdraw_cookie(cookie); -} -EXPORT_SYMBOL(fscache_withdraw_cookie); - -/* - * Allow the netfs to release a cookie back to the cache. - * - the object will be marked as recyclable on disk if retire is true - */ -void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) -{ - fscache_stat(&fscache_n_relinquishes); - if (retire) - fscache_stat(&fscache_n_relinquishes_retire); - - _enter("c=%08x{%d},%d", - cookie->debug_id, atomic_read(&cookie->n_active), retire); - - if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), - "Cookie c=%x already relinquished\n", cookie->debug_id)) - return; - - if (retire) - set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); - trace_fscache_relinquish(cookie, retire); - - ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); - ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); - atomic_dec(&cookie->volume->n_cookies); - - if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { - set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); - fscache_drop_withdraw_cookie(cookie); - } else { - fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); - fscache_unhash_cookie(cookie); - } - fscache_put_cookie(cookie, fscache_cookie_put_relinquish); -} -EXPORT_SYMBOL(__fscache_relinquish_cookie); - -/* - * Drop a reference to a cookie. - */ -void fscache_put_cookie(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - struct fscache_volume *volume = cookie->volume; - unsigned int cookie_debug_id = cookie->debug_id; - bool zero; - int ref; - - zero = __refcount_dec_and_test(&cookie->ref, &ref); - trace_fscache_cookie(cookie_debug_id, ref - 1, where); - if (zero) { - fscache_free_cookie(cookie); - fscache_put_volume(volume, fscache_volume_put_cookie); - } -} -EXPORT_SYMBOL(fscache_put_cookie); - -/* - * Get a reference to a cookie. - */ -struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - int ref; - - __refcount_inc(&cookie->ref, &ref); - trace_fscache_cookie(cookie->debug_id, ref + 1, where); - return cookie; -} -EXPORT_SYMBOL(fscache_get_cookie); - -/* - * Ask the cache to effect invalidation of a cookie. - */ -static void fscache_perform_invalidation(struct fscache_cookie *cookie) -{ - if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) - fscache_caching_failed(cookie); - fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); -} - -/* - * Invalidate an object. - */ -void __fscache_invalidate(struct fscache_cookie *cookie, - const void *aux_data, loff_t new_size, - unsigned int flags) -{ - bool is_caching; - - _enter("c=%x", cookie->debug_id); - - fscache_stat(&fscache_n_invalidates); - - if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), - "Trying to invalidate relinquished cookie\n")) - return; - - if ((flags & FSCACHE_INVAL_DIO_WRITE) && - test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) - return; - - spin_lock(&cookie->lock); - set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); - fscache_update_aux(cookie, aux_data, &new_size); - cookie->inval_counter++; - trace_fscache_invalidate(cookie, new_size); - - switch (cookie->state) { - case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ - default: - spin_unlock(&cookie->lock); - _leave(" [no %u]", cookie->state); - return; - - case FSCACHE_COOKIE_STATE_LOOKING_UP: - if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) - __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie); - fallthrough; - case FSCACHE_COOKIE_STATE_CREATING: - spin_unlock(&cookie->lock); - _leave(" [look %x]", cookie->inval_counter); - return; - - case FSCACHE_COOKIE_STATE_ACTIVE: - is_caching = fscache_begin_cookie_access( - cookie, fscache_access_invalidate_cookie); - if (is_caching) - __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); - spin_unlock(&cookie->lock); - wake_up_cookie_state(cookie); - - if (is_caching) - fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); - _leave(" [inv]"); - return; - } -} -EXPORT_SYMBOL(__fscache_invalidate); - -#ifdef CONFIG_PROC_FS -/* - * Generate a list of extant cookies in /proc/fs/fscache/cookies - */ -static int fscache_cookies_seq_show(struct seq_file *m, void *v) -{ - struct fscache_cookie *cookie; - unsigned int keylen = 0, auxlen = 0; - u8 *p; - - if (v == &fscache_cookies) { - seq_puts(m, - "COOKIE VOLUME REF ACT ACC S FL DEF \n" - "======== ======== === === === = == ================\n" - ); - return 0; - } - - cookie = list_entry(v, struct fscache_cookie, proc_link); - - seq_printf(m, - "%08x %08x %3d %3d %3d %c %02lx", - cookie->debug_id, - cookie->volume->debug_id, - refcount_read(&cookie->ref), - atomic_read(&cookie->n_active), - atomic_read(&cookie->n_accesses), - fscache_cookie_states[cookie->state], - cookie->flags); - - keylen = cookie->key_len; - auxlen = cookie->aux_len; - - if (keylen > 0 || auxlen > 0) { - seq_puts(m, " "); - p = keylen <= sizeof(cookie->inline_key) ? - cookie->inline_key : cookie->key; - for (; keylen > 0; keylen--) - seq_printf(m, "%02x", *p++); - if (auxlen > 0) { - seq_puts(m, ", "); - p = auxlen <= sizeof(cookie->inline_aux) ? - cookie->inline_aux : cookie->aux; - for (; auxlen > 0; auxlen--) - seq_printf(m, "%02x", *p++); - } - } - - seq_puts(m, "\n"); - return 0; -} - -static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) - __acquires(fscache_cookies_lock) -{ - read_lock(&fscache_cookies_lock); - return seq_list_start_head(&fscache_cookies, *_pos); -} - -static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) -{ - return seq_list_next(v, &fscache_cookies, _pos); -} - -static void fscache_cookies_seq_stop(struct seq_file *m, void *v) - __releases(rcu) -{ - read_unlock(&fscache_cookies_lock); -} - - -const struct seq_operations fscache_cookies_seq_ops = { - .start = fscache_cookies_seq_start, - .next = fscache_cookies_seq_next, - .stop = fscache_cookies_seq_stop, - .show = fscache_cookies_seq_show, -}; -#endif diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h deleted file mode 100644 index 1336f517e9b1..000000000000 --- a/fs/fscache/internal.h +++ /dev/null @@ -1,277 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Internal definitions for FS-Cache - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#ifdef pr_fmt -#undef pr_fmt -#endif - -#define pr_fmt(fmt) "FS-Cache: " fmt - -#include -#include -#include -#include -#include - -/* - * cache.c - */ -#ifdef CONFIG_PROC_FS -extern const struct seq_operations fscache_caches_seq_ops; -#endif -bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); -void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); -struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); -void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); - -static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) -{ - return smp_load_acquire(&cache->state); -} - -static inline bool fscache_cache_is_live(const struct fscache_cache *cache) -{ - return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; -} - -static inline void fscache_set_cache_state(struct fscache_cache *cache, - enum fscache_cache_state new_state) -{ - smp_store_release(&cache->state, new_state); - -} - -static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, - enum fscache_cache_state old_state, - enum fscache_cache_state new_state) -{ - return try_cmpxchg_release(&cache->state, &old_state, new_state); -} - -/* - * cookie.c - */ -extern struct kmem_cache *fscache_cookie_jar; -#ifdef CONFIG_PROC_FS -extern const struct seq_operations fscache_cookies_seq_ops; -#endif -extern struct timer_list fscache_cookie_lru_timer; - -extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); -extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, - enum fscache_access_trace why); - -static inline void fscache_see_cookie(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), - where); -} - -/* - * main.c - */ -extern unsigned fscache_debug; - -extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len); - -/* - * proc.c - */ -#ifdef CONFIG_PROC_FS -extern int __init fscache_proc_init(void); -extern void fscache_proc_cleanup(void); -#else -#define fscache_proc_init() (0) -#define fscache_proc_cleanup() do {} while (0) -#endif - -/* - * stats.c - */ -#ifdef CONFIG_FSCACHE_STATS -extern atomic_t fscache_n_volumes; -extern atomic_t fscache_n_volumes_collision; -extern atomic_t fscache_n_volumes_nomem; -extern atomic_t fscache_n_cookies; -extern atomic_t fscache_n_cookies_lru; -extern atomic_t fscache_n_cookies_lru_expired; -extern atomic_t fscache_n_cookies_lru_removed; -extern atomic_t fscache_n_cookies_lru_dropped; - -extern atomic_t fscache_n_acquires; -extern atomic_t fscache_n_acquires_ok; -extern atomic_t fscache_n_acquires_oom; - -extern atomic_t fscache_n_invalidates; - -extern atomic_t fscache_n_relinquishes; -extern atomic_t fscache_n_relinquishes_retire; -extern atomic_t fscache_n_relinquishes_dropped; - -extern atomic_t fscache_n_resizes; -extern atomic_t fscache_n_resizes_null; - -static inline void fscache_stat(atomic_t *stat) -{ - atomic_inc(stat); -} - -static inline void fscache_stat_d(atomic_t *stat) -{ - atomic_dec(stat); -} - -#define __fscache_stat(stat) (stat) - -int fscache_stats_show(struct seq_file *m, void *v); -#else - -#define __fscache_stat(stat) (NULL) -#define fscache_stat(stat) do {} while (0) -#define fscache_stat_d(stat) do {} while (0) -#endif - -/* - * volume.c - */ -#ifdef CONFIG_PROC_FS -extern const struct seq_operations fscache_volumes_seq_ops; -#endif - -struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, - enum fscache_volume_trace where); -void fscache_put_volume(struct fscache_volume *volume, - enum fscache_volume_trace where); -bool fscache_begin_volume_access(struct fscache_volume *volume, - struct fscache_cookie *cookie, - enum fscache_access_trace why); -void fscache_create_volume(struct fscache_volume *volume, bool wait); - - -/*****************************************************************************/ -/* - * debug tracing - */ -#define dbgprintk(FMT, ...) \ - printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) - -#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) -#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) -#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) - -#define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__) - -#ifdef __KDEBUG -#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) -#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) -#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) - -#elif defined(CONFIG_FSCACHE_DEBUG) -#define _enter(FMT, ...) \ -do { \ - if (__do_kdebug(ENTER)) \ - kenter(FMT, ##__VA_ARGS__); \ -} while (0) - -#define _leave(FMT, ...) \ -do { \ - if (__do_kdebug(LEAVE)) \ - kleave(FMT, ##__VA_ARGS__); \ -} while (0) - -#define _debug(FMT, ...) \ -do { \ - if (__do_kdebug(DEBUG)) \ - kdebug(FMT, ##__VA_ARGS__); \ -} while (0) - -#else -#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) -#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) -#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) -#endif - -/* - * determine whether a particular optional debugging point should be logged - * - we need to go through three steps to persuade cpp to correctly join the - * shorthand in FSCACHE_DEBUG_LEVEL with its prefix - */ -#define ____do_kdebug(LEVEL, POINT) \ - unlikely((fscache_debug & \ - (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3)))) -#define ___do_kdebug(LEVEL, POINT) \ - ____do_kdebug(LEVEL, POINT) -#define __do_kdebug(POINT) \ - ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT) - -#define FSCACHE_DEBUG_CACHE 0 -#define FSCACHE_DEBUG_COOKIE 1 -#define FSCACHE_DEBUG_OBJECT 2 -#define FSCACHE_DEBUG_OPERATION 3 - -#define FSCACHE_POINT_ENTER 1 -#define FSCACHE_POINT_LEAVE 2 -#define FSCACHE_POINT_DEBUG 4 - -#ifndef FSCACHE_DEBUG_LEVEL -#define FSCACHE_DEBUG_LEVEL CACHE -#endif - -/* - * assertions - */ -#if 1 /* defined(__KDEBUGALL) */ - -#define ASSERT(X) \ -do { \ - if (unlikely(!(X))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - BUG(); \ - } \ -} while (0) - -#define ASSERTCMP(X, OP, Y) \ -do { \ - if (unlikely(!((X) OP (Y)))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - pr_err("%lx " #OP " %lx is false\n", \ - (unsigned long)(X), (unsigned long)(Y)); \ - BUG(); \ - } \ -} while (0) - -#define ASSERTIF(C, X) \ -do { \ - if (unlikely((C) && !(X))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - BUG(); \ - } \ -} while (0) - -#define ASSERTIFCMP(C, X, OP, Y) \ -do { \ - if (unlikely((C) && !((X) OP (Y)))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - pr_err("%lx " #OP " %lx is false\n", \ - (unsigned long)(X), (unsigned long)(Y)); \ - BUG(); \ - } \ -} while (0) - -#else - -#define ASSERT(X) do {} while (0) -#define ASSERTCMP(X, OP, Y) do {} while (0) -#define ASSERTIF(C, X) do {} while (0) -#define ASSERTIFCMP(C, X, OP, Y) do {} while (0) - -#endif /* assert or not */ diff --git a/fs/fscache/io.c b/fs/fscache/io.c deleted file mode 100644 index 0d2b8dec8f82..000000000000 --- a/fs/fscache/io.c +++ /dev/null @@ -1,327 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Cache data I/O routines - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ -#define FSCACHE_DEBUG_LEVEL OPERATION -#include -#include -#include -#include -#include -#include "internal.h" - -/** - * fscache_wait_for_operation - Wait for an object become accessible - * @cres: The cache resources for the operation being performed - * @want_state: The minimum state the object must be at - * - * See if the target cache object is at the specified minimum state of - * accessibility yet, and if not, wait for it. - */ -bool fscache_wait_for_operation(struct netfs_cache_resources *cres, - enum fscache_want_state want_state) -{ - struct fscache_cookie *cookie = fscache_cres_cookie(cres); - enum fscache_cookie_state state; - -again: - if (!fscache_cache_is_live(cookie->volume->cache)) { - _leave(" [broken]"); - return false; - } - - state = fscache_cookie_state(cookie); - _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); - - switch (state) { - case FSCACHE_COOKIE_STATE_CREATING: - case FSCACHE_COOKIE_STATE_INVALIDATING: - if (want_state == FSCACHE_WANT_PARAMS) - goto ready; /* There can be no content */ - fallthrough; - case FSCACHE_COOKIE_STATE_LOOKING_UP: - case FSCACHE_COOKIE_STATE_LRU_DISCARDING: - wait_var_event(&cookie->state, - fscache_cookie_state(cookie) != state); - goto again; - - case FSCACHE_COOKIE_STATE_ACTIVE: - goto ready; - case FSCACHE_COOKIE_STATE_DROPPED: - case FSCACHE_COOKIE_STATE_RELINQUISHING: - default: - _leave(" [not live]"); - return false; - } - -ready: - if (!cres->cache_priv2) - return cookie->volume->cache->ops->begin_operation(cres, want_state); - return true; -} -EXPORT_SYMBOL(fscache_wait_for_operation); - -/* - * Begin an I/O operation on the cache, waiting till we reach the right state. - * - * Attaches the resources required to the operation resources record. - */ -static int fscache_begin_operation(struct netfs_cache_resources *cres, - struct fscache_cookie *cookie, - enum fscache_want_state want_state, - enum fscache_access_trace why) -{ - enum fscache_cookie_state state; - long timeo; - bool once_only = false; - - cres->ops = NULL; - cres->cache_priv = cookie; - cres->cache_priv2 = NULL; - cres->debug_id = cookie->debug_id; - cres->inval_counter = cookie->inval_counter; - - if (!fscache_begin_cookie_access(cookie, why)) - return -ENOBUFS; - -again: - spin_lock(&cookie->lock); - - state = fscache_cookie_state(cookie); - _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); - - switch (state) { - case FSCACHE_COOKIE_STATE_LOOKING_UP: - case FSCACHE_COOKIE_STATE_LRU_DISCARDING: - case FSCACHE_COOKIE_STATE_INVALIDATING: - goto wait_for_file_wrangling; - case FSCACHE_COOKIE_STATE_CREATING: - if (want_state == FSCACHE_WANT_PARAMS) - goto ready; /* There can be no content */ - goto wait_for_file_wrangling; - case FSCACHE_COOKIE_STATE_ACTIVE: - goto ready; - case FSCACHE_COOKIE_STATE_DROPPED: - case FSCACHE_COOKIE_STATE_RELINQUISHING: - WARN(1, "Can't use cookie in state %u\n", cookie->state); - goto not_live; - default: - goto not_live; - } - -ready: - spin_unlock(&cookie->lock); - if (!cookie->volume->cache->ops->begin_operation(cres, want_state)) - goto failed; - return 0; - -wait_for_file_wrangling: - spin_unlock(&cookie->lock); - trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), - atomic_read(&cookie->n_accesses), - fscache_access_io_wait); - timeo = wait_var_event_timeout(&cookie->state, - fscache_cookie_state(cookie) != state, 20 * HZ); - if (timeo <= 1 && !once_only) { - pr_warn("%s: cookie state change wait timed out: cookie->state=%u state=%u", - __func__, fscache_cookie_state(cookie), state); - fscache_print_cookie(cookie, 'O'); - once_only = true; - } - goto again; - -not_live: - spin_unlock(&cookie->lock); -failed: - cres->cache_priv = NULL; - cres->ops = NULL; - fscache_end_cookie_access(cookie, fscache_access_io_not_live); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} - -int __fscache_begin_read_operation(struct netfs_cache_resources *cres, - struct fscache_cookie *cookie) -{ - return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, - fscache_access_io_read); -} -EXPORT_SYMBOL(__fscache_begin_read_operation); - -int __fscache_begin_write_operation(struct netfs_cache_resources *cres, - struct fscache_cookie *cookie) -{ - return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, - fscache_access_io_write); -} -EXPORT_SYMBOL(__fscache_begin_write_operation); - -/** - * fscache_dirty_folio - Mark folio dirty and pin a cache object for writeback - * @mapping: The mapping the folio belongs to. - * @folio: The folio being dirtied. - * @cookie: The cookie referring to the cache object - * - * Set the dirty flag on a folio and pin an in-use cache object in memory - * so that writeback can later write to it. This is intended - * to be called from the filesystem's ->dirty_folio() method. - * - * Return: true if the dirty flag was set on the folio, false otherwise. - */ -bool fscache_dirty_folio(struct address_space *mapping, struct folio *folio, - struct fscache_cookie *cookie) -{ - struct inode *inode = mapping->host; - bool need_use = false; - - _enter(""); - - if (!filemap_dirty_folio(mapping, folio)) - return false; - if (!fscache_cookie_valid(cookie)) - return true; - - if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { - spin_lock(&inode->i_lock); - if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { - inode->i_state |= I_PINNING_FSCACHE_WB; - need_use = true; - } - spin_unlock(&inode->i_lock); - - if (need_use) - fscache_use_cookie(cookie, true); - } - return true; -} -EXPORT_SYMBOL(fscache_dirty_folio); - -struct fscache_write_request { - struct netfs_cache_resources cache_resources; - struct address_space *mapping; - loff_t start; - size_t len; - bool set_bits; - netfs_io_terminated_t term_func; - void *term_func_priv; -}; - -void __fscache_clear_page_bits(struct address_space *mapping, - loff_t start, size_t len) -{ - pgoff_t first = start / PAGE_SIZE; - pgoff_t last = (start + len - 1) / PAGE_SIZE; - struct page *page; - - if (len) { - XA_STATE(xas, &mapping->i_pages, first); - - rcu_read_lock(); - xas_for_each(&xas, page, last) { - end_page_fscache(page); - } - rcu_read_unlock(); - } -} -EXPORT_SYMBOL(__fscache_clear_page_bits); - -/* - * Deal with the completion of writing the data to the cache. - */ -static void fscache_wreq_done(void *priv, ssize_t transferred_or_error, - bool was_async) -{ - struct fscache_write_request *wreq = priv; - - fscache_clear_page_bits(wreq->mapping, wreq->start, wreq->len, - wreq->set_bits); - - if (wreq->term_func) - wreq->term_func(wreq->term_func_priv, transferred_or_error, - was_async); - fscache_end_operation(&wreq->cache_resources); - kfree(wreq); -} - -void __fscache_write_to_cache(struct fscache_cookie *cookie, - struct address_space *mapping, - loff_t start, size_t len, loff_t i_size, - netfs_io_terminated_t term_func, - void *term_func_priv, - bool cond) -{ - struct fscache_write_request *wreq; - struct netfs_cache_resources *cres; - struct iov_iter iter; - int ret = -ENOBUFS; - - if (len == 0) - goto abandon; - - _enter("%llx,%zx", start, len); - - wreq = kzalloc(sizeof(struct fscache_write_request), GFP_NOFS); - if (!wreq) - goto abandon; - wreq->mapping = mapping; - wreq->start = start; - wreq->len = len; - wreq->set_bits = cond; - wreq->term_func = term_func; - wreq->term_func_priv = term_func_priv; - - cres = &wreq->cache_resources; - if (fscache_begin_operation(cres, cookie, FSCACHE_WANT_WRITE, - fscache_access_io_write) < 0) - goto abandon_free; - - ret = cres->ops->prepare_write(cres, &start, &len, i_size, false); - if (ret < 0) - goto abandon_end; - - /* TODO: Consider clearing page bits now for space the write isn't - * covering. This is more complicated than it appears when THPs are - * taken into account. - */ - - iov_iter_xarray(&iter, ITER_SOURCE, &mapping->i_pages, start, len); - fscache_write(cres, start, &iter, fscache_wreq_done, wreq); - return; - -abandon_end: - return fscache_wreq_done(wreq, ret, false); -abandon_free: - kfree(wreq); -abandon: - fscache_clear_page_bits(mapping, start, len, cond); - if (term_func) - term_func(term_func_priv, ret, false); -} -EXPORT_SYMBOL(__fscache_write_to_cache); - -/* - * Change the size of a backing object. - */ -void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) -{ - struct netfs_cache_resources cres; - - trace_fscache_resize(cookie, new_size); - if (fscache_begin_operation(&cres, cookie, FSCACHE_WANT_WRITE, - fscache_access_io_resize) == 0) { - fscache_stat(&fscache_n_resizes); - set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); - - /* We cannot defer a resize as we need to do it inside the - * netfs's inode lock so that we're serialised with respect to - * writes. - */ - cookie->volume->cache->ops->resize_cookie(&cres, new_size); - fscache_end_operation(&cres); - } else { - fscache_stat(&fscache_n_resizes_null); - } -} -EXPORT_SYMBOL(__fscache_resize_cookie); diff --git a/fs/fscache/main.c b/fs/fscache/main.c deleted file mode 100644 index dad85fd84f6f..000000000000 --- a/fs/fscache/main.c +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* General filesystem local caching manager - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include -#define CREATE_TRACE_POINTS -#include "internal.h" - -MODULE_DESCRIPTION("FS Cache Manager"); -MODULE_AUTHOR("Red Hat, Inc."); -MODULE_LICENSE("GPL"); - -unsigned fscache_debug; -module_param_named(debug, fscache_debug, uint, - S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(fscache_debug, - "FS-Cache debugging mask"); - -EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache); -EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume); -EXPORT_TRACEPOINT_SYMBOL(fscache_access); - -struct workqueue_struct *fscache_wq; -EXPORT_SYMBOL(fscache_wq); - -/* - * Mixing scores (in bits) for (7,20): - * Input delta: 1-bit 2-bit - * 1 round: 330.3 9201.6 - * 2 rounds: 1246.4 25475.4 - * 3 rounds: 1907.1 31295.1 - * 4 rounds: 2042.3 31718.6 - * Perfect: 2048 31744 - * (32*64) (32*31/2 * 64) - */ -#define HASH_MIX(x, y, a) \ - ( x ^= (a), \ - y ^= x, x = rol32(x, 7),\ - x += y, y = rol32(y,20),\ - y *= 9 ) - -static inline unsigned int fold_hash(unsigned long x, unsigned long y) -{ - /* Use arch-optimized multiply if one exists */ - return __hash_32(y ^ __hash_32(x)); -} - -/* - * Generate a hash. This is derived from full_name_hash(), but we want to be - * sure it is arch independent and that it doesn't change as bits of the - * computed hash value might appear on disk. The caller must guarantee that - * the source data is a multiple of four bytes in size. - */ -unsigned int fscache_hash(unsigned int salt, const void *data, size_t len) -{ - const __le32 *p = data; - unsigned int a, x = 0, y = salt, n = len / sizeof(__le32); - - for (; n; n--) { - a = le32_to_cpu(*p++); - HASH_MIX(x, y, a); - } - return fold_hash(x, y); -} - -/* - * initialise the fs caching module - */ -static int __init fscache_init(void) -{ - int ret = -ENOMEM; - - fscache_wq = alloc_workqueue("fscache", WQ_UNBOUND | WQ_FREEZABLE, 0); - if (!fscache_wq) - goto error_wq; - - ret = fscache_proc_init(); - if (ret < 0) - goto error_proc; - - fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", - sizeof(struct fscache_cookie), - 0, 0, NULL); - if (!fscache_cookie_jar) { - pr_notice("Failed to allocate a cookie jar\n"); - ret = -ENOMEM; - goto error_cookie_jar; - } - - pr_notice("Loaded\n"); - return 0; - -error_cookie_jar: - fscache_proc_cleanup(); -error_proc: - destroy_workqueue(fscache_wq); -error_wq: - return ret; -} - -fs_initcall(fscache_init); - -/* - * clean up on module removal - */ -static void __exit fscache_exit(void) -{ - _enter(""); - - kmem_cache_destroy(fscache_cookie_jar); - fscache_proc_cleanup(); - destroy_workqueue(fscache_wq); - pr_notice("Unloaded\n"); -} - -module_exit(fscache_exit); diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c deleted file mode 100644 index dc3b0e9c8cce..000000000000 --- a/fs/fscache/proc.c +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache statistics viewing interface - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include -#include -#include "internal.h" - -/* - * initialise the /proc/fs/fscache/ directory - */ -int __init fscache_proc_init(void) -{ - if (!proc_mkdir("fs/fscache", NULL)) - goto error_dir; - - if (!proc_create_seq("fs/fscache/caches", S_IFREG | 0444, NULL, - &fscache_caches_seq_ops)) - goto error; - - if (!proc_create_seq("fs/fscache/volumes", S_IFREG | 0444, NULL, - &fscache_volumes_seq_ops)) - goto error; - - if (!proc_create_seq("fs/fscache/cookies", S_IFREG | 0444, NULL, - &fscache_cookies_seq_ops)) - goto error; - -#ifdef CONFIG_FSCACHE_STATS - if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, - fscache_stats_show)) - goto error; -#endif - - return 0; - -error: - remove_proc_entry("fs/fscache", NULL); -error_dir: - return -ENOMEM; -} - -/* - * clean up the /proc/fs/fscache/ directory - */ -void fscache_proc_cleanup(void) -{ - remove_proc_subtree("fs/fscache", NULL); -} diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c deleted file mode 100644 index fc94e5e79f1c..000000000000 --- a/fs/fscache/stats.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache statistics - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include -#include "internal.h" - -/* - * operation counters - */ -atomic_t fscache_n_volumes; -atomic_t fscache_n_volumes_collision; -atomic_t fscache_n_volumes_nomem; -atomic_t fscache_n_cookies; -atomic_t fscache_n_cookies_lru; -atomic_t fscache_n_cookies_lru_expired; -atomic_t fscache_n_cookies_lru_removed; -atomic_t fscache_n_cookies_lru_dropped; - -atomic_t fscache_n_acquires; -atomic_t fscache_n_acquires_ok; -atomic_t fscache_n_acquires_oom; - -atomic_t fscache_n_invalidates; - -atomic_t fscache_n_updates; -EXPORT_SYMBOL(fscache_n_updates); - -atomic_t fscache_n_relinquishes; -atomic_t fscache_n_relinquishes_retire; -atomic_t fscache_n_relinquishes_dropped; - -atomic_t fscache_n_resizes; -atomic_t fscache_n_resizes_null; - -atomic_t fscache_n_read; -EXPORT_SYMBOL(fscache_n_read); -atomic_t fscache_n_write; -EXPORT_SYMBOL(fscache_n_write); -atomic_t fscache_n_no_write_space; -EXPORT_SYMBOL(fscache_n_no_write_space); -atomic_t fscache_n_no_create_space; -EXPORT_SYMBOL(fscache_n_no_create_space); -atomic_t fscache_n_culled; -EXPORT_SYMBOL(fscache_n_culled); - -/* - * display the general statistics - */ -int fscache_stats_show(struct seq_file *m, void *v) -{ - seq_puts(m, "FS-Cache statistics\n"); - seq_printf(m, "Cookies: n=%d v=%d vcol=%u voom=%u\n", - atomic_read(&fscache_n_cookies), - atomic_read(&fscache_n_volumes), - atomic_read(&fscache_n_volumes_collision), - atomic_read(&fscache_n_volumes_nomem) - ); - - seq_printf(m, "Acquire: n=%u ok=%u oom=%u\n", - atomic_read(&fscache_n_acquires), - atomic_read(&fscache_n_acquires_ok), - atomic_read(&fscache_n_acquires_oom)); - - seq_printf(m, "LRU : n=%u exp=%u rmv=%u drp=%u at=%ld\n", - atomic_read(&fscache_n_cookies_lru), - atomic_read(&fscache_n_cookies_lru_expired), - atomic_read(&fscache_n_cookies_lru_removed), - atomic_read(&fscache_n_cookies_lru_dropped), - timer_pending(&fscache_cookie_lru_timer) ? - fscache_cookie_lru_timer.expires - jiffies : 0); - - seq_printf(m, "Invals : n=%u\n", - atomic_read(&fscache_n_invalidates)); - - seq_printf(m, "Updates: n=%u rsz=%u rsn=%u\n", - atomic_read(&fscache_n_updates), - atomic_read(&fscache_n_resizes), - atomic_read(&fscache_n_resizes_null)); - - seq_printf(m, "Relinqs: n=%u rtr=%u drop=%u\n", - atomic_read(&fscache_n_relinquishes), - atomic_read(&fscache_n_relinquishes_retire), - atomic_read(&fscache_n_relinquishes_dropped)); - - seq_printf(m, "NoSpace: nwr=%u ncr=%u cull=%u\n", - atomic_read(&fscache_n_no_write_space), - atomic_read(&fscache_n_no_create_space), - atomic_read(&fscache_n_culled)); - - seq_printf(m, "IO : rd=%u wr=%u\n", - atomic_read(&fscache_n_read), - atomic_read(&fscache_n_write)); - - netfs_stats_show(m); - return 0; -} diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c deleted file mode 100644 index cdf991bdd9de..000000000000 --- a/fs/fscache/volume.c +++ /dev/null @@ -1,520 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Volume-level cache cookie handling. - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include -#include -#include "internal.h" - -#define fscache_volume_hash_shift 10 -static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; -static atomic_t fscache_volume_debug_id; -static LIST_HEAD(fscache_volumes); - -static void fscache_create_volume_work(struct work_struct *work); - -struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, - enum fscache_volume_trace where) -{ - int ref; - - __refcount_inc(&volume->ref, &ref); - trace_fscache_volume(volume->debug_id, ref + 1, where); - return volume; -} - -static void fscache_see_volume(struct fscache_volume *volume, - enum fscache_volume_trace where) -{ - int ref = refcount_read(&volume->ref); - - trace_fscache_volume(volume->debug_id, ref, where); -} - -/* - * Pin the cache behind a volume so that we can access it. - */ -static void __fscache_begin_volume_access(struct fscache_volume *volume, - struct fscache_cookie *cookie, - enum fscache_access_trace why) -{ - int n_accesses; - - n_accesses = atomic_inc_return(&volume->n_accesses); - smp_mb__after_atomic(); - trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, - refcount_read(&volume->ref), - n_accesses, why); -} - -/** - * fscache_begin_volume_access - Pin a cache so a volume can be accessed - * @volume: The volume cookie - * @cookie: A datafile cookie for a tracing reference (or NULL) - * @why: An indication of the circumstances of the access for tracing - * - * Attempt to pin the cache to prevent it from going away whilst we're - * accessing a volume and returns true if successful. This works as follows: - * - * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), - * then we return false to indicate access was not permitted. - * - * (2) If the cache tests as live, then we increment the volume's n_accesses - * count and then recheck the cache liveness, ending the access if it - * ceased to be live. - * - * (3) When we end the access, we decrement the volume's n_accesses and wake - * up the any waiters if it reaches 0. - * - * (4) Whilst the cache is caching, the volume's n_accesses is kept - * artificially incremented to prevent wakeups from happening. - * - * (5) When the cache is taken offline, the state is changed to prevent new - * accesses, the volume's n_accesses is decremented and we wait for it to - * become 0. - * - * The datafile @cookie and the @why indicator are merely provided for tracing - * purposes. - */ -bool fscache_begin_volume_access(struct fscache_volume *volume, - struct fscache_cookie *cookie, - enum fscache_access_trace why) -{ - if (!fscache_cache_is_live(volume->cache)) - return false; - __fscache_begin_volume_access(volume, cookie, why); - if (!fscache_cache_is_live(volume->cache)) { - fscache_end_volume_access(volume, cookie, fscache_access_unlive); - return false; - } - return true; -} - -/** - * fscache_end_volume_access - Unpin a cache at the end of an access. - * @volume: The volume cookie - * @cookie: A datafile cookie for a tracing reference (or NULL) - * @why: An indication of the circumstances of the access for tracing - * - * Unpin a cache volume after we've accessed it. The datafile @cookie and the - * @why indicator are merely provided for tracing purposes. - */ -void fscache_end_volume_access(struct fscache_volume *volume, - struct fscache_cookie *cookie, - enum fscache_access_trace why) -{ - int n_accesses; - - smp_mb__before_atomic(); - n_accesses = atomic_dec_return(&volume->n_accesses); - trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, - refcount_read(&volume->ref), - n_accesses, why); - if (n_accesses == 0) - wake_up_var(&volume->n_accesses); -} -EXPORT_SYMBOL(fscache_end_volume_access); - -static bool fscache_volume_same(const struct fscache_volume *a, - const struct fscache_volume *b) -{ - size_t klen; - - if (a->key_hash != b->key_hash || - a->cache != b->cache || - a->key[0] != b->key[0]) - return false; - - klen = round_up(a->key[0] + 1, sizeof(__le32)); - return memcmp(a->key, b->key, klen) == 0; -} - -static bool fscache_is_acquire_pending(struct fscache_volume *volume) -{ - return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); -} - -static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, - unsigned int collidee_debug_id) -{ - wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, - TASK_UNINTERRUPTIBLE, 20 * HZ); - if (fscache_is_acquire_pending(candidate)) { - pr_notice("Potential volume collision new=%08x old=%08x", - candidate->debug_id, collidee_debug_id); - fscache_stat(&fscache_n_volumes_collision); - wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, - TASK_UNINTERRUPTIBLE); - } -} - -/* - * Attempt to insert the new volume into the hash. If there's a collision, we - * wait for the old volume to complete if it's being relinquished and an error - * otherwise. - */ -static bool fscache_hash_volume(struct fscache_volume *candidate) -{ - struct fscache_volume *cursor; - struct hlist_bl_head *h; - struct hlist_bl_node *p; - unsigned int bucket, collidee_debug_id = 0; - - bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); - h = &fscache_volume_hash[bucket]; - - hlist_bl_lock(h); - hlist_bl_for_each_entry(cursor, p, h, hash_link) { - if (fscache_volume_same(candidate, cursor)) { - if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) - goto collision; - fscache_see_volume(cursor, fscache_volume_get_hash_collision); - set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); - set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); - collidee_debug_id = cursor->debug_id; - break; - } - } - - hlist_bl_add_head(&candidate->hash_link, h); - hlist_bl_unlock(h); - - if (fscache_is_acquire_pending(candidate)) - fscache_wait_on_volume_collision(candidate, collidee_debug_id); - return true; - -collision: - fscache_see_volume(cursor, fscache_volume_collision); - hlist_bl_unlock(h); - return false; -} - -/* - * Allocate and initialise a volume representation cookie. - */ -static struct fscache_volume *fscache_alloc_volume(const char *volume_key, - const char *cache_name, - const void *coherency_data, - size_t coherency_len) -{ - struct fscache_volume *volume; - struct fscache_cache *cache; - size_t klen, hlen; - u8 *key; - - klen = strlen(volume_key); - if (klen > NAME_MAX) - return NULL; - - if (!coherency_data) - coherency_len = 0; - - cache = fscache_lookup_cache(cache_name, false); - if (IS_ERR(cache)) - return NULL; - - volume = kzalloc(struct_size(volume, coherency, coherency_len), - GFP_KERNEL); - if (!volume) - goto err_cache; - - volume->cache = cache; - volume->coherency_len = coherency_len; - if (coherency_data) - memcpy(volume->coherency, coherency_data, coherency_len); - INIT_LIST_HEAD(&volume->proc_link); - INIT_WORK(&volume->work, fscache_create_volume_work); - refcount_set(&volume->ref, 1); - spin_lock_init(&volume->lock); - - /* Stick the length on the front of the key and pad it out to make - * hashing easier. - */ - hlen = round_up(1 + klen + 1, sizeof(__le32)); - key = kzalloc(hlen, GFP_KERNEL); - if (!key) - goto err_vol; - key[0] = klen; - memcpy(key + 1, volume_key, klen); - - volume->key = key; - volume->key_hash = fscache_hash(0, key, hlen); - - volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); - down_write(&fscache_addremove_sem); - atomic_inc(&cache->n_volumes); - list_add_tail(&volume->proc_link, &fscache_volumes); - fscache_see_volume(volume, fscache_volume_new_acquire); - fscache_stat(&fscache_n_volumes); - up_write(&fscache_addremove_sem); - _leave(" = v=%x", volume->debug_id); - return volume; - -err_vol: - kfree(volume); -err_cache: - fscache_put_cache(cache, fscache_cache_put_alloc_volume); - fscache_stat(&fscache_n_volumes_nomem); - return NULL; -} - -/* - * Create a volume's representation on disk. Have a volume ref and a cache - * access we have to release. - */ -static void fscache_create_volume_work(struct work_struct *work) -{ - const struct fscache_cache_ops *ops; - struct fscache_volume *volume = - container_of(work, struct fscache_volume, work); - - fscache_see_volume(volume, fscache_volume_see_create_work); - - ops = volume->cache->ops; - if (ops->acquire_volume) - ops->acquire_volume(volume); - fscache_end_cache_access(volume->cache, - fscache_access_acquire_volume_end); - - clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags); - fscache_put_volume(volume, fscache_volume_put_create_work); -} - -/* - * Dispatch a worker thread to create a volume's representation on disk. - */ -void fscache_create_volume(struct fscache_volume *volume, bool wait) -{ - if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) - goto maybe_wait; - if (volume->cache_priv) - goto no_wait; /* We raced */ - if (!fscache_begin_cache_access(volume->cache, - fscache_access_acquire_volume)) - goto no_wait; - - fscache_get_volume(volume, fscache_volume_get_create_work); - if (!schedule_work(&volume->work)) - fscache_put_volume(volume, fscache_volume_put_create_work); - -maybe_wait: - if (wait) { - fscache_see_volume(volume, fscache_volume_wait_create_work); - wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, - TASK_UNINTERRUPTIBLE); - } - return; -no_wait: - clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); - wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); -} - -/* - * Acquire a volume representation cookie and link it to a (proposed) cache. - */ -struct fscache_volume *__fscache_acquire_volume(const char *volume_key, - const char *cache_name, - const void *coherency_data, - size_t coherency_len) -{ - struct fscache_volume *volume; - - volume = fscache_alloc_volume(volume_key, cache_name, - coherency_data, coherency_len); - if (!volume) - return ERR_PTR(-ENOMEM); - - if (!fscache_hash_volume(volume)) { - fscache_put_volume(volume, fscache_volume_put_hash_collision); - return ERR_PTR(-EBUSY); - } - - fscache_create_volume(volume, false); - return volume; -} -EXPORT_SYMBOL(__fscache_acquire_volume); - -static void fscache_wake_pending_volume(struct fscache_volume *volume, - struct hlist_bl_head *h) -{ - struct fscache_volume *cursor; - struct hlist_bl_node *p; - - hlist_bl_for_each_entry(cursor, p, h, hash_link) { - if (fscache_volume_same(cursor, volume)) { - fscache_see_volume(cursor, fscache_volume_see_hash_wake); - clear_and_wake_up_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, - &cursor->flags); - return; - } - } -} - -/* - * Remove a volume cookie from the hash table. - */ -static void fscache_unhash_volume(struct fscache_volume *volume) -{ - struct hlist_bl_head *h; - unsigned int bucket; - - bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); - h = &fscache_volume_hash[bucket]; - - hlist_bl_lock(h); - hlist_bl_del(&volume->hash_link); - if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) - fscache_wake_pending_volume(volume, h); - hlist_bl_unlock(h); -} - -/* - * Drop a cache's volume attachments. - */ -static void fscache_free_volume(struct fscache_volume *volume) -{ - struct fscache_cache *cache = volume->cache; - - if (volume->cache_priv) { - __fscache_begin_volume_access(volume, NULL, - fscache_access_relinquish_volume); - if (volume->cache_priv) - cache->ops->free_volume(volume); - fscache_end_volume_access(volume, NULL, - fscache_access_relinquish_volume_end); - } - - down_write(&fscache_addremove_sem); - list_del_init(&volume->proc_link); - atomic_dec(&volume->cache->n_volumes); - up_write(&fscache_addremove_sem); - - if (!hlist_bl_unhashed(&volume->hash_link)) - fscache_unhash_volume(volume); - - trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); - kfree(volume->key); - kfree(volume); - fscache_stat_d(&fscache_n_volumes); - fscache_put_cache(cache, fscache_cache_put_volume); -} - -/* - * Drop a reference to a volume cookie. - */ -void fscache_put_volume(struct fscache_volume *volume, - enum fscache_volume_trace where) -{ - if (volume) { - unsigned int debug_id = volume->debug_id; - bool zero; - int ref; - - zero = __refcount_dec_and_test(&volume->ref, &ref); - trace_fscache_volume(debug_id, ref - 1, where); - if (zero) - fscache_free_volume(volume); - } -} - -/* - * Relinquish a volume representation cookie. - */ -void __fscache_relinquish_volume(struct fscache_volume *volume, - const void *coherency_data, - bool invalidate) -{ - if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) - return; - - if (invalidate) { - set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); - } else if (coherency_data) { - memcpy(volume->coherency, coherency_data, volume->coherency_len); - } - - fscache_put_volume(volume, fscache_volume_put_relinquish); -} -EXPORT_SYMBOL(__fscache_relinquish_volume); - -/** - * fscache_withdraw_volume - Withdraw a volume from being cached - * @volume: Volume cookie - * - * Withdraw a cache volume from service, waiting for all accesses to complete - * before returning. - */ -void fscache_withdraw_volume(struct fscache_volume *volume) -{ - int n_accesses; - - _debug("withdraw V=%x", volume->debug_id); - - /* Allow wakeups on dec-to-0 */ - n_accesses = atomic_dec_return(&volume->n_accesses); - trace_fscache_access_volume(volume->debug_id, 0, - refcount_read(&volume->ref), - n_accesses, fscache_access_cache_unpin); - - wait_var_event(&volume->n_accesses, - atomic_read(&volume->n_accesses) == 0); -} -EXPORT_SYMBOL(fscache_withdraw_volume); - -#ifdef CONFIG_PROC_FS -/* - * Generate a list of volumes in /proc/fs/fscache/volumes - */ -static int fscache_volumes_seq_show(struct seq_file *m, void *v) -{ - struct fscache_volume *volume; - - if (v == &fscache_volumes) { - seq_puts(m, - "VOLUME REF nCOOK ACC FL CACHE KEY\n" - "======== ===== ===== === == =============== ================\n"); - return 0; - } - - volume = list_entry(v, struct fscache_volume, proc_link); - seq_printf(m, - "%08x %5d %5d %3d %02lx %-15.15s %s\n", - volume->debug_id, - refcount_read(&volume->ref), - atomic_read(&volume->n_cookies), - atomic_read(&volume->n_accesses), - volume->flags, - volume->cache->name ?: "-", - volume->key + 1); - return 0; -} - -static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) - __acquires(&fscache_addremove_sem) -{ - down_read(&fscache_addremove_sem); - return seq_list_start_head(&fscache_volumes, *_pos); -} - -static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) -{ - return seq_list_next(v, &fscache_volumes, _pos); -} - -static void fscache_volumes_seq_stop(struct seq_file *m, void *v) - __releases(&fscache_addremove_sem) -{ - up_read(&fscache_addremove_sem); -} - -const struct seq_operations fscache_volumes_seq_ops = { - .start = fscache_volumes_seq_start, - .next = fscache_volumes_seq_next, - .stop = fscache_volumes_seq_stop, - .show = fscache_volumes_seq_show, -}; -#endif /* CONFIG_PROC_FS */ diff --git a/fs/netfs/Kconfig b/fs/netfs/Kconfig index b4db21022cb4..b4378688357c 100644 --- a/fs/netfs/Kconfig +++ b/fs/netfs/Kconfig @@ -21,3 +21,42 @@ config NETFS_STATS multi-CPU system these may be on cachelines that keep bouncing between CPUs. On the other hand, the stats are very useful for debugging purposes. Saying 'Y' here is recommended. + +config FSCACHE + tristate "General filesystem local caching manager" + select NETFS_SUPPORT + help + This option enables a generic filesystem caching manager that can be + used by various network and other filesystems to cache data locally. + Different sorts of caches can be plugged in, depending on the + resources available. + + See Documentation/filesystems/caching/fscache.rst for more information. + +config FSCACHE_STATS + bool "Gather statistical information on local caching" + depends on FSCACHE && PROC_FS + select NETFS_STATS + help + This option causes statistical information to be gathered on local + caching and exported through file: + + /proc/fs/fscache/stats + + The gathering of statistics adds a certain amount of overhead to + execution as there are a quite a few stats gathered, and on a + multi-CPU system these may be on cachelines that keep bouncing + between CPUs. On the other hand, the stats are very useful for + debugging purposes. Saying 'Y' here is recommended. + + See Documentation/filesystems/caching/fscache.rst for more information. + +config FSCACHE_DEBUG + bool "Debug FS-Cache" + depends on FSCACHE + help + This permits debugging to be dynamically enabled in the local caching + management module. If this is set, the debugging output may be + enabled by setting bits in /sys/modules/fscache/parameter/debug. + + See Documentation/filesystems/caching/fscache.rst for more information. diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile index 386d6fb92793..bbb2b824bd5e 100644 --- a/fs/netfs/Makefile +++ b/fs/netfs/Makefile @@ -1,5 +1,17 @@ # SPDX-License-Identifier: GPL-2.0 +fscache-y := \ + fscache_cache.o \ + fscache_cookie.o \ + fscache_io.o \ + fscache_main.o \ + fscache_volume.o + +fscache-$(CONFIG_PROC_FS) += fscache_proc.o +fscache-$(CONFIG_FSCACHE_STATS) += fscache_stats.o + +obj-$(CONFIG_FSCACHE) := fscache.o + netfs-y := \ buffered_read.o \ io.o \ @@ -9,4 +21,4 @@ netfs-y := \ netfs-$(CONFIG_NETFS_STATS) += stats.o -obj-$(CONFIG_NETFS_SUPPORT) := netfs.o +obj-$(CONFIG_NETFS_SUPPORT) += netfs.o diff --git a/fs/netfs/fscache_cache.c b/fs/netfs/fscache_cache.c new file mode 100644 index 000000000000..d645f8b302a2 --- /dev/null +++ b/fs/netfs/fscache_cache.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache cache handling + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" + +static LIST_HEAD(fscache_caches); +DECLARE_RWSEM(fscache_addremove_sem); +EXPORT_SYMBOL(fscache_addremove_sem); +DECLARE_WAIT_QUEUE_HEAD(fscache_clearance_waiters); +EXPORT_SYMBOL(fscache_clearance_waiters); + +static atomic_t fscache_cache_debug_id; + +/* + * Allocate a cache cookie. + */ +static struct fscache_cache *fscache_alloc_cache(const char *name) +{ + struct fscache_cache *cache; + + cache = kzalloc(sizeof(*cache), GFP_KERNEL); + if (cache) { + if (name) { + cache->name = kstrdup(name, GFP_KERNEL); + if (!cache->name) { + kfree(cache); + return NULL; + } + } + refcount_set(&cache->ref, 1); + INIT_LIST_HEAD(&cache->cache_link); + cache->debug_id = atomic_inc_return(&fscache_cache_debug_id); + } + return cache; +} + +static bool fscache_get_cache_maybe(struct fscache_cache *cache, + enum fscache_cache_trace where) +{ + bool success; + int ref; + + success = __refcount_inc_not_zero(&cache->ref, &ref); + if (success) + trace_fscache_cache(cache->debug_id, ref + 1, where); + return success; +} + +/* + * Look up a cache cookie. + */ +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache) +{ + struct fscache_cache *candidate, *cache, *unnamed = NULL; + + /* firstly check for the existence of the cache under read lock */ + down_read(&fscache_addremove_sem); + + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + if (!cache->name && !name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + } + + up_read(&fscache_addremove_sem); + + /* the cache does not exist - create a candidate */ + candidate = fscache_alloc_cache(name); + if (!candidate) + return ERR_PTR(-ENOMEM); + + /* write lock, search again and add if still not present */ + down_write(&fscache_addremove_sem); + + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + if (!cache->name) { + unnamed = cache; + if (!name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } + + if (unnamed && is_cache && + fscache_get_cache_maybe(unnamed, fscache_cache_get_acquire)) + goto use_unnamed_cache; + + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } + + list_add_tail(&candidate->cache_link, &fscache_caches); + trace_fscache_cache(candidate->debug_id, + refcount_read(&candidate->ref), + fscache_cache_new_acquire); + up_write(&fscache_addremove_sem); + return candidate; + +got_cache_r: + up_read(&fscache_addremove_sem); + return cache; +use_unnamed_cache: + cache = unnamed; + cache->name = candidate->name; + candidate->name = NULL; +got_cache_w: + up_write(&fscache_addremove_sem); + kfree(candidate->name); + kfree(candidate); + return cache; +} + +/** + * fscache_acquire_cache - Acquire a cache-level cookie. + * @name: The name of the cache. + * + * Get a cookie to represent an actual cache. If a name is given and there is + * a nameless cache record available, this will acquire that and set its name, + * directing all the volumes using it to this cache. + * + * The cache will be switched over to the preparing state if not currently in + * use, otherwise -EBUSY will be returned. + */ +struct fscache_cache *fscache_acquire_cache(const char *name) +{ + struct fscache_cache *cache; + + ASSERT(name); + cache = fscache_lookup_cache(name, true); + if (IS_ERR(cache)) + return cache; + + if (!fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_NOT_PRESENT, + FSCACHE_CACHE_IS_PREPARING)) { + pr_warn("Cache tag %s in use\n", name); + fscache_put_cache(cache, fscache_cache_put_cache); + return ERR_PTR(-EBUSY); + } + + return cache; +} +EXPORT_SYMBOL(fscache_acquire_cache); + +/** + * fscache_put_cache - Release a cache-level cookie. + * @cache: The cache cookie to be released + * @where: An indication of where the release happened + * + * Release the caller's reference on a cache-level cookie. The @where + * indication should give information about the circumstances in which the call + * occurs and will be logged through a tracepoint. + */ +void fscache_put_cache(struct fscache_cache *cache, + enum fscache_cache_trace where) +{ + unsigned int debug_id = cache->debug_id; + bool zero; + int ref; + + if (IS_ERR_OR_NULL(cache)) + return; + + zero = __refcount_dec_and_test(&cache->ref, &ref); + trace_fscache_cache(debug_id, ref - 1, where); + + if (zero) { + down_write(&fscache_addremove_sem); + list_del_init(&cache->cache_link); + up_write(&fscache_addremove_sem); + kfree(cache->name); + kfree(cache); + } +} + +/** + * fscache_relinquish_cache - Reset cache state and release cookie + * @cache: The cache cookie to be released + * + * Reset the state of a cache and release the caller's reference on a cache + * cookie. + */ +void fscache_relinquish_cache(struct fscache_cache *cache) +{ + enum fscache_cache_trace where = + (cache->state == FSCACHE_CACHE_IS_PREPARING) ? + fscache_cache_put_prep_failed : + fscache_cache_put_relinquish; + + cache->ops = NULL; + cache->cache_priv = NULL; + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_NOT_PRESENT); + fscache_put_cache(cache, where); +} +EXPORT_SYMBOL(fscache_relinquish_cache); + +/** + * fscache_add_cache - Declare a cache as being open for business + * @cache: The cache-level cookie representing the cache + * @ops: Table of cache operations to use + * @cache_priv: Private data for the cache record + * + * Add a cache to the system, making it available for netfs's to use. + * + * See Documentation/filesystems/caching/backend-api.rst for a complete + * description. + */ +int fscache_add_cache(struct fscache_cache *cache, + const struct fscache_cache_ops *ops, + void *cache_priv) +{ + int n_accesses; + + _enter("{%s,%s}", ops->name, cache->name); + + BUG_ON(fscache_cache_state(cache) != FSCACHE_CACHE_IS_PREPARING); + + /* Get a ref on the cache cookie and keep its n_accesses counter raised + * by 1 to prevent wakeups from transitioning it to 0 until we're + * withdrawing caching services from it. + */ + n_accesses = atomic_inc_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_pin); + + down_write(&fscache_addremove_sem); + + cache->ops = ops; + cache->cache_priv = cache_priv; + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_ACTIVE); + + up_write(&fscache_addremove_sem); + pr_notice("Cache \"%s\" added (type %s)\n", cache->name, ops->name); + _leave(" = 0 [%s]", cache->name); + return 0; +} +EXPORT_SYMBOL(fscache_add_cache); + +/** + * fscache_begin_cache_access - Pin a cache so it can be accessed + * @cache: The cache-level cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing it and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the n_accesses count and + * then recheck the liveness, ending the access if it ceased to be live. + * + * (3) When we end the access, we decrement n_accesses and wake up the any + * waiters if it reaches 0. + * + * (4) Whilst the cache is caching, n_accesses is kept artificially + * incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, n_accesses is decremented and we wait for n_accesses to + * become 0. + */ +bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) +{ + int n_accesses; + + if (!fscache_cache_is_live(cache)) + return false; + + n_accesses = atomic_inc_return(&cache->n_accesses); + smp_mb__after_atomic(); /* Reread live flag after n_accesses */ + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, why); + if (!fscache_cache_is_live(cache)) { + fscache_end_cache_access(cache, fscache_access_unlive); + return false; + } + return true; +} + +/** + * fscache_end_cache_access - Unpin a cache at the end of an access. + * @cache: The cache-level cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache after we've accessed it. The @why indicator is merely + * provided for tracing purposes. + */ +void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&cache->n_accesses); +} + +/** + * fscache_io_error - Note a cache I/O error + * @cache: The record describing the cache + * + * Note that an I/O error occurred in a cache and that it should no longer be + * used for anything. This also reports the error into the kernel log. + * + * See Documentation/filesystems/caching/backend-api.rst for a complete + * description. + */ +void fscache_io_error(struct fscache_cache *cache) +{ + if (fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_ACTIVE, + FSCACHE_CACHE_GOT_IOERROR)) + pr_err("Cache '%s' stopped due to I/O error\n", + cache->name); +} +EXPORT_SYMBOL(fscache_io_error); + +/** + * fscache_withdraw_cache - Withdraw a cache from the active service + * @cache: The cache cookie + * + * Begin the process of withdrawing a cache from service. This stops new + * cache-level and volume-level accesses from taking place and waits for + * currently ongoing cache-level accesses to end. + */ +void fscache_withdraw_cache(struct fscache_cache *cache) +{ + int n_accesses; + + pr_notice("Withdrawing cache \"%s\" (%u objs)\n", + cache->name, atomic_read(&cache->object_count)); + + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_WITHDRAWN); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&cache->n_accesses, + atomic_read(&cache->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_cache); + +#ifdef CONFIG_PROC_FS +static const char fscache_cache_states[NR__FSCACHE_CACHE_STATE] = "-PAEW"; + +/* + * Generate a list of caches in /proc/fs/fscache/caches + */ +static int fscache_caches_seq_show(struct seq_file *m, void *v) +{ + struct fscache_cache *cache; + + if (v == &fscache_caches) { + seq_puts(m, + "CACHE REF VOLS OBJS ACCES S NAME\n" + "======== ===== ===== ===== ===== = ===============\n" + ); + return 0; + } + + cache = list_entry(v, struct fscache_cache, cache_link); + seq_printf(m, + "%08x %5d %5d %5d %5d %c %s\n", + cache->debug_id, + refcount_read(&cache->ref), + atomic_read(&cache->n_volumes), + atomic_read(&cache->object_count), + atomic_read(&cache->n_accesses), + fscache_cache_states[cache->state], + cache->name ?: "-"); + return 0; +} + +static void *fscache_caches_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_caches, *_pos); +} + +static void *fscache_caches_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_caches, _pos); +} + +static void fscache_caches_seq_stop(struct seq_file *m, void *v) + __releases(fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_caches_seq_ops = { + .start = fscache_caches_seq_start, + .next = fscache_caches_seq_next, + .stop = fscache_caches_seq_stop, + .show = fscache_caches_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/netfs/fscache_cookie.c b/fs/netfs/fscache_cookie.c new file mode 100644 index 000000000000..bce2492186d0 --- /dev/null +++ b/fs/netfs/fscache_cookie.c @@ -0,0 +1,1180 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* netfs cookie management + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * See Documentation/filesystems/caching/netfs-api.rst for more information on + * the netfs API. + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include +#include +#include "internal.h" + +struct kmem_cache *fscache_cookie_jar; + +static void fscache_cookie_lru_timed_out(struct timer_list *timer); +static void fscache_cookie_lru_worker(struct work_struct *work); +static void fscache_cookie_worker(struct work_struct *work); +static void fscache_unhash_cookie(struct fscache_cookie *cookie); +static void fscache_perform_invalidation(struct fscache_cookie *cookie); + +#define fscache_cookie_hash_shift 15 +static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; +static LIST_HEAD(fscache_cookies); +static DEFINE_RWLOCK(fscache_cookies_lock); +static LIST_HEAD(fscache_cookie_lru); +static DEFINE_SPINLOCK(fscache_cookie_lru_lock); +DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); +static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); +static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD"; +static unsigned int fscache_lru_cookie_timeout = 10 * HZ; + +void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) +{ + const u8 *k; + + pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", + prefix, + cookie->debug_id, + cookie->flags, + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state]); + pr_err("%c-cookie V=%08x [%s]\n", + prefix, + cookie->volume->debug_id, + cookie->volume->key); + + k = (cookie->key_len <= sizeof(cookie->inline_key)) ? + cookie->inline_key : cookie->key; + pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); +} + +static void fscache_free_cookie(struct fscache_cookie *cookie) +{ + if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { + spin_lock(&fscache_cookie_lru_lock); + list_del_init(&cookie->commit_link); + spin_unlock(&fscache_cookie_lru_lock); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_removed); + } + + if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { + fscache_print_cookie(cookie, 'F'); + return; + } + + write_lock(&fscache_cookies_lock); + list_del(&cookie->proc_link); + write_unlock(&fscache_cookies_lock); + if (cookie->aux_len > sizeof(cookie->inline_aux)) + kfree(cookie->aux); + if (cookie->key_len > sizeof(cookie->inline_key)) + kfree(cookie->key); + fscache_stat_d(&fscache_n_cookies); + kmem_cache_free(fscache_cookie_jar, cookie); +} + +static void __fscache_queue_cookie(struct fscache_cookie *cookie) +{ + if (!queue_work(fscache_wq, &cookie->work)) + fscache_put_cookie(cookie, fscache_cookie_put_over_queued); +} + +static void fscache_queue_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + fscache_get_cookie(cookie, where); + __fscache_queue_cookie(cookie); +} + +/* + * Initialise the access gate on a cookie by setting a flag to prevent the + * state machine from being queued when the access counter transitions to 0. + * We're only interested in this when we withdraw caching services from the + * cookie. + */ +static void fscache_init_access_gate(struct fscache_cookie *cookie) +{ + int n_accesses; + + n_accesses = atomic_read(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_pin); + set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); +} + +/** + * fscache_end_cookie_access - Unpin a cache at the end of an access. + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache cookie after we've accessed it and bring a deferred + * relinquishment or withdrawal state into effect. + * + * The @why indicator is provided for tracing purposes. + */ +void fscache_end_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); + if (n_accesses == 0 && + !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} +EXPORT_SYMBOL(fscache_end_cookie_access); + +/* + * Pin the cache behind a cookie so that we can access it. + */ +static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&cookie->n_accesses); + smp_mb__after_atomic(); /* (Future) read state after is-caching. + * Reread n_accesses after is-caching + */ + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); +} + +/** + * fscache_begin_cookie_access - Pin a cache so data can be accessed + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing data and returns true if successful. This works as follows: + * + * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not + * set), we return false to indicate access was not permitted. + * + * (2) If the cookie is being cached, we increment its n_accesses count and + * then recheck the IS_CACHING flag, ending the access if it got cleared. + * + * (3) When we end the access, we decrement the cookie's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cookie is actively being cached, its n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline or if the cookie is culled, the flag is + * cleared to prevent new accesses, the cookie's n_accesses is decremented + * and we wait for it to become 0. + * + * The @why indicator are merely provided for tracing purposes. + */ +bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return false; + __fscache_begin_cookie_access(cookie, why); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || + !fscache_cache_is_live(cookie->volume->cache)) { + fscache_end_cookie_access(cookie, fscache_access_unlive); + return false; + } + return true; +} + +static inline void wake_up_cookie_state(struct fscache_cookie *cookie) +{ + /* Use a barrier to ensure that waiters see the state variable + * change, as spin_unlock doesn't guarantee a barrier. + * + * See comments over wake_up_bit() and waitqueue_active(). + */ + smp_mb(); + wake_up_var(&cookie->state); +} + +/* + * Change the state a cookie is at and wake up anyone waiting for that. Impose + * an ordering between the stuff stored in the cookie and the state member. + * Paired with fscache_cookie_state(). + */ +static void __fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + smp_store_release(&cookie->state, state); +} + +static void fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + spin_lock(&cookie->lock); + __fscache_set_cookie_state(cookie, state); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); +} + +/** + * fscache_cookie_lookup_negative - Note negative lookup + * @cookie: The cookie that was being looked up + * + * Note that some part of the metadata path in the cache doesn't exist and so + * we can release any waiting readers in the certain knowledge that there's + * nothing for them to actually read. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); +} +EXPORT_SYMBOL(fscache_cookie_lookup_negative); + +/** + * fscache_resume_after_invalidation - Allow I/O to resume after invalidation + * @cookie: The cookie that was invalidated + * + * Tell fscache that invalidation is sufficiently complete that I/O can be + * allowed again. + */ +void fscache_resume_after_invalidation(struct fscache_cookie *cookie) +{ + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); +} +EXPORT_SYMBOL(fscache_resume_after_invalidation); + +/** + * fscache_caching_failed - Report that a failure stopped caching on a cookie + * @cookie: The cookie that was affected + * + * Tell fscache that caching on a cookie needs to be stopped due to some sort + * of failure. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_caching_failed(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); + trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), + fscache_cookie_failed); +} +EXPORT_SYMBOL(fscache_caching_failed); + +/* + * Set the index key in a cookie. The cookie struct has space for a 16-byte + * key plus length and hash, but if that's not big enough, it's instead a + * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then + * the key data. + */ +static int fscache_set_key(struct fscache_cookie *cookie, + const void *index_key, size_t index_key_len) +{ + void *buf; + size_t buf_size; + + buf_size = round_up(index_key_len, sizeof(__le32)); + + if (index_key_len > sizeof(cookie->inline_key)) { + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + cookie->key = buf; + } else { + buf = cookie->inline_key; + } + + memcpy(buf, index_key, index_key_len); + cookie->key_hash = fscache_hash(cookie->volume->key_hash, + buf, buf_size); + return 0; +} + +static bool fscache_cookie_same(const struct fscache_cookie *a, + const struct fscache_cookie *b) +{ + const void *ka, *kb; + + if (a->key_hash != b->key_hash || + a->volume != b->volume || + a->key_len != b->key_len) + return false; + + if (a->key_len <= sizeof(a->inline_key)) { + ka = &a->inline_key; + kb = &b->inline_key; + } else { + ka = a->key; + kb = b->key; + } + return memcmp(ka, kb, a->key_len) == 0; +} + +static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); + +/* + * Allocate a cookie. + */ +static struct fscache_cookie *fscache_alloc_cookie( + struct fscache_volume *volume, + u8 advice, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + /* allocate and initialise a cookie */ + cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); + if (!cookie) + return NULL; + fscache_stat(&fscache_n_cookies); + + cookie->volume = volume; + cookie->advice = advice; + cookie->key_len = index_key_len; + cookie->aux_len = aux_data_len; + cookie->object_size = object_size; + if (object_size == 0) + __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + + if (fscache_set_key(cookie, index_key, index_key_len) < 0) + goto nomem; + + if (cookie->aux_len <= sizeof(cookie->inline_aux)) { + memcpy(cookie->inline_aux, aux_data, cookie->aux_len); + } else { + cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); + if (!cookie->aux) + goto nomem; + } + + refcount_set(&cookie->ref, 1); + cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); + spin_lock_init(&cookie->lock); + INIT_LIST_HEAD(&cookie->commit_link); + INIT_WORK(&cookie->work, fscache_cookie_worker); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + + write_lock(&fscache_cookies_lock); + list_add_tail(&cookie->proc_link, &fscache_cookies); + write_unlock(&fscache_cookies_lock); + fscache_see_cookie(cookie, fscache_cookie_new_acquire); + return cookie; + +nomem: + fscache_free_cookie(cookie); + return NULL; +} + +static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie) +{ + return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED; +} + +static void fscache_wait_on_collision(struct fscache_cookie *candidate, + struct fscache_cookie *wait_for) +{ + enum fscache_cookie_state *statep = &wait_for->state; + + wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for), + 20 * HZ); + if (!fscache_cookie_is_dropped(wait_for)) { + pr_notice("Potential collision c=%08x old: c=%08x", + candidate->debug_id, wait_for->debug_id); + wait_var_event(statep, fscache_cookie_is_dropped(wait_for)); + } +} + +/* + * Attempt to insert the new cookie into the hash. If there's a collision, we + * wait for the old cookie to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_cookie(struct fscache_cookie *candidate) +{ + struct fscache_cookie *cursor, *wait_for = NULL; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_cookie_same(candidate, cursor)) { + if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) + goto collision; + wait_for = fscache_get_cookie(cursor, + fscache_cookie_get_hash_collision); + break; + } + } + + fscache_get_volume(candidate->volume, fscache_volume_get_cookie); + atomic_inc(&candidate->volume->n_cookies); + hlist_bl_add_head(&candidate->hash_link, h); + set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); + hlist_bl_unlock(h); + + if (wait_for) { + fscache_wait_on_collision(candidate, wait_for); + fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); + } + return true; + +collision: + trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), + fscache_cookie_collision); + pr_err("Duplicate cookie detected\n"); + fscache_print_cookie(cursor, 'O'); + fscache_print_cookie(candidate, 'N'); + hlist_bl_unlock(h); + return false; +} + +/* + * Request a cookie to represent a data storage object within a volume. + * + * We never let on to the netfs about errors. We may set a negative cookie + * pointer, but that's okay + */ +struct fscache_cookie *__fscache_acquire_cookie( + struct fscache_volume *volume, + u8 advice, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + _enter("V=%x", volume->debug_id); + + if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) + return NULL; + if (!aux_data || !aux_data_len) { + aux_data = NULL; + aux_data_len = 0; + } + + fscache_stat(&fscache_n_acquires); + + cookie = fscache_alloc_cookie(volume, advice, + index_key, index_key_len, + aux_data, aux_data_len, + object_size); + if (!cookie) { + fscache_stat(&fscache_n_acquires_oom); + return NULL; + } + + if (!fscache_hash_cookie(cookie)) { + fscache_see_cookie(cookie, fscache_cookie_discard); + fscache_free_cookie(cookie); + return NULL; + } + + trace_fscache_acquire(cookie); + fscache_stat(&fscache_n_acquires_ok); + _leave(" = c=%08x", cookie->debug_id); + return cookie; +} +EXPORT_SYMBOL(__fscache_acquire_cookie); + +/* + * Prepare a cache object to be written to. + */ +static void fscache_prepare_to_write(struct fscache_cookie *cookie) +{ + cookie->volume->cache->ops->prepare_to_write(cookie); +} + +/* + * Look up a cookie in the cache. + */ +static void fscache_perform_lookup(struct fscache_cookie *cookie) +{ + enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; + bool need_withdraw = false; + + _enter(""); + + if (!cookie->volume->cache_priv) { + fscache_create_volume(cookie->volume, true); + if (!cookie->volume->cache_priv) { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + goto out; + } + } + + if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { + if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + need_withdraw = true; + _leave(" [fail]"); + goto out; + } + + fscache_see_cookie(cookie, fscache_cookie_see_active); + spin_lock(&cookie->lock); + if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_INVALIDATING); + else + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); + trace = fscache_access_lookup_cookie_end; + +out: + fscache_end_cookie_access(cookie, trace); + if (need_withdraw) + fscache_withdraw_cookie(cookie); + fscache_end_volume_access(cookie->volume, cookie, trace); +} + +/* + * Begin the process of looking up a cookie. We offload the actual process to + * a worker thread. + */ +static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) +{ + if (will_modify) { + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + } + if (!fscache_begin_volume_access(cookie->volume, cookie, + fscache_access_lookup_cookie)) + return false; + + __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); + set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); + return true; +} + +/* + * Start using the cookie for I/O. This prevents the backing object from being + * reaped by VM pressure. + */ +void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) +{ + enum fscache_cookie_state state; + bool queue = false; + int n_active; + + _enter("c=%08x", cookie->debug_id); + + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to use relinquished cookie\n")) + return; + + spin_lock(&cookie->lock); + + n_active = atomic_inc_return(&cookie->n_active); + trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref), + n_active, atomic_read(&cookie->n_accesses), + will_modify ? + fscache_active_use_modify : fscache_active_use); + +again: + state = fscache_cookie_state(cookie); + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + queue = fscache_begin_lookup(cookie, will_modify); + break; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_CREATING: + if (will_modify) + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + break; + case FSCACHE_COOKIE_STATE_ACTIVE: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (will_modify && + !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + queue = true; + } + /* + * We could race with cookie_lru which may set LRU_DISCARD bit + * but has yet to run the cookie state machine. If this happens + * and another thread tries to use the cookie, clear LRU_DISCARD + * so we don't end up withdrawing the cookie while in use. + */ + if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear); + break; + + case FSCACHE_COOKIE_STATE_FAILED: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + break; + + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + spin_unlock(&cookie->lock); + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + spin_lock(&cookie->lock); + goto again; + + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", state); + break; + } + + spin_unlock(&cookie->lock); + if (queue) + fscache_queue_cookie(cookie, fscache_cookie_get_use_work); + _leave(""); +} +EXPORT_SYMBOL(__fscache_use_cookie); + +static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return; + + cookie->unused_at = jiffies; + spin_lock(&fscache_cookie_lru_lock); + if (list_empty(&cookie->commit_link)) { + fscache_get_cookie(cookie, fscache_cookie_get_lru); + fscache_stat(&fscache_n_cookies_lru); + } + list_move_tail(&cookie->commit_link, &fscache_cookie_lru); + + spin_unlock(&fscache_cookie_lru_lock); + timer_reduce(&fscache_cookie_lru_timer, + jiffies + fscache_lru_cookie_timeout); +} + +/* + * Stop using the cookie for I/O. + */ +void __fscache_unuse_cookie(struct fscache_cookie *cookie, + const void *aux_data, const loff_t *object_size) +{ + unsigned int debug_id = cookie->debug_id; + unsigned int r = refcount_read(&cookie->ref); + unsigned int a = atomic_read(&cookie->n_accesses); + unsigned int c; + + if (aux_data || object_size) + __fscache_update_cookie(cookie, aux_data, object_size); + + /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ + c = atomic_fetch_add_unless(&cookie->n_active, -1, 1); + if (c != 1) { + trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse); + return; + } + + spin_lock(&cookie->lock); + r = refcount_read(&cookie->ref); + a = atomic_read(&cookie->n_accesses); + c = atomic_dec_return(&cookie->n_active); + trace_fscache_active(debug_id, r, c, a, fscache_active_unuse); + if (c == 0) + fscache_unuse_cookie_locked(cookie); + spin_unlock(&cookie->lock); +} +EXPORT_SYMBOL(__fscache_unuse_cookie); + +/* + * Perform work upon the cookie, such as committing its cache state, + * relinquishing it or withdrawing the backing cache. We're protected from the + * cache going away under us as object withdrawal must come through this + * non-reentrant work item. + */ +static void fscache_cookie_state_machine(struct fscache_cookie *cookie) +{ + enum fscache_cookie_state state; + bool wake = false; + + _enter("c=%x", cookie->debug_id); + +again: + spin_lock(&cookie->lock); +again_locked: + state = cookie->state; + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + /* The QUIESCENT state is jumped to the LOOKING_UP state by + * fscache_use_cookie(). + */ + + if (atomic_read(&cookie->n_accesses) == 0 && + test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + break; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + spin_unlock(&cookie->lock); + fscache_init_access_gate(cookie); + fscache_perform_lookup(cookie); + goto again; + + case FSCACHE_COOKIE_STATE_INVALIDATING: + spin_unlock(&cookie->lock); + fscache_perform_invalidation(cookie); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { + spin_unlock(&cookie->lock); + fscache_prepare_to_write(cookie); + spin_lock(&cookie->lock); + } + if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + wake = true; + goto again_locked; + } + fallthrough; + + case FSCACHE_COOKIE_STATE_FAILED: + if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); + + if (atomic_read(&cookie->n_accesses) != 0) + break; + if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_WITHDRAWING); + wake = true; + goto again_locked; + } + break; + + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + if (cookie->cache_priv) { + spin_unlock(&cookie->lock); + cookie->volume->cache->ops->withdraw_cookie(cookie); + spin_lock(&cookie->lock); + } + + if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); + + switch (state) { + case FSCACHE_COOKIE_STATE_RELINQUISHING: + fscache_see_cookie(cookie, fscache_cookie_see_relinquish); + fscache_unhash_cookie(cookie); + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_DROPPED); + wake = true; + goto out; + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); + break; + case FSCACHE_COOKIE_STATE_WITHDRAWING: + fscache_see_cookie(cookie, fscache_cookie_see_withdraw); + break; + default: + BUG(); + } + + clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + wake = true; + goto again_locked; + + case FSCACHE_COOKIE_STATE_DROPPED: + break; + + default: + WARN_ONCE(1, "Cookie %x in unexpected state %u\n", + cookie->debug_id, state); + break; + } + +out: + spin_unlock(&cookie->lock); + if (wake) + wake_up_cookie_state(cookie); + _leave(""); +} + +static void fscache_cookie_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); + + fscache_see_cookie(cookie, fscache_cookie_see_work); + fscache_cookie_state_machine(cookie); + fscache_put_cookie(cookie, fscache_cookie_put_work); +} + +/* + * Wait for the object to become inactive. The cookie's work item will be + * scheduled when someone transitions n_accesses to 0 - but if someone's + * already done that, schedule it anyway. + */ +static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + int n_accesses; + bool unpinned; + + unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); + + /* Need to read the access count after unpinning */ + n_accesses = atomic_read(&cookie->n_accesses); + if (unpinned) + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_unpin); + if (n_accesses == 0) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} + +static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) +{ + fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); + + spin_lock(&cookie->lock); + if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || + time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || + atomic_read(&cookie->n_active) > 0) { + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_removed); + } else { + set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_expired); + _debug("lru c=%x", cookie->debug_id); + __fscache_withdraw_cookie(cookie); + } + + fscache_put_cookie(cookie, fscache_cookie_put_lru); +} + +static void fscache_cookie_lru_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie; + unsigned long unused_at; + + spin_lock(&fscache_cookie_lru_lock); + + while (!list_empty(&fscache_cookie_lru)) { + cookie = list_first_entry(&fscache_cookie_lru, + struct fscache_cookie, commit_link); + unused_at = cookie->unused_at + fscache_lru_cookie_timeout; + if (time_before(jiffies, unused_at)) { + timer_reduce(&fscache_cookie_lru_timer, unused_at); + break; + } + + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + spin_unlock(&fscache_cookie_lru_lock); + fscache_cookie_lru_do_one(cookie); + spin_lock(&fscache_cookie_lru_lock); + } + + spin_unlock(&fscache_cookie_lru_lock); +} + +static void fscache_cookie_lru_timed_out(struct timer_list *timer) +{ + queue_work(fscache_wq, &fscache_cookie_lru_work); +} + +static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) +{ + bool need_put = false; + + if (!list_empty(&cookie->commit_link)) { + spin_lock(&fscache_cookie_lru_lock); + if (!list_empty(&cookie->commit_link)) { + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_dropped); + need_put = true; + } + spin_unlock(&fscache_cookie_lru_lock); + if (need_put) + fscache_put_cookie(cookie, fscache_cookie_put_lru); + } +} + +/* + * Remove a cookie from the hash table. + */ +static void fscache_unhash_cookie(struct fscache_cookie *cookie) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&cookie->hash_link); + clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); + hlist_bl_unlock(h); + fscache_stat(&fscache_n_relinquishes_dropped); +} + +static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) +{ + fscache_cookie_drop_from_lru(cookie); + __fscache_withdraw_cookie(cookie); +} + +/** + * fscache_withdraw_cookie - Mark a cookie for withdrawal + * @cookie: The cookie to be withdrawn. + * + * Allow the cache backend to withdraw the backing for a cookie for its own + * reasons, even if that cookie is in active use. + */ +void fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); +} +EXPORT_SYMBOL(fscache_withdraw_cookie); + +/* + * Allow the netfs to release a cookie back to the cache. + * - the object will be marked as recyclable on disk if retire is true + */ +void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) +{ + fscache_stat(&fscache_n_relinquishes); + if (retire) + fscache_stat(&fscache_n_relinquishes_retire); + + _enter("c=%08x{%d},%d", + cookie->debug_id, atomic_read(&cookie->n_active), retire); + + if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Cookie c=%x already relinquished\n", cookie->debug_id)) + return; + + if (retire) + set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); + trace_fscache_relinquish(cookie, retire); + + ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); + ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); + atomic_dec(&cookie->volume->n_cookies); + + if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); + } else { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); + fscache_unhash_cookie(cookie); + } + fscache_put_cookie(cookie, fscache_cookie_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_cookie); + +/* + * Drop a reference to a cookie. + */ +void fscache_put_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + struct fscache_volume *volume = cookie->volume; + unsigned int cookie_debug_id = cookie->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&cookie->ref, &ref); + trace_fscache_cookie(cookie_debug_id, ref - 1, where); + if (zero) { + fscache_free_cookie(cookie); + fscache_put_volume(volume, fscache_volume_put_cookie); + } +} +EXPORT_SYMBOL(fscache_put_cookie); + +/* + * Get a reference to a cookie. + */ +struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + int ref; + + __refcount_inc(&cookie->ref, &ref); + trace_fscache_cookie(cookie->debug_id, ref + 1, where); + return cookie; +} +EXPORT_SYMBOL(fscache_get_cookie); + +/* + * Ask the cache to effect invalidation of a cookie. + */ +static void fscache_perform_invalidation(struct fscache_cookie *cookie) +{ + if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) + fscache_caching_failed(cookie); + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); +} + +/* + * Invalidate an object. + */ +void __fscache_invalidate(struct fscache_cookie *cookie, + const void *aux_data, loff_t new_size, + unsigned int flags) +{ + bool is_caching; + + _enter("c=%x", cookie->debug_id); + + fscache_stat(&fscache_n_invalidates); + + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to invalidate relinquished cookie\n")) + return; + + if ((flags & FSCACHE_INVAL_DIO_WRITE) && + test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) + return; + + spin_lock(&cookie->lock); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_update_aux(cookie, aux_data, &new_size); + cookie->inval_counter++; + trace_fscache_invalidate(cookie, new_size); + + switch (cookie->state) { + case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ + default: + spin_unlock(&cookie->lock); + _leave(" [no %u]", cookie->state); + return; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie); + fallthrough; + case FSCACHE_COOKIE_STATE_CREATING: + spin_unlock(&cookie->lock); + _leave(" [look %x]", cookie->inval_counter); + return; + + case FSCACHE_COOKIE_STATE_ACTIVE: + is_caching = fscache_begin_cookie_access( + cookie, fscache_access_invalidate_cookie); + if (is_caching) + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); + + if (is_caching) + fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); + _leave(" [inv]"); + return; + } +} +EXPORT_SYMBOL(__fscache_invalidate); + +#ifdef CONFIG_PROC_FS +/* + * Generate a list of extant cookies in /proc/fs/fscache/cookies + */ +static int fscache_cookies_seq_show(struct seq_file *m, void *v) +{ + struct fscache_cookie *cookie; + unsigned int keylen = 0, auxlen = 0; + u8 *p; + + if (v == &fscache_cookies) { + seq_puts(m, + "COOKIE VOLUME REF ACT ACC S FL DEF \n" + "======== ======== === === === = == ================\n" + ); + return 0; + } + + cookie = list_entry(v, struct fscache_cookie, proc_link); + + seq_printf(m, + "%08x %08x %3d %3d %3d %c %02lx", + cookie->debug_id, + cookie->volume->debug_id, + refcount_read(&cookie->ref), + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state], + cookie->flags); + + keylen = cookie->key_len; + auxlen = cookie->aux_len; + + if (keylen > 0 || auxlen > 0) { + seq_puts(m, " "); + p = keylen <= sizeof(cookie->inline_key) ? + cookie->inline_key : cookie->key; + for (; keylen > 0; keylen--) + seq_printf(m, "%02x", *p++); + if (auxlen > 0) { + seq_puts(m, ", "); + p = auxlen <= sizeof(cookie->inline_aux) ? + cookie->inline_aux : cookie->aux; + for (; auxlen > 0; auxlen--) + seq_printf(m, "%02x", *p++); + } + } + + seq_puts(m, "\n"); + return 0; +} + +static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_cookies_lock) +{ + read_lock(&fscache_cookies_lock); + return seq_list_start_head(&fscache_cookies, *_pos); +} + +static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_cookies, _pos); +} + +static void fscache_cookies_seq_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + read_unlock(&fscache_cookies_lock); +} + + +const struct seq_operations fscache_cookies_seq_ops = { + .start = fscache_cookies_seq_start, + .next = fscache_cookies_seq_next, + .stop = fscache_cookies_seq_stop, + .show = fscache_cookies_seq_show, +}; +#endif diff --git a/fs/netfs/fscache_internal.h b/fs/netfs/fscache_internal.h new file mode 100644 index 000000000000..1336f517e9b1 --- /dev/null +++ b/fs/netfs/fscache_internal.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Internal definitions for FS-Cache + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "FS-Cache: " fmt + +#include +#include +#include +#include +#include + +/* + * cache.c + */ +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_caches_seq_ops; +#endif +bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); +void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); +void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); + +static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) +{ + return smp_load_acquire(&cache->state); +} + +static inline bool fscache_cache_is_live(const struct fscache_cache *cache) +{ + return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; +} + +static inline void fscache_set_cache_state(struct fscache_cache *cache, + enum fscache_cache_state new_state) +{ + smp_store_release(&cache->state, new_state); + +} + +static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, + enum fscache_cache_state old_state, + enum fscache_cache_state new_state) +{ + return try_cmpxchg_release(&cache->state, &old_state, new_state); +} + +/* + * cookie.c + */ +extern struct kmem_cache *fscache_cookie_jar; +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_cookies_seq_ops; +#endif +extern struct timer_list fscache_cookie_lru_timer; + +extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); +extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why); + +static inline void fscache_see_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), + where); +} + +/* + * main.c + */ +extern unsigned fscache_debug; + +extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len); + +/* + * proc.c + */ +#ifdef CONFIG_PROC_FS +extern int __init fscache_proc_init(void); +extern void fscache_proc_cleanup(void); +#else +#define fscache_proc_init() (0) +#define fscache_proc_cleanup() do {} while (0) +#endif + +/* + * stats.c + */ +#ifdef CONFIG_FSCACHE_STATS +extern atomic_t fscache_n_volumes; +extern atomic_t fscache_n_volumes_collision; +extern atomic_t fscache_n_volumes_nomem; +extern atomic_t fscache_n_cookies; +extern atomic_t fscache_n_cookies_lru; +extern atomic_t fscache_n_cookies_lru_expired; +extern atomic_t fscache_n_cookies_lru_removed; +extern atomic_t fscache_n_cookies_lru_dropped; + +extern atomic_t fscache_n_acquires; +extern atomic_t fscache_n_acquires_ok; +extern atomic_t fscache_n_acquires_oom; + +extern atomic_t fscache_n_invalidates; + +extern atomic_t fscache_n_relinquishes; +extern atomic_t fscache_n_relinquishes_retire; +extern atomic_t fscache_n_relinquishes_dropped; + +extern atomic_t fscache_n_resizes; +extern atomic_t fscache_n_resizes_null; + +static inline void fscache_stat(atomic_t *stat) +{ + atomic_inc(stat); +} + +static inline void fscache_stat_d(atomic_t *stat) +{ + atomic_dec(stat); +} + +#define __fscache_stat(stat) (stat) + +int fscache_stats_show(struct seq_file *m, void *v); +#else + +#define __fscache_stat(stat) (NULL) +#define fscache_stat(stat) do {} while (0) +#define fscache_stat_d(stat) do {} while (0) +#endif + +/* + * volume.c + */ +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_volumes_seq_ops; +#endif + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why); +void fscache_create_volume(struct fscache_volume *volume, bool wait); + + +/*****************************************************************************/ +/* + * debug tracing + */ +#define dbgprintk(FMT, ...) \ + printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) + +#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) + +#define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__) + +#ifdef __KDEBUG +#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) +#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) +#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) + +#elif defined(CONFIG_FSCACHE_DEBUG) +#define _enter(FMT, ...) \ +do { \ + if (__do_kdebug(ENTER)) \ + kenter(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _leave(FMT, ...) \ +do { \ + if (__do_kdebug(LEAVE)) \ + kleave(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _debug(FMT, ...) \ +do { \ + if (__do_kdebug(DEBUG)) \ + kdebug(FMT, ##__VA_ARGS__); \ +} while (0) + +#else +#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) +#endif + +/* + * determine whether a particular optional debugging point should be logged + * - we need to go through three steps to persuade cpp to correctly join the + * shorthand in FSCACHE_DEBUG_LEVEL with its prefix + */ +#define ____do_kdebug(LEVEL, POINT) \ + unlikely((fscache_debug & \ + (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3)))) +#define ___do_kdebug(LEVEL, POINT) \ + ____do_kdebug(LEVEL, POINT) +#define __do_kdebug(POINT) \ + ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT) + +#define FSCACHE_DEBUG_CACHE 0 +#define FSCACHE_DEBUG_COOKIE 1 +#define FSCACHE_DEBUG_OBJECT 2 +#define FSCACHE_DEBUG_OPERATION 3 + +#define FSCACHE_POINT_ENTER 1 +#define FSCACHE_POINT_LEAVE 2 +#define FSCACHE_POINT_DEBUG 4 + +#ifndef FSCACHE_DEBUG_LEVEL +#define FSCACHE_DEBUG_LEVEL CACHE +#endif + +/* + * assertions + */ +#if 1 /* defined(__KDEBUGALL) */ + +#define ASSERT(X) \ +do { \ + if (unlikely(!(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTCMP(X, OP, Y) \ +do { \ + if (unlikely(!((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIF(C, X) \ +do { \ + if (unlikely((C) && !(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIFCMP(C, X, OP, Y) \ +do { \ + if (unlikely((C) && !((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#else + +#define ASSERT(X) do {} while (0) +#define ASSERTCMP(X, OP, Y) do {} while (0) +#define ASSERTIF(C, X) do {} while (0) +#define ASSERTIFCMP(C, X, OP, Y) do {} while (0) + +#endif /* assert or not */ diff --git a/fs/netfs/fscache_io.c b/fs/netfs/fscache_io.c new file mode 100644 index 000000000000..0d2b8dec8f82 --- /dev/null +++ b/fs/netfs/fscache_io.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Cache data I/O routines + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ +#define FSCACHE_DEBUG_LEVEL OPERATION +#include +#include +#include +#include +#include +#include "internal.h" + +/** + * fscache_wait_for_operation - Wait for an object become accessible + * @cres: The cache resources for the operation being performed + * @want_state: The minimum state the object must be at + * + * See if the target cache object is at the specified minimum state of + * accessibility yet, and if not, wait for it. + */ +bool fscache_wait_for_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state) +{ + struct fscache_cookie *cookie = fscache_cres_cookie(cres); + enum fscache_cookie_state state; + +again: + if (!fscache_cache_is_live(cookie->volume->cache)) { + _leave(" [broken]"); + return false; + } + + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); + + switch (state) { + case FSCACHE_COOKIE_STATE_CREATING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + fallthrough; + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != state); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + default: + _leave(" [not live]"); + return false; + } + +ready: + if (!cres->cache_priv2) + return cookie->volume->cache->ops->begin_operation(cres, want_state); + return true; +} +EXPORT_SYMBOL(fscache_wait_for_operation); + +/* + * Begin an I/O operation on the cache, waiting till we reach the right state. + * + * Attaches the resources required to the operation resources record. + */ +static int fscache_begin_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie, + enum fscache_want_state want_state, + enum fscache_access_trace why) +{ + enum fscache_cookie_state state; + long timeo; + bool once_only = false; + + cres->ops = NULL; + cres->cache_priv = cookie; + cres->cache_priv2 = NULL; + cres->debug_id = cookie->debug_id; + cres->inval_counter = cookie->inval_counter; + + if (!fscache_begin_cookie_access(cookie, why)) + return -ENOBUFS; + +again: + spin_lock(&cookie->lock); + + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); + + switch (state) { + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_CREATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", cookie->state); + goto not_live; + default: + goto not_live; + } + +ready: + spin_unlock(&cookie->lock); + if (!cookie->volume->cache->ops->begin_operation(cres, want_state)) + goto failed; + return 0; + +wait_for_file_wrangling: + spin_unlock(&cookie->lock); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + atomic_read(&cookie->n_accesses), + fscache_access_io_wait); + timeo = wait_var_event_timeout(&cookie->state, + fscache_cookie_state(cookie) != state, 20 * HZ); + if (timeo <= 1 && !once_only) { + pr_warn("%s: cookie state change wait timed out: cookie->state=%u state=%u", + __func__, fscache_cookie_state(cookie), state); + fscache_print_cookie(cookie, 'O'); + once_only = true; + } + goto again; + +not_live: + spin_unlock(&cookie->lock); +failed: + cres->cache_priv = NULL; + cres->ops = NULL; + fscache_end_cookie_access(cookie, fscache_access_io_not_live); + _leave(" = -ENOBUFS"); + return -ENOBUFS; +} + +int __fscache_begin_read_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_read); +} +EXPORT_SYMBOL(__fscache_begin_read_operation); + +int __fscache_begin_write_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_write); +} +EXPORT_SYMBOL(__fscache_begin_write_operation); + +/** + * fscache_dirty_folio - Mark folio dirty and pin a cache object for writeback + * @mapping: The mapping the folio belongs to. + * @folio: The folio being dirtied. + * @cookie: The cookie referring to the cache object + * + * Set the dirty flag on a folio and pin an in-use cache object in memory + * so that writeback can later write to it. This is intended + * to be called from the filesystem's ->dirty_folio() method. + * + * Return: true if the dirty flag was set on the folio, false otherwise. + */ +bool fscache_dirty_folio(struct address_space *mapping, struct folio *folio, + struct fscache_cookie *cookie) +{ + struct inode *inode = mapping->host; + bool need_use = false; + + _enter(""); + + if (!filemap_dirty_folio(mapping, folio)) + return false; + if (!fscache_cookie_valid(cookie)) + return true; + + if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { + spin_lock(&inode->i_lock); + if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { + inode->i_state |= I_PINNING_FSCACHE_WB; + need_use = true; + } + spin_unlock(&inode->i_lock); + + if (need_use) + fscache_use_cookie(cookie, true); + } + return true; +} +EXPORT_SYMBOL(fscache_dirty_folio); + +struct fscache_write_request { + struct netfs_cache_resources cache_resources; + struct address_space *mapping; + loff_t start; + size_t len; + bool set_bits; + netfs_io_terminated_t term_func; + void *term_func_priv; +}; + +void __fscache_clear_page_bits(struct address_space *mapping, + loff_t start, size_t len) +{ + pgoff_t first = start / PAGE_SIZE; + pgoff_t last = (start + len - 1) / PAGE_SIZE; + struct page *page; + + if (len) { + XA_STATE(xas, &mapping->i_pages, first); + + rcu_read_lock(); + xas_for_each(&xas, page, last) { + end_page_fscache(page); + } + rcu_read_unlock(); + } +} +EXPORT_SYMBOL(__fscache_clear_page_bits); + +/* + * Deal with the completion of writing the data to the cache. + */ +static void fscache_wreq_done(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct fscache_write_request *wreq = priv; + + fscache_clear_page_bits(wreq->mapping, wreq->start, wreq->len, + wreq->set_bits); + + if (wreq->term_func) + wreq->term_func(wreq->term_func_priv, transferred_or_error, + was_async); + fscache_end_operation(&wreq->cache_resources); + kfree(wreq); +} + +void __fscache_write_to_cache(struct fscache_cookie *cookie, + struct address_space *mapping, + loff_t start, size_t len, loff_t i_size, + netfs_io_terminated_t term_func, + void *term_func_priv, + bool cond) +{ + struct fscache_write_request *wreq; + struct netfs_cache_resources *cres; + struct iov_iter iter; + int ret = -ENOBUFS; + + if (len == 0) + goto abandon; + + _enter("%llx,%zx", start, len); + + wreq = kzalloc(sizeof(struct fscache_write_request), GFP_NOFS); + if (!wreq) + goto abandon; + wreq->mapping = mapping; + wreq->start = start; + wreq->len = len; + wreq->set_bits = cond; + wreq->term_func = term_func; + wreq->term_func_priv = term_func_priv; + + cres = &wreq->cache_resources; + if (fscache_begin_operation(cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_write) < 0) + goto abandon_free; + + ret = cres->ops->prepare_write(cres, &start, &len, i_size, false); + if (ret < 0) + goto abandon_end; + + /* TODO: Consider clearing page bits now for space the write isn't + * covering. This is more complicated than it appears when THPs are + * taken into account. + */ + + iov_iter_xarray(&iter, ITER_SOURCE, &mapping->i_pages, start, len); + fscache_write(cres, start, &iter, fscache_wreq_done, wreq); + return; + +abandon_end: + return fscache_wreq_done(wreq, ret, false); +abandon_free: + kfree(wreq); +abandon: + fscache_clear_page_bits(mapping, start, len, cond); + if (term_func) + term_func(term_func_priv, ret, false); +} +EXPORT_SYMBOL(__fscache_write_to_cache); + +/* + * Change the size of a backing object. + */ +void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +{ + struct netfs_cache_resources cres; + + trace_fscache_resize(cookie, new_size); + if (fscache_begin_operation(&cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_resize) == 0) { + fscache_stat(&fscache_n_resizes); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + + /* We cannot defer a resize as we need to do it inside the + * netfs's inode lock so that we're serialised with respect to + * writes. + */ + cookie->volume->cache->ops->resize_cookie(&cres, new_size); + fscache_end_operation(&cres); + } else { + fscache_stat(&fscache_n_resizes_null); + } +} +EXPORT_SYMBOL(__fscache_resize_cookie); diff --git a/fs/netfs/fscache_main.c b/fs/netfs/fscache_main.c new file mode 100644 index 000000000000..dad85fd84f6f --- /dev/null +++ b/fs/netfs/fscache_main.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* General filesystem local caching manager + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#define CREATE_TRACE_POINTS +#include "internal.h" + +MODULE_DESCRIPTION("FS Cache Manager"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +unsigned fscache_debug; +module_param_named(debug, fscache_debug, uint, + S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(fscache_debug, + "FS-Cache debugging mask"); + +EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache); +EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume); +EXPORT_TRACEPOINT_SYMBOL(fscache_access); + +struct workqueue_struct *fscache_wq; +EXPORT_SYMBOL(fscache_wq); + +/* + * Mixing scores (in bits) for (7,20): + * Input delta: 1-bit 2-bit + * 1 round: 330.3 9201.6 + * 2 rounds: 1246.4 25475.4 + * 3 rounds: 1907.1 31295.1 + * 4 rounds: 2042.3 31718.6 + * Perfect: 2048 31744 + * (32*64) (32*31/2 * 64) + */ +#define HASH_MIX(x, y, a) \ + ( x ^= (a), \ + y ^= x, x = rol32(x, 7),\ + x += y, y = rol32(y,20),\ + y *= 9 ) + +static inline unsigned int fold_hash(unsigned long x, unsigned long y) +{ + /* Use arch-optimized multiply if one exists */ + return __hash_32(y ^ __hash_32(x)); +} + +/* + * Generate a hash. This is derived from full_name_hash(), but we want to be + * sure it is arch independent and that it doesn't change as bits of the + * computed hash value might appear on disk. The caller must guarantee that + * the source data is a multiple of four bytes in size. + */ +unsigned int fscache_hash(unsigned int salt, const void *data, size_t len) +{ + const __le32 *p = data; + unsigned int a, x = 0, y = salt, n = len / sizeof(__le32); + + for (; n; n--) { + a = le32_to_cpu(*p++); + HASH_MIX(x, y, a); + } + return fold_hash(x, y); +} + +/* + * initialise the fs caching module + */ +static int __init fscache_init(void) +{ + int ret = -ENOMEM; + + fscache_wq = alloc_workqueue("fscache", WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!fscache_wq) + goto error_wq; + + ret = fscache_proc_init(); + if (ret < 0) + goto error_proc; + + fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", + sizeof(struct fscache_cookie), + 0, 0, NULL); + if (!fscache_cookie_jar) { + pr_notice("Failed to allocate a cookie jar\n"); + ret = -ENOMEM; + goto error_cookie_jar; + } + + pr_notice("Loaded\n"); + return 0; + +error_cookie_jar: + fscache_proc_cleanup(); +error_proc: + destroy_workqueue(fscache_wq); +error_wq: + return ret; +} + +fs_initcall(fscache_init); + +/* + * clean up on module removal + */ +static void __exit fscache_exit(void) +{ + _enter(""); + + kmem_cache_destroy(fscache_cookie_jar); + fscache_proc_cleanup(); + destroy_workqueue(fscache_wq); + pr_notice("Unloaded\n"); +} + +module_exit(fscache_exit); diff --git a/fs/netfs/fscache_proc.c b/fs/netfs/fscache_proc.c new file mode 100644 index 000000000000..dc3b0e9c8cce --- /dev/null +++ b/fs/netfs/fscache_proc.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache statistics viewing interface + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include +#include "internal.h" + +/* + * initialise the /proc/fs/fscache/ directory + */ +int __init fscache_proc_init(void) +{ + if (!proc_mkdir("fs/fscache", NULL)) + goto error_dir; + + if (!proc_create_seq("fs/fscache/caches", S_IFREG | 0444, NULL, + &fscache_caches_seq_ops)) + goto error; + + if (!proc_create_seq("fs/fscache/volumes", S_IFREG | 0444, NULL, + &fscache_volumes_seq_ops)) + goto error; + + if (!proc_create_seq("fs/fscache/cookies", S_IFREG | 0444, NULL, + &fscache_cookies_seq_ops)) + goto error; + +#ifdef CONFIG_FSCACHE_STATS + if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, + fscache_stats_show)) + goto error; +#endif + + return 0; + +error: + remove_proc_entry("fs/fscache", NULL); +error_dir: + return -ENOMEM; +} + +/* + * clean up the /proc/fs/fscache/ directory + */ +void fscache_proc_cleanup(void) +{ + remove_proc_subtree("fs/fscache", NULL); +} diff --git a/fs/netfs/fscache_stats.c b/fs/netfs/fscache_stats.c new file mode 100644 index 000000000000..fc94e5e79f1c --- /dev/null +++ b/fs/netfs/fscache_stats.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache statistics + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" + +/* + * operation counters + */ +atomic_t fscache_n_volumes; +atomic_t fscache_n_volumes_collision; +atomic_t fscache_n_volumes_nomem; +atomic_t fscache_n_cookies; +atomic_t fscache_n_cookies_lru; +atomic_t fscache_n_cookies_lru_expired; +atomic_t fscache_n_cookies_lru_removed; +atomic_t fscache_n_cookies_lru_dropped; + +atomic_t fscache_n_acquires; +atomic_t fscache_n_acquires_ok; +atomic_t fscache_n_acquires_oom; + +atomic_t fscache_n_invalidates; + +atomic_t fscache_n_updates; +EXPORT_SYMBOL(fscache_n_updates); + +atomic_t fscache_n_relinquishes; +atomic_t fscache_n_relinquishes_retire; +atomic_t fscache_n_relinquishes_dropped; + +atomic_t fscache_n_resizes; +atomic_t fscache_n_resizes_null; + +atomic_t fscache_n_read; +EXPORT_SYMBOL(fscache_n_read); +atomic_t fscache_n_write; +EXPORT_SYMBOL(fscache_n_write); +atomic_t fscache_n_no_write_space; +EXPORT_SYMBOL(fscache_n_no_write_space); +atomic_t fscache_n_no_create_space; +EXPORT_SYMBOL(fscache_n_no_create_space); +atomic_t fscache_n_culled; +EXPORT_SYMBOL(fscache_n_culled); + +/* + * display the general statistics + */ +int fscache_stats_show(struct seq_file *m, void *v) +{ + seq_puts(m, "FS-Cache statistics\n"); + seq_printf(m, "Cookies: n=%d v=%d vcol=%u voom=%u\n", + atomic_read(&fscache_n_cookies), + atomic_read(&fscache_n_volumes), + atomic_read(&fscache_n_volumes_collision), + atomic_read(&fscache_n_volumes_nomem) + ); + + seq_printf(m, "Acquire: n=%u ok=%u oom=%u\n", + atomic_read(&fscache_n_acquires), + atomic_read(&fscache_n_acquires_ok), + atomic_read(&fscache_n_acquires_oom)); + + seq_printf(m, "LRU : n=%u exp=%u rmv=%u drp=%u at=%ld\n", + atomic_read(&fscache_n_cookies_lru), + atomic_read(&fscache_n_cookies_lru_expired), + atomic_read(&fscache_n_cookies_lru_removed), + atomic_read(&fscache_n_cookies_lru_dropped), + timer_pending(&fscache_cookie_lru_timer) ? + fscache_cookie_lru_timer.expires - jiffies : 0); + + seq_printf(m, "Invals : n=%u\n", + atomic_read(&fscache_n_invalidates)); + + seq_printf(m, "Updates: n=%u rsz=%u rsn=%u\n", + atomic_read(&fscache_n_updates), + atomic_read(&fscache_n_resizes), + atomic_read(&fscache_n_resizes_null)); + + seq_printf(m, "Relinqs: n=%u rtr=%u drop=%u\n", + atomic_read(&fscache_n_relinquishes), + atomic_read(&fscache_n_relinquishes_retire), + atomic_read(&fscache_n_relinquishes_dropped)); + + seq_printf(m, "NoSpace: nwr=%u ncr=%u cull=%u\n", + atomic_read(&fscache_n_no_write_space), + atomic_read(&fscache_n_no_create_space), + atomic_read(&fscache_n_culled)); + + seq_printf(m, "IO : rd=%u wr=%u\n", + atomic_read(&fscache_n_read), + atomic_read(&fscache_n_write)); + + netfs_stats_show(m); + return 0; +} diff --git a/fs/netfs/fscache_volume.c b/fs/netfs/fscache_volume.c new file mode 100644 index 000000000000..cdf991bdd9de --- /dev/null +++ b/fs/netfs/fscache_volume.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Volume-level cache cookie handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include +#include +#include "internal.h" + +#define fscache_volume_hash_shift 10 +static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; +static atomic_t fscache_volume_debug_id; +static LIST_HEAD(fscache_volumes); + +static void fscache_create_volume_work(struct work_struct *work); + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref; + + __refcount_inc(&volume->ref, &ref); + trace_fscache_volume(volume->debug_id, ref + 1, where); + return volume; +} + +static void fscache_see_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref = refcount_read(&volume->ref); + + trace_fscache_volume(volume->debug_id, ref, where); +} + +/* + * Pin the cache behind a volume so that we can access it. + */ +static void __fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&volume->n_accesses); + smp_mb__after_atomic(); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); +} + +/** + * fscache_begin_volume_access - Pin a cache so a volume can be accessed + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing a volume and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the volume's n_accesses + * count and then recheck the cache liveness, ending the access if it + * ceased to be live. + * + * (3) When we end the access, we decrement the volume's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cache is caching, the volume's n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, the volume's n_accesses is decremented and we wait for it to + * become 0. + * + * The datafile @cookie and the @why indicator are merely provided for tracing + * purposes. + */ +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!fscache_cache_is_live(volume->cache)) + return false; + __fscache_begin_volume_access(volume, cookie, why); + if (!fscache_cache_is_live(volume->cache)) { + fscache_end_volume_access(volume, cookie, fscache_access_unlive); + return false; + } + return true; +} + +/** + * fscache_end_volume_access - Unpin a cache at the end of an access. + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache volume after we've accessed it. The datafile @cookie and the + * @why indicator are merely provided for tracing purposes. + */ +void fscache_end_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&volume->n_accesses); +} +EXPORT_SYMBOL(fscache_end_volume_access); + +static bool fscache_volume_same(const struct fscache_volume *a, + const struct fscache_volume *b) +{ + size_t klen; + + if (a->key_hash != b->key_hash || + a->cache != b->cache || + a->key[0] != b->key[0]) + return false; + + klen = round_up(a->key[0] + 1, sizeof(__le32)); + return memcmp(a->key, b->key, klen) == 0; +} + +static bool fscache_is_acquire_pending(struct fscache_volume *volume) +{ + return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); +} + +static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, + unsigned int collidee_debug_id) +{ + wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, + TASK_UNINTERRUPTIBLE, 20 * HZ); + if (fscache_is_acquire_pending(candidate)) { + pr_notice("Potential volume collision new=%08x old=%08x", + candidate->debug_id, collidee_debug_id); + fscache_stat(&fscache_n_volumes_collision); + wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, + TASK_UNINTERRUPTIBLE); + } +} + +/* + * Attempt to insert the new volume into the hash. If there's a collision, we + * wait for the old volume to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_volume(struct fscache_volume *candidate) +{ + struct fscache_volume *cursor; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket, collidee_debug_id = 0; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(candidate, cursor)) { + if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) + goto collision; + fscache_see_volume(cursor, fscache_volume_get_hash_collision); + set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); + set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); + collidee_debug_id = cursor->debug_id; + break; + } + } + + hlist_bl_add_head(&candidate->hash_link, h); + hlist_bl_unlock(h); + + if (fscache_is_acquire_pending(candidate)) + fscache_wait_on_volume_collision(candidate, collidee_debug_id); + return true; + +collision: + fscache_see_volume(cursor, fscache_volume_collision); + hlist_bl_unlock(h); + return false; +} + +/* + * Allocate and initialise a volume representation cookie. + */ +static struct fscache_volume *fscache_alloc_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + struct fscache_cache *cache; + size_t klen, hlen; + u8 *key; + + klen = strlen(volume_key); + if (klen > NAME_MAX) + return NULL; + + if (!coherency_data) + coherency_len = 0; + + cache = fscache_lookup_cache(cache_name, false); + if (IS_ERR(cache)) + return NULL; + + volume = kzalloc(struct_size(volume, coherency, coherency_len), + GFP_KERNEL); + if (!volume) + goto err_cache; + + volume->cache = cache; + volume->coherency_len = coherency_len; + if (coherency_data) + memcpy(volume->coherency, coherency_data, coherency_len); + INIT_LIST_HEAD(&volume->proc_link); + INIT_WORK(&volume->work, fscache_create_volume_work); + refcount_set(&volume->ref, 1); + spin_lock_init(&volume->lock); + + /* Stick the length on the front of the key and pad it out to make + * hashing easier. + */ + hlen = round_up(1 + klen + 1, sizeof(__le32)); + key = kzalloc(hlen, GFP_KERNEL); + if (!key) + goto err_vol; + key[0] = klen; + memcpy(key + 1, volume_key, klen); + + volume->key = key; + volume->key_hash = fscache_hash(0, key, hlen); + + volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); + down_write(&fscache_addremove_sem); + atomic_inc(&cache->n_volumes); + list_add_tail(&volume->proc_link, &fscache_volumes); + fscache_see_volume(volume, fscache_volume_new_acquire); + fscache_stat(&fscache_n_volumes); + up_write(&fscache_addremove_sem); + _leave(" = v=%x", volume->debug_id); + return volume; + +err_vol: + kfree(volume); +err_cache: + fscache_put_cache(cache, fscache_cache_put_alloc_volume); + fscache_stat(&fscache_n_volumes_nomem); + return NULL; +} + +/* + * Create a volume's representation on disk. Have a volume ref and a cache + * access we have to release. + */ +static void fscache_create_volume_work(struct work_struct *work) +{ + const struct fscache_cache_ops *ops; + struct fscache_volume *volume = + container_of(work, struct fscache_volume, work); + + fscache_see_volume(volume, fscache_volume_see_create_work); + + ops = volume->cache->ops; + if (ops->acquire_volume) + ops->acquire_volume(volume); + fscache_end_cache_access(volume->cache, + fscache_access_acquire_volume_end); + + clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags); + fscache_put_volume(volume, fscache_volume_put_create_work); +} + +/* + * Dispatch a worker thread to create a volume's representation on disk. + */ +void fscache_create_volume(struct fscache_volume *volume, bool wait) +{ + if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) + goto maybe_wait; + if (volume->cache_priv) + goto no_wait; /* We raced */ + if (!fscache_begin_cache_access(volume->cache, + fscache_access_acquire_volume)) + goto no_wait; + + fscache_get_volume(volume, fscache_volume_get_create_work); + if (!schedule_work(&volume->work)) + fscache_put_volume(volume, fscache_volume_put_create_work); + +maybe_wait: + if (wait) { + fscache_see_volume(volume, fscache_volume_wait_create_work); + wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, + TASK_UNINTERRUPTIBLE); + } + return; +no_wait: + clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); + wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); +} + +/* + * Acquire a volume representation cookie and link it to a (proposed) cache. + */ +struct fscache_volume *__fscache_acquire_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + + volume = fscache_alloc_volume(volume_key, cache_name, + coherency_data, coherency_len); + if (!volume) + return ERR_PTR(-ENOMEM); + + if (!fscache_hash_volume(volume)) { + fscache_put_volume(volume, fscache_volume_put_hash_collision); + return ERR_PTR(-EBUSY); + } + + fscache_create_volume(volume, false); + return volume; +} +EXPORT_SYMBOL(__fscache_acquire_volume); + +static void fscache_wake_pending_volume(struct fscache_volume *volume, + struct hlist_bl_head *h) +{ + struct fscache_volume *cursor; + struct hlist_bl_node *p; + + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(cursor, volume)) { + fscache_see_volume(cursor, fscache_volume_see_hash_wake); + clear_and_wake_up_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, + &cursor->flags); + return; + } + } +} + +/* + * Remove a volume cookie from the hash table. + */ +static void fscache_unhash_volume(struct fscache_volume *volume) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&volume->hash_link); + if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) + fscache_wake_pending_volume(volume, h); + hlist_bl_unlock(h); +} + +/* + * Drop a cache's volume attachments. + */ +static void fscache_free_volume(struct fscache_volume *volume) +{ + struct fscache_cache *cache = volume->cache; + + if (volume->cache_priv) { + __fscache_begin_volume_access(volume, NULL, + fscache_access_relinquish_volume); + if (volume->cache_priv) + cache->ops->free_volume(volume); + fscache_end_volume_access(volume, NULL, + fscache_access_relinquish_volume_end); + } + + down_write(&fscache_addremove_sem); + list_del_init(&volume->proc_link); + atomic_dec(&volume->cache->n_volumes); + up_write(&fscache_addremove_sem); + + if (!hlist_bl_unhashed(&volume->hash_link)) + fscache_unhash_volume(volume); + + trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); + kfree(volume->key); + kfree(volume); + fscache_stat_d(&fscache_n_volumes); + fscache_put_cache(cache, fscache_cache_put_volume); +} + +/* + * Drop a reference to a volume cookie. + */ +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + if (volume) { + unsigned int debug_id = volume->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&volume->ref, &ref); + trace_fscache_volume(debug_id, ref - 1, where); + if (zero) + fscache_free_volume(volume); + } +} + +/* + * Relinquish a volume representation cookie. + */ +void __fscache_relinquish_volume(struct fscache_volume *volume, + const void *coherency_data, + bool invalidate) +{ + if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) + return; + + if (invalidate) { + set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); + } else if (coherency_data) { + memcpy(volume->coherency, coherency_data, volume->coherency_len); + } + + fscache_put_volume(volume, fscache_volume_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_volume); + +/** + * fscache_withdraw_volume - Withdraw a volume from being cached + * @volume: Volume cookie + * + * Withdraw a cache volume from service, waiting for all accesses to complete + * before returning. + */ +void fscache_withdraw_volume(struct fscache_volume *volume) +{ + int n_accesses; + + _debug("withdraw V=%x", volume->debug_id); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, 0, + refcount_read(&volume->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&volume->n_accesses, + atomic_read(&volume->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_volume); + +#ifdef CONFIG_PROC_FS +/* + * Generate a list of volumes in /proc/fs/fscache/volumes + */ +static int fscache_volumes_seq_show(struct seq_file *m, void *v) +{ + struct fscache_volume *volume; + + if (v == &fscache_volumes) { + seq_puts(m, + "VOLUME REF nCOOK ACC FL CACHE KEY\n" + "======== ===== ===== === == =============== ================\n"); + return 0; + } + + volume = list_entry(v, struct fscache_volume, proc_link); + seq_printf(m, + "%08x %5d %5d %3d %02lx %-15.15s %s\n", + volume->debug_id, + refcount_read(&volume->ref), + atomic_read(&volume->n_cookies), + atomic_read(&volume->n_accesses), + volume->flags, + volume->cache->name ?: "-", + volume->key + 1); + return 0; +} + +static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(&fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_volumes, *_pos); +} + +static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_volumes, _pos); +} + +static void fscache_volumes_seq_stop(struct seq_file *m, void *v) + __releases(&fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_volumes_seq_ops = { + .start = fscache_volumes_seq_start, + .next = fscache_volumes_seq_next, + .stop = fscache_volumes_seq_stop, + .show = fscache_volumes_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 43fac1b14e40..e96432499eb2 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -5,9 +5,12 @@ * Written by David Howells (dhowells@redhat.com) */ +#include +#include #include #include #include +#include "fscache_internal.h" #ifdef pr_fmt #undef pr_fmt @@ -107,6 +110,7 @@ static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx) /* * debug tracing */ +#if 0 #define dbgprintk(FMT, ...) \ printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) @@ -143,3 +147,4 @@ do { \ #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) #endif +#endif diff --git a/fs/netfs/main.c b/fs/netfs/main.c index 068568702957..237c54a01d97 100644 --- a/fs/netfs/main.c +++ b/fs/netfs/main.c @@ -8,8 +8,8 @@ #include #include #include "internal.h" -#define CREATE_TRACE_POINTS -#include +//#define CREATE_TRACE_POINTS +//#include MODULE_DESCRIPTION("Network fs support"); MODULE_AUTHOR("Red Hat, Inc."); @@ -18,3 +18,4 @@ MODULE_LICENSE("GPL"); unsigned netfs_debug; module_param_named(debug, netfs_debug, uint, S_IWUSR | S_IRUGO); MODULE_PARM_DESC(netfs_debug, "Netfs support debugging mask"); + -- cgit