From b18def121f077857ccf92fc620366e19850bc297 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Tue, 20 Apr 2021 20:13:25 -0700 Subject: bitmap_parse: Support 'all' semantics RCU code supports an 'all' group as a special case when parsing rcu_nocbs parameter. This patch moves the 'all' support to the core bitmap_parse code, so that all bitmap users can enjoy this extension. Moving 'all' parsing to a bitmap_parse level also allows users to pass patterns together with 'all' in regular group:pattern format, for example, "rcu_nocbs=all:1/2" would offload all the even-numbered CPUs regardless of the number of CPUs on the system. Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov Signed-off-by: Paul E. McKenney --- lib/bitmap.c | 9 +++++++++ lib/test_bitmap.c | 7 +++++++ 2 files changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/bitmap.c b/lib/bitmap.c index 74ceb02f45e3..6e29b2aae6ba 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -581,6 +581,14 @@ static const char *bitmap_parse_region(const char *str, struct region *r) { unsigned int lastbit = r->nbits - 1; + if (!strncasecmp(str, "all", 3)) { + r->start = 0; + r->end = lastbit; + str += 3; + + goto check_pattern; + } + str = bitmap_getnum(str, &r->start, lastbit); if (IS_ERR(str)) return str; @@ -595,6 +603,7 @@ static const char *bitmap_parse_region(const char *str, struct region *r) if (IS_ERR(str)) return str; +check_pattern: if (end_of_region(*str)) goto no_pattern; diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 9cd575583180..4ea73f5aed41 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -366,6 +366,13 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = { {0, "0-31:1/3,1-31:1/3,2-31:1/3", &exp1[8 * step], 32, 0}, {0, "1-10:8/12,8-31:24/29,0-31:0/3", &exp1[9 * step], 32, 0}, + {0, "all", &exp1[8 * step], 32, 0}, + {0, "0, 1, all, ", &exp1[8 * step], 32, 0}, + {0, "all:1/2", &exp1[4 * step], 32, 0}, + {0, "ALL:1/2", &exp1[4 * step], 32, 0}, + {-EINVAL, "al", NULL, 8, 0}, + {-EINVAL, "alll", NULL, 8, 0}, + {-EINVAL, "-1", NULL, 8, 0}, {-EINVAL, "-0", NULL, 8, 0}, {-EINVAL, "10-1", NULL, 8, 0}, -- cgit From cde3d0f81e67f21f1f61fd895255f2e3b86f5bed Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 10 May 2021 22:46:29 +0300 Subject: bitmap: Make bitmap_remap() and bitmap_bitremap() available to users Currently the bitmap_remap() and bitmap_bitremap() are available only for CONFIG_NUMA=y case, while some users may benefit out of it and being independent to NUMA code. Make them available to users by moving out of ifdeffery and exporting for modules. Signed-off-by: Andy Shevchenko Tested-by: Neeli Srinivas Acked-by: Yury Norov Signed-off-by: Bartosz Golaszewski --- lib/bitmap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/bitmap.c b/lib/bitmap.c index 74ceb02f45e3..7b6b2a67a6a6 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -784,8 +784,6 @@ int bitmap_parse(const char *start, unsigned int buflen, } EXPORT_SYMBOL(bitmap_parse); - -#ifdef CONFIG_NUMA /** * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap * @buf: pointer to a bitmap @@ -894,6 +892,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); } } +EXPORT_SYMBOL(bitmap_remap); /** * bitmap_bitremap - Apply map defined by a pair of bitmaps to a single bit @@ -931,7 +930,9 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, else return bitmap_ord_to_pos(new, n % w, bits); } +EXPORT_SYMBOL(bitmap_bitremap); +#ifdef CONFIG_NUMA /** * bitmap_onto - translate one bitmap relative to another * @dst: resulting translated bitmap -- cgit From 0733d83905326baef3c25d8bd9a96fdc9eb71b86 Mon Sep 17 00:00:00 2001 From: Shawn Guo Date: Sun, 25 Apr 2021 10:00:24 +0800 Subject: firmware: replace HOTPLUG with UEVENT in FW_ACTION defines With commit 312c004d36ce ("[PATCH] driver core: replace "hotplug" by "uevent"") already in the tree over a decade, update the name of FW_ACTION defines to follow semantics, and reflect what the defines are really meant for, i.e. whether or not generate user space event. Acked-by: Lee Jones Signed-off-by: Shawn Guo Link: https://lore.kernel.org/r/20210425020024.28057-1-shawn.guo@linaro.org Signed-off-by: Greg Kroah-Hartman --- lib/test_firmware.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/test_firmware.c b/lib/test_firmware.c index b6fe89add9fe..1bccd6cd5f48 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -260,8 +260,8 @@ static ssize_t config_show(struct device *dev, len += scnprintf(buf + len, PAGE_SIZE - len, "send_uevent:\t\t%s\n", test_fw_config->send_uevent ? - "FW_ACTION_HOTPLUG" : - "FW_ACTION_NOHOTPLUG"); + "FW_ACTION_UEVENT" : + "FW_ACTION_NOUEVENT"); len += scnprintf(buf + len, PAGE_SIZE - len, "into_buf:\t\t%s\n", test_fw_config->into_buf ? "true" : "false"); @@ -729,7 +729,7 @@ static ssize_t trigger_custom_fallback_store(struct device *dev, mutex_lock(&test_fw_mutex); release_firmware(test_firmware); test_firmware = NULL; - rc = request_firmware_nowait(THIS_MODULE, FW_ACTION_NOHOTPLUG, name, + rc = request_firmware_nowait(THIS_MODULE, FW_ACTION_NOUEVENT, name, dev, GFP_KERNEL, NULL, trigger_async_request_cb); if (rc) { @@ -938,8 +938,8 @@ ssize_t trigger_batched_requests_async_store(struct device *dev, pr_info("batched loading '%s' custom fallback mechanism %u times\n", test_fw_config->name, test_fw_config->num_requests); - send_uevent = test_fw_config->send_uevent ? FW_ACTION_HOTPLUG : - FW_ACTION_NOHOTPLUG; + send_uevent = test_fw_config->send_uevent ? FW_ACTION_UEVENT : + FW_ACTION_NOUEVENT; for (i = 0; i < test_fw_config->num_requests; i++) { req = &test_fw_config->reqs[i]; -- cgit From 5c3e241f5246445da29bd03783ca61a18face968 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 28 Apr 2021 14:32:03 +0800 Subject: lib: devres: Add error information printing for __devm_ioremap_resource() Ensure that all error handling branches print error information. In this way, when this function fails, the upper-layer functions can directly return an error code without missing debugging information. Otherwise, the error message will be printed redundantly or missing. Reviewed-by: Vladimir Oltean Signed-off-by: Zhen Lei Link: https://lore.kernel.org/r/20210428063203.691-1-thunder.leizhen@huawei.com Signed-off-by: Greg Kroah-Hartman --- lib/devres.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/devres.c b/lib/devres.c index 4679dbb1bf5f..bdb06898a977 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -157,8 +157,10 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, dev_name(dev), res->name); else pretty_name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); - if (!pretty_name) + if (!pretty_name) { + dev_err(dev, "can't generate pretty name for resource %pR\n", res); return IOMEM_ERR_PTR(-ENOMEM); + } if (!devm_request_mem_region(dev, res->start, size, pretty_name)) { dev_err(dev, "can't request region for resource %pR\n", res); -- cgit From cf536e185869d4815d506e777bcca6edd9966a6e Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Thu, 6 May 2021 15:34:59 +0800 Subject: Makefile: extend 32B aligned debug option to 64B aligned Commit 09c60546f04f ("./Makefile: add debug option to enable function aligned on 32 bytes") was introduced to help debugging strange kernel performance changes caused by code alignment change. Recently we found 2 similar cases [1][2] caused by code-alignment changes, which can only be identified by forcing 64 bytes aligned for all functions. Originally, 32 bytes was used mainly for not wasting too much text space, but this option is only for debug anyway where text space is not a big concern. So extend the alignment to 64 bytes to cover more similar cases. [1].https://lore.kernel.org/lkml/20210427090013.GG32408@xsang-OptiPlex-9020/ [2].https://lore.kernel.org/lkml/20210420030837.GB31773@xsang-OptiPlex-9020/ Signed-off-by: Feng Tang Signed-off-by: Masahiro Yamada --- lib/Kconfig.debug | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 678c13967580..6ce26b82a35b 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -400,8 +400,8 @@ config SECTION_MISMATCH_WARN_ONLY If unsure, say Y. -config DEBUG_FORCE_FUNCTION_ALIGN_32B - bool "Force all function address 32B aligned" if EXPERT +config DEBUG_FORCE_FUNCTION_ALIGN_64B + bool "Force all function address 64B aligned" if EXPERT help There are cases that a commit from one domain changes the function address alignment of other domains, and cause magic performance -- cgit From 7af5662826f7b1639490c999e6bac14ea6a79a7c Mon Sep 17 00:00:00 2001 From: Jim Cromie Date: Mon, 24 May 2021 21:32:40 -0600 Subject: dyndbg: display KiB of data memory used. If booted with verbose>=1, dyndbg prints the memory usage in bytes, of builtin modules' prdebugs. KiB reads better. no functional changes. Signed-off-by: Jim Cromie Link: https://lore.kernel.org/r/20210525033240.35260-1-jim.cromie@gmail.com Signed-off-by: Greg Kroah-Hartman --- lib/dynamic_debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 641767b0dce2..d3ce78298832 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -1117,9 +1117,9 @@ static int __init dynamic_debug_init(void) goto out_err; ddebug_init_success = 1; - vpr_info("%d modules, %d entries and %d bytes in ddebug tables, %d bytes in __dyndbg section\n", - modct, entries, (int)(modct * sizeof(struct ddebug_table)), - (int)(entries * sizeof(struct _ddebug))); + vpr_info("%d prdebugs in %d modules, %d KiB in ddebug tables, %d kiB in __dyndbg section\n", + entries, modct, (int)((modct * sizeof(struct ddebug_table)) >> 10), + (int)((entries * sizeof(struct _ddebug)) >> 10)); /* apply ddebug_query boot param, dont unload tables on err */ if (ddebug_setup_string[0] != '\0') { -- cgit From 66cd071a1f839b4834d45aa7dde622151041b1a0 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 9 Apr 2021 19:10:53 +0100 Subject: iov_iter: Remove iov_iter_for_each_range() Remove iov_iter_for_each_range() as it's no longer used with the removal of lustre. Signed-off-by: David Howells Signed-off-by: Al Viro --- lib/iov_iter.c | 27 --------------------------- 1 file changed, 27 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index c701b7a187f2..8f5ce5b1ff91 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -2093,30 +2093,3 @@ int import_single_range(int rw, void __user *buf, size_t len, return 0; } EXPORT_SYMBOL(import_single_range); - -int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, - int (*f)(struct kvec *vec, void *context), - void *context) -{ - struct kvec w; - int err = -EINVAL; - if (!bytes) - return 0; - - iterate_all_kinds(i, bytes, v, -EINVAL, ({ - w.iov_base = kmap(v.bv_page) + v.bv_offset; - w.iov_len = v.bv_len; - err = f(&w, context); - kunmap(v.bv_page); - err;}), ({ - w = v; - err = f(&w, context);}), ({ - w.iov_base = kmap(v.bv_page) + v.bv_offset; - w.iov_len = v.bv_len; - err = f(&w, context); - kunmap(v.bv_page); - err;}) - ) - return err; -} -EXPORT_SYMBOL(iov_iter_for_each_range); -- cgit From 08aa64796016cb47b2ef3d0924653b4d944b0d65 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 29 Apr 2021 20:42:25 -0400 Subject: teach copy_page_to_iter() to handle compound pages In situation when copy_page_to_iter() got a compound page the current code would only work on systems with no CONFIG_HIGHMEM. It *is* the majority of real-world setups, or we would've drown in bug reports by now. Still needs fixing. Current variant works for solitary page; rename that to __copy_page_to_iter() and turn the handling of compound pages into a loop over subpages. Cc: stable@vger.kernel.org Signed-off-by: Al Viro --- lib/iov_iter.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 8f5ce5b1ff91..12fb04b23143 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -957,11 +957,9 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) return false; } -size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, +static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { - if (unlikely(!page_copy_sane(page, offset, bytes))) - return 0; if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) { void *kaddr = kmap_atomic(page); size_t wanted = copy_to_iter(kaddr + offset, bytes, i); @@ -974,6 +972,30 @@ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, else return copy_page_to_iter_pipe(page, offset, bytes, i); } + +size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, + struct iov_iter *i) +{ + size_t res = 0; + if (unlikely(!page_copy_sane(page, offset, bytes))) + return 0; + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + size_t n = __copy_page_to_iter(page, offset, + min(bytes, (size_t)PAGE_SIZE - offset), i); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } + } + return res; +} EXPORT_SYMBOL(copy_page_to_iter); size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, -- cgit From a506abc7b644d71966a75337d5a534f531b3cdc4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Apr 2021 12:34:04 -0400 Subject: copy_page_to_iter(): fix ITER_DISCARD case we need to advance the iterator... Cc: stable@vger.kernel.org Signed-off-by: Al Viro --- lib/iov_iter.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 12fb04b23143..c8877cffb7bc 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -965,9 +965,12 @@ static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes size_t wanted = copy_to_iter(kaddr + offset, bytes, i); kunmap_atomic(kaddr); return wanted; - } else if (unlikely(iov_iter_is_discard(i))) + } else if (unlikely(iov_iter_is_discard(i))) { + if (unlikely(i->count < bytes)) + bytes = i->count; + i->count -= bytes; return bytes; - else if (likely(!iov_iter_is_pipe(i))) + } else if (likely(!iov_iter_is_pipe(i))) return copy_page_to_iter_iovec(page, offset, bytes, i); else return copy_page_to_iter_pipe(page, offset, bytes, i); -- cgit From 0e8f0d67401589a141950856902c7d0ec8d9c985 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 2 Jun 2021 14:48:21 -0400 Subject: [xarray] iov_iter_fault_in_readable() should do nothing in xarray case ... and actually should just check it's given an iovec-backed iterator in the first place. Cc: stable@vger.kernel.org Signed-off-by: Al Viro --- lib/iov_iter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index c8877cffb7bc..a3aabeda945b 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -476,7 +476,7 @@ int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) int err; struct iovec v; - if (!(i->type & (ITER_BVEC|ITER_KVEC))) { + if (iter_is_iovec(i)) { iterate_iovec(i, bytes, v, iov, skip, ({ err = fault_in_pages_readable(v.iov_base, v.iov_len); if (unlikely(err)) -- cgit From 3b3fc051cd2cba42bf736fa62780857d251a1236 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 23 Apr 2021 22:24:08 -0400 Subject: iov_iter_advance(): use consistent semantics for move past the end asking to advance by more than we have left in the iov_iter should move to the very end; it should *not* leave negative i->count and it should not spew into syslog, etc. - it's a legitimate operation. Signed-off-by: Al Viro --- lib/iov_iter.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index a3aabeda945b..bdbe6691457d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1117,8 +1117,6 @@ static inline void pipe_truncate(struct iov_iter *i) static void pipe_advance(struct iov_iter *i, size_t size) { struct pipe_inode_info *pipe = i->pipe; - if (unlikely(i->count < size)) - size = i->count; if (size) { struct pipe_buffer *buf; unsigned int p_mask = pipe->ring_size - 1; @@ -1159,6 +1157,8 @@ static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) void iov_iter_advance(struct iov_iter *i, size_t size) { + if (unlikely(i->count < size)) + size = i->count; if (unlikely(iov_iter_is_pipe(i))) { pipe_advance(i, size); return; @@ -1168,7 +1168,6 @@ void iov_iter_advance(struct iov_iter *i, size_t size) return; } if (unlikely(iov_iter_is_xarray(i))) { - size = min(size, i->count); i->iov_offset += size; i->count -= size; return; -- cgit From 4b6c132b7da6430cf5dcc96948b04849dea0a32a Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 29 Apr 2021 21:16:56 -0400 Subject: iov_iter: switch ..._full() variants of primitives to use of iov_iter_revert() Use corresponding plain variants, revert on short copy. That's the way it should've been done from the very beginning, except that we didn't have iov_iter_revert() back then... [fixed another braino caught by Qian Cai ] Signed-off-by: Al Viro --- lib/iov_iter.c | 104 --------------------------------------------------------- 1 file changed, 104 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index bdbe6691457d..ce9f8b9168ea 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -819,35 +819,6 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) } EXPORT_SYMBOL(_copy_from_iter); -bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - - if (iter_is_iovec(i)) - might_fault(); - iterate_all_kinds(i, bytes, v, ({ - if (copyin((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full); - size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { char *to = addr; @@ -907,32 +878,6 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); #endif -bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full_nocache); - static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) { struct page *head; @@ -1740,55 +1685,6 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, } EXPORT_SYMBOL(csum_and_copy_from_iter); -bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, - struct iov_iter *i) -{ - char *to = addr; - __wsum sum, next; - size_t off = 0; - sum = *csum; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len); - if (!next) - return false; - sum = csum_block_add(sum, next, off); - off += v.iov_len; - 0; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }) - ) - *csum = sum; - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(csum_and_copy_from_iter_full); - size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, struct iov_iter *i) { -- cgit From 28f38db7edbfa6d7736cd7a3a7aec76660bfef57 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 2 Jun 2021 17:25:59 -0400 Subject: iov_iter: reorder handling of flavours in primitives iovec is the most common one; test it first and test explicitly, rather than "not anything else". Replace all flavour checks with use of iov_iter_is_...() helpers. Signed-off-by: Al Viro --- lib/iov_iter.c | 91 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 46 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index ce9f8b9168ea..bdcf1fbeb2db 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -117,22 +117,21 @@ #define iterate_all_kinds(i, n, v, I, B, K, X) { \ if (likely(n)) { \ size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ + if (likely(iter_is_iovec(i))) { \ + const struct iovec *iov; \ + struct iovec v; \ + iterate_iovec(i, n, v, iov, skip, (I)) \ + } else if (iov_iter_is_bvec(i)) { \ struct bio_vec v; \ struct bvec_iter __bi; \ iterate_bvec(i, n, v, __bi, skip, (B)) \ - } else if (unlikely(i->type & ITER_KVEC)) { \ + } else if (iov_iter_is_kvec(i)) { \ const struct kvec *kvec; \ struct kvec v; \ iterate_kvec(i, n, v, kvec, skip, (K)) \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - } else if (unlikely(i->type & ITER_XARRAY)) { \ + } else if (iov_iter_is_xarray(i)) { \ struct bio_vec v; \ iterate_xarray(i, n, v, skip, (X)); \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ } \ } \ } @@ -142,7 +141,17 @@ n = i->count; \ if (i->count) { \ size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ + if (likely(iter_is_iovec(i))) { \ + const struct iovec *iov; \ + struct iovec v; \ + iterate_iovec(i, n, v, iov, skip, (I)) \ + if (skip == iov->iov_len) { \ + iov++; \ + skip = 0; \ + } \ + i->nr_segs -= iov - i->iov; \ + i->iov = iov; \ + } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ struct bio_vec v; \ struct bvec_iter __bi; \ @@ -150,7 +159,7 @@ i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ i->nr_segs -= i->bvec - bvec; \ skip = __bi.bi_bvec_done; \ - } else if (unlikely(i->type & ITER_KVEC)) { \ + } else if (iov_iter_is_kvec(i)) { \ const struct kvec *kvec; \ struct kvec v; \ iterate_kvec(i, n, v, kvec, skip, (K)) \ @@ -160,21 +169,11 @@ } \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - skip += n; \ - } else if (unlikely(i->type & ITER_XARRAY)) { \ + } else if (iov_iter_is_xarray(i)) { \ struct bio_vec v; \ iterate_xarray(i, n, v, skip, (X)) \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ - if (skip == iov->iov_len) { \ - iov++; \ - skip = 0; \ - } \ - i->nr_segs -= iov - i->iov; \ - i->iov = iov; \ + } else if (iov_iter_is_discard(i)) { \ + skip += n; \ } \ i->count -= n; \ i->iov_offset = skip; \ @@ -905,20 +904,24 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { - if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) { + if (likely(iter_is_iovec(i))) + return copy_page_to_iter_iovec(page, offset, bytes, i); + if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { void *kaddr = kmap_atomic(page); size_t wanted = copy_to_iter(kaddr + offset, bytes, i); kunmap_atomic(kaddr); return wanted; - } else if (unlikely(iov_iter_is_discard(i))) { + } + if (iov_iter_is_pipe(i)) + return copy_page_to_iter_pipe(page, offset, bytes, i); + if (unlikely(iov_iter_is_discard(i))) { if (unlikely(i->count < bytes)) bytes = i->count; i->count -= bytes; return bytes; - } else if (likely(!iov_iter_is_pipe(i))) - return copy_page_to_iter_iovec(page, offset, bytes, i); - else - return copy_page_to_iter_pipe(page, offset, bytes, i); + } + WARN_ON(1); + return 0; } size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, @@ -951,17 +954,16 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, { if (unlikely(!page_copy_sane(page, offset, bytes))) return 0; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return 0; - } - if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) { + if (likely(iter_is_iovec(i))) + return copy_page_from_iter_iovec(page, offset, bytes, i); + if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { void *kaddr = kmap_atomic(page); size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); kunmap_atomic(kaddr); return wanted; - } else - return copy_page_from_iter_iovec(page, offset, bytes, i); + } + WARN_ON(1); + return 0; } EXPORT_SYMBOL(copy_page_from_iter); @@ -1203,16 +1205,13 @@ EXPORT_SYMBOL(iov_iter_revert); */ size_t iov_iter_single_seg_count(const struct iov_iter *i) { - if (unlikely(iov_iter_is_pipe(i))) - return i->count; // it is a silly place, anyway - if (i->nr_segs == 1) - return i->count; - if (unlikely(iov_iter_is_discard(i) || iov_iter_is_xarray(i))) - return i->count; - if (iov_iter_is_bvec(i)) - return min(i->count, i->bvec->bv_len - i->iov_offset); - else - return min(i->count, i->iov->iov_len - i->iov_offset); + if (i->nr_segs > 1) { + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return min(i->count, i->iov->iov_len - i->iov_offset); + if (iov_iter_is_bvec(i)) + return min(i->count, i->bvec->bv_len - i->iov_offset); + } + return i->count; } EXPORT_SYMBOL(iov_iter_single_seg_count); -- cgit From 556351c1c09ad6511bc2eaa2c214992192f50410 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 17:01:22 -0400 Subject: iov_iter_advance(): don't modify ->iov_offset for ITER_DISCARD the field is not used for that flavour Signed-off-by: Al Viro --- lib/iov_iter.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index bdcf1fbeb2db..e6c5834da32d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -172,8 +172,6 @@ } else if (iov_iter_is_xarray(i)) { \ struct bio_vec v; \ iterate_xarray(i, n, v, skip, (X)) \ - } else if (iov_iter_is_discard(i)) { \ - skip += n; \ } \ i->count -= n; \ i->iov_offset = skip; \ -- cgit From 8cd54c1c848031a87820e58d772166ffdf8c08c0 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 22 Apr 2021 14:50:39 -0400 Subject: iov_iter: separate direction from flavour Instead of having them mixed in iter->type, use separate ->iter_type and ->data_source (u8 and bool resp.) And don't bother with (pseudo-) bitmap for the former - microoptimizations from being able to check if the flavour is one of two values are not worth the confusion for optimizer. It can't prove that we never get e.g. ITER_IOVEC | ITER_PIPE, so we end up with extra headache. Signed-off-by: Al Viro --- lib/iov_iter.c | 85 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 37 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index e6c5834da32d..03c4e677b075 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -489,19 +489,15 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - direction &= READ | WRITE; - - /* It will get better. Eventually... */ - if (uaccess_kernel()) { - i->type = ITER_KVEC | direction; - i->kvec = (struct kvec *)iov; - } else { - i->type = ITER_IOVEC | direction; - i->iov = iov; - } - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + WARN_ON_ONCE(uaccess_kernel()); + *i = (struct iov_iter) { + .iter_type = ITER_IOVEC, + .data_source = direction, + .iov = iov, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_init); @@ -1218,11 +1214,14 @@ void iov_iter_kvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_KVEC | (direction & (READ | WRITE)); - i->kvec = kvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_KVEC, + .data_source = direction, + .kvec = kvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_kvec); @@ -1231,11 +1230,14 @@ void iov_iter_bvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_BVEC | (direction & (READ | WRITE)); - i->bvec = bvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_BVEC, + .data_source = direction, + .bvec = bvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_bvec); @@ -1245,12 +1247,15 @@ void iov_iter_pipe(struct iov_iter *i, unsigned int direction, { BUG_ON(direction != READ); WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size)); - i->type = ITER_PIPE | READ; - i->pipe = pipe; - i->head = pipe->head; - i->iov_offset = 0; - i->count = count; - i->start_head = i->head; + *i = (struct iov_iter){ + .iter_type = ITER_PIPE, + .data_source = false, + .pipe = pipe, + .head = pipe->head, + .start_head = pipe->head, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_pipe); @@ -1271,11 +1276,14 @@ void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, loff_t start, size_t count) { BUG_ON(direction & ~1); - i->type = ITER_XARRAY | (direction & (READ | WRITE)); - i->xarray = xarray; - i->xarray_start = start; - i->count = count; - i->iov_offset = 0; + *i = (struct iov_iter) { + .iter_type = ITER_XARRAY, + .data_source = direction, + .xarray = xarray, + .xarray_start = start, + .count = count, + .iov_offset = 0 + }; } EXPORT_SYMBOL(iov_iter_xarray); @@ -1291,9 +1299,12 @@ EXPORT_SYMBOL(iov_iter_xarray); void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) { BUG_ON(direction != READ); - i->type = ITER_DISCARD | READ; - i->count = count; - i->iov_offset = 0; + *i = (struct iov_iter){ + .iter_type = ITER_DISCARD, + .data_source = false, + .count = count, + .iov_offset = 0 + }; } EXPORT_SYMBOL(iov_iter_discard); -- cgit From 185ac4d43669314f31c9c27d1ffc5ebcad791351 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 23 Apr 2021 12:58:53 -0400 Subject: iov_iter: optimize iov_iter_advance() for iovec and kvec We can do better than generic iterate_and_advance() for this one; inspired by bvec_iter_advance() (and massaged into that form by equivalent transformations). [fixed a braino caught by kernel test robot ] Signed-off-by: Al Viro --- lib/iov_iter.c | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 03c4e677b075..cd23c79acb94 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1096,28 +1096,42 @@ static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) i->iov_offset = bi.bi_bvec_done; } +static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) +{ + const struct iovec *iov, *end; + + if (!i->count) + return; + i->count -= size; + + size += i->iov_offset; // from beginning of current segment + for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) { + if (likely(size < iov->iov_len)) + break; + size -= iov->iov_len; + } + i->iov_offset = size; + i->nr_segs -= iov - i->iov; + i->iov = iov; +} + void iov_iter_advance(struct iov_iter *i, size_t size) { if (unlikely(i->count < size)) size = i->count; - if (unlikely(iov_iter_is_pipe(i))) { + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { + /* iovec and kvec have identical layouts */ + iov_iter_iovec_advance(i, size); + } else if (iov_iter_is_bvec(i)) { + iov_iter_bvec_advance(i, size); + } else if (iov_iter_is_pipe(i)) { pipe_advance(i, size); - return; - } - if (unlikely(iov_iter_is_discard(i))) { - i->count -= size; - return; - } - if (unlikely(iov_iter_is_xarray(i))) { + } else if (unlikely(iov_iter_is_xarray(i))) { i->iov_offset += size; i->count -= size; - return; - } - if (iov_iter_is_bvec(i)) { - iov_iter_bvec_advance(i, size); - return; + } else if (iov_iter_is_discard(i)) { + i->count -= size; } - iterate_and_advance(i, size, v, 0, 0, 0, 0) } EXPORT_SYMBOL(iov_iter_advance); -- cgit From 8409a0d261e20180361e7afe6d89847d1bad4ce8 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 11:57:37 -0400 Subject: sanitize iov_iter_fault_in_readable() 1) constify iov_iter argument; we are not advancing it in this primitive. 2) cap the amount requested by the amount of data in iov_iter. All existing callers should've been safe, but the check is really cheap and doing it here makes for easier analysis, as well as more consistent semantics among the primitives. 3) don't bother with iterate_iovec(). Explicit loop is not any harder to follow, and we get rid of standalone iterate_iovec() users - it's only used by iterate_and_advance() and (soon to be gone) iterate_all_kinds(). Signed-off-by: Al Viro --- lib/iov_iter.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index cd23c79acb94..2b543bea1e0d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -466,19 +466,25 @@ out: * Return 0 on success, or non-zero if the memory could not be accessed (i.e. * because it is an invalid address). */ -int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) +int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes) { - size_t skip = i->iov_offset; - const struct iovec *iov; - int err; - struct iovec v; - if (iter_is_iovec(i)) { - iterate_iovec(i, bytes, v, iov, skip, ({ - err = fault_in_pages_readable(v.iov_base, v.iov_len); + const struct iovec *p; + size_t skip; + + if (bytes > i->count) + bytes = i->count; + for (p = i->iov, skip = i->iov_offset; bytes; p++, skip = 0) { + size_t len = min(bytes, p->iov_len - skip); + int err; + + if (unlikely(!len)) + continue; + err = fault_in_pages_readable(p->iov_base + skip, len); if (unlikely(err)) - return err; - 0;})) + return err; + bytes -= len; + } } return 0; } -- cgit From 9221d2e37b729077797e6d02012289892dbdb859 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Apr 2021 00:44:35 -0400 Subject: iov_iter_alignment(): don't bother with iterate_all_kinds() It's easier to go over the array manually. We need to watch out for truncated iov_iter, though - iovec array might cover more than i->count. Signed-off-by: Al Viro --- lib/iov_iter.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 2b543bea1e0d..ed9318358b68 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1328,27 +1328,70 @@ void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) } EXPORT_SYMBOL(iov_iter_discard); -unsigned long iov_iter_alignment(const struct iov_iter *i) +static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) { unsigned long res = 0; size_t size = i->count; + size_t skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->iov[k].iov_len - skip; + if (len) { + res |= (unsigned long)i->iov[k].iov_base + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + } + return res; +} - if (unlikely(iov_iter_is_pipe(i))) { +static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) +{ + unsigned res = 0; + size_t size = i->count; + unsigned skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->bvec[k].bv_len - skip; + res |= (unsigned long)i->bvec[k].bv_offset + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + return res; +} + +unsigned long iov_iter_alignment(const struct iov_iter *i) +{ + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_iter_alignment_iovec(i); + + if (iov_iter_is_bvec(i)) + return iov_iter_alignment_bvec(i); + + if (iov_iter_is_pipe(i)) { unsigned int p_mask = i->pipe->ring_size - 1; + size_t size = i->count; if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) return size | i->iov_offset; return size; } - if (unlikely(iov_iter_is_xarray(i))) + + if (iov_iter_is_xarray(i)) return (i->xarray_start + i->iov_offset) | i->count; - iterate_all_kinds(i, size, v, - (res |= (unsigned long)v.iov_base | v.iov_len, 0), - res |= v.bv_offset | v.bv_len, - res |= (unsigned long)v.iov_base | v.iov_len, - res |= v.bv_offset | v.bv_len - ) - return res; + + return 0; } EXPORT_SYMBOL(iov_iter_alignment); -- cgit From 610c7a71543df32fcecf64004f974905f5881fb3 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Apr 2021 01:03:16 -0400 Subject: iov_iter_gap_alignment(): get rid of iterate_all_kinds() For one thing, it's only used for iovec (and makes sense only for those). For another, here we don't care about iov_offset, since the beginning of the first segment and the end of the last one are ignored. So it makes a lot more sense to just walk through the iovec array... We need to deal with the case of truncated iov_iter, but unlike the situation with iov_iter_alignment() we don't care where the last segment ends - just which segment is the last one. [fixed a braino spotted by Qian Cai ] Signed-off-by: Al Viro --- lib/iov_iter.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index ed9318358b68..6569e3f5d01d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1398,23 +1398,24 @@ EXPORT_SYMBOL(iov_iter_alignment); unsigned long iov_iter_gap_alignment(const struct iov_iter *i) { unsigned long res = 0; + unsigned long v = 0; size_t size = i->count; + unsigned k; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); + if (WARN_ON(!iter_is_iovec(i))) return ~0U; - } - iterate_all_kinds(i, size, v, - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0), 0), - (res |= (!res ? 0 : (unsigned long)v.bv_offset) | - (size != v.bv_len ? size : 0)), - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0)), - (res |= (!res ? 0 : (unsigned long)v.bv_offset) | - (size != v.bv_len ? size : 0)) - ); + for (k = 0; k < i->nr_segs; k++) { + if (i->iov[k].iov_len) { + unsigned long base = (unsigned long)i->iov[k].iov_base; + if (v) // if not the first one + res |= base | v; // this start | previous end + v = base + i->iov[k].iov_len; + if (size <= i->iov[k].iov_len) + break; + size -= i->iov[k].iov_len; + } + } return res; } EXPORT_SYMBOL(iov_iter_gap_alignment); -- cgit From 3d671ca62a08114810321a2a5e9d3523de5fb1b4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Apr 2021 09:14:44 -0400 Subject: get rid of iterate_all_kinds() in iov_iter_get_pages()/iov_iter_get_pages_alloc() Here iterate_all_kinds() is used just to find the first (non-empty, in case of iovec) segment. Which can be easily done explicitly. Note that in bvec case we now can get more than PAGE_SIZE worth of them, in case when we have a compound page in bvec and a range that crosses a subpage boundary. Older behaviour had been to stop on that boundary; we used to get the right first page (for_each_bvec() took care of that), but that was all we'd got. Signed-off-by: Al Viro --- lib/iov_iter.c | 147 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 91 insertions(+), 56 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 6569e3f5d01d..3fd331b3b8f2 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1450,9 +1450,6 @@ static ssize_t pipe_get_pages(struct iov_iter *i, unsigned int iter_head, npages; size_t capacity; - if (!maxsize) - return 0; - if (!sanity(i)) return -EFAULT; @@ -1533,29 +1530,67 @@ static ssize_t iter_xarray_get_pages(struct iov_iter *i, return actual; } +/* must be done on non-empty ITER_IOVEC one */ +static unsigned long first_iovec_segment(const struct iov_iter *i, + size_t *size, size_t *start, + size_t maxsize, unsigned maxpages) +{ + size_t skip; + long k; + + for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { + unsigned long addr = (unsigned long)i->iov[k].iov_base + skip; + size_t len = i->iov[k].iov_len - skip; + + if (unlikely(!len)) + continue; + if (len > maxsize) + len = maxsize; + len += (*start = addr % PAGE_SIZE); + if (len > maxpages * PAGE_SIZE) + len = maxpages * PAGE_SIZE; + *size = len; + return addr & PAGE_MASK; + } + BUG(); // if it had been empty, we wouldn't get called +} + +/* must be done on non-empty ITER_BVEC one */ +static struct page *first_bvec_segment(const struct iov_iter *i, + size_t *size, size_t *start, + size_t maxsize, unsigned maxpages) +{ + struct page *page; + size_t skip = i->iov_offset, len; + + len = i->bvec->bv_len - skip; + if (len > maxsize) + len = maxsize; + skip += i->bvec->bv_offset; + page = i->bvec->bv_page + skip / PAGE_SIZE; + len += (*start = skip % PAGE_SIZE); + if (len > maxpages * PAGE_SIZE) + len = maxpages * PAGE_SIZE; + *size = len; + return page; +} + ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, size_t maxsize, unsigned maxpages, size_t *start) { + size_t len; + int n, res; + if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages(i, pages, maxsize, maxpages, start); - if (unlikely(iov_iter_is_xarray(i))) - return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; - - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (likely(iter_is_iovec(i))) { + unsigned long addr; - if (len > maxpages * PAGE_SIZE) - len = maxpages * PAGE_SIZE; - addr &= ~(PAGE_SIZE - 1); + addr = first_iovec_segment(i, &len, start, maxsize, maxpages); n = DIV_ROUND_UP(len, PAGE_SIZE); res = get_user_pages_fast(addr, n, iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, @@ -1563,17 +1598,21 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, if (unlikely(res < 0)) return res; return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - get_page(*pages = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }), - 0 - ) - return 0; + } + if (iov_iter_is_bvec(i)) { + struct page *page; + + page = first_bvec_segment(i, &len, start, maxsize, maxpages); + n = DIV_ROUND_UP(len, PAGE_SIZE); + while (n--) + get_page(*pages++ = page++); + return len - *start; + } + if (iov_iter_is_pipe(i)) + return pipe_get_pages(i, pages, maxsize, maxpages, start); + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); + return -EFAULT; } EXPORT_SYMBOL(iov_iter_get_pages); @@ -1590,9 +1629,6 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i, unsigned int iter_head, npages; ssize_t n; - if (!maxsize) - return 0; - if (!sanity(i)) return -EFAULT; @@ -1665,24 +1701,18 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, size_t *start) { struct page **p; + size_t len; + int n, res; if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages_alloc(i, pages, maxsize, start); - if (unlikely(iov_iter_is_xarray(i))) - return iter_xarray_get_pages_alloc(i, pages, maxsize, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; - - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (likely(iter_is_iovec(i))) { + unsigned long addr; - addr &= ~(PAGE_SIZE - 1); + addr = first_iovec_segment(i, &len, start, maxsize, ~0U); n = DIV_ROUND_UP(len, PAGE_SIZE); p = get_pages_array(n); if (!p) @@ -1695,19 +1725,24 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, } *pages = p; return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - *pages = p = get_pages_array(1); + } + if (iov_iter_is_bvec(i)) { + struct page *page; + + page = first_bvec_segment(i, &len, start, maxsize, ~0U); + n = DIV_ROUND_UP(len, PAGE_SIZE); + *pages = p = get_pages_array(n); if (!p) return -ENOMEM; - get_page(*p = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }), 0 - ) - return 0; + while (n--) + get_page(*p++ = page++); + return len - *start; + } + if (iov_iter_is_pipe(i)) + return pipe_get_pages_alloc(i, pages, maxsize, start); + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages_alloc(i, pages, maxsize, start); + return -EFAULT; } EXPORT_SYMBOL(iov_iter_get_pages_alloc); -- cgit From 66531c65aa254e77c935785036beb50985d0fe89 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Apr 2021 16:00:48 -0400 Subject: iov_iter_npages(): don't bother with iterate_all_kinds() note that in bvec case pages can be compound ones - we can't just assume that each segment is covered by one (sub)page Signed-off-by: Al Viro --- lib/iov_iter.c | 88 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 3fd331b3b8f2..1c65de175371 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1864,19 +1864,56 @@ size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, } EXPORT_SYMBOL(hash_and_copy_to_iter); -int iov_iter_npages(const struct iov_iter *i, int maxpages) +static int iov_npages(const struct iov_iter *i, int maxpages) { - size_t size = i->count; + size_t skip = i->iov_offset, size = i->count; + const struct iovec *p; int npages = 0; - if (!size) - return 0; - if (unlikely(iov_iter_is_discard(i))) - return 0; + for (p = i->iov; size; skip = 0, p++) { + unsigned offs = offset_in_page(p->iov_base + skip); + size_t len = min(p->iov_len - skip, size); - if (unlikely(iov_iter_is_pipe(i))) { - struct pipe_inode_info *pipe = i->pipe; + if (len) { + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + } + return npages; +} + +static int bvec_npages(const struct iov_iter *i, int maxpages) +{ + size_t skip = i->iov_offset, size = i->count; + const struct bio_vec *p; + int npages = 0; + + for (p = i->bvec; size; skip = 0, p++) { + unsigned offs = (p->bv_offset + skip) % PAGE_SIZE; + size_t len = min(p->bv_len - skip, size); + + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + return npages; +} + +int iov_iter_npages(const struct iov_iter *i, int maxpages) +{ + if (unlikely(!i->count)) + return 0; + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_npages(i, maxpages); + if (iov_iter_is_bvec(i)) + return bvec_npages(i, maxpages); + if (iov_iter_is_pipe(i)) { unsigned int iter_head; + int npages; size_t off; if (!sanity(i)) @@ -1884,11 +1921,13 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages) data_start(i, &iter_head, &off); /* some of this one + all after this one */ - npages = pipe_space_for_user(iter_head, pipe->tail, pipe); - if (npages >= maxpages) - return maxpages; - } else if (unlikely(iov_iter_is_xarray(i))) { + npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); + return min(npages, maxpages); + } + if (iov_iter_is_xarray(i)) { + size_t size = i->count; unsigned offset; + int npages; offset = (i->xarray_start + i->iov_offset) & ~PAGE_MASK; @@ -1900,28 +1939,9 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages) if (size) npages++; } - if (npages >= maxpages) - return maxpages; - } else iterate_all_kinds(i, size, v, ({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - 0;}),({ - npages++; - if (npages >= maxpages) - return maxpages; - }),({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - }), - 0 - ) - return npages; + return min(npages, maxpages); + } + return 0; } EXPORT_SYMBOL(iov_iter_npages); -- cgit From e4f8df86798aea60aff6cfff40252b709431f850 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 3 May 2021 11:05:29 -0400 Subject: [xarray] iov_iter_npages(): just use DIV_ROUND_UP() Compiler is capable of recognizing division by power of 2 and turning it into shifts. Signed-off-by: Al Viro --- lib/iov_iter.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 1c65de175371..72c5bb794e8d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1925,20 +1925,8 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages) return min(npages, maxpages); } if (iov_iter_is_xarray(i)) { - size_t size = i->count; - unsigned offset; - int npages; - - offset = (i->xarray_start + i->iov_offset) & ~PAGE_MASK; - - npages = 1; - if (size > PAGE_SIZE - offset) { - size -= PAGE_SIZE - offset; - npages += size >> PAGE_SHIFT; - size &= ~PAGE_MASK; - if (size) - npages++; - } + unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; + int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); return min(npages, maxpages); } return 0; -- cgit From f0b65f39ac505e8f1dcdaa165aa7b8c0bd6fd454 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 30 Apr 2021 10:26:41 -0400 Subject: iov_iter: replace iov_iter_copy_from_user_atomic() with iterator-advancing variant Replacement is called copy_page_from_iter_atomic(); unlike the old primitive the callers do *not* need to do iov_iter_advance() after it. In case when they end up consuming less than they'd been given they need to do iov_iter_revert() on everything they had not consumed. That, however, needs to be done only on slow paths. All in-tree callers converted. And that kills the last user of iterate_all_kinds() Signed-off-by: Al Viro --- lib/iov_iter.c | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 72c5bb794e8d..362e8b5a5dc5 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -114,28 +114,6 @@ n = wanted - n; \ } -#define iterate_all_kinds(i, n, v, I, B, K, X) { \ - if (likely(n)) { \ - size_t skip = i->iov_offset; \ - if (likely(iter_is_iovec(i))) { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ - } else if (iov_iter_is_bvec(i)) { \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - } else if (iov_iter_is_kvec(i)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - } else if (iov_iter_is_xarray(i)) { \ - struct bio_vec v; \ - iterate_xarray(i, n, v, skip, (X)); \ - } \ - } \ -} - #define iterate_and_advance(i, n, v, I, B, K, X) { \ if (unlikely(i->count < n)) \ n = i->count; \ @@ -1009,8 +987,8 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) } EXPORT_SYMBOL(iov_iter_zero); -size_t iov_iter_copy_from_user_atomic(struct page *page, - struct iov_iter *i, unsigned long offset, size_t bytes) +size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes, + struct iov_iter *i) { char *kaddr = kmap_atomic(page), *p = kaddr + offset; if (unlikely(!page_copy_sane(page, offset, bytes))) { @@ -1022,7 +1000,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, WARN_ON(1); return 0; } - iterate_all_kinds(i, bytes, v, + iterate_and_advance(i, bytes, v, copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, v.bv_offset, v.bv_len), @@ -1033,7 +1011,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, kunmap_atomic(kaddr); return bytes; } -EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); +EXPORT_SYMBOL(copy_page_from_iter_atomic); static inline void pipe_truncate(struct iov_iter *i) { -- cgit From 594e450b3f4435a9d663df3d48d7fa34e685cbd1 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 5 Jun 2021 10:19:30 -0400 Subject: csum_and_copy_to_iter(): massage into form closer to csum_and_copy_from_iter() Namely, have off counted starting from 0 rather than from csstate->off. To compensate we need to shift the initial value (csstate->sum) (rotate by 8 bits, as usual for csum) and do the same after we are finished adding the pieces up. What we get out of that is a bit more redundancy in our variables - from is always equal to addr + off, which will be useful several commits down the road. Signed-off-by: Al Viro --- lib/iov_iter.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 362e8b5a5dc5..93ae0c2c8d66 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1781,8 +1781,8 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, if (unlikely(iov_iter_is_pipe(i))) return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i); - sum = csstate->csum; - off = csstate->off; + sum = csum_shift(csstate->csum, csstate->off); + off = 0; if (unlikely(iov_iter_is_discard(i))) { WARN_ON(1); /* for now */ return 0; @@ -1817,8 +1817,8 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, off += v.bv_len; }) ) - csstate->csum = sum; - csstate->off = off; + csstate->csum = csum_shift(sum, csstate->off); + csstate->off += bytes; return bytes; } EXPORT_SYMBOL(csum_and_copy_to_iter); -- cgit From f5da83545f4ed2c1a1648b7d760a6fc358798e52 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 28 Apr 2021 20:59:08 -0400 Subject: iterate_and_advance(): get rid of magic in case when n is 0 iov_iter_advance() needs to do some non-trivial work when it's given 0 as argument (skip all empty iovecs, mostly). We used to implement it via iterate_and_advance(); we no longer do so and for all other users of iterate_and_advance() zero length is a no-op. Signed-off-by: Al Viro --- lib/iov_iter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 93ae0c2c8d66..763114a754c5 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -117,7 +117,7 @@ #define iterate_and_advance(i, n, v, I, B, K, X) { \ if (unlikely(i->count < n)) \ n = i->count; \ - if (i->count) { \ + if (likely(n)) { \ size_t skip = i->iov_offset; \ if (likely(iter_is_iovec(i))) { \ const struct iovec *iov; \ -- cgit From 7a1bcb5d255d4fd8b9725c3cf7ee0880a6369d2f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Apr 2021 23:46:09 -0400 Subject: iov_iter: massage iterate_iovec and iterate_kvec to logics similar to iterate_bvec Premature optimization is the root of all evil... Trying to unroll the first pass through the loop makes it harder to follow and not just for readers - compiler ends up generating worse code than it would on a "non-optimized" loop. Signed-off-by: Al Viro --- lib/iov_iter.c | 91 +++++++++++++++++++++++----------------------------------- 1 file changed, 36 insertions(+), 55 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 763114a754c5..2098059da64c 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -16,55 +16,44 @@ #define PIPE_PARANOIA /* for now */ -#define iterate_iovec(i, n, __v, __p, skip, STEP) { \ - size_t left; \ - size_t wanted = n; \ - __p = i->iov; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } else { \ - left = 0; \ - } \ - while (unlikely(!left && n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted - n; \ +#define iterate_iovec(i, n, __v, __p, skip, STEP) { \ + size_t left; \ + size_t wanted = n; \ + __p = i->iov; \ + do { \ + __v.iov_len = min(n, __p->iov_len - skip); \ + if (likely(__v.iov_len)) { \ + __v.iov_base = __p->iov_base + skip; \ + left = (STEP); \ + __v.iov_len -= left; \ + skip += __v.iov_len; \ + n -= __v.iov_len; \ + if (skip < __p->iov_len) \ + break; \ + } \ + __p++; \ + skip = 0; \ + } while (n); \ + n = wanted - n; \ } -#define iterate_kvec(i, n, __v, __p, skip, STEP) { \ - size_t wanted = n; \ - __p = i->kvec; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - (void)(STEP); \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } \ - while (unlikely(n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - (void)(STEP); \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted; \ +#define iterate_kvec(i, n, __v, __p, skip, STEP) { \ + size_t wanted = n; \ + __p = i->kvec; \ + do { \ + __v.iov_len = min(n, __p->iov_len - skip); \ + if (likely(__v.iov_len)) { \ + __v.iov_base = __p->iov_base + skip; \ + (void)(STEP); \ + skip += __v.iov_len; \ + n -= __v.iov_len; \ + if (skip < __p->iov_len) \ + break; \ + } \ + __p++; \ + skip = 0; \ + } while (n); \ + n = wanted - n; \ } #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ @@ -123,10 +112,6 @@ const struct iovec *iov; \ struct iovec v; \ iterate_iovec(i, n, v, iov, skip, (I)) \ - if (skip == iov->iov_len) { \ - iov++; \ - skip = 0; \ - } \ i->nr_segs -= iov - i->iov; \ i->iov = iov; \ } else if (iov_iter_is_bvec(i)) { \ @@ -141,10 +126,6 @@ const struct kvec *kvec; \ struct kvec v; \ iterate_kvec(i, n, v, kvec, skip, (K)) \ - if (skip == kvec->iov_len) { \ - kvec++; \ - skip = 0; \ - } \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ -- cgit From 5c67aa90cd5c59912ee71cff879e8f1ab237ad88 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Apr 2021 23:57:42 -0400 Subject: iov_iter: unify iterate_iovec and iterate_kvec The differences between iterate_iovec and iterate_kvec are minor: * kvec callback is treated as if it returned 0 * initialization of __p is with i->iov and i->kvec resp. which is trivially dealt with. No code generation changes - compiler is quite capable of turning left = ((void)(STEP), 0); __v.iov_len -= left; (with no accesses to left downstream) and (void)(STEP); into the same code. Signed-off-by: Al Viro --- lib/iov_iter.c | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 2098059da64c..fc071d7b4528 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -16,10 +16,10 @@ #define PIPE_PARANOIA /* for now */ +/* covers iovec and kvec alike */ #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ size_t left; \ size_t wanted = n; \ - __p = i->iov; \ do { \ __v.iov_len = min(n, __p->iov_len - skip); \ if (likely(__v.iov_len)) { \ @@ -37,25 +37,6 @@ n = wanted - n; \ } -#define iterate_kvec(i, n, __v, __p, skip, STEP) { \ - size_t wanted = n; \ - __p = i->kvec; \ - do { \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - (void)(STEP); \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - if (skip < __p->iov_len) \ - break; \ - } \ - __p++; \ - skip = 0; \ - } while (n); \ - n = wanted - n; \ -} - #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ struct bvec_iter __start; \ __start.bi_size = n; \ @@ -109,7 +90,7 @@ if (likely(n)) { \ size_t skip = i->iov_offset; \ if (likely(iter_is_iovec(i))) { \ - const struct iovec *iov; \ + const struct iovec *iov = i->iov; \ struct iovec v; \ iterate_iovec(i, n, v, iov, skip, (I)) \ i->nr_segs -= iov - i->iov; \ @@ -123,9 +104,10 @@ i->nr_segs -= i->bvec - bvec; \ skip = __bi.bi_bvec_done; \ } else if (iov_iter_is_kvec(i)) { \ - const struct kvec *kvec; \ + const struct kvec *kvec = i->kvec; \ struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ + iterate_iovec(i, n, v, kvec, skip, \ + ((void)(K),0)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ -- cgit From 7491a2bf64e3a4f1699deba97728cd9f8856bdf3 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 26 Apr 2021 20:19:14 -0400 Subject: iterate_bvec(): expand bvec.h macro forest, massage a bit ... incidentally, using pointer instead of index in an array (the only change here) trims half-kilobyte of .text... Signed-off-by: Al Viro --- lib/iov_iter.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index fc071d7b4528..58d53deb62b8 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -37,14 +37,23 @@ n = wanted - n; \ } -#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ - struct bvec_iter __start; \ - __start.bi_size = n; \ - __start.bi_bvec_done = skip; \ - __start.bi_idx = 0; \ - for_each_bvec(__v, i->bvec, __bi, __start) { \ - (void)(STEP); \ - } \ +#define iterate_bvec(i, n, __v, p, skip, STEP) { \ + size_t wanted = n; \ + while (n) { \ + unsigned offset = p->bv_offset + skip; \ + __v.bv_offset = offset % PAGE_SIZE; \ + __v.bv_page = p->bv_page + offset / PAGE_SIZE; \ + __v.bv_len = min(min(n, p->bv_len - skip), \ + (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ + (void)(STEP); \ + skip += __v.bv_len; \ + if (skip == p->bv_len) { \ + skip = 0; \ + p++; \ + } \ + n -= __v.bv_len; \ + } \ + n = wanted - n; \ } #define iterate_xarray(i, n, __v, skip, STEP) { \ @@ -98,11 +107,9 @@ } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ - i->nr_segs -= i->bvec - bvec; \ - skip = __bi.bi_bvec_done; \ + iterate_bvec(i, n, v, bvec, skip, (B)) \ + i->nr_segs -= bvec - i->bvec; \ + i->bvec = bvec; \ } else if (iov_iter_is_kvec(i)) { \ const struct kvec *kvec = i->kvec; \ struct kvec v; \ -- cgit From 1b4fb5ffd79bac27a7b9beda63c827c7d7457c45 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 26 Apr 2021 20:33:42 -0400 Subject: iov_iter: teach iterate_{bvec,xarray}() about possible short copies ... and now we finally can sort out the mess in _copy_mc_to_iter(). Provide a variant of iterate_and_advance() that does *NOT* ignore the return values of bvec, xarray and kvec callbacks, use that in _copy_mc_to_iter(). That gets rid of magic in those callbacks - we used to need it so we'd get at least the right return value in case of failure halfway through. As a bonus, now iterator is advanced by the amount actually copied for all flavours. That's what the callers expect and it used to do that correctly in iovec and xarray cases. However, in kvec and bvec cases the iterator had not been advanced on such failures, breaking the users. Fixed now... Signed-off-by: Al Viro --- lib/iov_iter.c | 65 ++++++++++++++++++++++------------------------------------ 1 file changed, 24 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 58d53deb62b8..ac7682734df2 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -41,22 +41,27 @@ size_t wanted = n; \ while (n) { \ unsigned offset = p->bv_offset + skip; \ + unsigned left; \ __v.bv_offset = offset % PAGE_SIZE; \ __v.bv_page = p->bv_page + offset / PAGE_SIZE; \ __v.bv_len = min(min(n, p->bv_len - skip), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ - (void)(STEP); \ + left = (STEP); \ + __v.bv_len -= left; \ skip += __v.bv_len; \ if (skip == p->bv_len) { \ skip = 0; \ p++; \ } \ n -= __v.bv_len; \ + if (left) \ + break; \ } \ n = wanted - n; \ } #define iterate_xarray(i, n, __v, skip, STEP) { \ + __label__ __out; \ struct page *head = NULL; \ size_t wanted = n, seg, offset; \ loff_t start = i->xarray_start + skip; \ @@ -67,6 +72,7 @@ \ rcu_read_lock(); \ xas_for_each(&xas, head, ULONG_MAX) { \ + unsigned left; \ if (xas_retry(&xas, head)) \ continue; \ if (WARN_ON(xa_is_value(head))) \ @@ -80,20 +86,20 @@ seg = PAGE_SIZE - offset; \ __v.bv_offset = offset; \ __v.bv_len = min(n, seg); \ - (void)(STEP); \ + left = (STEP); \ + __v.bv_len -= left; \ n -= __v.bv_len; \ skip += __v.bv_len; \ - if (n == 0) \ - break; \ + if (left || n == 0) \ + goto __out; \ } \ - if (n == 0) \ - break; \ } \ +__out: \ rcu_read_unlock(); \ n = wanted - n; \ } -#define iterate_and_advance(i, n, v, I, B, K, X) { \ +#define __iterate_and_advance(i, n, v, I, B, K, X) { \ if (unlikely(i->count < n)) \ n = i->count; \ if (likely(n)) { \ @@ -113,8 +119,7 @@ } else if (iov_iter_is_kvec(i)) { \ const struct kvec *kvec = i->kvec; \ struct kvec v; \ - iterate_iovec(i, n, v, kvec, skip, \ - ((void)(K),0)) \ + iterate_iovec(i, n, v, kvec, skip, (K)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ @@ -125,6 +130,9 @@ i->iov_offset = skip; \ } \ } +#define iterate_and_advance(i, n, v, I, B, K, X) \ + __iterate_and_advance(i, n, v, I, ((void)(B),0), \ + ((void)(K),0), ((void)(X),0)) static int copyout(void __user *to, const void *from, size_t n) { @@ -698,45 +706,20 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { const char *from = addr; - unsigned long rem, curr_addr, s_addr = (unsigned long) addr; if (unlikely(iov_iter_is_pipe(i))) return copy_mc_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, + __iterate_and_advance(i, bytes, v, copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - ({ - rem = copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }), - ({ - rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len) - - v.iov_len, v.iov_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }), - ({ - rem = copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - rcu_read_unlock(); - i->iov_offset += bytes; - i->count -= bytes; - return bytes; - } - }) + copy_mc_to_page(v.bv_page, v.bv_offset, + (from += v.bv_len) - v.bv_len, v.bv_len), + copy_mc_to_kernel(v.iov_base, (from += v.iov_len) + - v.iov_len, v.iov_len), + copy_mc_to_page(v.bv_page, v.bv_offset, + (from += v.bv_len) - v.bv_len, v.bv_len) ) return bytes; -- cgit From 21b56c84775351ac66354c9b09fb429e5cdeceac Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 26 Apr 2021 20:50:05 -0400 Subject: iov_iter: get rid of separate bvec and xarray callbacks After the previous commit we have * xarray and bvec callbacks idential in all cases * both equivalent to kvec callback wrapped into kmap_local_page()/kunmap_local() pair. So we can pass only two (iovec and kvec) callbacks to iterate_and_advance() and let iterate_{bvec,xarray} wrap it into kmap_local_page()/kunmap_local_page(). Signed-off-by: Al Viro --- lib/iov_iter.c | 112 ++++++++++++++++----------------------------------------- 1 file changed, 30 insertions(+), 82 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index ac7682734df2..74e20d6d6e3d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -42,18 +42,20 @@ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ - __v.bv_offset = offset % PAGE_SIZE; \ - __v.bv_page = p->bv_page + offset / PAGE_SIZE; \ - __v.bv_len = min(min(n, p->bv_len - skip), \ + void *kaddr = kmap_local_page(p->bv_page + \ + offset / PAGE_SIZE); \ + __v.iov_base = kaddr + offset % PAGE_SIZE; \ + __v.iov_len = min(min(n, p->bv_len - skip), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ left = (STEP); \ - __v.bv_len -= left; \ - skip += __v.bv_len; \ + kunmap_local(kaddr); \ + __v.iov_len -= left; \ + skip += __v.iov_len; \ if (skip == p->bv_len) { \ skip = 0; \ p++; \ } \ - n -= __v.bv_len; \ + n -= __v.iov_len; \ if (left) \ break; \ } \ @@ -81,15 +83,16 @@ break; \ for (j = (head->index < index) ? index - head->index : 0; \ j < thp_nr_pages(head); j++) { \ - __v.bv_page = head + j; \ - offset = (i->xarray_start + skip) & ~PAGE_MASK; \ + void *kaddr = kmap_local_page(head + j); \ + offset = (i->xarray_start + skip) % PAGE_SIZE; \ + __v.iov_base = kaddr + offset; \ seg = PAGE_SIZE - offset; \ - __v.bv_offset = offset; \ - __v.bv_len = min(n, seg); \ + __v.iov_len = min(n, seg); \ left = (STEP); \ - __v.bv_len -= left; \ - n -= __v.bv_len; \ - skip += __v.bv_len; \ + kunmap_local(kaddr); \ + __v.iov_len -= left; \ + n -= __v.iov_len; \ + skip += __v.iov_len; \ if (left || n == 0) \ goto __out; \ } \ @@ -99,7 +102,7 @@ __out: \ n = wanted - n; \ } -#define __iterate_and_advance(i, n, v, I, B, K, X) { \ +#define __iterate_and_advance(i, n, v, I, K) { \ if (unlikely(i->count < n)) \ n = i->count; \ if (likely(n)) { \ @@ -112,8 +115,8 @@ __out: \ i->iov = iov; \ } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ - struct bio_vec v; \ - iterate_bvec(i, n, v, bvec, skip, (B)) \ + struct kvec v; \ + iterate_bvec(i, n, v, bvec, skip, (K)) \ i->nr_segs -= bvec - i->bvec; \ i->bvec = bvec; \ } else if (iov_iter_is_kvec(i)) { \ @@ -123,16 +126,15 @@ __out: \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ - struct bio_vec v; \ - iterate_xarray(i, n, v, skip, (X)) \ + struct kvec v; \ + iterate_xarray(i, n, v, skip, (K)) \ } \ i->count -= n; \ i->iov_offset = skip; \ } \ } -#define iterate_and_advance(i, n, v, I, B, K, X) \ - __iterate_and_advance(i, n, v, I, ((void)(B),0), \ - ((void)(K),0), ((void)(X),0)) +#define iterate_and_advance(i, n, v, I, K) \ + __iterate_and_advance(i, n, v, I, ((void)(K),0)) static int copyout(void __user *to, const void *from, size_t n) { @@ -612,11 +614,7 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) might_fault(); iterate_and_advance(i, bytes, v, copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), - memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len) + memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) ) return bytes; @@ -714,12 +712,8 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) __iterate_and_advance(i, bytes, v, copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), copy_mc_to_kernel(v.iov_base, (from += v.iov_len) - - v.iov_len, v.iov_len), - copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len) + - v.iov_len, v.iov_len) ) return bytes; @@ -738,11 +732,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) might_fault(); iterate_and_advance(i, bytes, v, copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) return bytes; @@ -759,11 +749,7 @@ size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) iterate_and_advance(i, bytes, v, __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) return bytes; @@ -795,12 +781,8 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) iterate_and_advance(i, bytes, v, __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, - v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + v.iov_len) ) return bytes; @@ -931,9 +913,7 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) return pipe_zero(bytes, i); iterate_and_advance(i, bytes, v, clear_user(v.iov_base, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len), - memset(v.iov_base, 0, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len) + memset(v.iov_base, 0, v.iov_len) ) return bytes; @@ -955,11 +935,7 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt } iterate_and_advance(i, bytes, v, copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) kunmap_atomic(kaddr); return bytes; @@ -1698,24 +1674,10 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, } next ? 0 : v.iov_len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len, sum, off); off += v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; }) ) *csum = sum; @@ -1750,24 +1712,10 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, } next ? 0 : v.iov_len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ sum = csum_and_memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len, sum, off); off += v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; }) ) csstate->csum = csum_shift(sum, csstate->off); -- cgit From 622838f3fde2c3671a718dc6196c19087ebe9b11 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 11:13:09 -0400 Subject: iov_iter: make the amount already copied available to iterator callbacks Making iterator macros keep track of the amount of data copied is pretty easy and it has several benefits: 1) we no longer need the mess like (from += v.iov_len) - v.iov_len in the callbacks - initial value + total amount copied so far would do just fine. 2) less obviously, we no longer need to remember the initial amount of data we wanted to copy; the loops in iterator macros are along the lines of wanted = bytes; while (bytes) { copy some bytes -= copied if short copy break } bytes = wanted - bytes; Replacement is offs = 0; while (bytes) { copy some offs += copied bytes -= copied if short copy break } bytes = offs; That wouldn't be a win per se, but unlike the initial value of bytes, the amount copied so far *is* useful in callbacks. 3) in some cases (csum_and_copy_..._iter()) we already had offs manually maintained by the callbacks. With that change we can drop that. Less boilerplate and more readable code... Signed-off-by: Al Viro --- lib/iov_iter.c | 120 ++++++++++++++++++++++++--------------------------------- 1 file changed, 50 insertions(+), 70 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 74e20d6d6e3d..0e04abe9ec49 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -17,15 +17,14 @@ #define PIPE_PARANOIA /* for now */ /* covers iovec and kvec alike */ -#define iterate_iovec(i, n, __v, __p, skip, STEP) { \ - size_t left; \ - size_t wanted = n; \ +#define iterate_iovec(i, n, __v, __off, __p, skip, STEP) { \ + size_t __off = 0; \ do { \ __v.iov_len = min(n, __p->iov_len - skip); \ if (likely(__v.iov_len)) { \ __v.iov_base = __p->iov_base + skip; \ - left = (STEP); \ - __v.iov_len -= left; \ + __v.iov_len -= (STEP); \ + __off += __v.iov_len; \ skip += __v.iov_len; \ n -= __v.iov_len; \ if (skip < __p->iov_len) \ @@ -34,11 +33,11 @@ __p++; \ skip = 0; \ } while (n); \ - n = wanted - n; \ + n = __off; \ } -#define iterate_bvec(i, n, __v, p, skip, STEP) { \ - size_t wanted = n; \ +#define iterate_bvec(i, n, __v, __off, p, skip, STEP) { \ + size_t __off = 0; \ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ @@ -50,6 +49,7 @@ left = (STEP); \ kunmap_local(kaddr); \ __v.iov_len -= left; \ + __off += __v.iov_len; \ skip += __v.iov_len; \ if (skip == p->bv_len) { \ skip = 0; \ @@ -59,13 +59,14 @@ if (left) \ break; \ } \ - n = wanted - n; \ + n = __off; \ } -#define iterate_xarray(i, n, __v, skip, STEP) { \ +#define iterate_xarray(i, n, __v, __off, skip, STEP) { \ __label__ __out; \ + size_t __off = 0; \ struct page *head = NULL; \ - size_t wanted = n, seg, offset; \ + size_t seg, offset; \ loff_t start = i->xarray_start + skip; \ pgoff_t index = start >> PAGE_SHIFT; \ int j; \ @@ -84,25 +85,26 @@ for (j = (head->index < index) ? index - head->index : 0; \ j < thp_nr_pages(head); j++) { \ void *kaddr = kmap_local_page(head + j); \ - offset = (i->xarray_start + skip) % PAGE_SIZE; \ + offset = (start + __off) % PAGE_SIZE; \ __v.iov_base = kaddr + offset; \ seg = PAGE_SIZE - offset; \ __v.iov_len = min(n, seg); \ left = (STEP); \ kunmap_local(kaddr); \ __v.iov_len -= left; \ + __off += __v.iov_len; \ n -= __v.iov_len; \ - skip += __v.iov_len; \ if (left || n == 0) \ goto __out; \ } \ } \ __out: \ rcu_read_unlock(); \ - n = wanted - n; \ + skip += __off; \ + n = __off; \ } -#define __iterate_and_advance(i, n, v, I, K) { \ +#define __iterate_and_advance(i, n, v, off, I, K) { \ if (unlikely(i->count < n)) \ n = i->count; \ if (likely(n)) { \ @@ -110,31 +112,31 @@ __out: \ if (likely(iter_is_iovec(i))) { \ const struct iovec *iov = i->iov; \ struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ + iterate_iovec(i, n, v, off, iov, skip, (I)) \ i->nr_segs -= iov - i->iov; \ i->iov = iov; \ } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ struct kvec v; \ - iterate_bvec(i, n, v, bvec, skip, (K)) \ + iterate_bvec(i, n, v, off, bvec, skip, (K)) \ i->nr_segs -= bvec - i->bvec; \ i->bvec = bvec; \ } else if (iov_iter_is_kvec(i)) { \ const struct kvec *kvec = i->kvec; \ struct kvec v; \ - iterate_iovec(i, n, v, kvec, skip, (K)) \ + iterate_iovec(i, n, v, off, kvec, skip, (K)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ struct kvec v; \ - iterate_xarray(i, n, v, skip, (K)) \ + iterate_xarray(i, n, v, off, skip, (K)) \ } \ i->count -= n; \ i->iov_offset = skip; \ } \ } -#define iterate_and_advance(i, n, v, I, K) \ - __iterate_and_advance(i, n, v, I, ((void)(K),0)) +#define iterate_and_advance(i, n, v, off, I, K) \ + __iterate_and_advance(i, n, v, off, I, ((void)(K),0)) static int copyout(void __user *to, const void *from, size_t n) { @@ -607,14 +609,13 @@ static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; if (unlikely(iov_iter_is_pipe(i))) return copy_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) + iterate_and_advance(i, bytes, v, off, + copyout(v.iov_base, addr + off, v.iov_len), + memcpy(v.iov_base, addr + off, v.iov_len) ) return bytes; @@ -703,17 +704,13 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, */ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; - if (unlikely(iov_iter_is_pipe(i))) return copy_mc_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - __iterate_and_advance(i, bytes, v, - copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len, - v.iov_len), - copy_mc_to_kernel(v.iov_base, (from += v.iov_len) - - v.iov_len, v.iov_len) + __iterate_and_advance(i, bytes, v, off, + copyout_mc(v.iov_base, addr + off, v.iov_len), + copy_mc_to_kernel(v.iov_base, addr + off, v.iov_len) ) return bytes; @@ -723,16 +720,15 @@ EXPORT_SYMBOL_GPL(_copy_mc_to_iter); size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, v, off, + copyin(addr + off, v.iov_base, v.iov_len), + memcpy(addr + off, v.iov_base, v.iov_len) ) return bytes; @@ -741,15 +737,14 @@ EXPORT_SYMBOL(_copy_from_iter); size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, + iterate_and_advance(i, bytes, v, off, + __copy_from_user_inatomic_nocache(addr + off, v.iov_base, v.iov_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) + memcpy(addr + off, v.iov_base, v.iov_len) ) return bytes; @@ -773,16 +768,13 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); */ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, - v.iov_len) + iterate_and_advance(i, bytes, v, off, + __copy_from_user_flushcache(addr + off, v.iov_base, v.iov_len), + memcpy_flushcache(addr + off, v.iov_base, v.iov_len) ) return bytes; @@ -911,7 +903,7 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) { if (unlikely(iov_iter_is_pipe(i))) return pipe_zero(bytes, i); - iterate_and_advance(i, bytes, v, + iterate_and_advance(i, bytes, v, count, clear_user(v.iov_base, v.iov_len), memset(v.iov_base, 0, v.iov_len) ) @@ -933,9 +925,9 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, v, off, + copyin(p + off, v.iov_base, v.iov_len), + memcpy(p + off, v.iov_base, v.iov_len) ) kunmap_atomic(kaddr); return bytes; @@ -1656,28 +1648,22 @@ EXPORT_SYMBOL(iov_iter_get_pages_alloc); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i) { - char *to = addr; __wsum sum, next; - size_t off = 0; sum = *csum; if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, ({ + iterate_and_advance(i, bytes, v, off, ({ next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, + addr + off, v.iov_len); - if (next) { + if (next) sum = csum_block_add(sum, next, off); - off += v.iov_len; - } next ? 0 : v.iov_len; }), ({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, + sum = csum_and_memcpy(addr + off, v.iov_base, v.iov_len, sum, off); - off += v.iov_len; }) ) *csum = sum; @@ -1689,33 +1675,27 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, struct iov_iter *i) { struct csum_state *csstate = _csstate; - const char *from = addr; __wsum sum, next; - size_t off; if (unlikely(iov_iter_is_pipe(i))) return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i); sum = csum_shift(csstate->csum, csstate->off); - off = 0; if (unlikely(iov_iter_is_discard(i))) { WARN_ON(1); /* for now */ return 0; } - iterate_and_advance(i, bytes, v, ({ - next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, + iterate_and_advance(i, bytes, v, off, ({ + next = csum_and_copy_to_user(addr + off, v.iov_base, v.iov_len); - if (next) { + if (next) sum = csum_block_add(sum, next, off); - off += v.iov_len; - } next ? 0 : v.iov_len; }), ({ sum = csum_and_memcpy(v.iov_base, - (from += v.iov_len) - v.iov_len, + addr + off, v.iov_len, sum, off); - off += v.iov_len; }) ) csstate->csum = csum_shift(sum, csstate->off); -- cgit From 7baa5099002f2f2ea6c026890598ed1708e7cfd4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 11:35:03 -0400 Subject: iov_iter: make iterator callbacks use base and len instead of iovec Iterator macros used to provide the arguments for step callbacks in a structure matching the flavour - iovec for ITER_IOVEC, kvec for ITER_KVEC and bio_vec for ITER_BVEC. That already broke down for ITER_XARRAY (bio_vec there); now that we are using kvec callback for bvec and xarray cases, we are always passing a pointer + length (void __user * + size_t for ITER_IOVEC callback, void * + size_t for everything else). Note that the original reason for bio_vec (page + offset + len) in case of ITER_BVEC used to be that we did *not* want to kmap a page when all we wanted was e.g. to find the alignment of its subrange. Now all such users are gone and the ones that are left want the page mapped anyway for actually copying the data. So in all cases we have pointer + length, and there's no good reason for keeping those in struct iovec or struct kvec - we can just pass them to callback separately. Again, less boilerplate in callbacks... Signed-off-by: Al Viro --- lib/iov_iter.c | 182 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 91 insertions(+), 91 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0e04abe9ec49..f4ea04e24e06 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -17,86 +17,86 @@ #define PIPE_PARANOIA /* for now */ /* covers iovec and kvec alike */ -#define iterate_iovec(i, n, __v, __off, __p, skip, STEP) { \ - size_t __off = 0; \ +#define iterate_iovec(i, n, base, len, off, __p, skip, STEP) { \ + size_t off = 0; \ do { \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - __v.iov_len -= (STEP); \ - __off += __v.iov_len; \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ + len = min(n, __p->iov_len - skip); \ + if (likely(len)) { \ + base = __p->iov_base + skip; \ + len -= (STEP); \ + off += len; \ + skip += len; \ + n -= len; \ if (skip < __p->iov_len) \ break; \ } \ __p++; \ skip = 0; \ } while (n); \ - n = __off; \ + n = off; \ } -#define iterate_bvec(i, n, __v, __off, p, skip, STEP) { \ - size_t __off = 0; \ +#define iterate_bvec(i, n, base, len, off, p, skip, STEP) { \ + size_t off = 0; \ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ void *kaddr = kmap_local_page(p->bv_page + \ offset / PAGE_SIZE); \ - __v.iov_base = kaddr + offset % PAGE_SIZE; \ - __v.iov_len = min(min(n, p->bv_len - skip), \ + base = kaddr + offset % PAGE_SIZE; \ + len = min(min(n, p->bv_len - skip), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ left = (STEP); \ kunmap_local(kaddr); \ - __v.iov_len -= left; \ - __off += __v.iov_len; \ - skip += __v.iov_len; \ + len -= left; \ + off += len; \ + skip += len; \ if (skip == p->bv_len) { \ skip = 0; \ p++; \ } \ - n -= __v.iov_len; \ + n -= len; \ if (left) \ break; \ } \ - n = __off; \ + n = off; \ } -#define iterate_xarray(i, n, __v, __off, skip, STEP) { \ +#define iterate_xarray(i, n, base, len, __off, skip, STEP) { \ __label__ __out; \ size_t __off = 0; \ struct page *head = NULL; \ - size_t seg, offset; \ + size_t offset; \ loff_t start = i->xarray_start + skip; \ pgoff_t index = start >> PAGE_SHIFT; \ int j; \ \ XA_STATE(xas, i->xarray, index); \ \ - rcu_read_lock(); \ - xas_for_each(&xas, head, ULONG_MAX) { \ - unsigned left; \ - if (xas_retry(&xas, head)) \ - continue; \ - if (WARN_ON(xa_is_value(head))) \ - break; \ - if (WARN_ON(PageHuge(head))) \ - break; \ + rcu_read_lock(); \ + xas_for_each(&xas, head, ULONG_MAX) { \ + unsigned left; \ + if (xas_retry(&xas, head)) \ + continue; \ + if (WARN_ON(xa_is_value(head))) \ + break; \ + if (WARN_ON(PageHuge(head))) \ + break; \ for (j = (head->index < index) ? index - head->index : 0; \ - j < thp_nr_pages(head); j++) { \ + j < thp_nr_pages(head); j++) { \ void *kaddr = kmap_local_page(head + j); \ - offset = (start + __off) % PAGE_SIZE; \ - __v.iov_base = kaddr + offset; \ - seg = PAGE_SIZE - offset; \ - __v.iov_len = min(n, seg); \ - left = (STEP); \ - kunmap_local(kaddr); \ - __v.iov_len -= left; \ - __off += __v.iov_len; \ - n -= __v.iov_len; \ - if (left || n == 0) \ - goto __out; \ - } \ + offset = (start + __off) % PAGE_SIZE; \ + base = kaddr + offset; \ + len = PAGE_SIZE - offset; \ + len = min(n, len); \ + left = (STEP); \ + kunmap_local(kaddr); \ + len -= left; \ + __off += len; \ + n -= len; \ + if (left || n == 0) \ + goto __out; \ + } \ } \ __out: \ rcu_read_unlock(); \ @@ -104,39 +104,47 @@ __out: \ n = __off; \ } -#define __iterate_and_advance(i, n, v, off, I, K) { \ +#define __iterate_and_advance(i, n, base, len, off, I, K) { \ if (unlikely(i->count < n)) \ n = i->count; \ if (likely(n)) { \ size_t skip = i->iov_offset; \ if (likely(iter_is_iovec(i))) { \ const struct iovec *iov = i->iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, off, iov, skip, (I)) \ + void __user *base; \ + size_t len; \ + iterate_iovec(i, n, base, len, off, \ + iov, skip, (I)) \ i->nr_segs -= iov - i->iov; \ i->iov = iov; \ } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ - struct kvec v; \ - iterate_bvec(i, n, v, off, bvec, skip, (K)) \ + void *base; \ + size_t len; \ + iterate_bvec(i, n, base, len, off, \ + bvec, skip, (K)) \ i->nr_segs -= bvec - i->bvec; \ i->bvec = bvec; \ } else if (iov_iter_is_kvec(i)) { \ const struct kvec *kvec = i->kvec; \ - struct kvec v; \ - iterate_iovec(i, n, v, off, kvec, skip, (K)) \ + void *base; \ + size_t len; \ + iterate_iovec(i, n, base, len, off, \ + kvec, skip, (K)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ - struct kvec v; \ - iterate_xarray(i, n, v, off, skip, (K)) \ + void *base; \ + size_t len; \ + iterate_xarray(i, n, base, len, off, \ + skip, (K)) \ } \ i->count -= n; \ i->iov_offset = skip; \ } \ } -#define iterate_and_advance(i, n, v, off, I, K) \ - __iterate_and_advance(i, n, v, off, I, ((void)(K),0)) +#define iterate_and_advance(i, n, base, len, off, I, K) \ + __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0)) static int copyout(void __user *to, const void *from, size_t n) { @@ -613,9 +621,9 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) return copy_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, off, - copyout(v.iov_base, addr + off, v.iov_len), - memcpy(v.iov_base, addr + off, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + copyout(base, addr + off, len), + memcpy(base, addr + off, len) ) return bytes; @@ -708,9 +716,9 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) return copy_mc_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - __iterate_and_advance(i, bytes, v, off, - copyout_mc(v.iov_base, addr + off, v.iov_len), - copy_mc_to_kernel(v.iov_base, addr + off, v.iov_len) + __iterate_and_advance(i, bytes, base, len, off, + copyout_mc(base, addr + off, len), + copy_mc_to_kernel(base, addr + off, len) ) return bytes; @@ -726,9 +734,9 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) } if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, off, - copyin(addr + off, v.iov_base, v.iov_len), - memcpy(addr + off, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + copyin(addr + off, base, len), + memcpy(addr + off, base, len) ) return bytes; @@ -741,10 +749,9 @@ size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, off, - __copy_from_user_inatomic_nocache(addr + off, - v.iov_base, v.iov_len), - memcpy(addr + off, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + __copy_from_user_inatomic_nocache(addr + off, base, len), + memcpy(addr + off, base, len) ) return bytes; @@ -772,9 +779,9 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, off, - __copy_from_user_flushcache(addr + off, v.iov_base, v.iov_len), - memcpy_flushcache(addr + off, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + __copy_from_user_flushcache(addr + off, base, len), + memcpy_flushcache(addr + off, base, len) ) return bytes; @@ -903,9 +910,9 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) { if (unlikely(iov_iter_is_pipe(i))) return pipe_zero(bytes, i); - iterate_and_advance(i, bytes, v, count, - clear_user(v.iov_base, v.iov_len), - memset(v.iov_base, 0, v.iov_len) + iterate_and_advance(i, bytes, base, len, count, + clear_user(base, len), + memset(base, 0, len) ) return bytes; @@ -925,9 +932,9 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, off, - copyin(p + off, v.iov_base, v.iov_len), - memcpy(p + off, v.iov_base, v.iov_len) + iterate_and_advance(i, bytes, base, len, off, + copyin(p + off, base, len), + memcpy(p + off, base, len) ) kunmap_atomic(kaddr); return bytes; @@ -1654,16 +1661,13 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, off, ({ - next = csum_and_copy_from_user(v.iov_base, - addr + off, - v.iov_len); + iterate_and_advance(i, bytes, base, len, off, ({ + next = csum_and_copy_from_user(base, addr + off, len); if (next) sum = csum_block_add(sum, next, off); - next ? 0 : v.iov_len; + next ? 0 : len; }), ({ - sum = csum_and_memcpy(addr + off, v.iov_base, v.iov_len, - sum, off); + sum = csum_and_memcpy(addr + off, base, len, sum, off); }) ) *csum = sum; @@ -1685,17 +1689,13 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, WARN_ON(1); /* for now */ return 0; } - iterate_and_advance(i, bytes, v, off, ({ - next = csum_and_copy_to_user(addr + off, - v.iov_base, - v.iov_len); + iterate_and_advance(i, bytes, base, len, off, ({ + next = csum_and_copy_to_user(addr + off, base, len); if (next) sum = csum_block_add(sum, next, off); - next ? 0 : v.iov_len; + next ? 0 : len; }), ({ - sum = csum_and_memcpy(v.iov_base, - addr + off, - v.iov_len, sum, off); + sum = csum_and_memcpy(base, addr + off, len, sum, off); }) ) csstate->csum = csum_shift(sum, csstate->off); -- cgit From a6e4ec7bfd32f42ff37577c6b708153d19880b6e Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 13:03:41 -0400 Subject: pull handling of ->iov_offset into iterate_{iovec,bvec,xarray} fewer arguments (by one, but still...) for iterate_...() macros Signed-off-by: Al Viro --- lib/iov_iter.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index f4ea04e24e06..48a55de2a172 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -17,8 +17,9 @@ #define PIPE_PARANOIA /* for now */ /* covers iovec and kvec alike */ -#define iterate_iovec(i, n, base, len, off, __p, skip, STEP) { \ +#define iterate_iovec(i, n, base, len, off, __p, STEP) { \ size_t off = 0; \ + size_t skip = i->iov_offset; \ do { \ len = min(n, __p->iov_len - skip); \ if (likely(len)) { \ @@ -33,18 +34,20 @@ __p++; \ skip = 0; \ } while (n); \ + i->iov_offset = skip; \ n = off; \ } -#define iterate_bvec(i, n, base, len, off, p, skip, STEP) { \ +#define iterate_bvec(i, n, base, len, off, p, STEP) { \ size_t off = 0; \ + unsigned skip = i->iov_offset; \ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ void *kaddr = kmap_local_page(p->bv_page + \ offset / PAGE_SIZE); \ base = kaddr + offset % PAGE_SIZE; \ - len = min(min(n, p->bv_len - skip), \ + len = min(min(n, (size_t)(p->bv_len - skip)), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ left = (STEP); \ kunmap_local(kaddr); \ @@ -59,15 +62,16 @@ if (left) \ break; \ } \ + i->iov_offset = skip; \ n = off; \ } -#define iterate_xarray(i, n, base, len, __off, skip, STEP) { \ +#define iterate_xarray(i, n, base, len, __off, STEP) { \ __label__ __out; \ size_t __off = 0; \ struct page *head = NULL; \ size_t offset; \ - loff_t start = i->xarray_start + skip; \ + loff_t start = i->xarray_start + i->iov_offset; \ pgoff_t index = start >> PAGE_SHIFT; \ int j; \ \ @@ -100,7 +104,7 @@ } \ __out: \ rcu_read_unlock(); \ - skip += __off; \ + i->iov_offset += __off; \ n = __off; \ } @@ -108,13 +112,12 @@ __out: \ if (unlikely(i->count < n)) \ n = i->count; \ if (likely(n)) { \ - size_t skip = i->iov_offset; \ if (likely(iter_is_iovec(i))) { \ const struct iovec *iov = i->iov; \ void __user *base; \ size_t len; \ iterate_iovec(i, n, base, len, off, \ - iov, skip, (I)) \ + iov, (I)) \ i->nr_segs -= iov - i->iov; \ i->iov = iov; \ } else if (iov_iter_is_bvec(i)) { \ @@ -122,7 +125,7 @@ __out: \ void *base; \ size_t len; \ iterate_bvec(i, n, base, len, off, \ - bvec, skip, (K)) \ + bvec, (K)) \ i->nr_segs -= bvec - i->bvec; \ i->bvec = bvec; \ } else if (iov_iter_is_kvec(i)) { \ @@ -130,17 +133,16 @@ __out: \ void *base; \ size_t len; \ iterate_iovec(i, n, base, len, off, \ - kvec, skip, (K)) \ + kvec, (K)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ } else if (iov_iter_is_xarray(i)) { \ void *base; \ size_t len; \ iterate_xarray(i, n, base, len, off, \ - skip, (K)) \ + (K)) \ } \ i->count -= n; \ - i->iov_offset = skip; \ } \ } #define iterate_and_advance(i, n, base, len, off, I, K) \ -- cgit From 4b179e9a9c7c98550747b76405626dd59968f078 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 4 May 2021 17:50:07 -0400 Subject: iterate_xarray(): only of the first iteration we might get offset != 0 recalculating offset on each iteration is pointless - on all subsequent passes through the loop it will be zero anyway. Signed-off-by: Al Viro --- lib/iov_iter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 48a55de2a172..d5f750cc6f4a 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -70,9 +70,9 @@ __label__ __out; \ size_t __off = 0; \ struct page *head = NULL; \ - size_t offset; \ loff_t start = i->xarray_start + i->iov_offset; \ - pgoff_t index = start >> PAGE_SHIFT; \ + unsigned offset = start % PAGE_SIZE; \ + pgoff_t index = start / PAGE_SIZE; \ int j; \ \ XA_STATE(xas, i->xarray, index); \ @@ -89,7 +89,6 @@ for (j = (head->index < index) ? index - head->index : 0; \ j < thp_nr_pages(head); j++) { \ void *kaddr = kmap_local_page(head + j); \ - offset = (start + __off) % PAGE_SIZE; \ base = kaddr + offset; \ len = PAGE_SIZE - offset; \ len = min(n, len); \ @@ -100,6 +99,7 @@ n -= len; \ if (left || n == 0) \ goto __out; \ + offset = 0; \ } \ } \ __out: \ -- cgit From c1d4d6a9ae88b87262fb5426823930bc471f6034 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Apr 2021 12:29:53 -0400 Subject: copy_page_to_iter(): don't bother with kmap_atomic() for bvec/kvec cases kmap_local_page() is enough there. Moreover, we can use _copy_to_iter() for actual copying in those cases - no useful extra checks on the address we are copying from in that call. Signed-off-by: Al Viro --- lib/iov_iter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index d5f750cc6f4a..8aff4eb4fdfd 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -821,9 +821,9 @@ static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes if (likely(iter_is_iovec(i))) return copy_page_to_iter_iovec(page, offset, bytes, i); if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { - void *kaddr = kmap_atomic(page); - size_t wanted = copy_to_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); + void *kaddr = kmap_local_page(page); + size_t wanted = _copy_to_iter(kaddr + offset, bytes, i); + kunmap_local(kaddr); return wanted; } if (iov_iter_is_pipe(i)) -- cgit From 55ca375c5dcc7aebd89de42f00ff18f5c40d25f3 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Apr 2021 12:33:24 -0400 Subject: copy_page_from_iter(): don't need kmap_atomic() for kvec/bvec cases kmap_local_page() is enough. Signed-off-by: Al Viro --- lib/iov_iter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 8aff4eb4fdfd..ba7eb6557750 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -871,9 +871,9 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, if (likely(iter_is_iovec(i))) return copy_page_from_iter_iovec(page, offset, bytes, i); if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { - void *kaddr = kmap_atomic(page); + void *kaddr = kmap_local_page(page); size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); + kunmap_local(kaddr); return wanted; } WARN_ON(1); -- cgit From 2495bdcc86dc5e6b71b6785e1faa76452496c687 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 30 Apr 2021 13:40:48 -0400 Subject: iov_iter: clean csum_and_copy_...() primitives up a bit 1) kmap_atomic() is not needed here, kmap_local_page() is enough. 2) No need to make sum = csum_block_add(sum, next, off); conditional upon next != 0 - adding 0 is a no-op as far as csum_block_add() is concerned. Signed-off-by: Al Viro --- lib/iov_iter.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index ba7eb6557750..3b442d25a966 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -600,9 +600,9 @@ static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, return 0; do { size_t chunk = min_t(size_t, n, PAGE_SIZE - r); - char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); sum = csum_and_memcpy(p + r, addr, chunk, sum, off); - kunmap_atomic(p); + kunmap_local(p); i->head = i_head; i->iov_offset = r + chunk; n -= chunk; @@ -1665,8 +1665,7 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, } iterate_and_advance(i, bytes, base, len, off, ({ next = csum_and_copy_from_user(base, addr + off, len); - if (next) - sum = csum_block_add(sum, next, off); + sum = csum_block_add(sum, next, off); next ? 0 : len; }), ({ sum = csum_and_memcpy(addr + off, base, len, sum, off); @@ -1693,8 +1692,7 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, } iterate_and_advance(i, bytes, base, len, off, ({ next = csum_and_copy_to_user(addr + off, base, len); - if (next) - sum = csum_block_add(sum, next, off); + sum = csum_block_add(sum, next, off); next ? 0 : len; }), ({ sum = csum_and_memcpy(base, addr + off, len, sum, off); -- cgit From 893839fd57330ce226d4ee1b16fd5221a27fb6ec Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 30 Apr 2021 18:39:25 -0400 Subject: pipe_zero(): we don't need no stinkin' kmap_atomic()... FWIW, memcpy_to_page() itself almost certainly ought to use kmap_local_page()... Signed-off-by: Al Viro --- lib/iov_iter.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 3b442d25a966..a827991f2644 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -897,7 +897,9 @@ static size_t pipe_zero(size_t bytes, struct iov_iter *i) do { size_t chunk = min_t(size_t, n, PAGE_SIZE - off); - memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); + memset(p + off, 0, chunk); + kunmap_local(p); i->head = i_head; i->iov_offset = off + chunk; n -= chunk; -- cgit From 2a510a744bebc7f5d9e71ee094b62e28b5b43218 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 17:16:34 -0400 Subject: clean up copy_mc_pipe_to_iter() ... and we don't need kmap_atomic() there - kmap_local_page() is fine. Signed-off-by: Al Viro --- lib/iov_iter.c | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index a827991f2644..9ce83db26571 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -642,19 +642,6 @@ static int copyout_mc(void __user *to, const void *from, size_t n) return n; } -static unsigned long copy_mc_to_page(struct page *page, size_t offset, - const char *from, size_t len) -{ - unsigned long ret; - char *to; - - to = kmap_atomic(page); - ret = copy_mc_to_kernel(to + offset, from, len); - kunmap_atomic(to); - - return ret; -} - static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { @@ -666,25 +653,23 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, if (!sanity(i)) return 0; - bytes = n = push_pipe(i, bytes, &i_head, &off); - if (unlikely(!n)) - return 0; - do { + n = push_pipe(i, bytes, &i_head, &off); + while (n) { size_t chunk = min_t(size_t, n, PAGE_SIZE - off); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); unsigned long rem; - - rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page, - off, addr, chunk); + rem = copy_mc_to_kernel(p + off, addr + xfer, chunk); + chunk -= rem; + kunmap_local(p); i->head = i_head; - i->iov_offset = off + chunk - rem; - xfer += chunk - rem; + i->iov_offset = off + chunk; + xfer += chunk; if (rem) break; n -= chunk; - addr += chunk; off = 0; i_head++; - } while (n); + } i->count -= xfer; return xfer; } -- cgit From 6852df1266995c35b8621a95dcb7f91ca11ea409 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 2 May 2021 17:24:40 -0400 Subject: csum_and_copy_to_pipe_iter(): leave handling of csum_state to caller ... since all the logics is already there for use by iovec/kvec/etc. cases. Signed-off-by: Al Viro --- lib/iov_iter.c | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 9ce83db26571..97e04c5dbeef 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -582,39 +582,34 @@ static __wsum csum_and_memcpy(void *to, const void *from, size_t len, } static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, - struct csum_state *csstate, - struct iov_iter *i) + struct iov_iter *i, __wsum *sump) { struct pipe_inode_info *pipe = i->pipe; unsigned int p_mask = pipe->ring_size - 1; - __wsum sum = csstate->csum; - size_t off = csstate->off; + __wsum sum = *sump; + size_t off = 0; unsigned int i_head; - size_t n, r; + size_t r; if (!sanity(i)) return 0; - bytes = n = push_pipe(i, bytes, &i_head, &r); - if (unlikely(!n)) - return 0; - do { - size_t chunk = min_t(size_t, n, PAGE_SIZE - r); + bytes = push_pipe(i, bytes, &i_head, &r); + while (bytes) { + size_t chunk = min_t(size_t, bytes, PAGE_SIZE - r); char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); - sum = csum_and_memcpy(p + r, addr, chunk, sum, off); + sum = csum_and_memcpy(p + r, addr + off, chunk, sum, off); kunmap_local(p); i->head = i_head; i->iov_offset = r + chunk; - n -= chunk; + bytes -= chunk; off += chunk; - addr += chunk; r = 0; i_head++; - } while (n); - i->count -= bytes; - csstate->csum = sum; - csstate->off = off; - return bytes; + } + *sump = sum; + i->count -= off; + return off; } size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) @@ -1669,15 +1664,15 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, struct csum_state *csstate = _csstate; __wsum sum, next; - if (unlikely(iov_iter_is_pipe(i))) - return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i); - - sum = csum_shift(csstate->csum, csstate->off); if (unlikely(iov_iter_is_discard(i))) { WARN_ON(1); /* for now */ return 0; } - iterate_and_advance(i, bytes, base, len, off, ({ + + sum = csum_shift(csstate->csum, csstate->off); + if (unlikely(iov_iter_is_pipe(i))) + bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum); + else iterate_and_advance(i, bytes, base, len, off, ({ next = csum_and_copy_to_user(addr + off, base, len); sum = csum_block_add(sum, next, off); next ? 0 : len; -- cgit From ca24306d83a125df187ad53eddb038fe0cffb8ca Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 2 Jun 2021 17:18:58 +0900 Subject: bootconfig: Change array value to use child node It is not possible to put an array value with subkeys under a key node, because both of subkeys and the array elements are using "next" field of the xbc_node. Thus this changes the array values to use "child" field in the array case. The reason why split this change is to test it easily. Link: https://lkml.kernel.org/r/162262193838.264090.16044473274501498656.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- lib/bootconfig.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 9f8c70a98fcf..44dcdcbd746a 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -367,6 +367,14 @@ static inline __init struct xbc_node *xbc_last_sibling(struct xbc_node *node) return node; } +static inline __init struct xbc_node *xbc_last_child(struct xbc_node *node) +{ + while (node->child) + node = xbc_node_get_child(node); + + return node; +} + static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) { struct xbc_node *sib, *node = xbc_add_node(data, flag); @@ -517,17 +525,20 @@ static int __init xbc_parse_array(char **__v) char *next; int c = 0; + if (last_parent->child) + last_parent = xbc_node_get_child(last_parent); + do { c = __xbc_parse_value(__v, &next); if (c < 0) return c; - node = xbc_add_sibling(*__v, XBC_VALUE); + node = xbc_add_child(*__v, XBC_VALUE); if (!node) return -ENOMEM; *__v = next; } while (c == ','); - node->next = 0; + node->child = 0; return c; } @@ -615,8 +626,12 @@ static int __init xbc_parse_kv(char **k, char *v, int op) if (op == ':' && child) { xbc_init_node(child, v, XBC_VALUE); - } else if (!xbc_add_sibling(v, XBC_VALUE)) - return -ENOMEM; + } else { + if (op == '+' && child) + last_parent = xbc_last_child(child); + if (!xbc_add_sibling(v, XBC_VALUE)) + return -ENOMEM; + } if (c == ',') { /* Array */ c = xbc_parse_array(&next); -- cgit From e5efaeb8a8f527d6e91289ff1f67fbcae452b2ca Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 2 Jun 2021 17:19:07 +0900 Subject: bootconfig: Support mixing a value and subkeys under a key Support mixing a value and subkeys under a key. Since kernel cmdline options will support "aaa.bbb=value1 aaa.bbb.ccc=value2", it is better that the bootconfig supports such configuration too. Note that this does not change syntax itself but just accepts mixed value and subkeys e.g. key = value1 key.subkey = value2 But this is not accepted; key { value1 subkey = value2 } That will make value1 as a subkey. Also, the order of the value node under a key is fixed. If there are a value and subkeys, the value is always the first child node of the key. Thus if user specifies subkeys first, e.g. key.subkey = value1 key = value2 In the program (and /proc/bootconfig), it will be shown as below key = value2 key.subkey = value1 Link: https://lkml.kernel.org/r/162262194685.264090.7738574774030567419.stgit@devnote2 Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- lib/bootconfig.c | 65 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 44dcdcbd746a..927017431fb6 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -156,7 +156,7 @@ xbc_node_find_child(struct xbc_node *parent, const char *key) struct xbc_node *node; if (parent) - node = xbc_node_get_child(parent); + node = xbc_node_get_subkey(parent); else node = xbc_root_node(); @@ -164,7 +164,7 @@ xbc_node_find_child(struct xbc_node *parent, const char *key) if (!xbc_node_match_prefix(node, &key)) node = xbc_node_get_next(node); else if (*key != '\0') - node = xbc_node_get_child(node); + node = xbc_node_get_subkey(node); else break; } @@ -274,6 +274,8 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, struct xbc_node *node) { + struct xbc_node *next; + if (unlikely(!xbc_data)) return NULL; @@ -282,6 +284,13 @@ struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, if (!node) node = xbc_nodes; } else { + /* Leaf node may have a subkey */ + next = xbc_node_get_subkey(node); + if (next) { + node = next; + goto found; + } + if (node == root) /* @root was a leaf, no child node. */ return NULL; @@ -296,6 +305,7 @@ struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, node = xbc_node_get_next(node); } +found: while (node && !xbc_node_is_leaf(node)) node = xbc_node_get_child(node); @@ -375,18 +385,20 @@ static inline __init struct xbc_node *xbc_last_child(struct xbc_node *node) return node; } -static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) +static struct xbc_node * __init __xbc_add_sibling(char *data, u32 flag, bool head) { struct xbc_node *sib, *node = xbc_add_node(data, flag); if (node) { if (!last_parent) { + /* Ignore @head in this case */ node->parent = XBC_NODE_MAX; sib = xbc_last_sibling(xbc_nodes); sib->next = xbc_node_index(node); } else { node->parent = xbc_node_index(last_parent); - if (!last_parent->child) { + if (!last_parent->child || head) { + node->next = last_parent->child; last_parent->child = xbc_node_index(node); } else { sib = xbc_node_get_child(last_parent); @@ -400,6 +412,16 @@ static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) return node; } +static inline struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) +{ + return __xbc_add_sibling(data, flag, false); +} + +static inline struct xbc_node * __init xbc_add_head_sibling(char *data, u32 flag) +{ + return __xbc_add_sibling(data, flag, true); +} + static inline __init struct xbc_node *xbc_add_child(char *data, u32 flag) { struct xbc_node *node = xbc_add_sibling(data, flag); @@ -568,8 +590,9 @@ static int __init __xbc_add_key(char *k) node = find_match_node(xbc_nodes, k); else { child = xbc_node_get_child(last_parent); + /* Since the value node is the first child, skip it. */ if (child && xbc_node_is_value(child)) - return xbc_parse_error("Subkey is mixed with value", k); + child = xbc_node_get_next(child); node = find_match_node(child, k); } @@ -612,27 +635,29 @@ static int __init xbc_parse_kv(char **k, char *v, int op) if (ret) return ret; - child = xbc_node_get_child(last_parent); - if (child) { - if (xbc_node_is_key(child)) - return xbc_parse_error("Value is mixed with subkey", v); - else if (op == '=') - return xbc_parse_error("Value is redefined", v); - } - c = __xbc_parse_value(&v, &next); if (c < 0) return c; - if (op == ':' && child) { - xbc_init_node(child, v, XBC_VALUE); - } else { - if (op == '+' && child) - last_parent = xbc_last_child(child); - if (!xbc_add_sibling(v, XBC_VALUE)) - return -ENOMEM; + child = xbc_node_get_child(last_parent); + if (child && xbc_node_is_value(child)) { + if (op == '=') + return xbc_parse_error("Value is redefined", v); + if (op == ':') { + unsigned short nidx = child->next; + + xbc_init_node(child, v, XBC_VALUE); + child->next = nidx; /* keep subkeys */ + goto array; + } + /* op must be '+' */ + last_parent = xbc_last_child(child); } + /* The value node should always be the first child */ + if (!xbc_add_head_sibling(v, XBC_VALUE)) + return -ENOMEM; +array: if (c == ',') { /* Array */ c = xbc_parse_array(&next); if (c < 0) -- cgit From 384426bd101cb3cd580b18de19d4891ec5ca5bf9 Mon Sep 17 00:00:00 2001 From: David Gow Date: Thu, 10 Jun 2021 20:57:25 -0700 Subject: kunit: Fix result propagation for parameterised tests When one parameter of a parameterised test failed, its failure would be propagated to the overall test, but not to the suite result (unless it was the last parameter). This is because test_case->success was being reset to the test->success result after each parameter was used, so a failing test's result would be overwritten by a non-failing result. The overall test result was handled in a third variable, test_result, but this was discarded after the status line was printed. Instead, just propagate the result after each parameter run. Signed-off-by: David Gow Fixes: fadb08e7c750 ("kunit: Support for Parameterized Testing") Reviewed-by: Marco Elver Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/test.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 2f6cc0123232..17973a4a44c2 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -376,7 +376,7 @@ static void kunit_run_case_catch_errors(struct kunit_suite *suite, context.test_case = test_case; kunit_try_catch_run(try_catch, &context); - test_case->success = test->success; + test_case->success &= test->success; } int kunit_run_tests(struct kunit_suite *suite) @@ -388,7 +388,7 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_suite_for_each_test_case(suite, test_case) { struct kunit test = { .param_value = NULL, .param_index = 0 }; - bool test_success = true; + test_case->success = true; if (test_case->generate_params) { /* Get initial param. */ @@ -398,7 +398,6 @@ int kunit_run_tests(struct kunit_suite *suite) do { kunit_run_case_catch_errors(suite, test_case, &test); - test_success &= test_case->success; if (test_case->generate_params) { if (param_desc[0] == '\0') { @@ -420,7 +419,7 @@ int kunit_run_tests(struct kunit_suite *suite) } } while (test.param_value); - kunit_print_ok_not_ok(&test, true, test_success, + kunit_print_ok_not_ok(&test, true, test_case->success, kunit_test_case_num(suite, test_case), test_case->name); } -- cgit From b6d5799b0b5866dc63be7f032473dc536f865b4d Mon Sep 17 00:00:00 2001 From: David Gow Date: Wed, 26 May 2021 14:24:04 -0700 Subject: kunit: Add 'kunit_shutdown' option Add a new kernel command-line option, 'kunit_shutdown', which allows the user to specify that the kernel poweroff, halt, or reboot after completing all KUnit tests; this is very handy for running KUnit tests on UML or a VM so that the UML/VM process exits cleanly immediately after running all tests without needing a special initramfs. Signed-off-by: David Gow Signed-off-by: Brendan Higgins Reviewed-by: Stephen Boyd Tested-By: Daniel Latypov Reviewed-by: Daniel Latypov Signed-off-by: Shuah Khan --- lib/kunit/executor.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 15832ed44668..7db619624437 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -18,6 +19,9 @@ module_param(filter_glob, charp, 0); MODULE_PARM_DESC(filter_glob, "Filter which KUnit test suites run at boot-time, e.g. list*"); +static char *kunit_shutdown; +core_param(kunit_shutdown, kunit_shutdown, charp, 0644); + static struct kunit_suite * const * kunit_filter_subsuite(struct kunit_suite * const * const subsuite) { @@ -82,6 +86,20 @@ static struct suite_set kunit_filter_suites(void) return filtered; } +static void kunit_handle_shutdown(void) +{ + if (!kunit_shutdown) + return; + + if (!strcmp(kunit_shutdown, "poweroff")) + kernel_power_off(); + else if (!strcmp(kunit_shutdown, "halt")) + kernel_halt(); + else if (!strcmp(kunit_shutdown, "reboot")) + kernel_restart(NULL); + +} + static void kunit_print_tap_header(struct suite_set *suite_set) { struct kunit_suite * const * const *suites, * const *subsuite; @@ -112,6 +130,8 @@ int kunit_run_all_tests(void) kfree(suite_set.start); } + kunit_handle_shutdown(); + return 0; } -- cgit From ca2e334232b6cd4ae5af9da2df83c009d042aefb Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Fri, 5 Mar 2021 13:19:52 +0100 Subject: lib: add iomem emulation (logic_iomem) Add IO memory emulation that uses callbacks for read/write to the allocated regions. The callbacks can be registered by the users using logic_iomem_alloc(). To use, an architecture must 'select LOGIC_IOMEM' in Kconfig and then include into asm/io.h to get the __raw_read*/__raw_write* functions. Optionally, an architecture may 'select LOGIC_IOMEM_FALLBACK' in which case non-emulated regions will 'fall back' to the various real_* functions that must then be provided. Cc: Arnd Bergmann Signed-off-by: Johannes Berg Signed-off-by: Richard Weinberger --- lib/Kconfig | 14 +++ lib/Makefile | 2 + lib/logic_iomem.c | 318 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 lib/logic_iomem.c (limited to 'lib') diff --git a/lib/Kconfig b/lib/Kconfig index ac3b30697b2b..d241fe476fda 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -102,6 +102,20 @@ config INDIRECT_PIO When in doubt, say N. +config INDIRECT_IOMEM + bool + help + This is selected by other options/architectures to provide the + emulated iomem accessors. + +config INDIRECT_IOMEM_FALLBACK + bool + depends on INDIRECT_IOMEM + help + If INDIRECT_IOMEM is selected, this enables falling back to plain + mmio accesses when the IO memory address is not a registered + emulated region. + config CRC_CCITT tristate "CRC-CCITT functions" help diff --git a/lib/Makefile b/lib/Makefile index 2cc359ec1fdd..fc38a55e6fe4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -147,6 +147,8 @@ obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o lib-y += logic_pio.o +lib-$(CONFIG_INDIRECT_IOMEM) += logic_iomem.o + obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o obj-$(CONFIG_BTREE) += btree.o diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c new file mode 100644 index 000000000000..b76b92dd0f1f --- /dev/null +++ b/lib/logic_iomem.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Intel Corporation + * Author: Johannes Berg + */ +#include +#include +#include + +struct logic_iomem_region { + const struct resource *res; + const struct logic_iomem_region_ops *ops; + struct list_head list; +}; + +struct logic_iomem_area { + const struct logic_iomem_ops *ops; + void *priv; +}; + +#define AREA_SHIFT 24 +#define MAX_AREA_SIZE (1 << AREA_SHIFT) +#define MAX_AREAS ((1ULL<<32) / MAX_AREA_SIZE) +#define AREA_BITS ((MAX_AREAS - 1) << AREA_SHIFT) +#define AREA_MASK (MAX_AREA_SIZE - 1) +#ifdef CONFIG_64BIT +#define IOREMAP_BIAS 0xDEAD000000000000UL +#define IOREMAP_MASK 0xFFFFFFFF00000000UL +#else +#define IOREMAP_BIAS 0 +#define IOREMAP_MASK 0 +#endif + +static DEFINE_MUTEX(regions_mtx); +static LIST_HEAD(regions_list); +static struct logic_iomem_area mapped_areas[MAX_AREAS]; + +int logic_iomem_add_region(struct resource *resource, + const struct logic_iomem_region_ops *ops) +{ + struct logic_iomem_region *rreg; + int err; + + if (WARN_ON(!resource || !ops)) + return -EINVAL; + + if (WARN_ON((resource->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM)) + return -EINVAL; + + rreg = kzalloc(sizeof(*rreg), GFP_KERNEL); + if (!rreg) + return -ENOMEM; + + err = request_resource(&iomem_resource, resource); + if (err) { + kfree(rreg); + return -ENOMEM; + } + + mutex_lock(®ions_mtx); + rreg->res = resource; + rreg->ops = ops; + list_add_tail(&rreg->list, ®ions_list); + mutex_unlock(®ions_mtx); + + return 0; +} +EXPORT_SYMBOL(logic_iomem_add_region); + +#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +static void __iomem *real_ioremap(phys_addr_t offset, size_t size) +{ + WARN(1, "invalid ioremap(0x%llx, 0x%zx)\n", + (unsigned long long)offset, size); + return NULL; +} + +static void real_iounmap(void __iomem *addr) +{ + WARN(1, "invalid iounmap for addr 0x%llx\n", + (unsigned long long)addr); +} +#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ + +void __iomem *ioremap(phys_addr_t offset, size_t size) +{ + void __iomem *ret = NULL; + struct logic_iomem_region *rreg, *found = NULL; + int i; + + mutex_lock(®ions_mtx); + list_for_each_entry(rreg, ®ions_list, list) { + if (rreg->res->start > offset) + continue; + if (rreg->res->end < offset + size - 1) + continue; + found = rreg; + break; + } + + if (!found) + goto out; + + for (i = 0; i < MAX_AREAS; i++) { + long offs; + + if (mapped_areas[i].ops) + continue; + + offs = rreg->ops->map(offset - found->res->start, + size, &mapped_areas[i].ops, + &mapped_areas[i].priv); + if (offs < 0) { + mapped_areas[i].ops = NULL; + break; + } + + if (WARN_ON(!mapped_areas[i].ops)) { + mapped_areas[i].ops = NULL; + break; + } + + ret = (void __iomem *)(IOREMAP_BIAS + (i << AREA_SHIFT) + offs); + break; + } +out: + mutex_unlock(®ions_mtx); + if (ret) + return ret; + return real_ioremap(offset, size); +} +EXPORT_SYMBOL(ioremap); + +static inline struct logic_iomem_area * +get_area(const volatile void __iomem *addr) +{ + unsigned long a = (unsigned long)addr; + unsigned int idx; + + if (WARN_ON((a & IOREMAP_MASK) != IOREMAP_BIAS)) + return NULL; + + idx = (a & AREA_BITS) >> AREA_SHIFT; + + if (mapped_areas[idx].ops) + return &mapped_areas[idx]; + + return NULL; +} + +void iounmap(void __iomem *addr) +{ + struct logic_iomem_area *area = get_area(addr); + + if (!area) { + real_iounmap(addr); + return; + } + + if (area->ops->unmap) + area->ops->unmap(area->priv); + + mutex_lock(®ions_mtx); + area->ops = NULL; + area->priv = NULL; + mutex_unlock(®ions_mtx); +} +EXPORT_SYMBOL(iounmap); + +#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#define MAKE_FALLBACK(op, sz) \ +static u##sz real_raw_read ## op(const volatile void __iomem *addr) \ +{ \ + WARN(1, "Invalid read" #op " at address %llx\n", \ + (unsigned long long)addr); \ + return (u ## sz)~0ULL; \ +} \ + \ +void real_raw_write ## op(u ## sz val, volatile void __iomem *addr) \ +{ \ + WARN(1, "Invalid writeq" #op " of 0x%llx at address %llx\n", \ + (unsigned long long)val, (unsigned long long)addr); \ +} \ + +MAKE_FALLBACK(b, 8); +MAKE_FALLBACK(w, 16); +MAKE_FALLBACK(l, 32); +#ifdef CONFIG_64BIT +MAKE_FALLBACK(q, 64); +#endif + +static void real_memset_io(volatile void __iomem *addr, int value, size_t size) +{ + WARN(1, "Invalid memset_io at address 0x%llx\n", + (unsigned long long)addr); +} + +static void real_memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size) +{ + WARN(1, "Invalid memcpy_fromio at address 0x%llx\n", + (unsigned long long)addr); + + memset(buffer, 0xff, size); +} + +static void real_memcpy_toio(volatile void __iomem *addr, const void *buffer, + size_t size) +{ + WARN(1, "Invalid memcpy_toio at address 0x%llx\n", + (unsigned long long)addr); +} +#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ + +#define MAKE_OP(op, sz) \ +u##sz __raw_read ## op(const volatile void __iomem *addr) \ +{ \ + struct logic_iomem_area *area = get_area(addr); \ + \ + if (!area) \ + return real_raw_read ## op(addr); \ + \ + return (u ## sz) area->ops->read(area->priv, \ + (unsigned long)addr & AREA_MASK,\ + sz / 8); \ +} \ +EXPORT_SYMBOL(__raw_read ## op); \ + \ +void __raw_write ## op(u ## sz val, volatile void __iomem *addr) \ +{ \ + struct logic_iomem_area *area = get_area(addr); \ + \ + if (!area) { \ + real_raw_write ## op(val, addr); \ + return; \ + } \ + \ + area->ops->write(area->priv, \ + (unsigned long)addr & AREA_MASK, \ + sz / 8, val); \ +} \ +EXPORT_SYMBOL(__raw_write ## op) + +MAKE_OP(b, 8); +MAKE_OP(w, 16); +MAKE_OP(l, 32); +#ifdef CONFIG_64BIT +MAKE_OP(q, 64); +#endif + +void memset_io(volatile void __iomem *addr, int value, size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + unsigned long offs, start; + + if (!area) { + real_memset_io(addr, value, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->set) { + area->ops->set(area->priv, start, value, size); + return; + } + + for (offs = 0; offs < size; offs++) + area->ops->write(area->priv, start + offs, 1, value); +} +EXPORT_SYMBOL(memset_io); + +void memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + u8 *buf = buffer; + unsigned long offs, start; + + if (!area) { + real_memcpy_fromio(buffer, addr, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->copy_from) { + area->ops->copy_from(area->priv, buffer, start, size); + return; + } + + for (offs = 0; offs < size; offs++) + buf[offs] = area->ops->read(area->priv, start + offs, 1); +} +EXPORT_SYMBOL(memcpy_fromio); + +void memcpy_toio(volatile void __iomem *addr, const void *buffer, size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + const u8 *buf = buffer; + unsigned long offs, start; + + if (!area) { + real_memcpy_toio(addr, buffer, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->copy_to) { + area->ops->copy_to(area->priv, start, buffer, size); + return; + } + + for (offs = 0; offs < size; offs++) + area->ops->write(area->priv, start + offs, 1, buf[offs]); +} +EXPORT_SYMBOL(memcpy_toio); -- cgit From 1253b9b87e42ab6a3d5c2cb27af2bdd67d7e50ff Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 27 May 2021 12:01:23 -0700 Subject: clocksource: Provide kernel module to test clocksource watchdog When the clocksource watchdog marks a clock as unstable, this might be due to that clock being unstable or it might be due to delays that happen to occur between the reads of the two clocks. It would be good to have a way of testing the clocksource watchdog's ability to distinguish between these two causes of clock skew and instability. Therefore, provide a new clocksource-wdtest module selected by a new TEST_CLOCKSOURCE_WATCHDOG Kconfig option. This module has a single module parameter named "holdoff" that provides the number of seconds of delay before testing should start, which defaults to zero when built as a module and to 10 seconds when built directly into the kernel. Very large systems that boot slowly may need to increase the value of this module parameter. This module uses hand-crafted clocksource structures to do its testing, thus avoiding messing up timing for the rest of the kernel and for user applications. This module first verifies that the ->uncertainty_margin field of the clocksource structures are set sanely. It then tests the delay-detection capability of the clocksource watchdog, increasing the number of consecutive delays injected, first provoking console messages complaining about the delays and finally forcing a clock-skew event. Unexpected test results cause at least one WARN_ON_ONCE() console splat. If there are no splats, the test has passed. Finally, it fuzzes the value returned from a clocksource to test the clocksource watchdog's ability to detect time skew. This module checks the state of its clocksource after each test, and uses WARN_ON_ONCE() to emit a console splat if there are any failures. This should enable all types of test frameworks to detect any such failures. This facility is intended for diagnostic use only, and should be avoided on production systems. Reported-by: Chris Mason Suggested-by: Thomas Gleixner Signed-off-by: Paul E. McKenney Signed-off-by: Thomas Gleixner Tested-by: Feng Tang Link: https://lore.kernel.org/r/20210527190124.440372-5-paulmck@kernel.org --- lib/Kconfig.debug | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 678c13967580..0a5a70c742e6 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2571,6 +2571,18 @@ config TEST_FPU If unsure, say N. +config TEST_CLOCKSOURCE_WATCHDOG + tristate "Test clocksource watchdog in kernel space" + depends on CLOCKSOURCE_WATCHDOG + help + Enable this option to create a kernel module that will trigger + a test of the clocksource watchdog. This module may be loaded + via modprobe or insmod in which case it will run upon being + loaded, or it may be built in, in which case it will run + shortly after boot. + + If unsure, say N. + endif # RUNTIME_TESTING_MENU config ARCH_USE_MEMTEST -- cgit From 255ede3b129041eae4edfdce121cedbfabfdd30e Mon Sep 17 00:00:00 2001 From: David Gow Date: Thu, 13 May 2021 12:32:04 -0700 Subject: lib/cmdline_kunit: Remove a cast which are no-longer required With some of the stricter type checking in KUnit's EXPECT macros removed, a cast in cmdline_kunit is no longer required. Remove the unnecessary cast, using NULL instead of (int *) to make it clearer. Signed-off-by: David Gow Acked-by: Andy Shevchenko Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/cmdline_kunit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/cmdline_kunit.c b/lib/cmdline_kunit.c index 018bfc8113c4..a72a2c16066e 100644 --- a/lib/cmdline_kunit.c +++ b/lib/cmdline_kunit.c @@ -124,7 +124,7 @@ static void cmdline_do_one_range_test(struct kunit *test, const char *in, n, e[0], r[0]); p = memchr_inv(&r[1], 0, sizeof(r) - sizeof(r[0])); - KUNIT_EXPECT_PTR_EQ_MSG(test, p, (int *)0, "in test %u at %u out of bound", n, p - r); + KUNIT_EXPECT_PTR_EQ_MSG(test, p, NULL, "in test %u at %u out of bound", n, p - r); } static void cmdline_test_range(struct kunit *test) -- cgit From 44acdbb250a57240ec113f12bd6229854681ea5f Mon Sep 17 00:00:00 2001 From: David Gow Date: Thu, 13 May 2021 13:03:50 -0700 Subject: kunit: Add gnu_printf specifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some KUnit functions use variable arguments to implement a printf-like format string. Use the __printf() attribute to let the compiler warn if invalid format strings are passed in. If the kernel is build with W=1, it complained about the lack of these specifiers, e.g.: ../lib/kunit/test.c:72:2: warning: function ‘kunit_log_append’ might be a candidate for ‘gnu_printf’ format attribute [-Wsuggest-attribute=format] Signed-off-by: David Gow Reviewed-by: Daniel Latypov Acked-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/string-stream.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h index fe98a00b75a9..5e94b623454f 100644 --- a/lib/kunit/string-stream.h +++ b/lib/kunit/string-stream.h @@ -35,9 +35,9 @@ struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp); int __printf(2, 3) string_stream_add(struct string_stream *stream, const char *fmt, ...); -int string_stream_vadd(struct string_stream *stream, - const char *fmt, - va_list args); +int __printf(2, 0) string_stream_vadd(struct string_stream *stream, + const char *fmt, + va_list args); char *string_stream_get_string(struct string_stream *stream); -- cgit From 7122debb4367ee5c89237e5d36dcc0007d7ec43c Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Mon, 3 May 2021 13:58:34 -0700 Subject: kunit: introduce kunit_kmalloc_array/kunit_kcalloc() helpers Add in: * kunit_kmalloc_array() and wire up kunit_kmalloc() to be a special case of it. * kunit_kcalloc() for symmetry with kunit_kzalloc() This should using KUnit more natural by making it more similar to the existing *alloc() APIs. And while we shouldn't necessarily be writing unit tests where overflow should be a concern, it can't hurt to be safe. Signed-off-by: Daniel Latypov Reviewed-by: David Gow Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/test.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 17973a4a44c2..06f6cff2c0e7 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -572,41 +572,43 @@ int kunit_destroy_resource(struct kunit *test, kunit_resource_match_t match, } EXPORT_SYMBOL_GPL(kunit_destroy_resource); -struct kunit_kmalloc_params { +struct kunit_kmalloc_array_params { + size_t n; size_t size; gfp_t gfp; }; -static int kunit_kmalloc_init(struct kunit_resource *res, void *context) +static int kunit_kmalloc_array_init(struct kunit_resource *res, void *context) { - struct kunit_kmalloc_params *params = context; + struct kunit_kmalloc_array_params *params = context; - res->data = kmalloc(params->size, params->gfp); + res->data = kmalloc_array(params->n, params->size, params->gfp); if (!res->data) return -ENOMEM; return 0; } -static void kunit_kmalloc_free(struct kunit_resource *res) +static void kunit_kmalloc_array_free(struct kunit_resource *res) { kfree(res->data); } -void *kunit_kmalloc(struct kunit *test, size_t size, gfp_t gfp) +void *kunit_kmalloc_array(struct kunit *test, size_t n, size_t size, gfp_t gfp) { - struct kunit_kmalloc_params params = { + struct kunit_kmalloc_array_params params = { .size = size, + .n = n, .gfp = gfp }; return kunit_alloc_resource(test, - kunit_kmalloc_init, - kunit_kmalloc_free, + kunit_kmalloc_array_init, + kunit_kmalloc_array_free, gfp, ¶ms); } -EXPORT_SYMBOL_GPL(kunit_kmalloc); +EXPORT_SYMBOL_GPL(kunit_kmalloc_array); void kunit_kfree(struct kunit *test, const void *ptr) { -- cgit From ebd09577be6c15ee2d343cf60e5bb819946a5ee8 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Mon, 3 May 2021 13:58:35 -0700 Subject: lib/test: convert lib/test_list_sort.c to use KUnit Functionally, this just means that the test output will be slightly changed and it'll now depend on CONFIG_KUNIT=y/m. It'll still run at boot time and can still be built as a loadable module. There was a pre-existing patch to convert this test that I found later, here [1]. Compared to [1], this patch doesn't rename files and uses KUnit features more heavily (i.e. does more than converting pr_err() calls to KUNIT_FAIL()). What this conversion gives us: * a shorter test thanks to KUnit's macros * a way to run this a bit more easily via kunit.py (and CONFIG_KUNIT_ALL_TESTS=y) [2] * a structured way of reporting pass/fail * uses kunit-managed allocations to avoid the risk of memory leaks * more descriptive error messages: * i.e. it prints out which fields are invalid, what the expected values are, etc. What this conversion does not do: * change the name of the file (and thus the name of the module) * change the name of the config option Leaving these as-is for now to minimize the impact to people wanting to run this test. IMO, that concern trumps following KUnit's style guide for both names, at least for now. [1] https://lore.kernel.org/linux-kselftest/20201015014616.309000-1-vitor@massaru.org/ [2] Can be run via $ ./tools/testing/kunit/kunit.py run --kunitconfig /dev/stdin < Tested-by: David Gow Acked-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/Kconfig.debug | 5 +- lib/test_list_sort.c | 129 ++++++++++++++++++++------------------------------- 2 files changed, 54 insertions(+), 80 deletions(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 678c13967580..5553508080db 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2047,8 +2047,9 @@ config LKDTM Documentation/fault-injection/provoke-crashes.rst config TEST_LIST_SORT - tristate "Linked list sorting test" - depends on DEBUG_KERNEL || m + tristate "Linked list sorting test" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS help Enable this to turn on 'list_sort()' function test. This test is executed only once during system boot (so affects only boot time), diff --git a/lib/test_list_sort.c b/lib/test_list_sort.c index 00daaf23316f..ade7a1ea0c8e 100644 --- a/lib/test_list_sort.c +++ b/lib/test_list_sort.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#define pr_fmt(fmt) "list_sort_test: " fmt +#include #include #include @@ -23,68 +23,52 @@ struct debug_el { struct list_head list; unsigned int poison2; int value; - unsigned serial; + unsigned int serial; }; -/* Array, containing pointers to all elements in the test list */ -static struct debug_el **elts __initdata; - -static int __init check(struct debug_el *ela, struct debug_el *elb) +static void check(struct kunit *test, struct debug_el *ela, struct debug_el *elb) { - if (ela->serial >= TEST_LIST_LEN) { - pr_err("error: incorrect serial %d\n", ela->serial); - return -EINVAL; - } - if (elb->serial >= TEST_LIST_LEN) { - pr_err("error: incorrect serial %d\n", elb->serial); - return -EINVAL; - } - if (elts[ela->serial] != ela || elts[elb->serial] != elb) { - pr_err("error: phantom element\n"); - return -EINVAL; - } - if (ela->poison1 != TEST_POISON1 || ela->poison2 != TEST_POISON2) { - pr_err("error: bad poison: %#x/%#x\n", - ela->poison1, ela->poison2); - return -EINVAL; - } - if (elb->poison1 != TEST_POISON1 || elb->poison2 != TEST_POISON2) { - pr_err("error: bad poison: %#x/%#x\n", - elb->poison1, elb->poison2); - return -EINVAL; - } - return 0; + struct debug_el **elts = test->priv; + + KUNIT_EXPECT_LT_MSG(test, ela->serial, (unsigned int)TEST_LIST_LEN, "incorrect serial"); + KUNIT_EXPECT_LT_MSG(test, elb->serial, (unsigned int)TEST_LIST_LEN, "incorrect serial"); + + KUNIT_EXPECT_PTR_EQ_MSG(test, elts[ela->serial], ela, "phantom element"); + KUNIT_EXPECT_PTR_EQ_MSG(test, elts[elb->serial], elb, "phantom element"); + + KUNIT_EXPECT_EQ_MSG(test, ela->poison1, TEST_POISON1, "bad poison"); + KUNIT_EXPECT_EQ_MSG(test, ela->poison2, TEST_POISON2, "bad poison"); + + KUNIT_EXPECT_EQ_MSG(test, elb->poison1, TEST_POISON1, "bad poison"); + KUNIT_EXPECT_EQ_MSG(test, elb->poison2, TEST_POISON2, "bad poison"); } -static int __init cmp(void *priv, const struct list_head *a, - const struct list_head *b) +/* `priv` is the test pointer so check() can fail the test if the list is invalid. */ +static int cmp(void *priv, const struct list_head *a, const struct list_head *b) { struct debug_el *ela, *elb; ela = container_of(a, struct debug_el, list); elb = container_of(b, struct debug_el, list); - check(ela, elb); + check(priv, ela, elb); return ela->value - elb->value; } -static int __init list_sort_test(void) +static void list_sort_test(struct kunit *test) { - int i, count = 1, err = -ENOMEM; - struct debug_el *el; + int i, count = 1; + struct debug_el *el, **elts; struct list_head *cur; LIST_HEAD(head); - pr_debug("start testing list_sort()\n"); - - elts = kcalloc(TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); - if (!elts) - return err; + elts = kunit_kcalloc(test, TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, elts); + test->priv = elts; for (i = 0; i < TEST_LIST_LEN; i++) { - el = kmalloc(sizeof(*el), GFP_KERNEL); - if (!el) - goto exit; + el = kunit_kmalloc(test, sizeof(*el), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, el); /* force some equivalencies */ el->value = prandom_u32() % (TEST_LIST_LEN / 3); @@ -95,55 +79,44 @@ static int __init list_sort_test(void) list_add_tail(&el->list, &head); } - list_sort(NULL, &head, cmp); + list_sort(test, &head, cmp); - err = -EINVAL; for (cur = head.next; cur->next != &head; cur = cur->next) { struct debug_el *el1; int cmp_result; - if (cur->next->prev != cur) { - pr_err("error: list is corrupted\n"); - goto exit; - } + KUNIT_ASSERT_PTR_EQ_MSG(test, cur->next->prev, cur, + "list is corrupted"); - cmp_result = cmp(NULL, cur, cur->next); - if (cmp_result > 0) { - pr_err("error: list is not sorted\n"); - goto exit; - } + cmp_result = cmp(test, cur, cur->next); + KUNIT_ASSERT_LE_MSG(test, cmp_result, 0, "list is not sorted"); el = container_of(cur, struct debug_el, list); el1 = container_of(cur->next, struct debug_el, list); - if (cmp_result == 0 && el->serial >= el1->serial) { - pr_err("error: order of equivalent elements not " - "preserved\n"); - goto exit; + if (cmp_result == 0) { + KUNIT_ASSERT_LE_MSG(test, el->serial, el1->serial, + "order of equivalent elements not preserved"); } - if (check(el, el1)) { - pr_err("error: element check failed\n"); - goto exit; - } + check(test, el, el1); count++; } - if (head.prev != cur) { - pr_err("error: list is corrupted\n"); - goto exit; - } + KUNIT_EXPECT_PTR_EQ_MSG(test, head.prev, cur, "list is corrupted"); + KUNIT_EXPECT_EQ_MSG(test, count, TEST_LIST_LEN, + "list length changed after sorting!"); +} - if (count != TEST_LIST_LEN) { - pr_err("error: bad list length %d", count); - goto exit; - } +static struct kunit_case list_sort_cases[] = { + KUNIT_CASE(list_sort_test), + {} +}; + +static struct kunit_suite list_sort_suite = { + .name = "list_sort", + .test_cases = list_sort_cases, +}; + +kunit_test_suites(&list_sort_suite); - err = 0; -exit: - for (i = 0; i < TEST_LIST_LEN; i++) - kfree(elts[i]); - kfree(elts); - return err; -} -module_init(list_sort_test); MODULE_LICENSE("GPL"); -- cgit From 6d2426b2f258da19fbe5fa1c93a5695460390eac Mon Sep 17 00:00:00 2001 From: David Gow Date: Thu, 24 Jun 2021 23:58:12 -0700 Subject: kunit: Support skipped tests The kunit_mark_skipped() macro marks the current test as "skipped", with the provided reason. The kunit_skip() macro will mark the test as skipped, and abort the test. The TAP specification supports this "SKIP directive" as a comment after the "ok" / "not ok" for a test. See the "Directives" section of the TAP spec for details: https://testanything.org/tap-specification.html#directives The 'success' field for KUnit tests is replaced with a kunit_status enum, which can be SUCCESS, FAILURE, or SKIPPED, combined with a 'status_comment' containing information on why a test was skipped. A new 'kunit_status' test suite is added to test this. Signed-off-by: David Gow Tested-by: Marco Elver Reviewed-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/debugfs.c | 2 +- lib/kunit/kunit-test.c | 42 ++++++++++++++++++++++++++++++++++++++- lib/kunit/test.c | 54 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 77 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 9214c493d8b7..b71db0abc12b 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -64,7 +64,7 @@ static int debugfs_print_results(struct seq_file *seq, void *v) debugfs_print_result(seq, suite, test_case); seq_printf(seq, "%s %d - %s\n", - kunit_status_to_string(success), 1, suite->name); + kunit_status_to_ok_not_ok(success), 1, suite->name); return 0; } diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c index 69f902440a0e..d69efcbed624 100644 --- a/lib/kunit/kunit-test.c +++ b/lib/kunit/kunit-test.c @@ -437,7 +437,47 @@ static void kunit_log_test(struct kunit *test) #endif } +static void kunit_status_set_failure_test(struct kunit *test) +{ + struct kunit fake; + + kunit_init_test(&fake, "fake test", NULL); + + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SUCCESS); + kunit_set_failure(&fake); + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_FAILURE); +} + +static void kunit_status_mark_skipped_test(struct kunit *test) +{ + struct kunit fake; + + kunit_init_test(&fake, "fake test", NULL); + + /* Before: Should be SUCCESS with no comment. */ + KUNIT_EXPECT_EQ(test, fake.status, KUNIT_SUCCESS); + KUNIT_EXPECT_STREQ(test, fake.status_comment, ""); + + /* Mark the test as skipped. */ + kunit_mark_skipped(&fake, "Accepts format string: %s", "YES"); + + /* After: Should be SKIPPED with our comment. */ + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SKIPPED); + KUNIT_EXPECT_STREQ(test, fake.status_comment, "Accepts format string: YES"); +} + +static struct kunit_case kunit_status_test_cases[] = { + KUNIT_CASE(kunit_status_set_failure_test), + KUNIT_CASE(kunit_status_mark_skipped_test), + {} +}; + +static struct kunit_suite kunit_status_test_suite = { + .name = "kunit_status", + .test_cases = kunit_status_test_cases, +}; + kunit_test_suites(&kunit_try_catch_test_suite, &kunit_resource_test_suite, - &kunit_log_test_suite); + &kunit_log_test_suite, &kunit_status_test_suite); MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 06f6cff2c0e7..b3d0c8e4e339 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -98,12 +98,14 @@ static void kunit_print_subtest_start(struct kunit_suite *suite) static void kunit_print_ok_not_ok(void *test_or_suite, bool is_test, - bool is_ok, + enum kunit_status status, size_t test_number, - const char *description) + const char *description, + const char *directive) { struct kunit_suite *suite = is_test ? NULL : test_or_suite; struct kunit *test = is_test ? test_or_suite : NULL; + const char *directive_header = (status == KUNIT_SKIPPED) ? " # SKIP " : ""; /* * We do not log the test suite results as doing so would @@ -114,25 +116,31 @@ static void kunit_print_ok_not_ok(void *test_or_suite, * representation. */ if (suite) - pr_info("%s %zd - %s\n", - kunit_status_to_string(is_ok), - test_number, description); + pr_info("%s %zd - %s%s%s\n", + kunit_status_to_ok_not_ok(status), + test_number, description, directive_header, + (status == KUNIT_SKIPPED) ? directive : ""); else - kunit_log(KERN_INFO, test, KUNIT_SUBTEST_INDENT "%s %zd - %s", - kunit_status_to_string(is_ok), - test_number, description); + kunit_log(KERN_INFO, test, + KUNIT_SUBTEST_INDENT "%s %zd - %s%s%s", + kunit_status_to_ok_not_ok(status), + test_number, description, directive_header, + (status == KUNIT_SKIPPED) ? directive : ""); } -bool kunit_suite_has_succeeded(struct kunit_suite *suite) +enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) { const struct kunit_case *test_case; + enum kunit_status status = KUNIT_SKIPPED; kunit_suite_for_each_test_case(suite, test_case) { - if (!test_case->success) - return false; + if (test_case->status == KUNIT_FAILURE) + return KUNIT_FAILURE; + else if (test_case->status == KUNIT_SUCCESS) + status = KUNIT_SUCCESS; } - return true; + return status; } EXPORT_SYMBOL_GPL(kunit_suite_has_succeeded); @@ -143,7 +151,8 @@ static void kunit_print_subtest_end(struct kunit_suite *suite) kunit_print_ok_not_ok((void *)suite, false, kunit_suite_has_succeeded(suite), kunit_suite_counter++, - suite->name); + suite->name, + suite->status_comment); } unsigned int kunit_test_case_num(struct kunit_suite *suite, @@ -252,7 +261,8 @@ void kunit_init_test(struct kunit *test, const char *name, char *log) test->log = log; if (test->log) test->log[0] = '\0'; - test->success = true; + test->status = KUNIT_SUCCESS; + test->status_comment[0] = '\0'; } EXPORT_SYMBOL_GPL(kunit_init_test); @@ -376,7 +386,11 @@ static void kunit_run_case_catch_errors(struct kunit_suite *suite, context.test_case = test_case; kunit_try_catch_run(try_catch, &context); - test_case->success &= test->success; + /* Propagate the parameter result to the test case. */ + if (test->status == KUNIT_FAILURE) + test_case->status = KUNIT_FAILURE; + else if (test_case->status != KUNIT_FAILURE && test->status == KUNIT_SUCCESS) + test_case->status = KUNIT_SUCCESS; } int kunit_run_tests(struct kunit_suite *suite) @@ -388,7 +402,7 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_suite_for_each_test_case(suite, test_case) { struct kunit test = { .param_value = NULL, .param_index = 0 }; - test_case->success = true; + test_case->status = KUNIT_SKIPPED; if (test_case->generate_params) { /* Get initial param. */ @@ -409,7 +423,7 @@ int kunit_run_tests(struct kunit_suite *suite) KUNIT_SUBTEST_INDENT "# %s: %s %d - %s", test_case->name, - kunit_status_to_string(test.success), + kunit_status_to_ok_not_ok(test.status), test.param_index + 1, param_desc); /* Get next param. */ @@ -419,9 +433,10 @@ int kunit_run_tests(struct kunit_suite *suite) } } while (test.param_value); - kunit_print_ok_not_ok(&test, true, test_case->success, + kunit_print_ok_not_ok(&test, true, test_case->status, kunit_test_case_num(suite, test_case), - test_case->name); + test_case->name, + test.status_comment); } kunit_print_subtest_end(suite); @@ -433,6 +448,7 @@ EXPORT_SYMBOL_GPL(kunit_run_tests); static void kunit_init_suite(struct kunit_suite *suite) { kunit_debugfs_create_suite(suite); + suite->status_comment[0] = '\0'; } int __kunit_test_suites_init(struct kunit_suite * const * const suites) -- cgit From d99ea675141934a1ea5cd1b2adff34eafcb779bc Mon Sep 17 00:00:00 2001 From: David Gow Date: Thu, 24 Jun 2021 23:58:14 -0700 Subject: kunit: test: Add example tests which are always skipped Add two new tests to the example test suite, both of which are always skipped. This is used as an example for how to write tests which are skipped, and to demonstrate the difference between kunit_skip() and kunit_mark_skipped(). Note that these tests are enabled by default, so a default run of KUnit will have two skipped tests. Signed-off-by: David Gow Reviewed-by: Daniel Latypov Reviewed-by: Brendan Higgins Reviewed-by: Marco Elver Signed-off-by: Shuah Khan --- lib/kunit/kunit-example-test.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'lib') diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index be1164ecc476..51099b0ca29c 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -40,6 +40,35 @@ static int example_test_init(struct kunit *test) return 0; } +/* + * This test should always be skipped. + */ +static void example_skip_test(struct kunit *test) +{ + /* This line should run */ + kunit_info(test, "You should not see a line below."); + + /* Skip (and abort) the test */ + kunit_skip(test, "this test should be skipped"); + + /* This line should not execute */ + KUNIT_FAIL(test, "You should not see this line."); +} + +/* + * This test should always be marked skipped. + */ +static void example_mark_skipped_test(struct kunit *test) +{ + /* This line should run */ + kunit_info(test, "You should see a line below."); + + /* Skip (but do not abort) the test */ + kunit_mark_skipped(test, "this test should be skipped"); + + /* This line should run */ + kunit_info(test, "You should see this line."); +} /* * Here we make a list of all the test cases we want to add to the test suite * below. @@ -52,6 +81,8 @@ static struct kunit_case example_test_cases[] = { * test suite. */ KUNIT_CASE(example_simple_test), + KUNIT_CASE(example_skip_test), + KUNIT_CASE(example_mark_skipped_test), {} }; -- cgit From 40eb5cf4cc913dbb615eb97d05f2353f0404a464 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Thu, 24 Jun 2021 23:58:15 -0700 Subject: kasan: test: make use of kunit_skip() Make use of the recently added kunit_skip() to skip tests, as it permits TAP parsers to recognize if a test was deliberately skipped. Signed-off-by: Marco Elver Signed-off-by: David Gow Reviewed-by: Daniel Latypov Reviewed-by: Andrey Konovalov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/test_kasan.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/test_kasan.c b/lib/test_kasan.c index cacbbbdef768..0a2029d14c91 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -111,17 +111,13 @@ static void kasan_test_exit(struct kunit *test) } while (0) #define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ - if (!IS_ENABLED(config)) { \ - kunit_info((test), "skipping, " #config " required"); \ - return; \ - } \ + if (!IS_ENABLED(config)) \ + kunit_skip((test), "Test requires " #config "=y"); \ } while (0) #define KASAN_TEST_NEEDS_CONFIG_OFF(test, config) do { \ - if (IS_ENABLED(config)) { \ - kunit_info((test), "skipping, " #config " enabled"); \ - return; \ - } \ + if (IS_ENABLED(config)) \ + kunit_skip((test), "Test requires " #config "=n"); \ } while (0) static void kmalloc_oob_right(struct kunit *test) -- cgit From 1d71307a6f94df3750f8f884545a769e227172fe Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Tue, 20 Apr 2021 19:04:27 -0700 Subject: kunit: add unit test for filtering suites by names This adds unit tests for kunit_filter_subsuite() and kunit_filter_suites(). Note: what the executor means by "subsuite" is the array of suites corresponding to each test file. This patch lightly refactors executor.c to avoid the use of global variables to make it testable. It also includes a clever `kfree_at_end()` helper that makes this test easier to write than it otherwise would have been. Tested by running just the new tests using itself $ ./tools/testing/kunit/kunit.py run '*exec*' Signed-off-by: Daniel Latypov Reviewed-by: David Gow Acked-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- lib/kunit/executor.c | 33 +++++++----- lib/kunit/executor_test.c | 133 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 lib/kunit/executor_test.c (limited to 'lib') diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 7db619624437..acd1de436f59 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -14,8 +14,8 @@ extern struct kunit_suite * const * const __kunit_suites_end[]; #if IS_BUILTIN(CONFIG_KUNIT) -static char *filter_glob; -module_param(filter_glob, charp, 0); +static char *filter_glob_param; +module_param_named(filter_glob, filter_glob_param, charp, 0); MODULE_PARM_DESC(filter_glob, "Filter which KUnit test suites run at boot-time, e.g. list*"); @@ -23,7 +23,8 @@ static char *kunit_shutdown; core_param(kunit_shutdown, kunit_shutdown, charp, 0644); static struct kunit_suite * const * -kunit_filter_subsuite(struct kunit_suite * const * const subsuite) +kunit_filter_subsuite(struct kunit_suite * const * const subsuite, + const char *filter_glob) { int i, n = 0; struct kunit_suite **filtered; @@ -56,19 +57,14 @@ struct suite_set { struct kunit_suite * const * const *end; }; -static struct suite_set kunit_filter_suites(void) +static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, + const char *filter_glob) { int i; struct kunit_suite * const **copy, * const *filtered_subsuite; struct suite_set filtered; - const size_t max = __kunit_suites_end - __kunit_suites_start; - - if (!filter_glob) { - filtered.start = __kunit_suites_start; - filtered.end = __kunit_suites_end; - return filtered; - } + const size_t max = suite_set->end - suite_set->start; copy = kmalloc_array(max, sizeof(*filtered.start), GFP_KERNEL); filtered.start = copy; @@ -78,7 +74,7 @@ static struct suite_set kunit_filter_suites(void) } for (i = 0; i < max; ++i) { - filtered_subsuite = kunit_filter_subsuite(__kunit_suites_start[i]); + filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], filter_glob); if (filtered_subsuite) *copy++ = filtered_subsuite; } @@ -116,15 +112,20 @@ static void kunit_print_tap_header(struct suite_set *suite_set) int kunit_run_all_tests(void) { struct kunit_suite * const * const *suites; + struct suite_set suite_set = { + .start = __kunit_suites_start, + .end = __kunit_suites_end, + }; - struct suite_set suite_set = kunit_filter_suites(); + if (filter_glob_param) + suite_set = kunit_filter_suites(&suite_set, filter_glob_param); kunit_print_tap_header(&suite_set); for (suites = suite_set.start; suites < suite_set.end; suites++) __kunit_test_suites_init(*suites); - if (filter_glob) { /* a copy was made of each array */ + if (filter_glob_param) { /* a copy was made of each array */ for (suites = suite_set.start; suites < suite_set.end; suites++) kfree(*suites); kfree(suite_set.start); @@ -135,4 +136,8 @@ int kunit_run_all_tests(void) return 0; } +#if IS_BUILTIN(CONFIG_KUNIT_TEST) +#include "executor_test.c" +#endif + #endif /* IS_BUILTIN(CONFIG_KUNIT) */ diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c new file mode 100644 index 000000000000..cdbe54b16501 --- /dev/null +++ b/lib/kunit/executor_test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for the KUnit executor. + * + * Copyright (C) 2021, Google LLC. + * Author: Daniel Latypov + */ + +#include + +static void kfree_at_end(struct kunit *test, const void *to_free); +static struct kunit_suite *alloc_fake_suite(struct kunit *test, + const char *suite_name); + +static void filter_subsuite_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + + subsuite[0] = alloc_fake_suite(test, "suite1"); + subsuite[1] = alloc_fake_suite(test, "suite2"); + + /* Want: suite1, suite2, NULL -> suite2, NULL */ + filtered = kunit_filter_subsuite(subsuite, "suite2*"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered); + kfree_at_end(test, filtered); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2"); + + KUNIT_EXPECT_FALSE(test, filtered[1]); +} + +static void filter_subsuite_to_empty_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + + subsuite[0] = alloc_fake_suite(test, "suite1"); + subsuite[1] = alloc_fake_suite(test, "suite2"); + + filtered = kunit_filter_subsuite(subsuite, "not_found"); + kfree_at_end(test, filtered); /* just in case */ + + KUNIT_EXPECT_FALSE_MSG(test, filtered, + "should be NULL to indicate no match"); +} + +static void kfree_subsuites_at_end(struct kunit *test, struct suite_set *suite_set) +{ + struct kunit_suite * const * const *suites; + + kfree_at_end(test, suite_set->start); + for (suites = suite_set->start; suites < suite_set->end; suites++) + kfree_at_end(test, *suites); +} + +static void filter_suites_test(struct kunit *test) +{ + /* Suites per-file are stored as a NULL terminated array */ + struct kunit_suite *subsuites[2][2] = { + {NULL, NULL}, + {NULL, NULL}, + }; + /* Match the memory layout of suite_set */ + struct kunit_suite * const * const suites[2] = { + subsuites[0], subsuites[1], + }; + + const struct suite_set suite_set = { + .start = suites, + .end = suites + 2, + }; + struct suite_set filtered = {.start = NULL, .end = NULL}; + + /* Emulate two files, each having one suite */ + subsuites[0][0] = alloc_fake_suite(test, "suite0"); + subsuites[1][0] = alloc_fake_suite(test, "suite1"); + + /* Filter out suite1 */ + filtered = kunit_filter_suites(&suite_set, "suite0"); + kfree_subsuites_at_end(test, &filtered); /* let us use ASSERTs without leaking */ + KUNIT_ASSERT_EQ(test, filtered.end - filtered.start, (ptrdiff_t)1); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered.start[0][0]->name, "suite0"); +} + +static struct kunit_case executor_test_cases[] = { + KUNIT_CASE(filter_subsuite_test), + KUNIT_CASE(filter_subsuite_to_empty_test), + KUNIT_CASE(filter_suites_test), + {} +}; + +static struct kunit_suite executor_test_suite = { + .name = "kunit_executor_test", + .test_cases = executor_test_cases, +}; + +kunit_test_suites(&executor_test_suite); + +/* Test helpers */ + +static void kfree_res_free(struct kunit_resource *res) +{ + kfree(res->data); +} + +/* Use the resource API to register a call to kfree(to_free). + * Since we never actually use the resource, it's safe to use on const data. + */ +static void kfree_at_end(struct kunit *test, const void *to_free) +{ + /* kfree() handles NULL already, but avoid allocating a no-op cleanup. */ + if (IS_ERR_OR_NULL(to_free)) + return; + kunit_alloc_and_get_resource(test, NULL, kfree_res_free, GFP_KERNEL, + (void *)to_free); +} + +static struct kunit_suite *alloc_fake_suite(struct kunit *test, + const char *suite_name) +{ + struct kunit_suite *suite; + + /* We normally never expect to allocate suites, hence the non-const cast. */ + suite = kunit_kzalloc(test, sizeof(*suite), GFP_KERNEL); + strncpy((char *)suite->name, suite_name, sizeof(suite->name) - 1); + + return suite; +} -- cgit From d3b16034a24a112bb83aeb669ac5b9b01f744bb7 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Sat, 26 Jun 2021 11:21:55 +0800 Subject: seq_buf: Fix overflow in seq_buf_putmem_hex() There's two variables being increased in that loop (i and j), and i follows the raw data, and j follows what is being written into the buffer. We should compare 'i' to MAX_MEMHEX_BYTES or compare 'j' to HEX_CHARS. Otherwise, if 'j' goes bigger than HEX_CHARS, it will overflow the destination buffer. Link: https://lore.kernel.org/lkml/20210625122453.5e2fe304@oasis.local.home/ Link: https://lkml.kernel.org/r/20210626032156.47889-1-yun.zhou@windriver.com Cc: stable@vger.kernel.org Fixes: 5e3ca0ec76fce ("ftrace: introduce the "hex" output method") Signed-off-by: Yun Zhou Signed-off-by: Steven Rostedt (VMware) --- lib/seq_buf.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 707453f5d58e..62c20596ef42 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -229,8 +229,10 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, WARN_ON(s->size == 0); + BUILD_BUG_ON(MAX_MEMHEX_BYTES * 2 >= HEX_CHARS); + while (len) { - start_len = min(len, HEX_CHARS - 1); + start_len = min(len, MAX_MEMHEX_BYTES); #ifdef __BIG_ENDIAN for (i = 0, j = 0; i < start_len; i++) { #else -- cgit From 6a2cbc58d6c9d90cd74288cc497c2b45815bc064 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Sat, 26 Jun 2021 11:21:56 +0800 Subject: seq_buf: Make trace_seq_putmem_hex() support data longer than 8 Since the raw memory 'data' does not go forward, it will dump repeated data if the data length is more than 8. If we want to dump longer data blocks, we need to repeatedly call macro SEQ_PUT_HEX_FIELD. I think it is a bit redundant, and multiple function calls also affect the performance. Link: https://lore.kernel.org/lkml/20210625122453.5e2fe304@oasis.local.home/ Link: https://lkml.kernel.org/r/20210626032156.47889-2-yun.zhou@windriver.com Cc: stable@vger.kernel.org Fixes: 6d2289f3faa7 ("tracing: Make trace_seq_putmem_hex() more robust") Signed-off-by: Yun Zhou Signed-off-by: Steven Rostedt (VMware) --- lib/seq_buf.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 62c20596ef42..6dafde851333 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -245,12 +245,14 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, break; /* j increments twice per loop */ - len -= j / 2; hex[j++] = ' '; seq_buf_putmem(s, hex, j); if (seq_buf_has_overflowed(s)) return -1; + + len -= start_len; + data += start_len; } return 0; } -- cgit From 26c6cb7cf830349c6518a7efe1c32ac796cd192e Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 28 Jun 2021 19:34:30 -0700 Subject: kunit: make test->lock irq safe The upcoming SLUB kunit test will be calling kunit_find_named_resource() from a context with disabled interrupts. That means kunit's test->lock needs to be IRQ safe to avoid potential deadlocks and lockdep splats. This patch therefore changes the test->lock usage to spin_lock_irqsave() and spin_unlock_irqrestore(). Link: https://lkml.kernel.org/r/20210511150734.3492-1-glittao@gmail.com Signed-off-by: Vlastimil Babka Signed-off-by: Oliver Glitta Reviewed-by: Brendan Higgins Cc: Christoph Lameter Cc: Daniel Latypov Cc: David Rientjes Cc: Joonsoo Kim Cc: Marco Elver Cc: Pekka Enberg Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/kunit/test.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 2f6cc0123232..45f068864d76 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -475,6 +475,7 @@ int kunit_add_resource(struct kunit *test, void *data) { int ret = 0; + unsigned long flags; res->free = free; kref_init(&res->refcount); @@ -487,10 +488,10 @@ int kunit_add_resource(struct kunit *test, res->data = data; } - spin_lock(&test->lock); + spin_lock_irqsave(&test->lock, flags); list_add_tail(&res->node, &test->resources); /* refcount for list is established by kref_init() */ - spin_unlock(&test->lock); + spin_unlock_irqrestore(&test->lock, flags); return ret; } @@ -548,9 +549,11 @@ EXPORT_SYMBOL_GPL(kunit_alloc_and_get_resource); void kunit_remove_resource(struct kunit *test, struct kunit_resource *res) { - spin_lock(&test->lock); + unsigned long flags; + + spin_lock_irqsave(&test->lock, flags); list_del(&res->node); - spin_unlock(&test->lock); + spin_unlock_irqrestore(&test->lock, flags); kunit_put_resource(res); } EXPORT_SYMBOL_GPL(kunit_remove_resource); @@ -630,6 +633,7 @@ EXPORT_SYMBOL_GPL(kunit_kfree); void kunit_cleanup(struct kunit *test) { struct kunit_resource *res; + unsigned long flags; /* * test->resources is a stack - each allocation must be freed in the @@ -641,9 +645,9 @@ void kunit_cleanup(struct kunit *test) * protect against the current node being deleted, not the next. */ while (true) { - spin_lock(&test->lock); + spin_lock_irqsave(&test->lock, flags); if (list_empty(&test->resources)) { - spin_unlock(&test->lock); + spin_unlock_irqrestore(&test->lock, flags); break; } res = list_last_entry(&test->resources, @@ -654,7 +658,7 @@ void kunit_cleanup(struct kunit *test) * resource, and this can't happen if the test->lock * is held. */ - spin_unlock(&test->lock); + spin_unlock_irqrestore(&test->lock, flags); kunit_remove_resource(test, res); } current->kunit_test = NULL; -- cgit From 1f9f78b1b376f82cdd8ed73cc0abdb74d0453d43 Mon Sep 17 00:00:00 2001 From: Oliver Glitta Date: Mon, 28 Jun 2021 19:34:33 -0700 Subject: mm/slub, kunit: add a KUnit test for SLUB debugging functionality SLUB has resiliency_test() function which is hidden behind #ifdef SLUB_RESILIENCY_TEST that is not part of Kconfig, so nobody runs it. KUnit should be a proper replacement for it. Try changing byte in redzone after allocation and changing pointer to next free node, first byte, 50th byte and redzone byte. Check if validation finds errors. There are several differences from the original resiliency test: Tests create own caches with known state instead of corrupting shared kmalloc caches. The corruption of freepointer uses correct offset, the original resiliency test got broken with freepointer changes. Scratch changing random byte test, because it does not have meaning in this form where we need deterministic results. Add new option CONFIG_SLUB_KUNIT_TEST in Kconfig. Tests next_pointer, first_word and clobber_50th_byte do not run with KASAN option on. Because the test deliberately modifies non-allocated objects. Use kunit_resource to count errors in cache and silence bug reports. Count error whenever slab_bug() or slab_fix() is called or when the count of pages is wrong. [glittao@gmail.com: remove unused function test_exit(), from SLUB KUnit test] Link: https://lkml.kernel.org/r/20210512140656.12083-1-glittao@gmail.com [akpm@linux-foundation.org: export kasan_enable/disable_current to modules] Link: https://lkml.kernel.org/r/20210511150734.3492-2-glittao@gmail.com Signed-off-by: Oliver Glitta Reviewed-by: Vlastimil Babka Acked-by: Daniel Latypov Acked-by: Marco Elver Cc: Brendan Higgins Cc: Christoph Lameter Cc: David Rientjes Cc: Joonsoo Kim Cc: Pekka Enberg Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 12 +++++ lib/Makefile | 1 + lib/slub_kunit.c | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 lib/slub_kunit.c (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 678c13967580..7723f58a9394 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2429,6 +2429,18 @@ config BITS_TEST If unsure, say N. +config SLUB_KUNIT_TEST + tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS + depends on SLUB_DEBUG && KUNIT + default KUNIT_ALL_TESTS + help + This builds SLUB allocator unit test. + Tests SLUB cache debugging functionality. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config TEST_UDELAY tristate "udelay test driver" help diff --git a/lib/Makefile b/lib/Makefile index 2cc359ec1fdd..6d5ea8f5b52c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -354,5 +354,6 @@ obj-$(CONFIG_LIST_KUNIT_TEST) += list-test.o obj-$(CONFIG_LINEAR_RANGES_TEST) += test_linear_ranges.o obj-$(CONFIG_BITS_TEST) += test_bits.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o +obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o diff --git a/lib/slub_kunit.c b/lib/slub_kunit.c new file mode 100644 index 000000000000..8662dc6cb509 --- /dev/null +++ b/lib/slub_kunit.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "../mm/slab.h" + +static struct kunit_resource resource; +static int slab_errors; + +static void test_clobber_zone(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_RZ_alloc", 64, 0, + SLAB_RED_ZONE, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kasan_disable_current(); + p[64] = 0x12; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kasan_enable_current(); + kmem_cache_free(s, p); + kmem_cache_destroy(s); +} + +#ifndef CONFIG_KASAN +static void test_next_pointer(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_next_ptr_free", 64, 0, + SLAB_POISON, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + unsigned long tmp; + unsigned long *ptr_addr; + + kmem_cache_free(s, p); + + ptr_addr = (unsigned long *)(p + s->offset); + tmp = *ptr_addr; + p[s->offset] = 0x12; + + /* + * Expecting three errors. + * One for the corrupted freechain and the other one for the wrong + * count of objects in use. The third error is fixing broken cache. + */ + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 3, slab_errors); + + /* + * Try to repair corrupted freepointer. + * Still expecting two errors. The first for the wrong count + * of objects in use. + * The second error is for fixing broken cache. + */ + *ptr_addr = tmp; + slab_errors = 0; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + /* + * Previous validation repaired the count of objects in use. + * Now expecting no error. + */ + slab_errors = 0; + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 0, slab_errors); + + kmem_cache_destroy(s); +} + +static void test_first_word(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_1th_word_free", 64, 0, + SLAB_POISON, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kmem_cache_free(s, p); + *p = 0x78; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kmem_cache_destroy(s); +} + +static void test_clobber_50th_byte(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_50th_word_free", 64, 0, + SLAB_POISON, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kmem_cache_free(s, p); + p[50] = 0x9a; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kmem_cache_destroy(s); +} +#endif + +static void test_clobber_redzone_free(struct kunit *test) +{ + struct kmem_cache *s = kmem_cache_create("TestSlub_RZ_free", 64, 0, + SLAB_RED_ZONE, NULL); + u8 *p = kmem_cache_alloc(s, GFP_KERNEL); + + kasan_disable_current(); + kmem_cache_free(s, p); + p[64] = 0xab; + + validate_slab_cache(s); + KUNIT_EXPECT_EQ(test, 2, slab_errors); + + kasan_enable_current(); + kmem_cache_destroy(s); +} + +static int test_init(struct kunit *test) +{ + slab_errors = 0; + + kunit_add_named_resource(test, NULL, NULL, &resource, + "slab_errors", &slab_errors); + return 0; +} + +static struct kunit_case test_cases[] = { + KUNIT_CASE(test_clobber_zone), + +#ifndef CONFIG_KASAN + KUNIT_CASE(test_next_pointer), + KUNIT_CASE(test_first_word), + KUNIT_CASE(test_clobber_50th_byte), +#endif + + KUNIT_CASE(test_clobber_redzone_free), + {} +}; + +static struct kunit_suite test_suite = { + .name = "slub_test", + .init = test_init, + .test_cases = test_cases, +}; +kunit_test_suite(test_suite); + +MODULE_LICENSE("GPL"); -- cgit From 792702911f581f7793962fbeb99d5c3a1b28f4c3 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Mon, 28 Jun 2021 19:34:52 -0700 Subject: slub: force on no_hash_pointers when slub_debug is enabled Obscuring the pointers that slub shows when debugging makes for some confusing slub debug messages: Padding overwritten. 0x0000000079f0674a-0x000000000d4dce17 Those addresses are hashed for kernel security reasons. If we're trying to be secure with slub_debug on the commandline we have some big problems given that we dump whole chunks of kernel memory to the kernel logs. Let's force on the no_hash_pointers commandline flag when slub_debug is on the commandline. This makes slub debug messages more meaningful and if by chance a kernel address is in some slub debug object dump we will have a better chance of figuring out what went wrong. Note that we don't use %px in the slub code because we want to reduce the number of places that %px is used in the kernel. This also nicely prints a big fat warning at kernel boot if slub_debug is on the commandline so that we know that this kernel shouldn't be used on production systems. [akpm@linux-foundation.org: fix build with CONFIG_SLUB_DEBUG=n] Link: https://lkml.kernel.org/r/20210601182202.3011020-5-swboyd@chromium.org Signed-off-by: Stephen Boyd Acked-by: Vlastimil Babka Acked-by: Petr Mladek Cc: Joe Perches Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Cc: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/vsprintf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/vsprintf.c b/lib/vsprintf.c index f0c35d9b65bf..cc281f5895f9 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -2186,7 +2186,7 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, bool no_hash_pointers __ro_after_init; EXPORT_SYMBOL_GPL(no_hash_pointers); -static int __init no_hash_pointers_enable(char *str) +int __init no_hash_pointers_enable(char *str) { if (no_hash_pointers) return 0; -- cgit From 46e6b31d4617612e47daeb7b4b6350b116349f6d Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Mon, 28 Jun 2021 19:39:38 -0700 Subject: lib/test_hmm: use vma_lookup() in dmirror_migrate() Use vma_lookup() to find the VMA at a specific address. As vma_lookup() will return NULL if the address is not within any VMA, the start address no longer needs to be validated. Link: https://lkml.kernel.org/r/20210521174745.2219620-18-Liam.Howlett@Oracle.com Signed-off-by: Liam R. Howlett Reviewed-by: Laurent Dufour Acked-by: David Hildenbrand Acked-by: Davidlohr Bueso Cc: Geert Uytterhoeven Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_hmm.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 80a78877bd93..15f2e2db77bc 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -686,9 +686,8 @@ static int dmirror_migrate(struct dmirror *dmirror, mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { - vma = find_vma(mm, addr); - if (!vma || addr < vma->vm_start || - !(vma->vm_flags & VM_READ)) { + vma = vma_lookup(mm, addr); + if (!vma || !(vma->vm_flags & VM_READ)) { ret = -EINVAL; goto out; } -- cgit From 4469c0f17ec63dcc8c9ed512f4330b566c2c0d34 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 28 Jun 2021 19:40:30 -0700 Subject: printk: introduce dump_stack_lvl() dump_stack() is used for many different cases, which may require a log level consistent with other kernel messages surrounding the dump_stack() call. Without that, certain systems that are configured to ignore the default level messages will miss stack traces in critical error reports. This patch introduces dump_stack_lvl() that behaves similarly to dump_stack(), but accepts a custom log level. The old dump_stack() becomes equal to dump_stack_lvl(KERN_DEFAULT). A somewhat similar patch has been proposed in 2012: https://lore.kernel.org/lkml/1332493269.2359.9.camel@hebo/ , but wasn't merged. [elver@google.com: add missing dump_stack_lvl() stub if CONFIG_PRINTK=n] Link: https://lkml.kernel.org/r/YJ0KAM0hQev1AmWe@elver.google.com Link: https://lkml.kernel.org/r/20210506105405.3535023-1-glider@google.com Signed-off-by: Alexander Potapenko Reviewed-by: Marco Elver Cc: Petr Mladek Cc: Ingo Molnar Cc: he, bo Cc: Yanmin Zhang Cc: Prasad Sodagudi Cc: Dmitry Vyukov Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Andrey Ryabinin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/dump_stack.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/dump_stack.c b/lib/dump_stack.c index f5a33b6f773f..586e3f2c6a15 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -73,10 +73,10 @@ void show_regs_print_info(const char *log_lvl) dump_stack_print_info(log_lvl); } -static void __dump_stack(void) +static void __dump_stack(const char *log_lvl) { - dump_stack_print_info(KERN_DEFAULT); - show_stack(NULL, NULL, KERN_DEFAULT); + dump_stack_print_info(log_lvl); + show_stack(NULL, NULL, log_lvl); } /** @@ -87,7 +87,7 @@ static void __dump_stack(void) #ifdef CONFIG_SMP static atomic_t dump_lock = ATOMIC_INIT(-1); -asmlinkage __visible void dump_stack(void) +asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { unsigned long flags; int was_locked; @@ -117,7 +117,7 @@ retry: goto retry; } - __dump_stack(); + __dump_stack(log_lvl); if (!was_locked) atomic_set(&dump_lock, -1); @@ -125,9 +125,15 @@ retry: local_irq_restore(flags); } #else -asmlinkage __visible void dump_stack(void) +asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { - __dump_stack(); + __dump_stack(log_lvl); } #endif +EXPORT_SYMBOL(dump_stack_lvl); + +asmlinkage __visible void dump_stack(void) +{ + dump_stack_lvl(KERN_DEFAULT); +} EXPORT_SYMBOL(dump_stack); -- cgit From 3ff16d30f593d80a958104ee06a94562a12c5879 Mon Sep 17 00:00:00 2001 From: David Gow Date: Mon, 28 Jun 2021 19:40:36 -0700 Subject: kasan: test: improve failure message in KUNIT_EXPECT_KASAN_FAIL() The KUNIT_EXPECT_KASAN_FAIL() macro currently uses KUNIT_EXPECT_EQ() to compare fail_data.report_expected and fail_data.report_found. This always gave a somewhat useless error message on failure, but the addition of extra compile-time checking with READ_ONCE() has caused it to get much longer, and be truncated before anything useful is displayed. Instead, just check fail_data.report_found by hand (we've just set report_expected to 'true'), and print a better failure message with KUNIT_FAIL(). Because of this, report_expected is no longer used anywhere, and can be removed. Beforehand, a failure in: KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)area)[3100]); would have looked like: [22:00:34] [FAILED] vmalloc_oob [22:00:34] # vmalloc_oob: EXPECTATION FAILED at lib/test_kasan.c:991 [22:00:34] Expected ({ do { extern void __compiletime_assert_705(void) __attribute__((__error__("Unsupported access size for {READ,WRITE}_ONCE()."))); if (!((sizeof(fail_data.report_expected) == sizeof(char) || sizeof(fail_data.repp [22:00:34] not ok 45 - vmalloc_oob With this change, it instead looks like: [22:04:04] [FAILED] vmalloc_oob [22:04:04] # vmalloc_oob: EXPECTATION FAILED at lib/test_kasan.c:993 [22:04:04] KASAN failure expected in "((volatile char *)area)[3100]", but none occurred [22:04:04] not ok 45 - vmalloc_oob Also update the example failure in the documentation to reflect this. Link: https://lkml.kernel.org/r/20210606005531.165954-1-davidgow@google.com Signed-off-by: David Gow Reviewed-by: Andrey Konovalov Reviewed-by: Marco Elver Acked-by: Brendan Higgins Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Daniel Axtens Cc: David Gow Cc: Jonathan Corbet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/test_kasan.c b/lib/test_kasan.c index cacbbbdef768..44e08f4d9c52 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -55,7 +55,6 @@ static int kasan_test_init(struct kunit *test) multishot = kasan_save_enable_multi_shot(); kasan_set_tagging_report_once(false); fail_data.report_found = false; - fail_data.report_expected = false; kunit_add_named_resource(test, NULL, NULL, &resource, "kasan_data", &fail_data); return 0; @@ -94,20 +93,20 @@ static void kasan_test_exit(struct kunit *test) !kasan_async_mode_enabled()) \ migrate_disable(); \ KUNIT_EXPECT_FALSE(test, READ_ONCE(fail_data.report_found)); \ - WRITE_ONCE(fail_data.report_expected, true); \ barrier(); \ expression; \ barrier(); \ - KUNIT_EXPECT_EQ(test, \ - READ_ONCE(fail_data.report_expected), \ - READ_ONCE(fail_data.report_found)); \ + if (!READ_ONCE(fail_data.report_found)) { \ + KUNIT_FAIL(test, KUNIT_SUBTEST_INDENT "KASAN failure " \ + "expected in \"" #expression \ + "\", but none occurred"); \ + } \ if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) { \ if (READ_ONCE(fail_data.report_found)) \ kasan_enable_tagging_sync(); \ migrate_enable(); \ } \ WRITE_ONCE(fail_data.report_found, false); \ - WRITE_ONCE(fail_data.report_expected, false); \ } while (0) #define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ -- cgit From 158f25522ca8cc87f512a03ed5e2a5923bd37eb3 Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Mon, 28 Jun 2021 19:40:39 -0700 Subject: kasan: allow an architecture to disable inline instrumentation Patch series "KASAN core changes for ppc64 radix KASAN", v16. Building on the work of Christophe, Aneesh and Balbir, I've ported KASAN to 64-bit Book3S kernels running on the Radix MMU. I've been trying this for a while, but we keep having collisions between the kasan code in the mm tree and the code I want to put in to the ppc tree. This series just contains the kasan core changes that we need. There should be no noticeable changes to other platforms. This patch (of 4): For annoying architectural reasons, it's very difficult to support inline instrumentation on powerpc64.* Add a Kconfig flag to allow an arch to disable inline. (It's a bit annoying to be 'backwards', but I'm not aware of any way to have an arch force a symbol to be 'n', rather than 'y'.) We also disable stack instrumentation in this case as it does things that are functionally equivalent to inline instrumentation, namely adding code that touches the shadow directly without going through a C helper. * on ppc64 atm, the shadow lives in virtual memory and isn't accessible in real mode. However, before we turn on virtual memory, we parse the device tree to determine which platform and MMU we're running under. That calls generic DT code, which is instrumented. Inline instrumentation in DT would unconditionally attempt to touch the shadow region, which we won't have set up yet, and would crash. We can make outline mode wait for the arch to be ready, but we can't change what the compiler inserts for inline mode. Link: https://lkml.kernel.org/r/20210624034050.511391-1-dja@axtens.net Link: https://lkml.kernel.org/r/20210624034050.511391-2-dja@axtens.net Signed-off-by: Daniel Axtens Reviewed-by: Marco Elver Reviewed-by: Andrey Konovalov Cc: Christophe Leroy Cc: Aneesh Kumar K.V Cc: Balbir Singh Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Dmitry Vyukov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.kasan | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index cffc2ebbf185..c3b228828a80 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -12,6 +12,13 @@ config HAVE_ARCH_KASAN_HW_TAGS config HAVE_ARCH_KASAN_VMALLOC bool +config ARCH_DISABLE_KASAN_INLINE + bool + help + An architecture might not support inline instrumentation. + When this option is selected, inline and stack instrumentation are + disabled. + config CC_HAS_KASAN_GENERIC def_bool $(cc-option, -fsanitize=kernel-address) @@ -130,6 +137,7 @@ config KASAN_OUTLINE config KASAN_INLINE bool "Inline instrumentation" + depends on !ARCH_DISABLE_KASAN_INLINE help Compiler directly inserts code checking shadow memory before memory accesses. This is faster than outline (in some workloads @@ -141,6 +149,7 @@ endchoice config KASAN_STACK bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST depends on KASAN_GENERIC || KASAN_SW_TAGS + depends on !ARCH_DISABLE_KASAN_INLINE default y if CC_IS_GCC help The LLVM stack address sanitizer has a know problem that @@ -154,6 +163,9 @@ config KASAN_STACK but clang users can still enable it for builds without CONFIG_COMPILE_TEST. On gcc it is assumed to always be safe to use and enabled by default. + If the architecture disables inline instrumentation, stack + instrumentation is also disabled as it adds inline-style + instrumentation that is run unconditionally. config KASAN_SW_TAGS_IDENTIFY bool "Enable memory corruption identification" -- cgit From f06f78ab48fb90cfbef5289e5556704b74c46b7a Mon Sep 17 00:00:00 2001 From: Kuan-Ying Lee Date: Mon, 28 Jun 2021 19:40:52 -0700 Subject: kasan: rename CONFIG_KASAN_SW_TAGS_IDENTIFY to CONFIG_KASAN_TAGS_IDENTIFY Patch series "kasan: add memory corruption identification support for hw tag-based kasan", v4. Add memory corruption identification for hardware tag-based KASAN mode. This patch (of 3): Rename CONFIG_KASAN_SW_TAGS_IDENTIFY to CONFIG_KASAN_TAGS_IDENTIFY in order to be compatible with hardware tag-based mode. Link: https://lkml.kernel.org/r/20210626100931.22794-1-Kuan-Ying.Lee@mediatek.com Link: https://lkml.kernel.org/r/20210626100931.22794-2-Kuan-Ying.Lee@mediatek.com Signed-off-by: Kuan-Ying Lee Suggested-by: Marco Elver Reviewed-by: Alexander Potapenko Reviewed-by: Andrey Konovalov Reviewed-by: Marco Elver Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Matthias Brugger Cc: Chinwen Chang Cc: Nicholas Tang Cc: Kuan-Ying Lee Cc: Greg Kroah-Hartman Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.kasan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index c3b228828a80..fdb4a08dba83 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -167,7 +167,7 @@ config KASAN_STACK instrumentation is also disabled as it adds inline-style instrumentation that is run unconditionally. -config KASAN_SW_TAGS_IDENTIFY +config KASAN_TAGS_IDENTIFY bool "Enable memory corruption identification" depends on KASAN_SW_TAGS help -- cgit From 7a22bdc3c443d5abc420df1381e425b49e8901a3 Mon Sep 17 00:00:00 2001 From: Kuan-Ying Lee Date: Mon, 28 Jun 2021 19:40:58 -0700 Subject: kasan: add memory corruption identification support for hardware tag-based mode Add memory corruption identification support for hardware tag-based mode. We store one old free pointer tag and free backtrace instead of five because hardware tag-based kasan only has 16 different tags. If we store as many stacks as SW tag-based kasan does(5 stacks), there is high probability to find the same tag in the stacks when out-of-bound issues happened and we will mistake out-of-bound issue for use-after-free. Link: https://lkml.kernel.org/r/20210626100931.22794-4-Kuan-Ying.Lee@mediatek.com Signed-off-by: Kuan-Ying Lee Suggested-by: Marco Elver Reviewed-by: Alexander Potapenko Reviewed-by: Andrey Konovalov Reviewed-by: Marco Elver Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Chinwen Chang Cc: Greg Kroah-Hartman Cc: Matthias Brugger Cc: Nicholas Tang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.kasan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index fdb4a08dba83..1e2d10f86011 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -169,7 +169,7 @@ config KASAN_STACK config KASAN_TAGS_IDENTIFY bool "Enable memory corruption identification" - depends on KASAN_SW_TAGS + depends on KASAN_SW_TAGS || KASAN_HW_TAGS help This option enables best-effort identification of bug type (use-after-free or out-of-bounds) at the cost of increased -- cgit From dbbee9d5cd83f9d0a29639e260516907ceb2ac3d Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Mon, 28 Jun 2021 19:41:41 -0700 Subject: mm/page_alloc: convert per-cpu list protection to local_lock There is a lack of clarity of what exactly local_irq_save/local_irq_restore protects in page_alloc.c . It conflates the protection of per-cpu page allocation structures with per-cpu vmstat deltas. This patch protects the PCP structure using local_lock which for most configurations is identical to IRQ enabling/disabling. The scope of the lock is still wider than it should be but this is decreased later. It is possible for the local_lock to be embedded safely within struct per_cpu_pages but it adds complexity to free_unref_page_list. [akpm@linux-foundation.org: coding style fixes] [mgorman@techsingularity.net: work around a pahole limitation with zero-sized struct pagesets] Link: https://lkml.kernel.org/r/20210526080741.GW30378@techsingularity.net [lkp@intel.com: Make pagesets static] Link: https://lkml.kernel.org/r/20210512095458.30632-3-mgorman@techsingularity.net Signed-off-by: Mel Gorman Acked-by: Vlastimil Babka Acked-by: Peter Zijlstra (Intel) Cc: Chuck Lever Cc: Ingo Molnar Cc: Jesper Dangaard Brouer Cc: Michal Hocko Cc: Sebastian Andrzej Siewior Cc: Thomas Gleixner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 7723f58a9394..deca67d28abb 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -313,6 +313,9 @@ config DEBUG_INFO_BTF config PAHOLE_HAS_SPLIT_BTF def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119") +config PAHOLE_HAS_ZEROSIZE_PERCPU_SUPPORT + def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "122") + config DEBUG_INFO_BTF_MODULES def_bool y depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF -- cgit From 6b49bf6ddbb0d7992c816846acfa5fd1cf751c36 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 30 Jun 2021 18:54:19 -0700 Subject: mm: rename migrate_pgmap_owner MMU notifier ranges have a migrate_pgmap_owner field which is used by drivers to store a pointer. This is subsequently used by the driver callback to filter MMU_NOTIFY_MIGRATE events. Other notifier event types can also benefit from this filtering, so rename the 'migrate_pgmap_owner' field to 'owner' and create a new notifier initialisation function to initialise this field. Link: https://lkml.kernel.org/r/20210616105937.23201-6-apopple@nvidia.com Signed-off-by: Alistair Popple Suggested-by: Peter Xu Reviewed-by: Peter Xu Cc: Ben Skeggs Cc: Christoph Hellwig Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: John Hubbard Cc: "Matthew Wilcox (Oracle)" Cc: Ralph Campbell Cc: Shakeel Butt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_hmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 15f2e2db77bc..fc7a20bc9b42 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -218,7 +218,7 @@ static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, * the invalidation is handled as part of the migration process. */ if (range->event == MMU_NOTIFY_MIGRATE && - range->migrate_pgmap_owner == dmirror->mdevice) + range->owner == dmirror->mdevice) return true; if (mmu_notifier_range_blockable(range)) -- cgit From b659baea75469f0c5bd26f18461dfcdc1bbbac82 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Wed, 30 Jun 2021 18:54:28 -0700 Subject: mm: selftests for exclusive device memory Adds some selftests for exclusive device memory. Link: https://lkml.kernel.org/r/20210616105937.23201-9-apopple@nvidia.com Signed-off-by: Alistair Popple Acked-by: Jason Gunthorpe Tested-by: Ralph Campbell Reviewed-by: Ralph Campbell Cc: Ben Skeggs Cc: Christoph Hellwig Cc: Hugh Dickins Cc: John Hubbard Cc: "Matthew Wilcox (Oracle)" Cc: Peter Xu Cc: Shakeel Butt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_hmm.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/test_hmm_uapi.h | 2 + 2 files changed, 127 insertions(+) (limited to 'lib') diff --git a/lib/test_hmm.c b/lib/test_hmm.c index fc7a20bc9b42..8c55c4723692 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "test_hmm_uapi.h" @@ -46,6 +47,7 @@ struct dmirror_bounce { unsigned long cpages; }; +#define DPT_XA_TAG_ATOMIC 1UL #define DPT_XA_TAG_WRITE 3UL /* @@ -619,6 +621,54 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, } } +static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, + unsigned long end) +{ + unsigned long pfn; + + for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { + void *entry; + struct page *page; + + entry = xa_load(&dmirror->pt, pfn); + page = xa_untag_pointer(entry); + if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) + return -EPERM; + } + + return 0; +} + +static int dmirror_atomic_map(unsigned long start, unsigned long end, + struct page **pages, struct dmirror *dmirror) +{ + unsigned long pfn, mapped = 0; + int i; + + /* Map the migrated pages into the device's page tables. */ + mutex_lock(&dmirror->mutex); + + for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) { + void *entry; + + if (!pages[i]) + continue; + + entry = pages[i]; + entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); + entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); + if (xa_is_err(entry)) { + mutex_unlock(&dmirror->mutex); + return xa_err(entry); + } + + mapped++; + } + + mutex_unlock(&dmirror->mutex); + return mapped; +} + static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, struct dmirror *dmirror) { @@ -661,6 +711,72 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, return 0; } +static int dmirror_exclusive(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + unsigned long start, end, addr; + unsigned long size = cmd->npages << PAGE_SHIFT; + struct mm_struct *mm = dmirror->notifier.mm; + struct page *pages[64]; + struct dmirror_bounce bounce; + unsigned long next; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + /* Since the mm is for the mirrored process, get a reference first. */ + if (!mmget_not_zero(mm)) + return -EINVAL; + + mmap_read_lock(mm); + for (addr = start; addr < end; addr = next) { + unsigned long mapped; + int i; + + if (end < addr + (ARRAY_SIZE(pages) << PAGE_SHIFT)) + next = end; + else + next = addr + (ARRAY_SIZE(pages) << PAGE_SHIFT); + + ret = make_device_exclusive_range(mm, addr, next, pages, NULL); + mapped = dmirror_atomic_map(addr, next, pages, dmirror); + for (i = 0; i < ret; i++) { + if (pages[i]) { + unlock_page(pages[i]); + put_page(pages[i]); + } + } + + if (addr + (mapped << PAGE_SHIFT) < next) { + mmap_read_unlock(mm); + mmput(mm); + return -EBUSY; + } + } + mmap_read_unlock(mm); + mmput(mm); + + /* Return the migrated data for verification. */ + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_migrate(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { @@ -948,6 +1064,15 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, ret = dmirror_migrate(dmirror, &cmd); break; + case HMM_DMIRROR_EXCLUSIVE: + ret = dmirror_exclusive(dmirror, &cmd); + break; + + case HMM_DMIRROR_CHECK_EXCLUSIVE: + ret = dmirror_check_atomic(dmirror, cmd.addr, + cmd.addr + (cmd.npages << PAGE_SHIFT)); + break; + case HMM_DMIRROR_SNAPSHOT: ret = dmirror_snapshot(dmirror, &cmd); break; diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index 670b4ef2a5b6..f14dea5dcd06 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -33,6 +33,8 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_WRITE _IOWR('H', 0x01, struct hmm_dmirror_cmd) #define HMM_DMIRROR_MIGRATE _IOWR('H', 0x02, struct hmm_dmirror_cmd) #define HMM_DMIRROR_SNAPSHOT _IOWR('H', 0x03, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_EXCLUSIVE _IOWR('H', 0x04, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x05, struct hmm_dmirror_cmd) /* * Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT. -- cgit From 92aeda50d4a96b7a30fc87960497d5e15b7428f7 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 30 Jun 2021 18:55:02 -0700 Subject: lib: decompress_bunzip2: remove an unneeded semicolon The semicolon immediately following '}' is unneeded. Link: https://lkml.kernel.org/r/20210508094926.2889-1-thunder.leizhen@huawei.com Signed-off-by: Zhen Lei Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/decompress_bunzip2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c index c72c865032fa..0d619cc129dd 100644 --- a/lib/decompress_bunzip2.c +++ b/lib/decompress_bunzip2.c @@ -385,7 +385,7 @@ static int INIT get_next_block(struct bunzip_data *bd) bd->inbufBits = (bd->inbufBits << 8)|bd->inbuf[bd->inbufPos++]; bd->inbufBitCount += 8; - }; + } bd->inbufBitCount -= hufGroup->maxLen; j = (bd->inbufBits >> bd->inbufBitCount)& ((1 << hufGroup->maxLen)-1); -- cgit From 62519b882d7485bae4c0a7e1e0adb576610400a9 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:08 -0700 Subject: lib/string_helpers: move ESCAPE_NP check inside 'else' branch in a loop Refactor code to have better readability by moving ESCAPE_NP handling inside 'else' branch in the loop. No functional change intended. Link: https://lkml.kernel.org/r/20210504180819.73127-3-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string_helpers.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 7f2d5fbaf243..b10a18b4663b 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -452,10 +452,10 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * The process of escaping byte buffer includes several parts. They are applied * in the following sequence. * - * 1. The character is matched to the printable class, if asked, and in - * case of match it passes through to the output. - * 2. The character is not matched to the one from @only string and thus + * 1. The character is not matched to the one from @only string and thus * must go as-is to the output. + * 2. The character is matched to the printable class, if asked, and in + * case of match it passes through to the output. * 3. The character is checked if it falls into the class given by @flags. * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any * character. Note that they actually can't go together, otherwise @@ -506,19 +506,22 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, /* * Apply rules in the following sequence: - * - the character is printable, when @flags has - * %ESCAPE_NP bit set * - the @only string is supplied and does not contain a * character under question + * - the character is printable, when @flags has + * %ESCAPE_NP bit set * - the character doesn't fall into a class of symbols * defined by given @flags * In these cases we just pass through a character to the * output buffer. */ - if ((flags & ESCAPE_NP && isprint(c)) || - (is_dict && !strchr(only, c))) { + if (is_dict && !strchr(only, c)) { /* do nothing */ } else { + if (isprint(c) && + flags & ESCAPE_NP && escape_passthrough(c, &p, end)) + continue; + if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) continue; -- cgit From 7e5969aeb7f1e7d6f68d5501a6c040605272763e Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:11 -0700 Subject: lib/string_helpers: drop indentation level in string_escape_mem() The only one conditional is left on the upper level, move the rest to the same level and drop indentation level. No functional changes. Link: https://lkml.kernel.org/r/20210504180819.73127-4-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string_helpers.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index b10a18b4663b..e3ef9f86cc34 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -515,29 +515,29 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, * In these cases we just pass through a character to the * output buffer. */ - if (is_dict && !strchr(only, c)) { - /* do nothing */ - } else { - if (isprint(c) && - flags & ESCAPE_NP && escape_passthrough(c, &p, end)) - continue; + if (is_dict && !strchr(only, c) && + escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) - continue; + if (isprint(c) && + flags & ESCAPE_NP && escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end)) - continue; + if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) + continue; - if (flags & ESCAPE_NULL && escape_null(c, &p, end)) - continue; + if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end)) + continue; - /* ESCAPE_OCTAL and ESCAPE_HEX always go last */ - if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end)) - continue; + if (flags & ESCAPE_NULL && escape_null(c, &p, end)) + continue; - if (flags & ESCAPE_HEX && escape_hex(c, &p, end)) - continue; - } + /* ESCAPE_OCTAL and ESCAPE_HEX always go last */ + if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end)) + continue; + + if (flags & ESCAPE_HEX && escape_hex(c, &p, end)) + continue; escape_passthrough(c, &p, end); } -- cgit From a0809783355cfe1cc1b2fa7f881c3a79df0b2a27 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:14 -0700 Subject: lib/string_helpers: introduce ESCAPE_NA for escaping non-ASCII Some users may want to have an ASCII based filter, provided by isascii() function. Here is the addition of a such. Link: https://lkml.kernel.org/r/20210504180819.73127-5-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string_helpers.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index e3ef9f86cc34..a963404b8c16 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -454,8 +454,8 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * * 1. The character is not matched to the one from @only string and thus * must go as-is to the output. - * 2. The character is matched to the printable class, if asked, and in - * case of match it passes through to the output. + * 2. The character is matched to the printable or ASCII class, if asked, + * and in case of match it passes through to the output. * 3. The character is checked if it falls into the class given by @flags. * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any * character. Note that they actually can't go together, otherwise @@ -463,7 +463,7 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * * Caller must provide valid source and destination pointers. Be aware that * destination buffer will not be NULL-terminated, thus caller have to append - * it if needs. The supported flags are:: + * it if needs. The supported flags are:: * * %ESCAPE_SPACE: (special white space, not space itself) * '\f' - form feed @@ -482,11 +482,18 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * %ESCAPE_ANY: * all previous together * %ESCAPE_NP: - * escape only non-printable characters (checked by isprint) + * escape only non-printable characters, checked by isprint() * %ESCAPE_ANY_NP: * all previous together * %ESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (2 digits) + * %ESCAPE_NA: + * escape only non-ascii characters, checked by isascii() + * + * One notable caveat, the %ESCAPE_NP and %ESCAPE_NA have higher priority + * than the rest of the flags (%ESCAPE_NP is higher than %ESCAPE_NA). + * It doesn't make much sense to use either of them without %ESCAPE_OCTAL + * or %ESCAPE_HEX, because they cover most of the other character classes. * * Return: * The total size of the escaped output that would be generated for @@ -510,6 +517,8 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, * character under question * - the character is printable, when @flags has * %ESCAPE_NP bit set + * - the character is ASCII, when @flags has + * %ESCAPE_NA bit set * - the character doesn't fall into a class of symbols * defined by given @flags * In these cases we just pass through a character to the @@ -523,6 +532,10 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, flags & ESCAPE_NP && escape_passthrough(c, &p, end)) continue; + if (isascii(c) && + flags & ESCAPE_NA && escape_passthrough(c, &p, end)) + continue; + if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) continue; -- cgit From 0362c27fb373ea04eace9e7a70e61036ab81f09f Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:17 -0700 Subject: lib/string_helpers: introduce ESCAPE_NAP to escape non-ASCII and non-printable Some users may want to have an ASCII based filter for printable only characters, provided by conjunction of isascii() and isprint() functions. Here is the addition of a such. Link: https://lkml.kernel.org/r/20210504180819.73127-6-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string_helpers.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index a963404b8c16..ceca5fbbd92c 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -454,9 +454,11 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * * 1. The character is not matched to the one from @only string and thus * must go as-is to the output. - * 2. The character is matched to the printable or ASCII class, if asked, + * 2. The character is matched to the printable and ASCII classes, if asked, * and in case of match it passes through to the output. - * 3. The character is checked if it falls into the class given by @flags. + * 3. The character is matched to the printable or ASCII class, if asked, + * and in case of match it passes through to the output. + * 4. The character is checked if it falls into the class given by @flags. * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any * character. Note that they actually can't go together, otherwise * %ESCAPE_HEX will be ignored. @@ -489,11 +491,15 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * '\xHH' - byte with hexadecimal value HH (2 digits) * %ESCAPE_NA: * escape only non-ascii characters, checked by isascii() + * %ESCAPE_NAP: + * escape only non-printable or non-ascii characters * - * One notable caveat, the %ESCAPE_NP and %ESCAPE_NA have higher priority - * than the rest of the flags (%ESCAPE_NP is higher than %ESCAPE_NA). + * One notable caveat, the %ESCAPE_NAP, %ESCAPE_NP and %ESCAPE_NA have the + * higher priority than the rest of the flags (%ESCAPE_NAP is the highest). * It doesn't make much sense to use either of them without %ESCAPE_OCTAL * or %ESCAPE_HEX, because they cover most of the other character classes. + * %ESCAPE_NAP can utilize %ESCAPE_SPACE or %ESCAPE_SPECIAL in addition to + * the above. * * Return: * The total size of the escaped output that would be generated for @@ -515,6 +521,8 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, * Apply rules in the following sequence: * - the @only string is supplied and does not contain a * character under question + * - the character is printable and ASCII, when @flags has + * %ESCAPE_NAP bit set * - the character is printable, when @flags has * %ESCAPE_NP bit set * - the character is ASCII, when @flags has @@ -528,6 +536,10 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, escape_passthrough(c, &p, end)) continue; + if (isascii(c) && isprint(c) && + flags & ESCAPE_NAP && escape_passthrough(c, &p, end)) + continue; + if (isprint(c) && flags & ESCAPE_NP && escape_passthrough(c, &p, end)) continue; -- cgit From aec0d0966f20d131cc4ff6927b02d448a478a6d4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:20 -0700 Subject: lib/string_helpers: allow to append additional characters to be escaped Introduce a new flag to append additional characters, passed in 'only' parameter, to be escaped if they fall in the corresponding class. Link: https://lkml.kernel.org/r/20210504180819.73127-7-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string_helpers.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index ceca5fbbd92c..c15aea7a82e9 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -493,6 +493,11 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * escape only non-ascii characters, checked by isascii() * %ESCAPE_NAP: * escape only non-printable or non-ascii characters + * %ESCAPE_APPEND: + * append characters from @only to be escaped by the given classes + * + * %ESCAPE_APPEND would help to pass additional characters to the escaped, when + * one of %ESCAPE_NP, %ESCAPE_NA, or %ESCAPE_NAP is provided. * * One notable caveat, the %ESCAPE_NAP, %ESCAPE_NP and %ESCAPE_NA have the * higher priority than the rest of the flags (%ESCAPE_NAP is the highest). @@ -513,9 +518,11 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, char *p = dst; char *end = p + osz; bool is_dict = only && *only; + bool is_append = flags & ESCAPE_APPEND; while (isz--) { unsigned char c = *src++; + bool in_dict = is_dict && strchr(only, c); /* * Apply rules in the following sequence: @@ -531,20 +538,24 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, * defined by given @flags * In these cases we just pass through a character to the * output buffer. + * + * When %ESCAPE_APPEND is passed, the characters from @only + * have been excluded from the %ESCAPE_NAP, %ESCAPE_NP, and + * %ESCAPE_NA cases. */ - if (is_dict && !strchr(only, c) && + if (!(is_append || in_dict) && is_dict && escape_passthrough(c, &p, end)) continue; - if (isascii(c) && isprint(c) && + if (!(is_append && in_dict) && isascii(c) && isprint(c) && flags & ESCAPE_NAP && escape_passthrough(c, &p, end)) continue; - if (isprint(c) && + if (!(is_append && in_dict) && isprint(c) && flags & ESCAPE_NP && escape_passthrough(c, &p, end)) continue; - if (isascii(c) && + if (!(is_append && in_dict) && isascii(c) && flags & ESCAPE_NA && escape_passthrough(c, &p, end)) continue; -- cgit From 229563b196ed3ce36036a18b6bdfe4cce9dcbbd4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:23 -0700 Subject: lib/test-string_helpers: print flags in hexadecimal format Since flags are bitmapped, it's better to print them in hexadecimal format. Link: https://lkml.kernel.org/r/20210504180819.73127-8-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test-string_helpers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 10360d4ea273..079fb12d59cc 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -19,7 +19,7 @@ static __init bool test_string_check_buf(const char *name, unsigned int flags, if (q_real == q_test && !memcmp(out_test, out_real, q_test)) return true; - pr_warn("Test '%s' failed: flags = %u\n", name, flags); + pr_warn("Test '%s' failed: flags = %#x\n", name, flags); print_hex_dump(KERN_WARNING, "Input: ", DUMP_PREFIX_NONE, 16, 1, in, p, true); @@ -290,7 +290,7 @@ test_string_escape_overflow(const char *in, int p, unsigned int flags, const cha q_real = string_escape_mem(in, p, NULL, 0, flags, esc); if (q_real != q_test) - pr_warn("Test '%s' failed: flags = %u, osz = 0, expected %d, got %d\n", + pr_warn("Test '%s' failed: flags = %#x, osz = 0, expected %d, got %d\n", name, flags, q_test, q_real); } -- cgit From 69325698df55c609da96ebbd592e59d88c4d335d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:26 -0700 Subject: lib/test-string_helpers: get rid of trailing comma in terminators Terminators by definition shouldn't accept anything behind. Make them robust by removing trailing commas. Link: https://lkml.kernel.org/r/20210504180819.73127-9-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test-string_helpers.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 079fb12d59cc..3e2def9ccfac 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -136,7 +136,7 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_SPACE | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\\h\\\"\a\e\\", .s1 = {{ @@ -150,7 +150,7 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_SPECIAL | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\eb \\C\007\"\x90\r]", .s1 = {{ @@ -201,7 +201,7 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_NP | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ /* terminator */ }}; @@ -217,7 +217,7 @@ static const struct test_string_2 escape1[] __initconst = {{ .flags = ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\\h\\\"\a\e\\", .s1 = {{ @@ -225,7 +225,7 @@ static const struct test_string_2 escape1[] __initconst = {{ .flags = ESCAPE_OCTAL, },{ /* terminator */ - }}, + }} },{ .in = "\eb \\C\007\"\x90\r]", .s1 = {{ @@ -233,7 +233,7 @@ static const struct test_string_2 escape1[] __initconst = {{ .flags = ESCAPE_OCTAL, },{ /* terminator */ - }}, + }} },{ /* terminator */ }}; -- cgit From 259fa5d7d825122c30ad4122c6a1cc937eb74c2d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:29 -0700 Subject: lib/test-string_helpers: add test cases for new features We have got new flags and hence new features of string_escape_mem(). Add test cases for that. Link: https://lkml.kernel.org/r/20210504180819.73127-10-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test-string_helpers.c | 141 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 3e2def9ccfac..2185d71704f0 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -202,11 +202,25 @@ static const struct test_string_2 escape0[] __initconst = {{ },{ /* terminator */ }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\\220\\317\r", + .flags = ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\\x90\\xcf\r", + .flags = ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NA, + },{ + /* terminator */ + }} },{ /* terminator */ }}; -#define TEST_STRING_2_DICT_1 "b\\ \t\r" +#define TEST_STRING_2_DICT_1 "b\\ \t\r\xCF" static const struct test_string_2 escape1[] __initconst = {{ .in = "\f\\ \n\r\t\v", .s1 = {{ @@ -215,14 +229,38 @@ static const struct test_string_2 escape1[] __initconst = {{ },{ .out = "\f\\x5c\\x20\n\\x0d\\x09\v", .flags = ESCAPE_HEX, + },{ + .out = "\f\\134\\040\n\\015\\011\v", + .flags = ESCAPE_ANY | ESCAPE_APPEND, + },{ + .out = "\\014\\134\\040\\012\\015\\011\\013", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\\x0c\\x5c\\x20\\x0a\\x0d\\x09\\x0b", + .flags = ESCAPE_HEX | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\f\\134\\040\n\\015\\011\v", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NA, + },{ + .out = "\f\\x5c\\x20\n\\x0d\\x09\v", + .flags = ESCAPE_HEX | ESCAPE_APPEND | ESCAPE_NA, },{ /* terminator */ }} },{ - .in = "\\h\\\"\a\e\\", + .in = "\\h\\\"\a\xCF\e\\", .s1 = {{ - .out = "\\134h\\134\"\a\e\\134", + .out = "\\134h\\134\"\a\\317\e\\134", .flags = ESCAPE_OCTAL, + },{ + .out = "\\134h\\134\"\a\\317\e\\134", + .flags = ESCAPE_ANY | ESCAPE_APPEND, + },{ + .out = "\\134h\\134\"\\007\\317\\033\\134", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\\134h\\134\"\a\\317\e\\134", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NA, },{ /* terminator */ }} @@ -234,6 +272,88 @@ static const struct test_string_2 escape1[] __initconst = {{ },{ /* terminator */ }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPACE | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPECIAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_SPACE | ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_ANY | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPACE | ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NA, + },{ + /* terminator */ + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\\r", + .flags = ESCAPE_SPACE | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPECIAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\015", + .flags = ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\r", + .flags = ESCAPE_SPACE | ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\015", + .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_ANY | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\x0d", + .flags = ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\r", + .flags = ESCAPE_SPACE | ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\x0d", + .flags = ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NAP, + },{ + /* terminator */ + }} },{ /* terminator */ }}; @@ -315,8 +435,13 @@ static __init void test_string_escape(const char *name, /* NULL injection */ if (flags & ESCAPE_NULL) { in[p++] = '\0'; - out_test[q_test++] = '\\'; - out_test[q_test++] = '0'; + /* '\0' passes isascii() test */ + if (flags & ESCAPE_NA && !(flags & ESCAPE_APPEND && esc)) { + out_test[q_test++] = '\0'; + } else { + out_test[q_test++] = '\\'; + out_test[q_test++] = '0'; + } } /* Don't try strings that have no output */ @@ -459,17 +584,17 @@ static int __init test_string_helpers_init(void) unsigned int i; pr_info("Running tests...\n"); - for (i = 0; i < UNESCAPE_ANY + 1; i++) + for (i = 0; i < UNESCAPE_ALL_MASK + 1; i++) test_string_unescape("unescape", i, false); test_string_unescape("unescape inplace", get_random_int() % (UNESCAPE_ANY + 1), true); /* Without dictionary */ - for (i = 0; i < (ESCAPE_ANY_NP | ESCAPE_HEX) + 1; i++) + for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) test_string_escape("escape 0", escape0, i, TEST_STRING_2_DICT_0); /* With dictionary */ - for (i = 0; i < (ESCAPE_ANY_NP | ESCAPE_HEX) + 1; i++) + for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) test_string_escape("escape 1", escape1, i, TEST_STRING_2_DICT_1); /* Test string_get_size() */ -- cgit From cc72181a65990193f54284417efa01d4580014e6 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:55:46 -0700 Subject: seq_file: drop unused *_escape_mem_ascii() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are no more users of the seq_escape_mem_ascii() followed by string_escape_mem_ascii(). Remove them for good. Link: https://lkml.kernel.org/r/20210504180819.73127-16-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alexander Viro Cc: Chuck Lever Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string_helpers.c | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index c15aea7a82e9..5a35c7e16e96 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -582,25 +582,6 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, } EXPORT_SYMBOL(string_escape_mem); -int string_escape_mem_ascii(const char *src, size_t isz, char *dst, - size_t osz) -{ - char *p = dst; - char *end = p + osz; - - while (isz--) { - unsigned char c = *src++; - - if (!isprint(c) || !isascii(c) || c == '"' || c == '\\') - escape_hex(c, &p, end); - else - escape_passthrough(c, &p, end); - } - - return p - dst; -} -EXPORT_SYMBOL(string_escape_mem_ascii); - /* * Return an allocated string that has been escaped of special characters * and double quotes, making it safe to log in quotes. -- cgit From 65a0d3c14685663ba111038a35db70f559e39336 Mon Sep 17 00:00:00 2001 From: Trent Piepho Date: Wed, 30 Jun 2021 18:55:49 -0700 Subject: lib/math/rational.c: fix divide by zero If the input is out of the range of the allowed values, either larger than the largest value or closer to zero than the smallest non-zero allowed value, then a division by zero would occur. In the case of input too large, the division by zero will occur on the first iteration. The best result (largest allowed value) will be found by always choosing the semi-convergent and excluding the denominator based limit when finding it. In the case of the input too small, the division by zero will occur on the second iteration. The numerator based semi-convergent should not be calculated to avoid the division by zero. But the semi-convergent vs previous convergent test is still needed, which effectively chooses between 0 (the previous convergent) vs the smallest allowed fraction (best semi-convergent) as the result. Link: https://lkml.kernel.org/r/20210525144250.214670-1-tpiepho@gmail.com Fixes: 323dd2c3ed0 ("lib/math/rational.c: fix possible incorrect result from rational fractions helper") Signed-off-by: Trent Piepho Reported-by: Yiyuan Guo Reviewed-by: Andy Shevchenko Cc: Oskar Schirmer Cc: Daniel Latypov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/math/rational.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/math/rational.c b/lib/math/rational.c index 9781d521963d..c0ab51d8fbb9 100644 --- a/lib/math/rational.c +++ b/lib/math/rational.c @@ -12,6 +12,7 @@ #include #include #include +#include /* * calculate best rational approximation for a given fraction @@ -78,13 +79,18 @@ void rational_best_approximation( * found below as 't'. */ if ((n2 > max_numerator) || (d2 > max_denominator)) { - unsigned long t = min((max_numerator - n0) / n1, - (max_denominator - d0) / d1); + unsigned long t = ULONG_MAX; - /* This tests if the semi-convergent is closer - * than the previous convergent. + if (d1) + t = (max_denominator - d0) / d1; + if (n1) + t = min(t, (max_numerator - n0) / n1); + + /* This tests if the semi-convergent is closer than the previous + * convergent. If d1 is zero there is no previous convergent as this + * is the 1st iteration, so always choose the semi-convergent. */ - if (2u * t > a || (2u * t == a && d0 * dp > d1 * d)) { + if (!d1 || 2u * t > a || (2u * t == a && d0 * dp > d1 * d)) { n1 = n0 + t * n1; d1 = d0 + t * d1; } -- cgit From b6c75c4afceb8bc065a4ebb5c6c381452bf96f53 Mon Sep 17 00:00:00 2001 From: Trent Piepho Date: Wed, 30 Jun 2021 18:55:52 -0700 Subject: lib/math/rational: add Kunit test cases Adds a number of test cases that cover a range of possible code paths. [akpm@linux-foundation.org: remove non-ascii characters, fix whitespace] [colin.king@canonical.com: fix spelling mistake "demominator" -> "denominator"] Link: https://lkml.kernel.org/r/20210526085049.6393-1-colin.king@canonical.com Link: https://lkml.kernel.org/r/20210525144250.214670-2-tpiepho@gmail.com Signed-off-by: Trent Piepho Signed-off-by: Colin Ian King Reviewed-by: Andy Shevchenko Cc: Daniel Latypov Cc: Oskar Schirmer Cc: Yiyuan Guo Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 12 +++++++++++ lib/math/Makefile | 1 + lib/math/rational-test.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 lib/math/rational-test.c (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index deca67d28abb..ea9a4096007d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2444,6 +2444,18 @@ config SLUB_KUNIT_TEST If unsure, say N. +config RATIONAL_KUNIT_TEST + tristate "KUnit test for rational.c" if !KUNIT_ALL_TESTS + depends on KUNIT + select RATIONAL + default KUNIT_ALL_TESTS + help + This builds the rational math unit test. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config TEST_UDELAY tristate "udelay test driver" help diff --git a/lib/math/Makefile b/lib/math/Makefile index 7456edb864fc..bfac26ddfc22 100644 --- a/lib/math/Makefile +++ b/lib/math/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o obj-$(CONFIG_RATIONAL) += rational.o obj-$(CONFIG_TEST_DIV64) += test_div64.o +obj-$(CONFIG_RATIONAL_KUNIT_TEST) += rational-test.o diff --git a/lib/math/rational-test.c b/lib/math/rational-test.c new file mode 100644 index 000000000000..01611ddff420 --- /dev/null +++ b/lib/math/rational-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include + +struct rational_test_param { + unsigned long num, den; + unsigned long max_num, max_den; + unsigned long exp_num, exp_den; + + const char *name; +}; + +static const struct rational_test_param test_parameters[] = { + { 1230, 10, 100, 20, 100, 1, "Exceeds bounds, semi-convergent term > 1/2 last term" }, + { 34567,100, 120, 20, 120, 1, "Exceeds bounds, semi-convergent term < 1/2 last term" }, + { 1, 30, 100, 10, 0, 1, "Closest to zero" }, + { 1, 19, 100, 10, 1, 10, "Closest to smallest non-zero" }, + { 27,32, 16, 16, 11, 13, "Use convergent" }, + { 1155, 7735, 255, 255, 33, 221, "Exact answer" }, + { 87, 32, 70, 32, 68, 25, "Semiconvergent, numerator limit" }, + { 14533, 4626, 15000, 2400, 7433, 2366, "Semiconvergent, denominator limit" }, +}; + +static void get_desc(const struct rational_test_param *param, char *desc) +{ + strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE); +} + +/* Creates function rational_gen_params */ +KUNIT_ARRAY_PARAM(rational, test_parameters, get_desc); + +static void rational_test(struct kunit *test) +{ + const struct rational_test_param *param = (const struct rational_test_param *)test->param_value; + unsigned long n = 0, d = 0; + + rational_best_approximation(param->num, param->den, param->max_num, param->max_den, &n, &d); + KUNIT_EXPECT_EQ(test, n, param->exp_num); + KUNIT_EXPECT_EQ(test, d, param->exp_den); +} + +static struct kunit_case rational_test_cases[] = { + KUNIT_CASE_PARAM(rational_test, rational_gen_params), + {} +}; + +static struct kunit_suite rational_test_suite = { + .name = "rational", + .test_cases = rational_test_cases, +}; + +kunit_test_suites(&rational_test_suite); + +MODULE_LICENSE("GPL v2"); -- cgit From 05911c5d964956442d17fe21db239de5a1dace4a Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 30 Jun 2021 18:55:55 -0700 Subject: lib/decompressors: fix spelling mistakes Fix some spelling mistakes in comments: sentinal ==> sentinel compresed ==> compressed dependeny ==> dependency immediatelly ==> immediately dervied ==> derived splitted ==> split nore ==> not independed ==> independent asumed ==> assumed Link: https://lkml.kernel.org/r/20210604085656.12257-1-thunder.leizhen@huawei.com Signed-off-by: Zhen Lei Cc: Jiri Kosina Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/decompress_bunzip2.c | 4 ++-- lib/decompress_unxz.c | 2 +- lib/decompress_unzstd.c | 4 ++-- lib/xz/xz_dec_bcj.c | 2 +- lib/xz/xz_dec_lzma2.c | 8 ++++---- lib/zlib_inflate/inffast.c | 2 +- lib/zstd/huf.h | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c index 0d619cc129dd..3518e7394eca 100644 --- a/lib/decompress_bunzip2.c +++ b/lib/decompress_bunzip2.c @@ -80,7 +80,7 @@ /* This is what we know about each Huffman coding group */ struct group_data { - /* We have an extra slot at the end of limit[] for a sentinal value. */ + /* We have an extra slot at the end of limit[] for a sentinel value. */ int limit[MAX_HUFCODE_BITS+1]; int base[MAX_HUFCODE_BITS]; int permute[MAX_SYMBOLS]; @@ -337,7 +337,7 @@ static int INIT get_next_block(struct bunzip_data *bd) pp <<= 1; base[i+1] = pp-(t += temp[i]); } - limit[maxLen+1] = INT_MAX; /* Sentinal value for + limit[maxLen+1] = INT_MAX; /* Sentinel value for * reading next sym. */ limit[maxLen] = pp+temp[maxLen]-1; base[minLen] = 0; diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 25d59a95bd66..a2f38e23004a 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -23,7 +23,7 @@ * uncompressible. Thus, we must look for worst-case expansion when the * compressor is encoding uncompressible data. * - * The structure of the .xz file in case of a compresed kernel is as follows. + * The structure of the .xz file in case of a compressed kernel is as follows. * Sizes (as bytes) of the fields are in parenthesis. * * Stream Header (12) diff --git a/lib/decompress_unzstd.c b/lib/decompress_unzstd.c index 790abc472f5b..6b629ab31c1e 100644 --- a/lib/decompress_unzstd.c +++ b/lib/decompress_unzstd.c @@ -16,7 +16,7 @@ * uncompressible. Thus, we must look for worst-case expansion when the * compressor is encoding uncompressible data. * - * The structure of the .zst file in case of a compresed kernel is as follows. + * The structure of the .zst file in case of a compressed kernel is as follows. * Maximum sizes (as bytes) of the fields are in parenthesis. * * Frame Header: (18) @@ -56,7 +56,7 @@ /* * Preboot environments #include "path/to/decompress_unzstd.c". * All of the source files we depend on must be #included. - * zstd's only source dependeny is xxhash, which has no source + * zstd's only source dependency is xxhash, which has no source * dependencies. * * When UNZSTD_PREBOOT is defined we declare __decompress(), which is diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index 72ddac6ef2ec..ef449e97d1a1 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -422,7 +422,7 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, /* * Flush pending already filtered data to the output buffer. Return - * immediatelly if we couldn't flush everything, or if the next + * immediately if we couldn't flush everything, or if the next * filter in the chain had already returned XZ_STREAM_END. */ if (s->temp.filtered > 0) { diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index ca2603abee08..7a6781e3f47b 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -147,8 +147,8 @@ struct lzma_dec { /* * LZMA properties or related bit masks (number of literal - * context bits, a mask dervied from the number of literal - * position bits, and a mask dervied from the number + * context bits, a mask derived from the number of literal + * position bits, and a mask derived from the number * position bits) */ uint32_t lc; @@ -484,7 +484,7 @@ static __always_inline void rc_normalize(struct rc_dec *rc) } /* - * Decode one bit. In some versions, this function has been splitted in three + * Decode one bit. In some versions, this function has been split in three * functions so that the compiler is supposed to be able to more easily avoid * an extra branch. In this particular version of the LZMA decoder, this * doesn't seem to be a good idea (tested with GCC 3.3.6, 3.4.6, and 4.3.3 @@ -761,7 +761,7 @@ static bool lzma_main(struct xz_dec_lzma2 *s) } /* - * Reset the LZMA decoder and range decoder state. Dictionary is nore reset + * Reset the LZMA decoder and range decoder state. Dictionary is not reset * here, because LZMA state may be reset without resetting the dictionary. */ static void lzma_reset(struct xz_dec_lzma2 *s) diff --git a/lib/zlib_inflate/inffast.c b/lib/zlib_inflate/inffast.c index ed1f3df27260..f19c4fbe1be7 100644 --- a/lib/zlib_inflate/inffast.c +++ b/lib/zlib_inflate/inffast.c @@ -15,7 +15,7 @@ union uu { unsigned char b[2]; }; -/* Endian independed version */ +/* Endian independent version */ static inline unsigned short get_unaligned16(const unsigned short *p) { diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h index 2143da28d952..923218d12e28 100644 --- a/lib/zstd/huf.h +++ b/lib/zstd/huf.h @@ -134,7 +134,7 @@ typedef enum { HUF_repeat_none, /**< Cannot use the previous table */ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ - HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. -- cgit From 478485f6c0e5936b62c0c9393a865bfb00f037a5 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 30 Jun 2021 18:55:58 -0700 Subject: lib/mpi: fix spelling mistakes Fix some spelling mistakes in comments: flaged ==> flagged bufer ==> buffer multipler ==> multiplier MULTIPLER ==> MULTIPLIER leaset ==> least chnage ==> change Link: https://lkml.kernel.org/r/20210604074401.12198-1-thunder.leizhen@huawei.com Signed-off-by: Zhen Lei Cc: Herbert Xu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/mpi/longlong.h | 4 ++-- lib/mpi/mpicoder.c | 6 +++--- lib/mpi/mpiutil.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h index afbd99987cf8..b6fa1d08fb55 100644 --- a/lib/mpi/longlong.h +++ b/lib/mpi/longlong.h @@ -48,8 +48,8 @@ /* Define auxiliary asm macros. * - * 1) umul_ppmm(high_prod, low_prod, multipler, multiplicand) multiplies two - * UWtype integers MULTIPLER and MULTIPLICAND, and generates a two UWtype + * 1) umul_ppmm(high_prod, low_prod, multiplier, multiplicand) multiplies two + * UWtype integers MULTIPLIER and MULTIPLICAND, and generates a two UWtype * word product in HIGH_PROD and LOW_PROD. * * 2) __umulsidi3(a,b) multiplies two UWtype integers A and B, and returns a diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c index 7ea225b2204f..39c4c6731094 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/mpi/mpicoder.c @@ -234,11 +234,11 @@ static int count_lzeros(MPI a) } /** - * mpi_read_buffer() - read MPI to a bufer provided by user (msb first) + * mpi_read_buffer() - read MPI to a buffer provided by user (msb first) * * @a: a multi precision integer - * @buf: bufer to which the output will be written to. Needs to be at - * leaset mpi_get_size(a) long. + * @buf: buffer to which the output will be written to. Needs to be at + * least mpi_get_size(a) long. * @buf_len: size of the buf. * @nbytes: receives the actual length of the data written on success and * the data to-be-written on -EOVERFLOW in case buf_len was too diff --git a/lib/mpi/mpiutil.c b/lib/mpi/mpiutil.c index 3c63710c20c6..9a75ca3f7edf 100644 --- a/lib/mpi/mpiutil.c +++ b/lib/mpi/mpiutil.c @@ -80,7 +80,7 @@ EXPORT_SYMBOL_GPL(mpi_const); /**************** * Note: It was a bad idea to use the number of limbs to allocate * because on a alpha the limbs are large but we normally need - * integers of n bits - So we should chnage this to bits (or bytes). + * integers of n bits - So we should change this to bits (or bytes). * * But mpi_alloc is used in a lot of places :-) */ -- cgit From 1a58be6277e4324c853babfd35890c2d5e171e8f Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Wed, 30 Jun 2021 18:56:01 -0700 Subject: lib: memscan() fixlet Generic version doesn't trucate second argument to char. Older brother memchr() does as do s390, sparc and i386 assembly versions. Fortunately, no code passes c >= 256. Link: https://lkml.kernel.org/r/YLv4cCf0t5UPdyK+@localhost.localdomain Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/string.c b/lib/string.c index 7548eb715ddb..77bd0b1d3296 100644 --- a/lib/string.c +++ b/lib/string.c @@ -977,7 +977,7 @@ void *memscan(void *addr, int c, size_t size) unsigned char *p = addr; while (size) { - if (*p == c) + if (*p == (unsigned char)c) return (void *)p; p++; size--; -- cgit From ad65dcef3a87c24d6c6156eae5e7b47311d6e3cf Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Wed, 30 Jun 2021 18:56:04 -0700 Subject: lib: uninline simple_strtoull() Gcc inlines simple_strtoull() too agressively. Given that all 4 signatures match, everything very efficiently calls or tailcalls into simple_strtoull(): ffffffff81da0240 : ffffffff81da0240: 80 3f 2d cmp BYTE PTR [rdi],0x2d ffffffff81da0243: 74 05 je ffffffff81da024a ffffffff81da0245: e9 76 ff ff ff jmp simple_strtoull ffffffff81da024a: 48 83 c7 01 add rdi,0x1 ffffffff81da024e: e8 6d ff ff ff call simple_strtoull ffffffff81da0253: 48 f7 d8 neg rax ffffffff81da0256: c3 ret Space savings (on F34-ish .config) add/remove: 0/0 grow/shrink: 1/3 up/down: 52/-313 (-261) Function old new delta vsscanf 2167 2219 +52 simple_strtoul 72 2 -70 simple_strtoll 143 23 -120 simple_strtol 143 20 -123 Link: https://lkml.kernel.org/r/YMO2zoOQk2eF34tn@localhost.localdomain Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/vsprintf.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/vsprintf.c b/lib/vsprintf.c index cc281f5895f9..30e1bc22105c 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -61,6 +61,7 @@ * * This function has caveats. Please use kstrtoull instead. */ +noinline unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) { unsigned long long result; -- cgit From ce71efd03916ea8fe45e9ef6bd6abe4c20734a57 Mon Sep 17 00:00:00 2001 From: Matteo Croce Date: Wed, 30 Jun 2021 18:56:07 -0700 Subject: lib/test_string.c: allow module removal The test_string module can't be removed because it lacks an exit hook. Since there is no reason for it to be permanent, add an empty one to allow module removal. Link: https://lkml.kernel.org/r/20210616234503.28678-1-mcroce@linux.microsoft.com Signed-off-by: Matteo Croce Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_string.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/test_string.c b/lib/test_string.c index 7b31f4a505bf..9dfd6f52de92 100644 --- a/lib/test_string.c +++ b/lib/test_string.c @@ -179,6 +179,10 @@ static __init int strnchr_selftest(void) return 0; } +static __exit void string_selftest_remove(void) +{ +} + static __init int string_selftest_init(void) { int test, subtest; @@ -216,4 +220,5 @@ fail: } module_init(string_selftest_init); +module_exit(string_selftest_remove); MODULE_LICENSE("GPL v2"); -- cgit From 4c52729377eab025b238caeed48994a39c3b73f2 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Jun 2021 18:56:10 -0700 Subject: kernel.h: split out kstrtox() and simple_strtox() to a separate header kernel.h is being used as a dump for all kinds of stuff for a long time. Here is the attempt to start cleaning it up by splitting out kstrtox() and simple_strtox() helpers. At the same time convert users in header and lib folders to use new header. Though for time being include new header back to kernel.h to avoid twisted indirected includes for existing users. [andy.shevchenko@gmail.com: fix documentation references] Link: https://lkml.kernel.org/r/20210615220003.377901-1-andy.shevchenko@gmail.com Link: https://lkml.kernel.org/r/20210611185815.44103-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Acked-by: Jonathan Cameron Cc: Francis Laniel Cc: Randy Dunlap Cc: Kars Mulder Cc: Trond Myklebust Cc: Anna Schumaker Cc: "J. Bruce Fields" Cc: Chuck Lever Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/kstrtox.c | 5 +++-- lib/parser.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/kstrtox.c b/lib/kstrtox.c index a118b0b1e9b2..a0cc8dcf4a35 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -14,11 +14,12 @@ */ #include #include -#include -#include #include +#include +#include #include #include + #include "kstrtox.h" const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) diff --git a/lib/parser.c b/lib/parser.c index f1a6d90b8c34..bcb23484100e 100644 --- a/lib/parser.c +++ b/lib/parser.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include -- cgit From 7fde9d6e839db604569ad5de5fbe7dd3cd8e2136 Mon Sep 17 00:00:00 2001 From: Rajat Asthana Date: Wed, 30 Jun 2021 18:56:13 -0700 Subject: lz4_decompress: declare LZ4_decompress_safe_withPrefix64k static Declare LZ4_decompress_safe_withPrefix64k as static to fix sparse warning: > warning: symbol 'LZ4_decompress_safe_withPrefix64k' was not declared. > Should it be static? Link: https://lkml.kernel.org/r/20210511154345.610569-1-thisisrast7@gmail.com Signed-off-by: Rajat Asthana Reviewed-by: Nick Terrell Cc: Gao Xiang Cc: Joe Perches Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/lz4/lz4_decompress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index 8a7724a6ce2f..926f4823d5ea 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -481,7 +481,7 @@ int LZ4_decompress_fast(const char *source, char *dest, int originalSize) /* ===== Instantiate a few more decoding cases, used more than once. ===== */ -int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, +static int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, -- cgit From 2c484419efc09e7234c667aa72698cb79ba8d8ed Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 30 Jun 2021 18:56:16 -0700 Subject: lib/decompress_unlz4.c: correctly handle zero-padding around initrds. lz4 compatible decompressor is simple. The format is underspecified and relies on EOF notification to determine when to stop. Initramfs buffer format[1] explicitly states that it can have arbitrary number of zero padding. Thus when operating without a fill function, be extra careful to ensure that sizes less than 4, or apperantly empty chunksizes are treated as EOF. To test this I have created two cpio initrds, first a normal one, main.cpio. And second one with just a single /test-file with content "second" second.cpio. Then i compressed both of them with gzip, and with lz4 -l. Then I created a padding of 4 bytes (dd if=/dev/zero of=pad4 bs=1 count=4). To create four testcase initrds: 1) main.cpio.gzip + extra.cpio.gzip = pad0.gzip 2) main.cpio.lz4 + extra.cpio.lz4 = pad0.lz4 3) main.cpio.gzip + pad4 + extra.cpio.gzip = pad4.gzip 4) main.cpio.lz4 + pad4 + extra.cpio.lz4 = pad4.lz4 The pad4 test-cases replicate the initrd load by grub, as it pads and aligns every initrd it loads. All of the above boot, however /test-file was not accessible in the initrd for the testcase #4, as decoding in lz4 decompressor failed. Also an error message printed which usually is harmless. Whith a patched kernel, all of the above testcases now pass, and /test-file is accessible. This fixes lz4 initrd decompress warning on every boot with grub. And more importantly this fixes inability to load multiple lz4 compressed initrds with grub. This patch has been shipping in Ubuntu kernels since January 2021. [1] ./Documentation/driver-api/early-userspace/buffer-format.rst BugLink: https://bugs.launchpad.net/bugs/1835660 Link: https://lore.kernel.org/lkml/20210114200256.196589-1-xnox@ubuntu.com/ # v0 Link: https://lkml.kernel.org/r/20210513104831.432975-1-dimitri.ledkov@canonical.com Signed-off-by: Dimitri John Ledkov Cc: Kyungsik Lee Cc: Yinghai Lu Cc: Bongkyu Kim Cc: Kees Cook Cc: Sven Schmidt <4sschmid@informatik.uni-hamburg.de> Cc: Rajat Asthana Cc: Nick Terrell Cc: Gao Xiang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/decompress_unlz4.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c index c0cfcfd486be..e6327391b6b6 100644 --- a/lib/decompress_unlz4.c +++ b/lib/decompress_unlz4.c @@ -112,6 +112,9 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, error("data corrupted"); goto exit_2; } + } else if (size < 4) { + /* empty or end-of-file */ + goto exit_3; } chunksize = get_unaligned_le32(inp); @@ -125,6 +128,10 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, continue; } + if (!fill && chunksize == 0) { + /* empty or end-of-file */ + goto exit_3; + } if (posp) *posp += 4; @@ -184,6 +191,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, } } +exit_3: ret = 0; exit_2: if (!input) -- cgit From 3b52348345b2cfe038d317de52bcdef788c6520d Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Wed, 30 Jun 2021 18:57:06 -0700 Subject: lib/decompressors: remove set but not used variabled 'level' Fixes gcc '-Wunused-but-set-variable' warning: lib/decompress_unlzo.c:46:5: warning: variable `level' set but not used [-Wunused-but-set-variable] It is never used and so can be removed. [akpm@linux-foundation.org: warning: value computed is not used] Link: https://lkml.kernel.org/r/20210514062050.3532344-1-yukuai3@huawei.com Fixes: 7dd65feb6c60 ("lib: add support for LZO-compressed kernels") Signed-off-by: Yu Kuai Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/decompress_unlzo.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c index 1f439a622076..64c1358500ce 100644 --- a/lib/decompress_unlzo.c +++ b/lib/decompress_unlzo.c @@ -43,7 +43,6 @@ STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len) int l; u8 *parse = input; u8 *end = input + in_len; - u8 level = 0; u16 version; /* @@ -65,7 +64,7 @@ STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len) version = get_unaligned_be16(parse); parse += 7; if (version >= 0x0940) - level = *parse++; + parse++; if (get_unaligned_be32(parse) & HEADER_HAS_FILTER) parse += 8; /* flags + filter info */ else -- cgit From a180bd1d7e16173d965b263c5a536aa40afa2a2a Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 4 Jul 2021 16:12:42 -0700 Subject: iov_iter: remove uaccess_kernel() warning from iov_iter_init() This warning was there to catch any architectures that still use CONFIG_SET_FS, and that would mis-use iov_iter_init() for anything that wasn't a proper user space pointer. So that WARN_ON_ONCE(uaccess_kernel()); makes perfect conceptual sense: you really shouldn't use a kernel pointer with set_fs(KERNEL_DS) and then pass it to iov_iter_init(). HOWEVER. Guenter Roeck reports that this warning actually triggers in no-mmu configurations of both ARM and m68k. And the reason isn't that they pass in a kernel pointer under set_fs(KERNEL_DS) at all: the reason is that in those configurations, "uaccess_kernel()" is simply not reliable. Those no-mmu setups set USER_DS and KERNEL_DS to the same values, so you can't test for the difference. In particular, the no-mmu case for ARM does #define USER_DS KERNEL_DS #define uaccess_kernel() (true) so USER_DS and KERNEL_DS have the same value, and uaccess_kernel() is always trivially true. The m68k case is slightly different and not quite as obvious. It does (spread out over multiple header files just to be extra exciting: asm/processor.h, asm/segment.h and asm-generic/uaccess.h): #define TASK_SIZE (0xFFFFFFFFUL) #define USER_DS MAKE_MM_SEG(TASK_SIZE) #define KERNEL_DS MAKE_MM_SEG(~0UL) #define get_fs() (current_thread_info()->addr_limit) #define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) but the end result is the same: uaccess_kernel() will always be true, because USER_DS and KERNEL_DS end up having the same value, even if that value is defined differently. This is very arguably a misfeature in those implementations, but in the end we don't really care. All modern architectures have gotten rid of set_fs() already, and generic kernel code never uses it. And while the sanity check was a nice idea, an architecture would have to go the extra mile to actually break this. So this well-intentioned warning isn't really all that likely to find anything but these known false positives, and as such just isn't worth maintaining. Reported-by: Guenter Roeck Fixes: 8cd54c1c8480 ("iov_iter: separate direction from flavour") Cc: Matthew Wilcox Cc: Al Viro Signed-off-by: Linus Torvalds --- lib/iov_iter.c | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 97e04c5dbeef..e23123ae3a13 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -465,7 +465,6 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - WARN_ON_ONCE(uaccess_kernel()); *i = (struct iov_iter) { .iter_type = ITER_IOVEC, .data_source = direction, -- cgit From 53b0fe36ab7c6eb3ce8ca711e636806649273463 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 7 Jul 2021 18:07:28 -0700 Subject: lib/test: fix spelling mistakes Fix some spelling mistakes in comments found by "codespell": thats ==> that's unitialized ==> uninitialized panicing ==> panicking sucess ==> success possitive ==> positive intepreted ==> interpreted Link: https://lkml.kernel.org/r/20210607133036.12525-2-thunder.leizhen@huawei.com Signed-off-by: Zhen Lei Acked-by: Yonghong Song [test_bfp.c] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_bitops.c | 2 +- lib/test_bpf.c | 2 +- lib/test_kasan.c | 2 +- lib/test_kmod.c | 6 +++--- lib/test_scanf.c | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/test_bitops.c b/lib/test_bitops.c index 471141ddd691..3b7bcbee84db 100644 --- a/lib/test_bitops.c +++ b/lib/test_bitops.c @@ -15,7 +15,7 @@ * get_count_order/long */ -/* use an enum because thats the most common BITMAP usage */ +/* use an enum because that's the most common BITMAP usage */ enum bitops_fun { BITOPS_4 = 4, BITOPS_7 = 7, diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 4dc4dcbecd12..d500320778c7 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -1095,7 +1095,7 @@ static struct bpf_test tests[] = { { "RET_A", .u.insns = { - /* check that unitialized X and A contain zeros */ + /* check that uninitialized X and A contain zeros */ BPF_STMT(BPF_MISC | BPF_TXA, 0), BPF_STMT(BPF_RET | BPF_A, 0) }, diff --git a/lib/test_kasan.c b/lib/test_kasan.c index f5f54766cbc2..8f7b0b2f6e11 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -651,7 +651,7 @@ static void kasan_global_oob(struct kunit *test) { /* * Deliberate out-of-bounds access. To prevent CONFIG_UBSAN_LOCAL_BOUNDS - * from failing here and panicing the kernel, access the array via a + * from failing here and panicking the kernel, access the array via a * volatile pointer, which will prevent the compiler from being able to * determine the array bounds. * diff --git a/lib/test_kmod.c b/lib/test_kmod.c index 38c250fbace3..ce1589391413 100644 --- a/lib/test_kmod.c +++ b/lib/test_kmod.c @@ -286,7 +286,7 @@ static int tally_work_test(struct kmod_test_device_info *info) * If this ran it means *all* tasks were created fine and we * are now just collecting results. * - * Only propagate errors, do not override with a subsequent sucess case. + * Only propagate errors, do not override with a subsequent success case. */ static void tally_up_work(struct kmod_test_device *test_dev) { @@ -543,7 +543,7 @@ static int trigger_config_run(struct kmod_test_device *test_dev) * wrong with the setup of the test. If the test setup went fine * then userspace must just check the result of config->test_result. * One issue with relying on the return from a call in the kernel - * is if the kernel returns a possitive value using this trigger + * is if the kernel returns a positive value using this trigger * will not return the value to userspace, it would be lost. * * By not relying on capturing the return value of tests we are using @@ -585,7 +585,7 @@ trigger_config_store(struct device *dev, * Note: any return > 0 will be treated as success * and the error value will not be available to userspace. * Do not rely on trying to send to userspace a test value - * return value as possitive return errors will be lost. + * return value as positive return errors will be lost. */ if (WARN_ON(ret > 0)) return -EINVAL; diff --git a/lib/test_scanf.c b/lib/test_scanf.c index 48ff5747a4da..84fe09eaf55e 100644 --- a/lib/test_scanf.c +++ b/lib/test_scanf.c @@ -600,7 +600,7 @@ static void __init numbers_prefix_overflow(void) /* * 0x prefix in a field of width 2 using %i conversion: first field * converts to 0. Next field scan starts at the character after "0x", - * which will convert if can be intepreted as decimal but will fail + * which will convert if can be interpreted as decimal but will fail * if it contains any hex digits (since no 0x prefix). */ test_number_prefix(long long, "0x67", "%2lli%lli", 0, 67, 2, check_ll); -- cgit From 9dbbc3b9d09d6deba9f3b9e1d5b355032ed46a75 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 7 Jul 2021 18:07:31 -0700 Subject: lib: fix spelling mistakes Fix some spelling mistakes in comments: permanentely ==> permanently wont ==> won't remaning ==> remaining succed ==> succeed shouldnt ==> shouldn't alpha-numeric ==> alphanumeric storeing ==> storing funtion ==> function documenation ==> documentation Determin ==> Determine intepreted ==> interpreted ammount ==> amount obious ==> obvious interupts ==> interrupts occured ==> occurred asssociated ==> associated taking into acount ==> taking into account squence ==> sequence stil ==> still contiguos ==> contiguous matchs ==> matches Link: https://lkml.kernel.org/r/20210607072555.12416-1-thunder.leizhen@huawei.com Signed-off-by: Zhen Lei Reviewed-by: Jacob Keller Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 6 +++--- lib/asn1_encoder.c | 2 +- lib/devres.c | 2 +- lib/dynamic_debug.c | 2 +- lib/fonts/font_pearl_8x8.c | 2 +- lib/kfifo.c | 2 +- lib/list_sort.c | 2 +- lib/nlattr.c | 4 ++-- lib/oid_registry.c | 2 +- lib/pldmfw/pldmfw.c | 2 +- lib/reed_solomon/test_rslib.c | 2 +- lib/refcount.c | 2 +- lib/rhashtable.c | 2 +- lib/sbitmap.c | 2 +- lib/scatterlist.c | 4 ++-- lib/seq_buf.c | 2 +- lib/sort.c | 2 +- lib/stackdepot.c | 2 +- lib/vsprintf.c | 2 +- 19 files changed, 23 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8acc01d7d816..4cd6af3f89b3 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1282,7 +1282,7 @@ config PROVE_RAW_LOCK_NESTING option expect lockdep splats until these problems have been fully addressed which is work in progress. This config switch allows to identify and analyze these problems. It will be removed and the - check permanentely enabled once the main issues have been fixed. + check permanently enabled once the main issues have been fixed. If unsure, select N. @@ -1448,7 +1448,7 @@ config DEBUG_LOCKING_API_SELFTESTS Say Y here if you want the kernel to run a short self-test during bootup. The self-test checks whether common types of locking bugs are detected by debugging mechanisms or not. (if you disable - lock debugging then those bugs wont be detected of course.) + lock debugging then those bugs won't be detected of course.) The following locking APIs are covered: spinlocks, rwlocks, mutexes and rwsems. @@ -1928,7 +1928,7 @@ config FAIL_IO_TIMEOUT thus exercising the error handling. Only works with drivers that use the generic timeout handling, - for others it wont do anything. + for others it won't do anything. config FAIL_FUTEX bool "Fault-injection capability for futexes" diff --git a/lib/asn1_encoder.c b/lib/asn1_encoder.c index 41e71aae3ef6..27bbe891714f 100644 --- a/lib/asn1_encoder.c +++ b/lib/asn1_encoder.c @@ -181,7 +181,7 @@ EXPORT_SYMBOL_GPL(asn1_encode_oid); /** * asn1_encode_length() - encode a length to follow an ASN.1 tag * @data: pointer to encode at - * @data_len: pointer to remaning length (adjusted by routine) + * @data_len: pointer to remaining length (adjusted by routine) * @len: length to encode * * This routine can encode lengths up to 65535 using the ASN.1 rules. diff --git a/lib/devres.c b/lib/devres.c index bdb06898a977..b0e1c6702c71 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -355,7 +355,7 @@ static void pcim_iomap_release(struct device *gendev, void *res) * detach. * * This function might sleep when the table is first allocated but can - * be safely called without context and guaranteed to succed once + * be safely called without context and guaranteed to succeed once * allocated. */ void __iomem * const *pcim_iomap_table(struct pci_dev *pdev) diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index d3ce78298832..cb5abb42c16a 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -991,7 +991,7 @@ static int ddebug_dyndbg_param_cb(char *param, char *val, ddebug_exec_queries((val ? val : "+p"), modname); - return 0; /* query failure shouldnt stop module load */ + return 0; /* query failure shouldn't stop module load */ } /* handle both dyndbg and $module.dyndbg params at boot */ diff --git a/lib/fonts/font_pearl_8x8.c b/lib/fonts/font_pearl_8x8.c index b1678ed0017c..ae98ca17982e 100644 --- a/lib/fonts/font_pearl_8x8.c +++ b/lib/fonts/font_pearl_8x8.c @@ -3,7 +3,7 @@ /* */ /* Font file generated by cpi2fnt */ /* ------------------------------ */ -/* Combined with the alpha-numeric */ +/* Combined with the alphanumeric */ /* portion of Greg Harp's old PEARL */ /* font (from earlier versions of */ /* linux-m86k) by John Shifflett */ diff --git a/lib/kfifo.c b/lib/kfifo.c index 70dab9ac7827..12f5a347aa13 100644 --- a/lib/kfifo.c +++ b/lib/kfifo.c @@ -415,7 +415,7 @@ static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize) ) /* - * __kfifo_poke_n internal helper function for storeing the length of + * __kfifo_poke_n internal helper function for storing the length of * the record into the fifo */ static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize) diff --git a/lib/list_sort.c b/lib/list_sort.c index 1e1e37762799..0fb59e92ca2d 100644 --- a/lib/list_sort.c +++ b/lib/list_sort.c @@ -104,7 +104,7 @@ static void merge_final(void *priv, list_cmp_func_t cmp, struct list_head *head, * @head: the list to sort * @cmp: the elements comparison function * - * The comparison funtion @cmp must return > 0 if @a should sort after + * The comparison function @cmp must return > 0 if @a should sort after * @b ("@a > @b" if you want an ascending sort), and <= 0 if @a should * sort before @b *or* their original order should be preserved. It is * always called with the element that came first in the input in @a, diff --git a/lib/nlattr.c b/lib/nlattr.c index 1d051ef66afe..86029ad5ead4 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -619,7 +619,7 @@ static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype, * Validates all attributes in the specified attribute stream against the * specified policy. Validation depends on the validate flags passed, see * &enum netlink_validation for more details on that. - * See documenation of struct nla_policy for more details. + * See documentation of struct nla_policy for more details. * * Returns 0 on success or a negative error code. */ @@ -633,7 +633,7 @@ int __nla_validate(const struct nlattr *head, int len, int maxtype, EXPORT_SYMBOL(__nla_validate); /** - * nla_policy_len - Determin the max. length of a policy + * nla_policy_len - Determine the max. length of a policy * @policy: policy to use * @n: number of policies * diff --git a/lib/oid_registry.c b/lib/oid_registry.c index 3dfaa836e7c5..e592d48b1974 100644 --- a/lib/oid_registry.c +++ b/lib/oid_registry.c @@ -124,7 +124,7 @@ EXPORT_SYMBOL_GPL(parse_OID); * @bufsize: The size of the buffer * * The OID is rendered into the buffer in "a.b.c.d" format and the number of - * bytes is returned. -EBADMSG is returned if the data could not be intepreted + * bytes is returned. -EBADMSG is returned if the data could not be interpreted * and -ENOBUFS if the buffer was too small. */ int sprint_oid(const void *data, size_t datasize, char *buffer, size_t bufsize) diff --git a/lib/pldmfw/pldmfw.c b/lib/pldmfw/pldmfw.c index e5d4b3b2af81..6e77eb6d8e72 100644 --- a/lib/pldmfw/pldmfw.c +++ b/lib/pldmfw/pldmfw.c @@ -82,7 +82,7 @@ pldm_check_fw_space(struct pldmfw_priv *data, size_t offset, size_t length) * @bytes_to_move: number of bytes to move the offset forward by * * Check that there is enough space past the current offset, and then move the - * offset forward by this ammount. + * offset forward by this amount. * * Returns: zero on success, or -EFAULT if the image is too small to fit the * expected length. diff --git a/lib/reed_solomon/test_rslib.c b/lib/reed_solomon/test_rslib.c index 4eb29f365ece..d9d1c33aebda 100644 --- a/lib/reed_solomon/test_rslib.c +++ b/lib/reed_solomon/test_rslib.c @@ -385,7 +385,7 @@ static void test_bc(struct rs_control *rs, int len, int errs, /* * We check that the returned word is actually a - * codeword. The obious way to do this would be to + * codeword. The obvious way to do this would be to * compute the syndrome, but we don't want to replicate * that code here. However, all the codes are in * systematic form, and therefore we can encode the diff --git a/lib/refcount.c b/lib/refcount.c index ebac8b7d15a7..a207a8f22b3c 100644 --- a/lib/refcount.c +++ b/lib/refcount.c @@ -164,7 +164,7 @@ EXPORT_SYMBOL(refcount_dec_and_lock); * @flags: saved IRQ-flags if the is acquired * * Same as refcount_dec_and_lock() above except that the spinlock is acquired - * with disabled interupts. + * with disabled interrupts. * * Return: true and hold spinlock if able to decrement refcount to 0, false * otherwise diff --git a/lib/rhashtable.c b/lib/rhashtable.c index c949c1e3b87c..e12bbfb240b8 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -703,7 +703,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit); * * Returns zero if successful. * - * Returns -EAGAIN if resize event occured. Note that the iterator + * Returns -EAGAIN if resize event occurred. Note that the iterator * will rewind back to the beginning and you may use it immediately * by calling rhashtable_walk_next. * diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 47b3691058eb..b25db9be938a 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -583,7 +583,7 @@ void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, /* * Once the clear bit is set, the bit may be allocated out. * - * Orders READ/WRITE on the asssociated instance(such as request + * Orders READ/WRITE on the associated instance(such as request * of blk_mq) by this bit for avoiding race with re-allocation, * and its pair is the memory barrier implied in __sbitmap_get_word. * diff --git a/lib/scatterlist.c b/lib/scatterlist.c index a59778946404..27efa6178153 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -38,7 +38,7 @@ EXPORT_SYMBOL(sg_next); * @sg: The scatterlist * * Description: - * Allows to know how many entries are in sg, taking into acount + * Allows to know how many entries are in sg, taking into account * chaining as well * **/ @@ -59,7 +59,7 @@ EXPORT_SYMBOL(sg_nents); * * Description: * Determines the number of entries in sg that are required to meet - * the supplied length, taking into acount chaining as well + * the supplied length, taking into account chaining as well * * Returns: * the number of sg entries needed, negative error on failure diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 6dafde851333..0a68f7aa85d6 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -289,7 +289,7 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) } /** - * seq_buf_to_user - copy the squence buffer to user space + * seq_buf_to_user - copy the sequence buffer to user space * @s: seq_buf descriptor * @ubuf: The userspace memory location to copy to * @cnt: The amount to copy diff --git a/lib/sort.c b/lib/sort.c index 3ad454411997..aa18153864d2 100644 --- a/lib/sort.c +++ b/lib/sort.c @@ -51,7 +51,7 @@ static bool is_aligned(const void *base, size_t size, unsigned char align) * which basically all CPUs have, to minimize loop overhead computations. * * For some reason, on x86 gcc 7.3.0 adds a redundant test of n at the - * bottom of the loop, even though the zero flag is stil valid from the + * bottom of the loop, even though the zero flag is still valid from the * subtract (since the intervening mov instructions don't alter the flags). * Gcc 8.1.0 doesn't have that problem. */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index df9179f4f441..0a2e417f83cb 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -11,7 +11,7 @@ * Instead, stack depot maintains a hashtable of unique stacktraces. Since alloc * and free stacks repeat a lot, we save about 100x space. * Stacks are never removed from depot, so we store them contiguously one after - * another in a contiguos memory allocation. + * another in a contiguous memory allocation. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 2926cc27623f..77ba1c40a99d 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -3417,7 +3417,7 @@ int vsscanf(const char *buf, const char *fmt, va_list args) while (*fmt) { /* skip any white space in format */ - /* white space in format matchs any amount of + /* white space in format matches any amount of * white space, including none, in the input. */ if (isspace(*fmt)) { -- cgit From a010d79b6683b6b1e66be2ea7204944f1323661c Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:06 -0700 Subject: buildid: only consider GNU notes for build ID parsing Patch series "Add build ID to stacktraces", v6. This series adds the kernel's build ID[1] to the stacktrace header printed in oops messages, warnings, etc. and the build ID for any module that appears in the stacktrace after the module name. The goal is to make the stacktrace more self-contained and descriptive by including the relevant build IDs in the kernel logs when something goes wrong. This can be used by post processing tools like script/decode_stacktrace.sh and kernel developers to easily locate the debug info associated with a kernel crash and line up what line and file things started falling apart at. To show how this can be used I've included a patch to decode_stacktrace.sh that downloads the debuginfo from a debuginfod server. This also includes some patches to make the buildid.c file use more const arguments and consolidate logic into buildid.c from kdump. These are left to the end as they were mostly cleanup patches. Here's an example lkdtm stacktrace on arm64. WARNING: CPU: 4 PID: 3255 at drivers/misc/lkdtm/bugs.c:83 lkdtm_WARNING+0x28/0x30 [lkdtm] Modules linked in: lkdtm rfcomm algif_hash algif_skcipher af_alg xt_cgroup uinput xt_MASQUERADE CPU: 4 PID: 3255 Comm: bash Not tainted 5.11 #3 aa23f7a1231c229de205662d5a9e0d4c580f19a1 Hardware name: Google Lazor (rev3+) with KB Backlight (DT) pstate: 00400009 (nzcv daif +PAN -UAO -TCO BTYPE=--) pc : lkdtm_WARNING+0x28/0x30 [lkdtm] lr : lkdtm_do_action+0x24/0x40 [lkdtm] sp : ffffffc0134fbca0 x29: ffffffc0134fbca0 x28: ffffff92d53ba240 x27: 0000000000000000 x26: 0000000000000000 x25: 0000000000000000 x24: ffffffe3622352c0 x23: 0000000000000020 x22: ffffffe362233366 x21: ffffffe3622352e0 x20: ffffffc0134fbde0 x19: 0000000000000008 x18: 0000000000000000 x17: ffffff929b6536fc x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000012 x13: ffffffe380ed892c x12: ffffffe381d05068 x11: 0000000000000000 x10: 0000000000000000 x9 : 0000000000000001 x8 : ffffffe362237000 x7 : aaaaaaaaaaaaaaaa x6 : 0000000000000000 x5 : 0000000000000000 x4 : 0000000000000001 x3 : 0000000000000008 x2 : ffffff93fef25a70 x1 : ffffff93fef15788 x0 : ffffffe3622352e0 Call trace: lkdtm_WARNING+0x28/0x30 [lkdtm ed5019fdf5e53be37cb1ba7899292d7e143b259e] direct_entry+0x16c/0x1b4 [lkdtm ed5019fdf5e53be37cb1ba7899292d7e143b259e] full_proxy_write+0x74/0xa4 vfs_write+0xec/0x2e8 ksys_write+0x84/0xf0 __arm64_sys_write+0x24/0x30 el0_svc_common+0xf4/0x1c0 do_el0_svc_compat+0x28/0x3c el0_svc_compat+0x10/0x1c el0_sync_compat_handler+0xa8/0xcc el0_sync_compat+0x178/0x180 ---[ end trace 3d95032303e59e68 ]--- This patch (of 13): Some kernel elf files have various notes that also happen to have an elf note type of '3', which matches NT_GNU_BUILD_ID but the note name isn't "GNU". For example, this note trips up the existing logic: Owner Data size Description Xen 0x00000008 Unknown note type: (0x00000003) description data: 00 00 00 ffffff80 ffffffff ffffffff ffffffff ffffffff Let's make sure that it is a GNU note when parsing the build ID so that we can use this function to parse a vmlinux's build ID too. Link: https://lkml.kernel.org/r/20210511003845.2429846-1-swboyd@chromium.org Link: https://lkml.kernel.org/r/20210511003845.2429846-2-swboyd@chromium.org Fixes: bd7525dacd7e ("bpf: Move stack_map_get_build_id into lib") Signed-off-by: Stephen Boyd Reported-by: Petr Mladek Tested-by: Petr Mladek Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Andy Shevchenko Cc: Baoquan He Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Young Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Matthew Wilcox Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Vivek Goyal Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/buildid.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/buildid.c b/lib/buildid.c index 6156997c3895..e014636ec3eb 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -31,6 +31,7 @@ static inline int parse_build_id(void *page_addr, if (nhdr->n_type == BUILD_ID && nhdr->n_namesz == sizeof("GNU") && + !strcmp((char *)(nhdr + 1), "GNU") && nhdr->n_descsz > 0 && nhdr->n_descsz <= BUILD_ID_SIZE_MAX) { memcpy(build_id, -- cgit From 7eaf3cf3b7c5a49b3ca60e1ceb3d1d7430cc9d0e Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:10 -0700 Subject: buildid: add API to parse build ID out of buffer Add an API that can parse the build ID out of a buffer, instead of a vma, to support printing a kernel module's build ID for stack traces. Link: https://lkml.kernel.org/r/20210511003845.2429846-3-swboyd@chromium.org Signed-off-by: Stephen Boyd Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Andy Shevchenko Cc: Baoquan He Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Young Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Matthew Wilcox Cc: Petr Mladek Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Vivek Goyal Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/buildid.c | 50 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/buildid.c b/lib/buildid.c index e014636ec3eb..6aea1c4e5e85 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -2,30 +2,23 @@ #include #include +#include #include #define BUILD_ID 3 + /* * Parse build id from the note segment. This logic can be shared between * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are * identical. */ -static inline int parse_build_id(void *page_addr, - unsigned char *build_id, - __u32 *size, - void *note_start, - Elf32_Word note_size) +static int parse_build_id_buf(unsigned char *build_id, + __u32 *size, + const void *note_start, + Elf32_Word note_size) { Elf32_Word note_offs = 0, new_offs; - /* check for overflow */ - if (note_start < page_addr || note_start + note_size < note_start) - return -EINVAL; - - /* only supports note that fits in the first page */ - if (note_start + note_size > page_addr + PAGE_SIZE) - return -EINVAL; - while (note_offs + sizeof(Elf32_Nhdr) < note_size) { Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); @@ -50,9 +43,27 @@ static inline int parse_build_id(void *page_addr, break; note_offs = new_offs; } + return -EINVAL; } +static inline int parse_build_id(void *page_addr, + unsigned char *build_id, + __u32 *size, + void *note_start, + Elf32_Word note_size) +{ + /* check for overflow */ + if (note_start < page_addr || note_start + note_size < note_start) + return -EINVAL; + + /* only supports note that fits in the first page */ + if (note_start + note_size > page_addr + PAGE_SIZE) + return -EINVAL; + + return parse_build_id_buf(build_id, size, note_start, note_size); +} + /* Parse build ID from 32-bit ELF */ static int get_build_id_32(void *page_addr, unsigned char *build_id, __u32 *size) @@ -148,3 +159,16 @@ out: put_page(page); return ret; } + +/** + * build_id_parse_buf - Get build ID from a buffer + * @buf: Elf note section(s) to parse + * @buf_size: Size of @buf in bytes + * @build_id: Build ID parsed from @buf, at least BUILD_ID_SIZE_MAX long + * + * Return: 0 on success, -EINVAL otherwise + */ +int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) +{ + return parse_build_id_buf(build_id, NULL, buf, buf_size); +} -- cgit From 83cc6fa0049d7c5333a53f4d959a9457340284ea Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:13 -0700 Subject: buildid: stash away kernels build ID on init Parse the kernel's build ID at initialization so that other code can print a hex format string representation of the running kernel's build ID. This will be used in the kdump and dump_stack code so that developers can easily locate the vmlinux debug symbols for a crash/stacktrace. [swboyd@chromium.org: fix implicit declaration of init_vmlinux_build_id()] Link: https://lkml.kernel.org/r/CAE-0n51UjTbay8N9FXAyE7_aR2+ePrQnKSRJ0gbmRsXtcLBVaw@mail.gmail.com Link: https://lkml.kernel.org/r/20210511003845.2429846-4-swboyd@chromium.org Signed-off-by: Stephen Boyd Acked-by: Baoquan He Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Dave Young Cc: Vivek Goyal Cc: Andy Shevchenko Cc: Borislav Petkov Cc: Catalin Marinas Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Matthew Wilcox Cc: Petr Mladek Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/buildid.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'lib') diff --git a/lib/buildid.c b/lib/buildid.c index 6aea1c4e5e85..1103ed46214f 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -172,3 +173,17 @@ int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) { return parse_build_id_buf(build_id, NULL, buf, buf_size); } + +unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; + +/** + * init_vmlinux_build_id - Compute and stash the running kernel's build ID + */ +void __init init_vmlinux_build_id(void) +{ + extern const void __start_notes __weak; + extern const void __stop_notes __weak; + unsigned int size = &__stop_notes - &__start_notes; + + build_id_parse_buf(&__start_notes, vmlinux_build_id, size); +} -- cgit From 22f4e66df79d0a730fcd6c17f3403b5ab8c72ced Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:17 -0700 Subject: dump_stack: add vmlinux build ID to stack traces Add the running kernel's build ID[1] to the stacktrace information header. This makes it simpler for developers to locate the vmlinux with full debuginfo for a particular kernel stacktrace. Combined with scripts/decode_stracktrace.sh, a developer can download the correct vmlinux from a debuginfod[2] server and find the exact file and line number for the functions plus offsets in a stacktrace. This is especially useful for pstore crash debugging where the kernel crashes are recorded in the pstore logs and the recovery kernel is different or the debuginfo doesn't exist on the device due to space concerns (the data can be large and a security concern). The stacktrace can be analyzed after the crash by using the build ID to find the matching vmlinux and understand where in the function something went wrong. Example stacktrace from lkdtm: WARNING: CPU: 4 PID: 3255 at drivers/misc/lkdtm/bugs.c:83 lkdtm_WARNING+0x28/0x30 [lkdtm] Modules linked in: lkdtm rfcomm algif_hash algif_skcipher af_alg xt_cgroup uinput xt_MASQUERADE CPU: 4 PID: 3255 Comm: bash Not tainted 5.11 #3 aa23f7a1231c229de205662d5a9e0d4c580f19a1 Hardware name: Google Lazor (rev3+) with KB Backlight (DT) pstate: 00400009 (nzcv daif +PAN -UAO -TCO BTYPE=--) pc : lkdtm_WARNING+0x28/0x30 [lkdtm] The hex string aa23f7a1231c229de205662d5a9e0d4c580f19a1 is the build ID, following the kernel version number. Put it all behind a config option, STACKTRACE_BUILD_ID, so that kernel developers can remove this information if they decide it is too much. Link: https://lkml.kernel.org/r/20210511003845.2429846-5-swboyd@chromium.org Link: https://fedoraproject.org/wiki/Releases/FeatureBuildId [1] Link: https://sourceware.org/elfutils/Debuginfod.html [2] Signed-off-by: Stephen Boyd Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Petr Mladek Cc: Steven Rostedt Cc: Andy Shevchenko Cc: Matthew Wilcox Cc: Baoquan He Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Young Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Thomas Gleixner Cc: Vivek Goyal Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 11 +++++++++++ lib/buildid.c | 2 ++ lib/dump_stack.c | 13 +++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 4cd6af3f89b3..2987925efe7d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -35,6 +35,17 @@ config PRINTK_CALLER no option to enable/disable at the kernel command line parameter or sysfs interface. +config STACKTRACE_BUILD_ID + bool "Show build ID information in stacktraces" + depends on PRINTK + help + Selecting this option adds build ID information for symbols in + stacktraces printed with the printk format '%p[SR]b'. + + This option is intended for distros where debuginfo is not easily + accessible but can be downloaded given the build ID of the vmlinux or + kernel module where the function is located. + config CONSOLE_LOGLEVEL_DEFAULT int "Default console loglevel (1-15)" range 1 15 diff --git a/lib/buildid.c b/lib/buildid.c index 1103ed46214f..6f1e2903740b 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -174,6 +174,7 @@ int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) return parse_build_id_buf(build_id, NULL, buf, buf_size); } +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; /** @@ -187,3 +188,4 @@ void __init init_vmlinux_build_id(void) build_id_parse_buf(&__start_notes, vmlinux_build_id, size); } +#endif diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 27f16872320d..cd3387bb34e5 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -36,6 +37,14 @@ void __init dump_stack_set_arch_desc(const char *fmt, ...) va_end(args); } +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) +#define BUILD_ID_FMT " %20phN" +#define BUILD_ID_VAL vmlinux_build_id +#else +#define BUILD_ID_FMT "%s" +#define BUILD_ID_VAL "" +#endif + /** * dump_stack_print_info - print generic debug info for dump_stack() * @log_lvl: log level @@ -45,13 +54,13 @@ void __init dump_stack_set_arch_desc(const char *fmt, ...) */ void dump_stack_print_info(const char *log_lvl) { - printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s\n", + printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s" BUILD_ID_FMT "\n", log_lvl, raw_smp_processor_id(), current->pid, current->comm, kexec_crash_loaded() ? "Kdump: loaded " : "", print_tainted(), init_utsname()->release, (int)strcspn(init_utsname()->version, " "), - init_utsname()->version); + init_utsname()->version, BUILD_ID_VAL); if (dump_stack_arch_desc_str[0] != '\0') printk("%sHardware name: %s\n", -- cgit From 9294523e3768030ae8afb84110bcecc66425a647 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:20 -0700 Subject: module: add printk formats to add module build ID to stacktraces Let's make kernel stacktraces easier to identify by including the build ID[1] of a module if the stacktrace is printing a symbol from a module. This makes it simpler for developers to locate a kernel module's full debuginfo for a particular stacktrace. Combined with scripts/decode_stracktrace.sh, a developer can download the matching debuginfo from a debuginfod[2] server and find the exact file and line number for the functions plus offsets in a stacktrace that match the module. This is especially useful for pstore crash debugging where the kernel crashes are recorded in something like console-ramoops and the recovery kernel/modules are different or the debuginfo doesn't exist on the device due to space concerns (the debuginfo can be too large for space limited devices). Originally, I put this on the %pS format, but that was quickly rejected given that %pS is used in other places such as ftrace where build IDs aren't meaningful. There was some discussions on the list to put every module build ID into the "Modules linked in:" section of the stacktrace message but that quickly becomes very hard to read once you have more than three or four modules linked in. It also provides too much information when we don't expect each module to be traversed in a stacktrace. Having the build ID for modules that aren't important just makes things messy. Splitting it to multiple lines for each module quickly explodes the number of lines printed in an oops too, possibly wrapping the warning off the console. And finally, trying to stash away each module used in a callstack to provide the ID of each symbol printed is cumbersome and would require changes to each architecture to stash away modules and return their build IDs once unwinding has completed. Instead, we opt for the simpler approach of introducing new printk formats '%pS[R]b' for "pointer symbolic backtrace with module build ID" and '%pBb' for "pointer backtrace with module build ID" and then updating the few places in the architecture layer where the stacktrace is printed to use this new format. Before: Call trace: lkdtm_WARNING+0x28/0x30 [lkdtm] direct_entry+0x16c/0x1b4 [lkdtm] full_proxy_write+0x74/0xa4 vfs_write+0xec/0x2e8 After: Call trace: lkdtm_WARNING+0x28/0x30 [lkdtm 6c2215028606bda50de823490723dc4bc5bf46f9] direct_entry+0x16c/0x1b4 [lkdtm 6c2215028606bda50de823490723dc4bc5bf46f9] full_proxy_write+0x74/0xa4 vfs_write+0xec/0x2e8 [akpm@linux-foundation.org: fix build with CONFIG_MODULES=n, tweak code layout] [rdunlap@infradead.org: fix build when CONFIG_MODULES is not set] Link: https://lkml.kernel.org/r/20210513171510.20328-1-rdunlap@infradead.org [akpm@linux-foundation.org: make kallsyms_lookup_buildid() static] [cuibixuan@huawei.com: fix build error when CONFIG_SYSFS is disabled] Link: https://lkml.kernel.org/r/20210525105049.34804-1-cuibixuan@huawei.com Link: https://lkml.kernel.org/r/20210511003845.2429846-6-swboyd@chromium.org Link: https://fedoraproject.org/wiki/Releases/FeatureBuildId [1] Link: https://sourceware.org/elfutils/Debuginfod.html [2] Signed-off-by: Stephen Boyd Signed-off-by: Bixuan Cui Signed-off-by: Randy Dunlap Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Petr Mladek Cc: Steven Rostedt Cc: Sergey Senozhatsky Cc: Andy Shevchenko Cc: Rasmus Villemoes Cc: Matthew Wilcox Cc: Baoquan He Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Young Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Sasha Levin Cc: Thomas Gleixner Cc: Vivek Goyal Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/vsprintf.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 77ba1c40a99d..26c83943748a 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -993,8 +993,12 @@ char *symbol_string(char *buf, char *end, void *ptr, value = (unsigned long)ptr; #ifdef CONFIG_KALLSYMS - if (*fmt == 'B') + if (*fmt == 'B' && fmt[1] == 'b') + sprint_backtrace_build_id(sym, value); + else if (*fmt == 'B') sprint_backtrace(sym, value); + else if (*fmt == 'S' && (fmt[1] == 'b' || (fmt[1] == 'R' && fmt[2] == 'b'))) + sprint_symbol_build_id(sym, value); else if (*fmt != 's') sprint_symbol(sym, value); else @@ -2263,9 +2267,11 @@ early_param("no_hash_pointers", no_hash_pointers_enable); * - 'S' For symbolic direct pointers (or function descriptors) with offset * - 's' For symbolic direct pointers (or function descriptors) without offset * - '[Ss]R' as above with __builtin_extract_return_addr() translation + * - 'S[R]b' as above with module build ID (for use in backtraces) * - '[Ff]' %pf and %pF were obsoleted and later removed in favor of * %ps and %pS. Be careful when re-using these specifiers. * - 'B' For backtraced symbolic direct pointers with offset + * - 'Bb' as above with module build ID (for use in backtraces) * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref] * - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201] * - 'b[l]' For a bitmap, the number of bits is determined by the field -- cgit From 60eec32637161ca4455dfab6080215abe6b86a2a Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:42 -0700 Subject: buildid: mark some arguments const These arguments are never modified so they can be marked const to indicate as such. Link: https://lkml.kernel.org/r/20210511003845.2429846-12-swboyd@chromium.org Signed-off-by: Stephen Boyd Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Andy Shevchenko Cc: Baoquan He Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Young Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Matthew Wilcox Cc: Petr Mladek Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Vivek Goyal Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/buildid.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/buildid.c b/lib/buildid.c index 6f1e2903740b..8fe24d6c0925 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -48,10 +48,10 @@ static int parse_build_id_buf(unsigned char *build_id, return -EINVAL; } -static inline int parse_build_id(void *page_addr, +static inline int parse_build_id(const void *page_addr, unsigned char *build_id, __u32 *size, - void *note_start, + const void *note_start, Elf32_Word note_size) { /* check for overflow */ @@ -66,7 +66,7 @@ static inline int parse_build_id(void *page_addr, } /* Parse build ID from 32-bit ELF */ -static int get_build_id_32(void *page_addr, unsigned char *build_id, +static int get_build_id_32(const void *page_addr, unsigned char *build_id, __u32 *size) { Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; @@ -91,7 +91,7 @@ static int get_build_id_32(void *page_addr, unsigned char *build_id, } /* Parse build ID from 64-bit ELF */ -static int get_build_id_64(void *page_addr, unsigned char *build_id, +static int get_build_id_64(const void *page_addr, unsigned char *build_id, __u32 *size) { Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; -- cgit From 3f14d029f98f0d4f369d64458084cf31e66f820f Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:46 -0700 Subject: buildid: fix kernel-doc notation Kernel doc should use "Return:" instead of "Returns" to properly reflect the return values. Link: https://lkml.kernel.org/r/20210511003845.2429846-13-swboyd@chromium.org Signed-off-by: Stephen Boyd Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Andy Shevchenko Cc: Baoquan He Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Young Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Matthew Wilcox Cc: Petr Mladek Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Vivek Goyal Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/buildid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/buildid.c b/lib/buildid.c index 8fe24d6c0925..180a1a9b3b76 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -121,7 +121,7 @@ static int get_build_id_64(const void *page_addr, unsigned char *build_id, * @build_id: buffer to store build id, at least BUILD_ID_SIZE long * @size: returns actual build id size in case of success * - * Returns 0 on success, otherwise error (< 0). + * Return: 0 on success, -EINVAL otherwise */ int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size) -- cgit From 44e8a5e9120bf4fc1ab046b648b0598e6652c36e Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Wed, 7 Jul 2021 18:09:49 -0700 Subject: kdump: use vmlinux_build_id to simplify We can use the vmlinux_build_id array here now instead of open coding it. This mostly consolidates code. Link: https://lkml.kernel.org/r/20210511003845.2429846-14-swboyd@chromium.org Signed-off-by: Stephen Boyd Cc: Jiri Olsa Cc: Alexei Starovoitov Cc: Jessica Yu Cc: Evan Green Cc: Hsin-Yi Wang Cc: Dave Young Cc: Baoquan He Cc: Vivek Goyal Cc: Andy Shevchenko Cc: Borislav Petkov Cc: Catalin Marinas Cc: Ingo Molnar Cc: Konstantin Khlebnikov Cc: Matthew Wilcox Cc: Petr Mladek Cc: Rasmus Villemoes Cc: Sasha Levin Cc: Sergey Senozhatsky Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/buildid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/buildid.c b/lib/buildid.c index 180a1a9b3b76..dfc62625cae4 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -174,7 +174,7 @@ int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) return parse_build_id_buf(build_id, NULL, buf, buf_size); } -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_CRASH_CORE) unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; /** -- cgit From 6bce244390a8bad89536ae0ea5c03c59ae155a12 Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Mon, 5 Jul 2021 10:51:17 +0100 Subject: mm/page_alloc: Revert pahole zero-sized workaround Commit dbbee9d5cd83 ("mm/page_alloc: convert per-cpu list protection to local_lock") folded in a workaround patch for pahole that was unable to deal with zero-sized percpu structures. A superior workaround is achieved with commit a0b8200d06ad ("kbuild: skip per-CPU BTF generation for pahole v1.18-v1.21"). This patch reverts the dummy field and the pahole version check. Fixes: dbbee9d5cd83 ("mm/page_alloc: convert per-cpu list protection to local_lock") Signed-off-by: Mel Gorman Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 83a931c972f5..831212722924 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -324,9 +324,6 @@ config DEBUG_INFO_BTF config PAHOLE_HAS_SPLIT_BTF def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119") -config PAHOLE_HAS_ZEROSIZE_PERCPU_SUPPORT - def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "122") - config DEBUG_INFO_BTF_MODULES def_bool y depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF -- cgit From 91027d0a7a0e309b94674923dc1b245b709b5c1e Mon Sep 17 00:00:00 2001 From: Chris Down Date: Tue, 15 Jun 2021 17:52:45 +0100 Subject: string_helpers: Escape double quotes in escape_special From an abstract point of view, escape_special's counterpart, unescape_special, already handles the unescaping of blackslashed double quote sequences. As a more practical example, printk indexing is an example case where this is already practically useful. Compare an example with `ESCAPE_SPECIAL | ESCAPE_SPACE`, with quotes not escaped: [root@ktst ~]# grep drivers/pci/pci-stub.c:69 /sys/kernel/debug/printk/index/vmlinux <4> drivers/pci/pci-stub.c:69 pci_stub_init "pci-stub: invalid ID string "%s"\n" ...and the same after this patch: [root@ktst ~]# grep drivers/pci/pci-stub.c:69 /sys/kernel/debug/printk/index/vmlinux <4> drivers/pci/pci-stub.c:69 pci_stub_init "pci-stub: invalid ID string \"%s\"\n" One can of course, alternatively, use ESCAPE_APPEND with a quote in @only, but without this patch quotes are coerced into hex or octal which can hurt readability quite significantly. I've checked uses of ESCAPE_SPECIAL and %pE across the codebase, and I'm pretty confident that this shouldn't affect any stable interfaces. Signed-off-by: Chris Down Reviewed-by: Andy Shevchenko Reviewed-by: Petr Mladek Cc: Rasmus Villemoes Acked-by: Andy Shevchenko Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/af144c5b75e41ce417386253ba2694456bc04118.1623775748.git.chris@chrisdown.name --- lib/string_helpers.c | 4 ++++ lib/test-string_helpers.c | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 5a35c7e16e96..3806a52ce697 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -361,6 +361,9 @@ static bool escape_special(unsigned char c, char **dst, char *end) case '\e': to = 'e'; break; + case '"': + to = '"'; + break; default: return false; } @@ -474,6 +477,7 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * '\t' - horizontal tab * '\v' - vertical tab * %ESCAPE_SPECIAL: + * '\"' - double quote * '\\' - backslash * '\a' - alert (BEL) * '\e' - escape diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 2185d71704f0..437d8e6b7cb1 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -140,13 +140,13 @@ static const struct test_string_2 escape0[] __initconst = {{ },{ .in = "\\h\\\"\a\e\\", .s1 = {{ - .out = "\\\\h\\\\\"\\a\\e\\\\", + .out = "\\\\h\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL, },{ - .out = "\\\\\\150\\\\\\042\\a\\e\\\\", + .out = "\\\\\\150\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ - .out = "\\\\\\x68\\\\\\x22\\a\\e\\\\", + .out = "\\\\\\x68\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL | ESCAPE_HEX, },{ /* terminator */ @@ -157,10 +157,10 @@ static const struct test_string_2 escape0[] __initconst = {{ .out = "\eb \\C\007\"\x90\\r]", .flags = ESCAPE_SPACE, },{ - .out = "\\eb \\\\C\\a\"\x90\r]", + .out = "\\eb \\\\C\\a\\\"\x90\r]", .flags = ESCAPE_SPECIAL, },{ - .out = "\\eb \\\\C\\a\"\x90\\r]", + .out = "\\eb \\\\C\\a\\\"\x90\\r]", .flags = ESCAPE_SPACE | ESCAPE_SPECIAL, },{ .out = "\\033\\142\\040\\134\\103\\007\\042\\220\\015\\135", @@ -169,10 +169,10 @@ static const struct test_string_2 escape0[] __initconst = {{ .out = "\\033\\142\\040\\134\\103\\007\\042\\220\\r\\135", .flags = ESCAPE_SPACE | ESCAPE_OCTAL, },{ - .out = "\\e\\142\\040\\\\\\103\\a\\042\\220\\015\\135", + .out = "\\e\\142\\040\\\\\\103\\a\\\"\\220\\015\\135", .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ - .out = "\\e\\142\\040\\\\\\103\\a\\042\\220\\r\\135", + .out = "\\e\\142\\040\\\\\\103\\a\\\"\\220\\r\\135", .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ .out = "\eb \\C\007\"\x90\r]", -- cgit